Arbitrary callback_data (#1844)

.github/pull_request_template.md

@@ -28,3 +28,4 @@ Hey! You're PRing? Cool! Please have a look at the below checklist. It's here to
     - [ ] Added new filters for new message (sub)types
     - [ ] Added or updated documentation for the changed class(es) and/or method(s)
     - [ ] Updated the Bot API version number in all places: `README.rst` and `README_RAW.rst` (including the badge), as well as `telegram.constants.BOT_API_VERSION`
+    - [ ] Added logic for arbitrary callback data in `tg.ext.Bot` for new methods that either accept a `reply_markup` in some form or have a return type that is/contains `telegram.Message`

.pre-commit-config.yaml

@@ -10,11 +10,11 @@ repos:
         - --diff
         - --check
 -   repo: https://gitlab.com/pycqa/flake8
-    rev: 3.9.1
+    rev: 3.9.2
     hooks:
     -   id: flake8
 -   repo: https://github.com/PyCQA/pylint
-    rev: v2.8.2
+    rev: v2.8.3
     hooks:
     -   id: pylint
         files: ^(telegram|examples)/.*\.py$
@@ -24,6 +24,7 @@ repos:
           - certifi
           - tornado>=6.1
           - APScheduler==3.6.3
+          - cachetools==4.2.2
           - . # this basically does `pip install -e .`
 -   repo: https://github.com/pre-commit/mirrors-mypy
     rev: v0.812
@@ -35,6 +36,7 @@ repos:
           - certifi
           - tornado>=6.1
           - APScheduler==3.6.3
+          - cachetools==4.2.2
           - . # this basically does `pip install -e .`
     -   id: mypy
         name: mypy-examples
@@ -46,9 +48,10 @@ repos:
           - certifi
           - tornado>=6.1
           - APScheduler==3.6.3
+          - cachetools==4.2.2
           - . # this basically does `pip install -e .`
 -   repo: https://github.com/asottile/pyupgrade
-    rev: v2.13.0
+    rev: v2.19.1
     hooks:
     -   id: pyupgrade
         files: ^(telegram|examples|tests)/.*\.py$

docs/source/telegram.ext.callbackdatacache.rst

@@ -0,0 +1,8 @@
+:github_url: https://github.com/python-telegram-bot/python-telegram-bot/blob/master/telegram/ext/callbackdatacache.py
+
+telegram.ext.CallbackDataCache
+==============================
+
+.. autoclass:: telegram.ext.CallbackDataCache
+    :members:
+    :show-inheritance:

docs/source/telegram.ext.extbot.rst

@@ -0,0 +1,7 @@
+:github_url: https://github.com/python-telegram-bot/python-telegram-bot/blob/master/telegram/ext/extbot.py
+
+telegram.ext.ExtBot
+===================
+
+.. autoclass:: telegram.ext.ExtBot
+    :show-inheritance:

docs/source/telegram.ext.invalidcallbackdata.rst

@@ -0,0 +1,8 @@
+:github_url: https://github.com/python-telegram-bot/python-telegram-bot/blob/master/telegram/ext/callbackdatacache.py
+
+telegram.ext.InvalidCallbackData
+================================
+
+.. autoclass:: telegram.ext.InvalidCallbackData
+    :members:
+    :show-inheritance:

docs/source/telegram.ext.rst

@@ -3,6 +3,7 @@ telegram.ext package
 
 .. toctree::
 
+    telegram.ext.extbot
     telegram.ext.updater
     telegram.ext.dispatcher
     telegram.ext.dispatcherhandlerstop
@@ -47,6 +48,14 @@ Persistence
     telegram.ext.picklepersistence
     telegram.ext.dictpersistence
 
+Arbitrary Callback Data
+-----------------------
+
+.. toctree::
+
+    telegram.ext.callbackdatacache
+    telegram.ext.invalidcallbackdata
+
 utils
 -----
 

examples/README.md

@@ -52,5 +52,8 @@ A basic example on how `(my_)chat_member` updates can be used.
 ### [`contexttypesbot.py`](https://github.com/python-telegram-bot/python-telegram-bot/blob/master/examples/contexttypesbot.py)
 This example showcases how `telegram.ext.ContextTypes` can be used to customize the `context` argument of handler and job callbacks.
 
+### [`arbitrarycallbackdatabot.py`](https://github.com/python-telegram-bot/python-telegram-bot/blob/master/examples/arbitrarycallbackdatabot.py)
+This example showcases how PTBs "arbitrary callback data" feature can be used.
+
 ## Pure API
 The [`rawapibot.py`](https://github.com/python-telegram-bot/python-telegram-bot/blob/master/examples/rawapibot.py) example uses only the pure, "bare-metal" API wrapper.

examples/arbitrarycallbackdatabot.py

@@ -0,0 +1,110 @@
+#!/usr/bin/env python
+# pylint: disable=C0116,W0613
+# This program is dedicated to the public domain under the CC0 license.
+
+"""This example showcases how PTBs "arbitrary callback data" feature can be used.
+
+For detailed info on arbitrary callback data, see the wiki page at https://git.io/JGBDI
+"""
+import logging
+from typing import List, Tuple, cast
+
+from telegram import InlineKeyboardButton, InlineKeyboardMarkup, Update
+from telegram.ext import (
+    Updater,
+    CommandHandler,
+    CallbackQueryHandler,
+    CallbackContext,
+    InvalidCallbackData,
+    PicklePersistence,
+)
+
+logging.basicConfig(
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.INFO
+)
+logger = logging.getLogger(__name__)
+
+
+def start(update: Update, context: CallbackContext) -> None:
+    """Sends a message with 5 inline buttons attached."""
+    number_list: List[int] = []
+    update.message.reply_text('Please choose:', reply_markup=build_keyboard(number_list))
+
+
+def help_command(update: Update, context: CallbackContext) -> None:
+    """Displays info on how to use the bot."""
+    update.message.reply_text(
+        "Use /start to test this bot. Use /clear to clear the stored data so that you can see "
+        "what happens, if the button data is not available. "
+    )
+
+
+def clear(update: Update, context: CallbackContext) -> None:
+    """Clears the callback data cache"""
+    context.bot.callback_data_cache.clear_callback_data()  # type: ignore[attr-defined]
+    context.bot.callback_data_cache.clear_callback_queries()  # type: ignore[attr-defined]
+    update.effective_message.reply_text('All clear!')
+
+
+def build_keyboard(current_list: List[int]) -> InlineKeyboardMarkup:
+    """Helper function to build the next inline keyboard."""
+    return InlineKeyboardMarkup.from_column(
+        [InlineKeyboardButton(str(i), callback_data=(i, current_list)) for i in range(1, 6)]
+    )
+
+
+def list_button(update: Update, context: CallbackContext) -> None:
+    """Parses the CallbackQuery and updates the message text."""
+    query = update.callback_query
+    query.answer()
+    # Get the data from the callback_data.
+    # If you're using a type checker like MyPy, you'll have to use typing.cast
+    # to make the checker get the expected type of the callback_data
+    number, number_list = cast(Tuple[int, List[int]], query.data)
+    # append the number to the list
+    number_list.append(number)
+
+    query.edit_message_text(
+        text=f"So far you've selected {number_list}. Choose the next item:",
+        reply_markup=build_keyboard(number_list),
+    )
+
+    # we can delete the data stored for the query, because we've replaced the buttons
+    context.drop_callback_data(query)
+
+
+def handle_invalid_button(update: Update, context: CallbackContext) -> None:
+    """Informs the user that the button is no longer available."""
+    update.callback_query.answer()
+    update.effective_message.edit_text(
+        'Sorry, I could not process this button click 😕 Please send /start to get a new keyboard.'
+    )
+
+
+def main() -> None:
+    """Run the bot."""
+    # We use persistence to demonstrate how buttons can still work after the bot was restarted
+    persistence = PicklePersistence(
+        filename='arbitrarycallbackdatabot.pickle', store_callback_data=True
+    )
+    # Create the Updater and pass it your bot's token.
+    updater = Updater("TOKEN", persistence=persistence, arbitrary_callback_data=True)
+
+    updater.dispatcher.add_handler(CommandHandler('start', start))
+    updater.dispatcher.add_handler(CommandHandler('help', help_command))
+    updater.dispatcher.add_handler(CommandHandler('clear', clear))
+    updater.dispatcher.add_handler(
+        CallbackQueryHandler(handle_invalid_button, pattern=InvalidCallbackData)
+    )
+    updater.dispatcher.add_handler(CallbackQueryHandler(list_button))
+
+    # Start the Bot
+    updater.start_polling()
+
+    # Run the bot until the user presses Ctrl-C or the process receives SIGINT,
+    # SIGTERM or SIGABRT
+    updater.idle()
+
+
+if __name__ == '__main__':
+    main()

requirements-dev.txt

@@ -4,12 +4,12 @@ cryptography!=3.4,!=3.4.1,!=3.4.2,!=3.4.3
 pre-commit
 # Make sure that the versions specified here match the pre-commit settings!
 black==20.8b1
-flake8==3.9.1
-pylint==2.8.2
+flake8==3.9.2
+pylint==2.8.3
 mypy==0.812
-pyupgrade==2.13.0
+pyupgrade==2.19.1
 
-pytest==6.2.3
+pytest==6.2.4
 
 flaky
 beautifulsoup4

requirements.txt

@@ -5,3 +5,4 @@ certifi
 tornado>=6.1
 APScheduler==3.6.3
 pytz>=2018.6
+cachetools==4.2.2

telegram/bot.py

@@ -21,6 +21,7 @@
 
 import functools
 import logging
+import warnings
 from datetime import datetime
 
 from typing import (
@@ -89,6 +90,7 @@ from telegram import (
 )
 from telegram.constants import MAX_INLINE_QUERY_RESULTS
 from telegram.error import InvalidToken, TelegramError
+from telegram.utils.deprecate import TelegramDeprecationWarning
 from telegram.utils.helpers import (
     DEFAULT_NONE,
     DefaultValue,
@@ -156,6 +158,11 @@ class Bot(TelegramObject):
         defaults (:class:`telegram.ext.Defaults`, optional): An object containing default values to
             be used if not set explicitly in the bot methods.
 
+            .. deprecated:: 13.6
+               Passing :class:`telegram.ext.Defaults` to :class:`telegram.Bot` is deprecated. If
+               you want to use :class:`telegram.ext.Defaults`, please use
+               :class:`telegram.ext.ExtBot` instead.
+
     """
 
     __slots__ = (
@@ -185,6 +192,13 @@ class Bot(TelegramObject):
         # Gather default
         self.defaults = defaults
 
+        if self.defaults:
+            warnings.warn(
+                'Passing Defaults to telegram.Bot is deprecated. Use telegram.ext.ExtBot instead.',
+                TelegramDeprecationWarning,
+                stacklevel=3,
+            )
+
         if base_url is None:
             base_url = 'https://api.telegram.org/bot'
 
@@ -209,8 +223,10 @@ class Bot(TelegramObject):
                 private_key, password=private_key_password, backend=default_backend()
             )
 
-    def __setattr__(self, key: str, value: object) -> None:
-        if issubclass(self.__class__, Bot) and self.__class__ is not Bot:
+    # The ext_bot argument is a little hack to get warnings handled correctly.
+    # It's not very clean, but the warnings will be dropped at some point anyway.
+    def __setattr__(self, key: str, value: object, ext_bot: bool = False) -> None:
+        if issubclass(self.__class__, Bot) and self.__class__ is not Bot and not ext_bot:
             object.__setattr__(self, key, value)
             return
         super().__setattr__(key, value)
@@ -1994,6 +2010,62 @@ class Bot(TelegramObject):
 
         return result  # type: ignore[return-value]
 
+    def _effective_inline_results(  # pylint: disable=R0201
+        self,
+        results: Union[
+            Sequence['InlineQueryResult'], Callable[[int], Optional[Sequence['InlineQueryResult']]]
+        ],
+        next_offset: str = None,
+        current_offset: str = None,
+    ) -> Tuple[Sequence['InlineQueryResult'], Optional[str]]:
+        """
+        Builds the effective results from the results input.
+        We make this a stand-alone method so tg.ext.ExtBot can wrap it.
+
+        Returns:
+            Tuple of 1. the effective results and 2. correct the next_offset
+
+        """
+        if current_offset is not None and next_offset is not None:
+            raise ValueError('`current_offset` and `next_offset` are mutually exclusive!')
+
+        if current_offset is not None:
+            # Convert the string input to integer
+            if current_offset == '':
+                current_offset_int = 0
+            else:
+                current_offset_int = int(current_offset)
+
+            # for now set to empty string, stating that there are no more results
+            # might change later
+            next_offset = ''
+
+            if callable(results):
+                callable_output = results(current_offset_int)
+                if not callable_output:
+                    effective_results: Sequence['InlineQueryResult'] = []
+                else:
+                    effective_results = callable_output
+                    # the callback *might* return more results on the next call, so we increment
+                    # the page count
+                    next_offset = str(current_offset_int + 1)
+            else:
+                if len(results) > (current_offset_int + 1) * MAX_INLINE_QUERY_RESULTS:
+                    # we expect more results for the next page
+                    next_offset_int = current_offset_int + 1
+                    next_offset = str(next_offset_int)
+                    effective_results = results[
+                        current_offset_int
+                        * MAX_INLINE_QUERY_RESULTS : next_offset_int
+                        * MAX_INLINE_QUERY_RESULTS
+                    ]
+                else:
+                    effective_results = results[current_offset_int * MAX_INLINE_QUERY_RESULTS :]
+        else:
+            effective_results = results  # type: ignore[assignment]
+
+        return effective_results, next_offset
+
     @log
     def answer_inline_query(
         self,
@@ -2098,38 +2170,11 @@ class Bot(TelegramObject):
                     else:
                         res.input_message_content.disable_web_page_preview = None
 
-        if current_offset is not None and next_offset is not None:
-            raise ValueError('`current_offset` and `next_offset` are mutually exclusive!')
-
-        if current_offset is not None:
-            if current_offset == '':
-                current_offset_int = 0
-            else:
-                current_offset_int = int(current_offset)
-
-            next_offset = ''
-
-            if callable(results):
-                callable_output = results(current_offset_int)
-                if not callable_output:
-                    effective_results: Sequence['InlineQueryResult'] = []
-                else:
-                    effective_results = callable_output
-                    next_offset = str(current_offset_int + 1)
-            else:
-                if len(results) > (current_offset_int + 1) * MAX_INLINE_QUERY_RESULTS:
-                    next_offset_int = current_offset_int + 1
-                    next_offset = str(next_offset_int)
-                    effective_results = results[
-                        current_offset_int
-                        * MAX_INLINE_QUERY_RESULTS : next_offset_int
-                        * MAX_INLINE_QUERY_RESULTS
-                    ]
-                else:
-                    effective_results = results[current_offset_int * MAX_INLINE_QUERY_RESULTS :]
-        else:
-            effective_results = results  # type: ignore[assignment]
+        effective_results, next_offset = self._effective_inline_results(
+            results=results, next_offset=next_offset, current_offset=current_offset
+        )
 
+        # Apply defaults
         for result in effective_results:
             _set_defaults(result)
 
@@ -2765,18 +2810,22 @@ class Bot(TelegramObject):
         # * Long polling poses a different problem: the connection might have been dropped while
         #   waiting for the server to return and there's no way of knowing the connection had been
         #   dropped in real time.
-        result = self._post(
-            'getUpdates', data, timeout=float(read_latency) + float(timeout), api_kwargs=api_kwargs
+        result = cast(
+            List[JSONDict],
+            self._post(
+                'getUpdates',
+                data,
+                timeout=float(read_latency) + float(timeout),
+                api_kwargs=api_kwargs,
+            ),
         )
 
         if result:
-            self.logger.debug(
-                'Getting updates: %s', [u['update_id'] for u in result]  # type: ignore
-            )
+            self.logger.debug('Getting updates: %s', [u['update_id'] for u in result])
         else:
             self.logger.debug('No new updates found.')
 
-        return [Update.de_json(u, self) for u in result]  # type: ignore
+        return Update.de_list(result, self)  # type: ignore[return-value]
 
     @log
     def set_webhook(

telegram/callbackquery.py

@@ -53,6 +53,13 @@ class CallbackQuery(TelegramObject):
           until you call :attr:`answer`. It is, therefore, necessary to react
           by calling :attr:`telegram.Bot.answer_callback_query` even if no notification to the user
           is needed (e.g., without specifying any of the optional parameters).
+        * If you're using :attr:`Bot.arbitrary_callback_data`, :attr:`data` may be an instance
+          of :class:`telegram.ext.InvalidCallbackData`. This will be the case, if the data
+          associated with the button triggering the :class:`telegram.CallbackQuery` was already
+          deleted or if :attr:`data` was manipulated by a malicious client.
+
+          .. versionadded:: 13.6
+
 
     Args:
         id (:obj:`str`): Unique identifier for this query.
@@ -77,7 +84,7 @@ class CallbackQuery(TelegramObject):
             the message with the callback button was sent.
         message (:class:`telegram.Message`): Optional. Message with the callback button that
             originated the query.
-        data (:obj:`str`): Optional. Data associated with the callback button.
+        data (:obj:`str` | :obj:`object`): Optional. Data associated with the callback button.
         inline_message_id (:obj:`str`): Optional. Identifier of the message sent via the bot in
                 inline mode, that originated the query.
         game_short_name (:obj:`str`): Optional. Short name of a Game to be returned.

telegram/ext/__init__.py

@@ -19,6 +19,7 @@
 # pylint: disable=C0413
 """Extensions over the Telegram Bot API to facilitate bot making"""
 
+from .extbot import ExtBot
 from .basepersistence import BasePersistence
 from .picklepersistence import PicklePersistence
 from .dictpersistence import DictPersistence
@@ -59,11 +60,13 @@ from .pollanswerhandler import PollAnswerHandler
 from .pollhandler import PollHandler
 from .chatmemberhandler import ChatMemberHandler
 from .defaults import Defaults
+from .callbackdatacache import CallbackDataCache, InvalidCallbackData
 
 __all__ = (
     'BaseFilter',
     'BasePersistence',
     'CallbackContext',
+    'CallbackDataCache',
     'CallbackQueryHandler',
     'ChatMemberHandler',
     'ChosenInlineResultHandler',
@@ -75,9 +78,11 @@ __all__ = (
     'DictPersistence',
     'Dispatcher',
     'DispatcherHandlerStop',
+    'ExtBot',
     'Filters',
     'Handler',
     'InlineQueryHandler',
+    'InvalidCallbackData',
     'Job',
     'JobQueue',
     'MessageFilter',

telegram/ext/basepersistence.py

@@ -26,9 +26,9 @@ from typing import Dict, Optional, Tuple, cast, ClassVar, Generic, DefaultDict
 from telegram.utils.deprecate import set_new_attribute_deprecated
 
 from telegram import Bot
+import telegram.ext.extbot
 
-from telegram.utils.types import ConversationDict
-from telegram.ext.utils.types import UD, CD, BD
+from telegram.ext.utils.types import UD, CD, BD, ConversationDict, CDCData
 
 
 class BasePersistence(Generic[UD, CD, BD], ABC):
@@ -46,6 +46,8 @@ class BasePersistence(Generic[UD, CD, BD], ABC):
     * :meth:`get_user_data`
     * :meth:`update_user_data`
     * :meth:`refresh_user_data`
+    * :meth:`get_callback_data`
+    * :meth:`update_callback_data`
     * :meth:`get_conversations`
     * :meth:`update_conversation`
     * :meth:`flush`
@@ -72,7 +74,11 @@ class BasePersistence(Generic[UD, CD, BD], ABC):
         store_chat_data (:obj:`bool`, optional): Whether chat_data should be saved by this
             persistence class. Default is :obj:`True` .
         store_bot_data (:obj:`bool`, optional): Whether bot_data should be saved by this
-            persistence class. Default is :obj:`True` .
+            persistence class. Default is :obj:`True`.
+        store_callback_data (:obj:`bool`, optional): Whether callback_data should be saved by this
+            persistence class. Default is :obj:`False`.
+
+            .. versionadded:: 13.6
 
     Attributes:
         store_user_data (:obj:`bool`): Optional, Whether user_data should be saved by this
@@ -81,16 +87,27 @@ class BasePersistence(Generic[UD, CD, BD], ABC):
             persistence class.
         store_bot_data (:obj:`bool`): Optional. Whether bot_data should be saved by this
             persistence class.
+        store_callback_data (:obj:`bool`): Optional. Whether callback_data should be saved by this
+            persistence class.
+
+            .. versionadded:: 13.6
     """
 
     # Apparently Py 3.7 and below have '__dict__' in ABC
     if py_ver < (3, 7):
-        __slots__ = ('store_user_data', 'store_chat_data', 'store_bot_data', 'bot')
+        __slots__ = (
+            'store_user_data',
+            'store_chat_data',
+            'store_bot_data',
+            'store_callback_data',
+            'bot',
+        )
     else:
         __slots__ = (
             'store_user_data',  # type: ignore[assignment]
             'store_chat_data',
             'store_bot_data',
+            'store_callback_data',
             'bot',
             '__dict__',
         )
@@ -101,14 +118,19 @@ class BasePersistence(Generic[UD, CD, BD], ABC):
         """This overrides the get_* and update_* methods to use insert/replace_bot.
         That has the side effect that we always pass deepcopied data to those methods, so in
         Pickle/DictPersistence we don't have to worry about copying the data again.
+
+        Note: This doesn't hold for second tuple-entry of callback_data. That's a Dict[str, str],
+        so no bots to replace anyway.
         """
         instance = super().__new__(cls)
         get_user_data = instance.get_user_data
         get_chat_data = instance.get_chat_data
         get_bot_data = instance.get_bot_data
+        get_callback_data = instance.get_callback_data
         update_user_data = instance.update_user_data
         update_chat_data = instance.update_chat_data
         update_bot_data = instance.update_bot_data
+        update_callback_data = instance.update_callback_data
 
         def get_user_data_insert_bot() -> DefaultDict[int, UD]:
             return instance.insert_bot(get_user_data())
@@ -119,6 +141,12 @@ class BasePersistence(Generic[UD, CD, BD], ABC):
         def get_bot_data_insert_bot() -> BD:
             return instance.insert_bot(get_bot_data())
 
+        def get_callback_data_insert_bot() -> Optional[CDCData]:
+            cdc_data = get_callback_data()
+            if cdc_data is None:
+                return None
+            return instance.insert_bot(cdc_data[0]), cdc_data[1]
+
         def update_user_data_replace_bot(user_id: int, data: UD) -> None:
             return update_user_data(user_id, instance.replace_bot(data))
 
@@ -128,13 +156,19 @@ class BasePersistence(Generic[UD, CD, BD], ABC):
         def update_bot_data_replace_bot(data: BD) -> None:
             return update_bot_data(instance.replace_bot(data))
 
+        def update_callback_data_replace_bot(data: CDCData) -> None:
+            obj_data, queue = data
+            return update_callback_data((instance.replace_bot(obj_data), queue))
+
         # We want to ignore TGDeprecation warnings so we use obj.__setattr__. Adds to __dict__
         object.__setattr__(instance, 'get_user_data', get_user_data_insert_bot)
         object.__setattr__(instance, 'get_chat_data', get_chat_data_insert_bot)
         object.__setattr__(instance, 'get_bot_data', get_bot_data_insert_bot)
+        object.__setattr__(instance, 'get_callback_data', get_callback_data_insert_bot)
         object.__setattr__(instance, 'update_user_data', update_user_data_replace_bot)
         object.__setattr__(instance, 'update_chat_data', update_chat_data_replace_bot)
         object.__setattr__(instance, 'update_bot_data', update_bot_data_replace_bot)
+        object.__setattr__(instance, 'update_callback_data', update_callback_data_replace_bot)
         return instance
 
     def __init__(
@@ -142,10 +176,12 @@ class BasePersistence(Generic[UD, CD, BD], ABC):
         store_user_data: bool = True,
         store_chat_data: bool = True,
         store_bot_data: bool = True,
+        store_callback_data: bool = False,
     ):
         self.store_user_data = store_user_data
         self.store_chat_data = store_chat_data
         self.store_bot_data = store_bot_data
+        self.store_callback_data = store_callback_data
         self.bot: Bot = None  # type: ignore[assignment]
 
     def __setattr__(self, key: str, value: object) -> None:
@@ -164,6 +200,9 @@ class BasePersistence(Generic[UD, CD, BD], ABC):
         Args:
             bot (:class:`telegram.Bot`): The bot.
         """
+        if self.store_callback_data and not isinstance(bot, telegram.ext.extbot.ExtBot):
+            raise TypeError('store_callback_data can only be used with telegram.ext.ExtBot.')
+
         self.bot = bot
 
     @classmethod
@@ -372,6 +411,18 @@ class BasePersistence(Generic[UD, CD, BD], ABC):
             :class:`telegram.ext.utils.types.BD`: The restored bot data.
         """
 
+    def get_callback_data(self) -> Optional[CDCData]:
+        """Will be called by :class:`telegram.ext.Dispatcher` upon creation with a
+        persistence object. If callback data was stored, it should be returned.
+
+        .. versionadded:: 13.6
+
+        Returns:
+            Optional[:class:`telegram.ext.utils.types.CDCData`]:  The restored meta data or
+                :obj:`None`, if no data was stored.
+        """
+        raise NotImplementedError
+
     @abstractmethod
     def get_conversations(self, name: str) -> ConversationDict:
         """Will be called by :class:`telegram.ext.Dispatcher` when a
@@ -466,6 +517,18 @@ class BasePersistence(Generic[UD, CD, BD], ABC):
             bot_data (:class:`telegram.ext.utils.types.BD`): The ``bot_data``.
         """
 
+    def update_callback_data(self, data: CDCData) -> None:
+        """Will be called by the :class:`telegram.ext.Dispatcher` after a handler has
+        handled an update.
+
+        .. versionadded:: 13.6
+
+        Args:
+            data (:class:`telegram.ext.utils.types.CDCData`:): The relevant data to restore
+                :attr:`telegram.ext.dispatcher.bot.callback_data_cache`.
+        """
+        raise NotImplementedError
+
     def flush(self) -> None:
         """Will be called by :class:`telegram.ext.Updater` upon receiving a stop signal. Gives the
         persistence a chance to finish up saving or close a database connection gracefully.

telegram/ext/callbackcontext.py

@@ -33,7 +33,8 @@ from typing import (
     TypeVar,
 )
 
-from telegram import Update
+from telegram import Update, CallbackQuery
+from telegram.ext import ExtBot
 from telegram.ext.utils.types import UD, CD, BD
 
 if TYPE_CHECKING:
@@ -194,6 +195,34 @@ class CallbackContext(Generic[UD, CD, BD]):
             if self.dispatcher.persistence.store_user_data and self._user_id_and_data is not None:
                 self.dispatcher.persistence.refresh_user_data(*self._user_id_and_data)
 
+    def drop_callback_data(self, callback_query: CallbackQuery) -> None:
+        """
+        Deletes the cached data for the specified callback query.
+
+        .. versionadded:: 13.6
+
+        Note:
+            Will *not* raise exceptions in case the data is not found in the cache.
+            *Will* raise :class:`KeyError` in case the callback query can not be found in the
+            cache.
+
+        Args:
+            callback_query (:class:`telegram.CallbackQuery`): The callback query.
+
+        Raises:
+            KeyError | RuntimeError: :class:`KeyError`, if the callback query can not be found in
+                the cache and :class:`RuntimeError`, if the bot doesn't allow for arbitrary
+                callback data.
+        """
+        if isinstance(self.bot, ExtBot):
+            if not self.bot.arbitrary_callback_data:
+                raise RuntimeError(
+                    'This telegram.ext.ExtBot instance does not use arbitrary callback data.'
+                )
+            self.bot.callback_data_cache.drop_data(callback_query)
+        else:
+            raise RuntimeError('telegram.Bot does not allow for arbitrary callback data.')
+
     @classmethod
     def from_error(
         cls: Type[CC],

telegram/ext/callbackdatacache.py

@@ -0,0 +1,427 @@
+#!/usr/bin/env python
+
+#
+#  A library that provides a Python interface to the Telegram Bot API
+#  Copyright (C) 2015-2021
+#  Leandro Toledo de Souza <devs@python-telegram-bot.org>
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU Lesser Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU Lesser Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser Public License
+#  along with this program.  If not, see [http://www.gnu.org/licenses/].
+
+#
+# A library that provides a Python interface to the Telegram Bot API
+# Copyright (C) 2015-2021
+# Leandro Toledo de Souza <devs@python-telegram-bot.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser Public License for more details.
+#
+# You should have received a copy of the GNU Lesser Public License
+# along with this program.  If not, see [http://www.gnu.org/licenses/].
+"""This module contains the CallbackDataCache class."""
+import logging
+import time
+from datetime import datetime
+from threading import Lock
+from typing import Dict, Tuple, Union, Optional, MutableMapping, TYPE_CHECKING, cast
+from uuid import uuid4
+
+from cachetools import LRUCache  # pylint: disable=E0401
+
+from telegram import (
+    InlineKeyboardMarkup,
+    InlineKeyboardButton,
+    TelegramError,
+    CallbackQuery,
+    Message,
+    User,
+)
+from telegram.utils.helpers import to_float_timestamp
+from telegram.ext.utils.types import CDCData
+
+if TYPE_CHECKING:
+    from telegram.ext import ExtBot
+
+
+class InvalidCallbackData(TelegramError):
+    """
+    Raised when the received callback data has been tempered with or deleted from cache.
+
+    .. versionadded:: 13.6
+
+    Args:
+        callback_data (:obj:`int`, optional): The button data of which the callback data could not
+            be found.
+
+    Attributes:
+        callback_data (:obj:`int`): Optional. The button data of which the callback data could not
+            be found.
+    """
+
+    __slots__ = ('callback_data',)
+
+    def __init__(self, callback_data: str = None) -> None:
+        super().__init__(
+            'The object belonging to this callback_data was deleted or the callback_data was '
+            'manipulated.'
+        )
+        self.callback_data = callback_data
+
+    def __reduce__(self) -> Tuple[type, Tuple[Optional[str]]]:  # type: ignore[override]
+        return self.__class__, (self.callback_data,)
+
+
+class _KeyboardData:
+    __slots__ = ('keyboard_uuid', 'button_data', 'access_time')
+
+    def __init__(
+        self, keyboard_uuid: str, access_time: float = None, button_data: Dict[str, object] = None
+    ):
+        self.keyboard_uuid = keyboard_uuid
+        self.button_data = button_data or {}
+        self.access_time = access_time or time.time()
+
+    def update_access_time(self) -> None:
+        """Updates the access time with the current time."""
+        self.access_time = time.time()
+
+    def to_tuple(self) -> Tuple[str, float, Dict[str, object]]:
+        """Gives a tuple representation consisting of the keyboard uuid, the access time and the
+        button data.
+        """
+        return self.keyboard_uuid, self.access_time, self.button_data
+
+
+class CallbackDataCache:
+    """A custom cache for storing the callback data of a :class:`telegram.ext.ExtBot`. Internally,
+    it keeps two mappings with fixed maximum size:
+
+        * One for mapping the data received in callback queries to the cached objects
+        * One for mapping the IDs of received callback queries to the cached objects
+
+    The second mapping allows to manually drop data that has been cached for keyboards of messages
+    sent via inline mode.
+    If necessary, will drop the least recently used items.
+
+    .. versionadded:: 13.6
+
+    Args:
+        bot (:class:`telegram.ext.ExtBot`): The bot this cache is for.
+        maxsize (:obj:`int`, optional): Maximum number of items in each of the internal mappings.
+            Defaults to 1024.
+        persistent_data (:obj:`telegram.ext.utils.types.CDCData`, optional): Data to initialize
+            the cache with, as returned by :meth:`telegram.ext.BasePersistence.get_callback_data`.
+
+    Attributes:
+        bot (:class:`telegram.ext.ExtBot`): The bot this cache is for.
+        maxsize (:obj:`int`): maximum size of the cache.
+
+    """
+
+    __slots__ = ('bot', 'maxsize', '_keyboard_data', '_callback_queries', '__lock', 'logger')
+
+    def __init__(
+        self,
+        bot: 'ExtBot',
+        maxsize: int = 1024,
+        persistent_data: CDCData = None,
+    ):
+        self.logger = logging.getLogger(__name__)
+
+        self.bot = bot
+        self.maxsize = maxsize
+        self._keyboard_data: MutableMapping[str, _KeyboardData] = LRUCache(maxsize=maxsize)
+        self._callback_queries: MutableMapping[str, str] = LRUCache(maxsize=maxsize)
+        self.__lock = Lock()
+
+        if persistent_data:
+            keyboard_data, callback_queries = persistent_data
+            for key, value in callback_queries.items():
+                self._callback_queries[key] = value
+            for uuid, access_time, data in keyboard_data:
+                self._keyboard_data[uuid] = _KeyboardData(
+                    keyboard_uuid=uuid, access_time=access_time, button_data=data
+                )
+
+    @property
+    def persistence_data(self) -> CDCData:
+        """:obj:`telegram.ext.utils.types.CDCData`: The data that needs to be persisted to allow
+        caching callback data across bot reboots.
+        """
+        # While building a list/dict from the LRUCaches has linear runtime (in the number of
+        # entries), the runtime is bounded by maxsize and it has the big upside of not throwing a
+        # highly customized data structure at users trying to implement a custom persistence class
+        with self.__lock:
+            return [data.to_tuple() for data in self._keyboard_data.values()], dict(
+                self._callback_queries.items()
+            )
+
+    def process_keyboard(self, reply_markup: InlineKeyboardMarkup) -> InlineKeyboardMarkup:
+        """Registers the reply markup to the cache. If any of the buttons have
+        :attr:`callback_data`, stores that data and builds a new keyboard with the correspondingly
+        replaced buttons. Otherwise does nothing and returns the original reply markup.
+
+        Args:
+            reply_markup (:class:`telegram.InlineKeyboardMarkup`): The keyboard.
+
+        Returns:
+            :class:`telegram.InlineKeyboardMarkup`: The keyboard to be passed to Telegram.
+
+        """
+        with self.__lock:
+            return self.__process_keyboard(reply_markup)
+
+    def __process_keyboard(self, reply_markup: InlineKeyboardMarkup) -> InlineKeyboardMarkup:
+        keyboard_uuid = uuid4().hex
+        keyboard_data = _KeyboardData(keyboard_uuid)
+
+        # Built a new nested list of buttons by replacing the callback data if needed
+        buttons = [
+            [
+                # We create a new button instead of replacing callback_data in case the
+                # same object is used elsewhere
+                InlineKeyboardButton(
+                    btn.text,
+                    callback_data=self.__put_button(btn.callback_data, keyboard_data),
+                )
+                if btn.callback_data
+                else btn
+                for btn in column
+            ]
+            for column in reply_markup.inline_keyboard
+        ]
+
+        if not keyboard_data.button_data:
+            # If we arrive here, no data had to be replaced and we can return the input
+            return reply_markup
+
+        self._keyboard_data[keyboard_uuid] = keyboard_data
+        return InlineKeyboardMarkup(buttons)
+
+    @staticmethod
+    def __put_button(callback_data: object, keyboard_data: _KeyboardData) -> str:
+        """Stores the data for a single button in :attr:`keyboard_data`.
+        Returns the string that should be passed instead of the callback_data, which is
+        ``keyboard_uuid + button_uuids``.
+        """
+        uuid = uuid4().hex
+        keyboard_data.button_data[uuid] = callback_data
+        return f'{keyboard_data.keyboard_uuid}{uuid}'
+
+    def __get_keyboard_uuid_and_button_data(
+        self, callback_data: str
+    ) -> Union[Tuple[str, object], Tuple[None, InvalidCallbackData]]:
+        keyboard, button = self.extract_uuids(callback_data)
+        try:
+            # we get the values before calling update() in case KeyErrors are raised
+            # we don't want to update in that case
+            keyboard_data = self._keyboard_data[keyboard]
+            button_data = keyboard_data.button_data[button]
+            # Update the timestamp for the LRU
+            keyboard_data.update_access_time()
+            return keyboard, button_data
+        except KeyError:
+            return None, InvalidCallbackData(callback_data)
+
+    @staticmethod
+    def extract_uuids(callback_data: str) -> Tuple[str, str]:
+        """Extracts the keyboard uuid and the button uuid from the given ``callback_data``.
+
+        Args:
+            callback_data (:obj:`str`): The ``callback_data`` as present in the button.
+
+        Returns:
+            (:obj:`str`, :obj:`str`): Tuple of keyboard and button uuid
+
+        """
+        # Extract the uuids as put in __put_button
+        return callback_data[:32], callback_data[32:]
+
+    def process_message(self, message: Message) -> None:
+        """Replaces the data in the inline keyboard attached to the message with the cached
+        objects, if necessary. If the data could not be found,
+        :class:`telegram.ext.InvalidCallbackData` will be inserted.
+
+        Note:
+            Checks :attr:`telegram.Message.via_bot` and :attr:`telegram.Message.from_user` to check
+            if the reply markup (if any) was actually sent by this caches bot. If it was not, the
+            message will be returned unchanged.
+
+            Note that this will fail for channel posts, as :attr:`telegram.Message.from_user` is
+            :obj:`None` for those! In the corresponding reply markups the callback data will be
+            replaced by :class:`telegram.ext.InvalidCallbackData`.
+
+        Warning:
+            * Does *not* consider :attr:`telegram.Message.reply_to_message` and
+              :attr:`telegram.Message.pinned_message`. Pass them to these method separately.
+            * *In place*, i.e. the passed :class:`telegram.Message` will be changed!
+
+        Args:
+            message (:class:`telegram.Message`): The message.
+
+        """
+        with self.__lock:
+            self.__process_message(message)
+
+    def __process_message(self, message: Message) -> Optional[str]:
+        """As documented in process_message, but returns the uuid of the attached keyboard, if any,
+        which is relevant for process_callback_query.
+
+        **IN PLACE**
+        """
+        if not message.reply_markup:
+            return None
+
+        if message.via_bot:
+            sender: Optional[User] = message.via_bot
+        elif message.from_user:
+            sender = message.from_user
+        else:
+            sender = None
+
+        if sender is not None and sender != self.bot.bot:
+            return None
+
+        keyboard_uuid = None
+
+        for row in message.reply_markup.inline_keyboard:
+            for button in row:
+                if button.callback_data:
+                    button_data = cast(str, button.callback_data)
+                    keyboard_id, callback_data = self.__get_keyboard_uuid_and_button_data(
+                        button_data
+                    )
+                    # update_callback_data makes sure that the _id_attrs are updated
+                    button.update_callback_data(callback_data)
+
+                    # This is lazy loaded. The firsts time we find a button
+                    # we load the associated keyboard - afterwards, there is
+                    if not keyboard_uuid and not isinstance(callback_data, InvalidCallbackData):
+                        keyboard_uuid = keyboard_id
+
+        return keyboard_uuid
+
+    def process_callback_query(self, callback_query: CallbackQuery) -> None:
+        """Replaces the data in the callback query and the attached messages keyboard with the
+        cached objects, if necessary. If the data could not be found,
+        :class:`telegram.ext.InvalidCallbackData` will be inserted.
+        If :attr:`callback_query.data` or :attr:`callback_query.message` is present, this also
+        saves the callback queries ID in order to be able to resolve it to the stored data.
+
+        Note:
+            Also considers inserts data into the buttons of
+            :attr:`telegram.Message.reply_to_message` and :attr:`telegram.Message.pinned_message`
+            if necessary.
+
+        Warning:
+            *In place*, i.e. the passed :class:`telegram.CallbackQuery` will be changed!
+
+        Args:
+            callback_query (:class:`telegram.CallbackQuery`): The callback query.
+
+        """
+        with self.__lock:
+            mapped = False
+
+            if callback_query.data:
+                data = callback_query.data
+
+                # Get the cached callback data for the CallbackQuery
+                keyboard_uuid, button_data = self.__get_keyboard_uuid_and_button_data(data)
+                callback_query.data = button_data  # type: ignore[assignment]
+
+                # Map the callback queries ID to the keyboards UUID for later use
+                if not mapped and not isinstance(button_data, InvalidCallbackData):
+                    self._callback_queries[callback_query.id] = keyboard_uuid  # type: ignore
+                    mapped = True
+
+            # Get the cached callback data for the inline keyboard attached to the
+            # CallbackQuery.
+            if callback_query.message:
+                self.__process_message(callback_query.message)
+                for message in (
+                    callback_query.message.pinned_message,
+                    callback_query.message.reply_to_message,
+                ):
+                    if message:
+                        self.__process_message(message)
+
+    def drop_data(self, callback_query: CallbackQuery) -> None:
+        """Deletes the data for the specified callback query.
+
+        Note:
+            Will *not* raise exceptions in case the callback data is not found in the cache.
+            *Will* raise :class:`KeyError` in case the callback query can not be found in the
+            cache.
+
+        Args:
+            callback_query (:class:`telegram.CallbackQuery`): The callback query.
+
+        Raises:
+            KeyError: If the callback query can not be found in the cache
+        """
+        with self.__lock:
+            try:
+                keyboard_uuid = self._callback_queries.pop(callback_query.id)
+                self.__drop_keyboard(keyboard_uuid)
+            except KeyError as exc:
+                raise KeyError('CallbackQuery was not found in cache.') from exc
+
+    def __drop_keyboard(self, keyboard_uuid: str) -> None:
+        try:
+            self._keyboard_data.pop(keyboard_uuid)
+        except KeyError:
+            return
+
+    def clear_callback_data(self, time_cutoff: Union[float, datetime] = None) -> None:
+        """Clears the stored callback data.
+
+        Args:
+            time_cutoff (:obj:`float` | :obj:`datetime.datetime`, optional): Pass a UNIX timestamp
+                or a :obj:`datetime.datetime` to clear only entries which are older.
+                For timezone naive :obj:`datetime.datetime` objects, the default timezone of the
+                bot will be used.
+
+        """
+        with self.__lock:
+            self.__clear(self._keyboard_data, time_cutoff=time_cutoff)
+
+    def clear_callback_queries(self) -> None:
+        """Clears the stored callback query IDs."""
+        with self.__lock:
+            self.__clear(self._callback_queries)
+
+    def __clear(self, mapping: MutableMapping, time_cutoff: Union[float, datetime] = None) -> None:
+        if not time_cutoff:
+            mapping.clear()
+            return
+
+        if isinstance(time_cutoff, datetime):
+            effective_cutoff = to_float_timestamp(
+                time_cutoff, tzinfo=self.bot.defaults.tzinfo if self.bot.defaults else None
+            )
+        else:
+            effective_cutoff = time_cutoff
+
+        # We need a list instead of a generator here, as the list doesn't change it's size
+        # during the iteration
+        to_drop = [key for key, data in mapping.items() if data.access_time < effective_cutoff]
+        for key in to_drop:
+            mapping.pop(key)

telegram/ext/callbackqueryhandler.py

@@ -49,13 +49,21 @@ class CallbackQueryHandler(Handler[Update, CCT]):
     Read the documentation of the ``re`` module for more information.
 
     Note:
-        :attr:`pass_user_data` and :attr:`pass_chat_data` determine whether a ``dict`` you
-        can use to keep any data in will be sent to the :attr:`callback` function. Related to
-        either the user or the chat that the update was sent in. For each update from the same user
-        or in the same chat, it will be the same ``dict``.
+        * :attr:`pass_user_data` and :attr:`pass_chat_data` determine whether a ``dict`` you
+          can use to keep any data in will be sent to the :attr:`callback` function. Related to
+          either the user or the chat that the update was sent in. For each update from the same
+          user or in the same chat, it will be the same ``dict``.
 
-        Note that this is DEPRECATED, and you should use context based callbacks. See
-        https://git.io/fxJuV for more info.
+          Note that this is DEPRECATED, and you should use context based callbacks. See
+          https://git.io/fxJuV for more info.
+        * If your bot allows arbitrary objects as ``callback_data``, it may happen that the
+          original ``callback_data`` for the incoming :class:`telegram.CallbackQuery`` can not be
+          found. This is the case when either a malicious client tempered with the
+          ``callback_data`` or the data was simply dropped from cache or not persisted. In these
+          cases, an instance of :class:`telegram.ext.InvalidCallbackData` will be set as
+          ``callback_data``.
+
+          .. versionadded:: 13.6
 
     Warning:
         When setting ``run_async`` to :obj:`True`, you cannot rely on adding custom
@@ -80,10 +88,24 @@ class CallbackQueryHandler(Handler[Update, CCT]):
             :class:`telegram.ext.JobQueue` instance created by the :class:`telegram.ext.Updater`
             which can be used to schedule new jobs. Default is :obj:`False`.
             DEPRECATED: Please switch to context based callbacks.
-        pattern (:obj:`str` | `Pattern`, optional): Regex pattern. If not :obj:`None`, ``re.match``
-            is used on :attr:`telegram.CallbackQuery.data` to determine if an update should be
-            handled by this handler. If :attr:`telegram.CallbackQuery.data` is not present, the
+        pattern (:obj:`str` | `Pattern` | :obj:`callable` | :obj:`type`, optional):
+            Pattern to test :attr:`telegram.CallbackQuery.data` against. If a string or a regex
+            pattern is passed, :meth:`re.match` is used on :attr:`telegram.CallbackQuery.data` to
+            determine if an update should be handled by this handler. If your bot allows arbitrary
+            objects as ``callback_data``, non-strings will be accepted. To filter arbitrary
+            objects you may pass
+
+                * a callable, accepting exactly one argument, namely the
+                  :attr:`telegram.CallbackQuery.data`. It must return :obj:`True` or
+                  :obj:`False`/:obj:`None` to indicate, whether the update should be handled.
+                * a :obj:`type`. If :attr:`telegram.CallbackQuery.data` is an instance of that type
+                  (or a subclass), the update will be handled.
+
+            If :attr:`telegram.CallbackQuery.data` is :obj:`None`, the
             :class:`telegram.CallbackQuery` update will not be handled.
+
+            .. versionchanged:: 13.6
+               Added support for arbitrary callback data.
         pass_groups (:obj:`bool`, optional): If the callback should be passed the result of
             ``re.match(pattern, data).groups()`` as a keyword argument called ``groups``.
             Default is :obj:`False`
@@ -107,8 +129,11 @@ class CallbackQueryHandler(Handler[Update, CCT]):
             passed to the callback function.
         pass_job_queue (:obj:`bool`): Determines whether ``job_queue`` will be passed to
             the callback function.
-        pattern (:obj:`str` | `Pattern`): Optional. Regex pattern to test
-            :attr:`telegram.CallbackQuery.data` against.
+        pattern (`Pattern` | :obj:`callable` | :obj:`type`): Optional. Regex pattern, callback or
+            type to test :attr:`telegram.CallbackQuery.data` against.
+
+            .. versionchanged:: 13.6
+               Added support for arbitrary callback data.
         pass_groups (:obj:`bool`): Determines whether ``groups`` will be passed to the
             callback function.
         pass_groupdict (:obj:`bool`): Determines whether ``groupdict``. will be passed to
@@ -128,7 +153,7 @@ class CallbackQueryHandler(Handler[Update, CCT]):
         callback: Callable[[Update, CCT], RT],
         pass_update_queue: bool = False,
         pass_job_queue: bool = False,
-        pattern: Union[str, Pattern] = None,
+        pattern: Union[str, Pattern, type, Callable[[object], Optional[bool]]] = None,
         pass_groups: bool = False,
         pass_groupdict: bool = False,
         pass_user_data: bool = False,
@@ -162,11 +187,17 @@ class CallbackQueryHandler(Handler[Update, CCT]):
 
         """
         if isinstance(update, Update) and update.callback_query:
+            callback_data = update.callback_query.data
             if self.pattern:
-                if update.callback_query.data:
-                    match = re.match(self.pattern, update.callback_query.data)
-                    if match:
-                        return match
+                if callback_data is None:
+                    return False
+                if isinstance(self.pattern, type):
+                    return isinstance(callback_data, self.pattern)
+                if callable(self.pattern):
+                    return self.pattern(callback_data)
+                match = re.match(self.pattern, callback_data)
+                if match:
+                    return match
             else:
                 return True
         return None
@@ -182,7 +213,7 @@ class CallbackQueryHandler(Handler[Update, CCT]):
         needed.
         """
         optional_args = super().collect_optional_args(dispatcher, update, check_result)
-        if self.pattern:
+        if self.pattern and not callable(self.pattern):
             check_result = cast(Match, check_result)
             if self.pass_groups:
                 optional_args['groups'] = check_result.groups()

telegram/ext/conversationhandler.py

@@ -37,7 +37,7 @@ from telegram.ext import (
     InlineQueryHandler,
 )
 from telegram.ext.utils.promise import Promise
-from telegram.utils.types import ConversationDict
+from telegram.ext.utils.types import ConversationDict
 from telegram.ext.utils.types import CCT
 
 if TYPE_CHECKING:

telegram/ext/dictpersistence.py

@@ -18,7 +18,7 @@
 # along with this program.  If not, see [http://www.gnu.org/licenses/].
 """This module contains the DictPersistence class."""
 
-from typing import DefaultDict, Dict, Optional, Tuple
+from typing import DefaultDict, Dict, Optional, Tuple, cast
 from collections import defaultdict
 
 from telegram.utils.helpers import (
@@ -27,7 +27,7 @@ from telegram.utils.helpers import (
     encode_conversations_to_json,
 )
 from telegram.ext import BasePersistence
-from telegram.utils.types import ConversationDict
+from telegram.ext.utils.types import ConversationDict, CDCData
 
 try:
     import ujson as json
@@ -59,13 +59,21 @@ class DictPersistence(BasePersistence):
         store_chat_data (:obj:`bool`, optional): Whether user_data should be saved by this
             persistence class. Default is :obj:`True`.
         store_bot_data (:obj:`bool`, optional): Whether bot_data should be saved by this
-            persistence class. Default is :obj:`True` .
+            persistence class. Default is :obj:`True`.
+        store_callback_data (:obj:`bool`, optional): Whether callback_data should be saved by this
+            persistence class. Default is :obj:`False`.
+
+            .. versionadded:: 13.6
         user_data_json (:obj:`str`, optional): Json string that will be used to reconstruct
             user_data on creating this persistence. Default is ``""``.
         chat_data_json (:obj:`str`, optional): Json string that will be used to reconstruct
             chat_data on creating this persistence. Default is ``""``.
         bot_data_json (:obj:`str`, optional): Json string that will be used to reconstruct
             bot_data on creating this persistence. Default is ``""``.
+        callback_data_json (:obj:`str`, optional): Json string that will be used to reconstruct
+            callback_data on creating this persistence. Default is ``""``.
+
+            .. versionadded:: 13.6
         conversations_json (:obj:`str`, optional): Json string that will be used to reconstruct
             conversation on creating this persistence. Default is ``""``.
 
@@ -76,16 +84,22 @@ class DictPersistence(BasePersistence):
             persistence class.
         store_bot_data (:obj:`bool`): Whether bot_data should be saved by this
             persistence class.
+        store_callback_data (:obj:`bool`): Whether callback_data be saved by this
+            persistence class.
+
+            .. versionadded:: 13.6
     """
 
     __slots__ = (
         '_user_data',
         '_chat_data',
         '_bot_data',
+        '_callback_data',
         '_conversations',
         '_user_data_json',
         '_chat_data_json',
         '_bot_data_json',
+        '_callback_data_json',
         '_conversations_json',
     )
 
@@ -98,19 +112,24 @@ class DictPersistence(BasePersistence):
         chat_data_json: str = '',
         bot_data_json: str = '',
         conversations_json: str = '',
+        store_callback_data: bool = False,
+        callback_data_json: str = '',
     ):
         super().__init__(
             store_user_data=store_user_data,
             store_chat_data=store_chat_data,
             store_bot_data=store_bot_data,
+            store_callback_data=store_callback_data,
         )
         self._user_data = None
         self._chat_data = None
         self._bot_data = None
+        self._callback_data = None
         self._conversations = None
         self._user_data_json = None
         self._chat_data_json = None
         self._bot_data_json = None
+        self._callback_data_json = None
         self._conversations_json = None
         if user_data_json:
             try:
@@ -132,6 +151,34 @@ class DictPersistence(BasePersistence):
                 raise TypeError("Unable to deserialize bot_data_json. Not valid JSON") from exc
             if not isinstance(self._bot_data, dict):
                 raise TypeError("bot_data_json must be serialized dict")
+        if callback_data_json:
+            try:
+                data = json.loads(callback_data_json)
+            except (ValueError, AttributeError) as exc:
+                raise TypeError(
+                    "Unable to deserialize callback_data_json. Not valid JSON"
+                ) from exc
+            # We are a bit more thorough with the checking of the format here, because it's
+            # more complicated than for the other things
+            try:
+                if data is None:
+                    self._callback_data = None
+                else:
+                    self._callback_data = cast(
+                        CDCData,
+                        ([(one, float(two), three) for one, two, three in data[0]], data[1]),
+                    )
+                self._callback_data_json = callback_data_json
+            except (ValueError, IndexError) as exc:
+                raise TypeError("callback_data_json is not in the required format") from exc
+            if self._callback_data is not None and (
+                not all(
+                    isinstance(entry[2], dict) and isinstance(entry[0], str)
+                    for entry in self._callback_data[0]
+                )
+                or not isinstance(self._callback_data[1], dict)
+            ):
+                raise TypeError("callback_data_json is not in the required format")
 
         if conversations_json:
             try:
@@ -179,7 +226,25 @@ class DictPersistence(BasePersistence):
         return json.dumps(self.bot_data)
 
     @property
-    def conversations(self) -> Optional[Dict[str, Dict[Tuple, object]]]:
+    def callback_data(self) -> Optional[CDCData]:
+        """:class:`telegram.ext.utils.types.CDCData`: The meta data on the stored callback data.
+
+        .. versionadded:: 13.6
+        """
+        return self._callback_data
+
+    @property
+    def callback_data_json(self) -> str:
+        """:obj:`str`: The meta data on the stored callback data as a JSON-string.
+
+        .. versionadded:: 13.6
+        """
+        if self._callback_data_json:
+            return self._callback_data_json
+        return json.dumps(self.callback_data)
+
+    @property
+    def conversations(self) -> Optional[Dict[str, ConversationDict]]:
         """:obj:`dict`: The conversations as a dict."""
         return self._conversations
 
@@ -197,9 +262,7 @@ class DictPersistence(BasePersistence):
         Returns:
             :obj:`defaultdict`: The restored user data.
         """
-        if self.user_data:
-            pass
-        else:
+        if self.user_data is None:
             self._user_data = defaultdict(dict)
         return self.user_data  # type: ignore[return-value]
 
@@ -210,9 +273,7 @@ class DictPersistence(BasePersistence):
         Returns:
             :obj:`defaultdict`: The restored chat data.
         """
-        if self.chat_data:
-            pass
-        else:
+        if self.chat_data is None:
             self._chat_data = defaultdict(dict)
         return self.chat_data  # type: ignore[return-value]
 
@@ -222,12 +283,24 @@ class DictPersistence(BasePersistence):
         Returns:
             :obj:`dict`: The restored bot data.
         """
-        if self.bot_data:
-            pass
-        else:
+        if self.bot_data is None:
             self._bot_data = {}
         return self.bot_data  # type: ignore[return-value]
 
+    def get_callback_data(self) -> Optional[CDCData]:
+        """Returns the callback_data created from the ``callback_data_json`` or :obj:`None`.
+
+        .. versionadded:: 13.6
+
+        Returns:
+            Optional[:class:`telegram.ext.utils.types.CDCData`]: The restored meta data or
+                :obj:`None`, if no data was stored.
+        """
+        if self.callback_data is None:
+            self._callback_data = None
+            return None
+        return self.callback_data[0], self.callback_data[1].copy()
+
     def get_conversations(self, name: str) -> ConversationDict:
         """Returns the conversations created from the ``conversations_json`` or an empty
         :obj:`dict`.
@@ -235,9 +308,7 @@ class DictPersistence(BasePersistence):
         Returns:
             :obj:`dict`: The restored conversations data.
         """
-        if self.conversations:
-            pass
-        else:
+        if self.conversations is None:
             self._conversations = {}
         return self.conversations.get(name, {}).copy()  # type: ignore[union-attr]
 
@@ -297,6 +368,20 @@ class DictPersistence(BasePersistence):
         self._bot_data = data
         self._bot_data_json = None
 
+    def update_callback_data(self, data: CDCData) -> None:
+        """Will update the callback_data (if changed).
+
+        .. versionadded:: 13.6
+
+        Args:
+            data (:class:`telegram.ext.utils.types.CDCData`:): The relevant data to restore
+                :attr:`telegram.ext.dispatcher.bot.callback_data_cache`.
+        """
+        if self._callback_data == data:
+            return
+        self._callback_data = (data[0], data[1].copy())
+        self._callback_data_json = None
+
     def refresh_user_data(self, user_id: int, user_data: Dict) -> None:
         """Does nothing.
 

telegram/ext/dispatcher.py

@@ -29,6 +29,7 @@ from time import sleep
 from typing import (
     TYPE_CHECKING,
     Callable,
+    DefaultDict,
     Dict,
     List,
     Optional,
@@ -38,7 +39,6 @@ from typing import (
     TypeVar,
     overload,
     cast,
-    DefaultDict,
 )
 from uuid import uuid4
 
@@ -46,6 +46,8 @@ from telegram import TelegramError, Update
 from telegram.ext import BasePersistence, ContextTypes
 from telegram.ext.callbackcontext import CallbackContext
 from telegram.ext.handler import Handler
+import telegram.ext.extbot
+from telegram.ext.callbackdatacache import CallbackDataCache
 from telegram.utils.deprecate import TelegramDeprecationWarning, set_new_attribute_deprecated
 from telegram.ext.utils.promise import Promise
 from telegram.utils.helpers import DefaultValue, DEFAULT_FALSE
@@ -273,7 +275,17 @@ class Dispatcher(Generic[CCT, UD, CD, BD]):
                     raise ValueError(
                         f"bot_data must be of type {self.context_types.bot_data.__name__}"
                     )
-
+            if self.persistence.store_callback_data:
+                self.bot = cast(telegram.ext.extbot.ExtBot, self.bot)
+                persistent_data = self.persistence.get_callback_data()
+                if persistent_data is not None:
+                    if not isinstance(persistent_data, tuple) and len(persistent_data) != 2:
+                        raise ValueError('callback_data must be a 2-tuple')
+                    self.bot.callback_data_cache = CallbackDataCache(
+                        self.bot,
+                        self.bot.callback_data_cache.maxsize,
+                        persistent_data=persistent_data,
+                    )
         else:
             self.persistence = None
 
@@ -667,6 +679,22 @@ class Dispatcher(Generic[CCT, UD, CD, BD]):
                 else:
                     user_ids = []
 
+            if self.persistence.store_callback_data:
+                self.bot = cast(telegram.ext.extbot.ExtBot, self.bot)
+                try:
+                    self.persistence.update_callback_data(
+                        self.bot.callback_data_cache.persistence_data
+                    )
+                except Exception as exc:
+                    try:
+                        self.dispatch_error(update, exc)
+                    except Exception:
+                        message = (
+                            'Saving callback data raised an error and an '
+                            'uncaught error was raised while handling '
+                            'the error with an error_handler'
+                        )
+                        self.logger.exception(message)
             if self.persistence.store_bot_data:
                 try:
                     self.persistence.update_bot_data(self.bot_data)

telegram/ext/extbot.py

@@ -0,0 +1,326 @@
+#!/usr/bin/env python
+# pylint: disable=E0611,E0213,E1102,C0103,E1101,R0913,R0904
+#
+# A library that provides a Python interface to the Telegram Bot API
+# Copyright (C) 2015-2021
+# Leandro Toledo de Souza <devs@python-telegram-bot.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser Public License for more details.
+#
+# You should have received a copy of the GNU Lesser Public License
+# along with this program.  If not, see [http://www.gnu.org/licenses/].
+"""This module contains an object that represents a Telegram Bot with convenience extensions."""
+from copy import copy
+from typing import Union, cast, List, Callable, Optional, Tuple, TypeVar, TYPE_CHECKING, Sequence
+
+import telegram.bot
+from telegram import (
+    ReplyMarkup,
+    Message,
+    InlineKeyboardMarkup,
+    Poll,
+    MessageId,
+    Update,
+    Chat,
+    CallbackQuery,
+)
+
+from telegram.ext.callbackdatacache import CallbackDataCache
+from telegram.utils.types import JSONDict, ODVInput, DVInput
+from ..utils.helpers import DEFAULT_NONE
+
+if TYPE_CHECKING:
+    from telegram import InlineQueryResult, MessageEntity
+    from telegram.utils.request import Request
+    from .defaults import Defaults
+
+HandledTypes = TypeVar('HandledTypes', bound=Union[Message, CallbackQuery, Chat])
+
+
+class ExtBot(telegram.bot.Bot):
+    """This object represents a Telegram Bot with convenience extensions.
+
+    Warning:
+        Not to be confused with :class:`telegram.Bot`.
+
+    For the documentation of the arguments, methods and attributes, please see
+    :class:`telegram.Bot`.
+
+    .. versionadded:: 13.6
+
+    Args:
+        defaults (:class:`telegram.ext.Defaults`, optional): An object containing default values to
+            be used if not set explicitly in the bot methods.
+        arbitrary_callback_data (:obj:`bool` | :obj:`int`, optional): Whether to
+            allow arbitrary objects as callback data for :class:`telegram.InlineKeyboardButton`.
+            Pass an integer to specify the maximum number of objects cached in memory. For more
+            details, please see our `wiki <https://git.io/JGBDI>`_. Defaults to :obj:`False`.
+
+    Attributes:
+        arbitrary_callback_data (:obj:`bool` | :obj:`int`): Whether this bot instance
+            allows to use arbitrary objects as callback data for
+            :class:`telegram.InlineKeyboardButton`.
+        callback_data_cache (:class:`telegram.ext.CallbackDataCache`): The cache for objects passed
+            as callback data for :class:`telegram.InlineKeyboardButton`.
+
+    """
+
+    __slots__ = ('arbitrary_callback_data', 'callback_data_cache')
+
+    # The ext_bot argument is a little hack to get warnings handled correctly.
+    # It's not very clean, but the warnings will be dropped at some point anyway.
+    def __setattr__(self, key: str, value: object, ext_bot: bool = True) -> None:
+        if issubclass(self.__class__, ExtBot) and self.__class__ is not ExtBot:
+            object.__setattr__(self, key, value)
+            return
+        super().__setattr__(key, value, ext_bot=ext_bot)  # type: ignore[call-arg]
+
+    def __init__(
+        self,
+        token: str,
+        base_url: str = None,
+        base_file_url: str = None,
+        request: 'Request' = None,
+        private_key: bytes = None,
+        private_key_password: bytes = None,
+        defaults: 'Defaults' = None,
+        arbitrary_callback_data: Union[bool, int] = False,
+    ):
+        super().__init__(
+            token=token,
+            base_url=base_url,
+            base_file_url=base_file_url,
+            request=request,
+            private_key=private_key,
+            private_key_password=private_key_password,
+            defaults=defaults,
+        )
+
+        # set up callback_data
+        if not isinstance(arbitrary_callback_data, bool):
+            maxsize = cast(int, arbitrary_callback_data)
+            self.arbitrary_callback_data = True
+        else:
+            maxsize = 1024
+            self.arbitrary_callback_data = arbitrary_callback_data
+        self.callback_data_cache: CallbackDataCache = CallbackDataCache(bot=self, maxsize=maxsize)
+
+    def _replace_keyboard(self, reply_markup: Optional[ReplyMarkup]) -> Optional[ReplyMarkup]:
+        # If the reply_markup is an inline keyboard and we allow arbitrary callback data, let the
+        # CallbackDataCache build a new keyboard with the data replaced. Otherwise return the input
+        if isinstance(reply_markup, InlineKeyboardMarkup) and self.arbitrary_callback_data:
+            return self.callback_data_cache.process_keyboard(reply_markup)
+
+        return reply_markup
+
+    def insert_callback_data(self, update: Update) -> None:
+        """If this bot allows for arbitrary callback data, this inserts the cached data into all
+        corresponding buttons within this update.
+
+        Note:
+            Checks :attr:`telegram.Message.via_bot` and :attr:`telegram.Message.from_user` to check
+            if the reply markup (if any) was actually sent by this caches bot. If it was not, the
+            message will be returned unchanged.
+
+            Note that this will fail for channel posts, as :attr:`telegram.Message.from_user` is
+            :obj:`None` for those! In the corresponding reply markups the callback data will be
+            replaced by :class:`telegram.ext.InvalidCallbackData`.
+
+        Warning:
+            *In place*, i.e. the passed :class:`telegram.Message` will be changed!
+
+        Args:
+            update (:class`telegram.Update`): The update.
+
+        """
+        # The only incoming updates that can directly contain a message sent by the bot itself are:
+        # * CallbackQueries
+        # * Messages where the pinned_message is sent by the bot
+        # * Messages where the reply_to_message is sent by the bot
+        # * Messages where via_bot is the bot
+        # Finally there is effective_chat.pinned message, but that's only returned in get_chat
+        if update.callback_query:
+            self._insert_callback_data(update.callback_query)
+        # elif instead of if, as effective_message includes callback_query.message
+        # and that has already been processed
+        elif update.effective_message:
+            self._insert_callback_data(update.effective_message)
+
+    def _insert_callback_data(self, obj: HandledTypes) -> HandledTypes:
+        if not self.arbitrary_callback_data:
+            return obj
+
+        if isinstance(obj, CallbackQuery):
+            self.callback_data_cache.process_callback_query(obj)
+            return obj  # type: ignore[return-value]
+
+        if isinstance(obj, Message):
+            if obj.reply_to_message:
+                # reply_to_message can't contain further reply_to_messages, so no need to check
+                self.callback_data_cache.process_message(obj.reply_to_message)
+                if obj.reply_to_message.pinned_message:
+                    # pinned messages can't contain reply_to_message, no need to check
+                    self.callback_data_cache.process_message(obj.reply_to_message.pinned_message)
+            if obj.pinned_message:
+                # pinned messages can't contain reply_to_message, no need to check
+                self.callback_data_cache.process_message(obj.pinned_message)
+
+            # Finally, handle the message itself
+            self.callback_data_cache.process_message(message=obj)
+            return obj  # type: ignore[return-value]
+
+        if isinstance(obj, Chat) and obj.pinned_message:
+            self.callback_data_cache.process_message(obj.pinned_message)
+
+        return obj
+
+    def _message(
+        self,
+        endpoint: str,
+        data: JSONDict,
+        reply_to_message_id: int = None,
+        disable_notification: ODVInput[bool] = DEFAULT_NONE,
+        reply_markup: ReplyMarkup = None,
+        allow_sending_without_reply: ODVInput[bool] = DEFAULT_NONE,
+        timeout: ODVInput[float] = DEFAULT_NONE,
+        api_kwargs: JSONDict = None,
+    ) -> Union[bool, Message]:
+        # We override this method to call self._replace_keyboard and self._insert_callback_data.
+        # This covers most methods that have a reply_markup
+        result = super()._message(
+            endpoint=endpoint,
+            data=data,
+            reply_to_message_id=reply_to_message_id,
+            disable_notification=disable_notification,
+            reply_markup=self._replace_keyboard(reply_markup),
+            allow_sending_without_reply=allow_sending_without_reply,
+            timeout=timeout,
+            api_kwargs=api_kwargs,
+        )
+        if isinstance(result, Message):
+            self._insert_callback_data(result)
+        return result
+
+    def get_updates(
+        self,
+        offset: int = None,
+        limit: int = 100,
+        timeout: float = 0,
+        read_latency: float = 2.0,
+        allowed_updates: List[str] = None,
+        api_kwargs: JSONDict = None,
+    ) -> List[Update]:
+        updates = super().get_updates(
+            offset=offset,
+            limit=limit,
+            timeout=timeout,
+            read_latency=read_latency,
+            allowed_updates=allowed_updates,
+            api_kwargs=api_kwargs,
+        )
+
+        for update in updates:
+            self.insert_callback_data(update)
+
+        return updates
+
+    def _effective_inline_results(  # pylint: disable=R0201
+        self,
+        results: Union[
+            Sequence['InlineQueryResult'], Callable[[int], Optional[Sequence['InlineQueryResult']]]
+        ],
+        next_offset: str = None,
+        current_offset: str = None,
+    ) -> Tuple[Sequence['InlineQueryResult'], Optional[str]]:
+        """
+        This method is called by Bot.answer_inline_query to build the actual results list.
+        Overriding this to call self._replace_keyboard suffices
+        """
+        effective_results, next_offset = super()._effective_inline_results(
+            results=results, next_offset=next_offset, current_offset=current_offset
+        )
+
+        # Process arbitrary callback
+        if not self.arbitrary_callback_data:
+            return effective_results, next_offset
+        results = []
+        for result in effective_results:
+            # All currently existingInlineQueryResults have a reply_markup, but future ones
+            # might not have. Better be save than sorry
+            if not hasattr(result, 'reply_markup'):
+                results.append(result)
+            else:
+                # We build a new result in case the user wants to use the same object in
+                # different places
+                new_result = copy(result)
+                markup = self._replace_keyboard(result.reply_markup)  # type: ignore[attr-defined]
+                new_result.reply_markup = markup
+                results.append(new_result)
+
+        return results, next_offset
+
+    def stop_poll(
+        self,
+        chat_id: Union[int, str],
+        message_id: int,
+        reply_markup: InlineKeyboardMarkup = None,
+        timeout: ODVInput[float] = DEFAULT_NONE,
+        api_kwargs: JSONDict = None,
+    ) -> Poll:
+        # We override this method to call self._replace_keyboard
+        return super().stop_poll(
+            chat_id=chat_id,
+            message_id=message_id,
+            reply_markup=self._replace_keyboard(reply_markup),
+            timeout=timeout,
+            api_kwargs=api_kwargs,
+        )
+
+    def copy_message(
+        self,
+        chat_id: Union[int, str],
+        from_chat_id: Union[str, int],
+        message_id: int,
+        caption: str = None,
+        parse_mode: ODVInput[str] = DEFAULT_NONE,
+        caption_entities: Union[Tuple['MessageEntity', ...], List['MessageEntity']] = None,
+        disable_notification: DVInput[bool] = DEFAULT_NONE,
+        reply_to_message_id: int = None,
+        allow_sending_without_reply: DVInput[bool] = DEFAULT_NONE,
+        reply_markup: ReplyMarkup = None,
+        timeout: ODVInput[float] = DEFAULT_NONE,
+        api_kwargs: JSONDict = None,
+    ) -> MessageId:
+        # We override this method to call self._replace_keyboard
+        return super().copy_message(
+            chat_id=chat_id,
+            from_chat_id=from_chat_id,
+            message_id=message_id,
+            caption=caption,
+            parse_mode=parse_mode,
+            caption_entities=caption_entities,
+            disable_notification=disable_notification,
+            reply_to_message_id=reply_to_message_id,
+            allow_sending_without_reply=allow_sending_without_reply,
+            reply_markup=self._replace_keyboard(reply_markup),
+            timeout=timeout,
+            api_kwargs=api_kwargs,
+        )
+
+    def get_chat(
+        self,
+        chat_id: Union[str, int],
+        timeout: ODVInput[float] = DEFAULT_NONE,
+        api_kwargs: JSONDict = None,
+    ) -> Chat:
+        # We override this method to call self._insert_callback_data
+        result = super().get_chat(chat_id=chat_id, timeout=timeout, api_kwargs=api_kwargs)
+        return self._insert_callback_data(result)

telegram/ext/jobqueue.py

@@ -78,7 +78,7 @@ class JobQueue:
     def _tz_now(self) -> datetime.datetime:
         return datetime.datetime.now(self.scheduler.timezone)
 
-    def _update_persistence(self, event: JobEvent) -> None:  # pylint: disable=W0613
+    def _update_persistence(self, _: JobEvent) -> None:
         self._dispatcher.update_persistence()
 
     def _dispatch_error(self, event: JobEvent) -> None:

telegram/ext/picklepersistence.py

@@ -30,8 +30,7 @@ from typing import (
 )
 
 from telegram.ext import BasePersistence
-from telegram.utils.types import ConversationDict  # pylint: disable=W0611
-from .utils.types import UD, CD, BD
+from .utils.types import UD, CD, BD, ConversationDict, CDCData
 from .contexttypes import ContextTypes
 
 
@@ -55,10 +54,14 @@ class PicklePersistence(BasePersistence[UD, CD, BD]):
         store_chat_data (:obj:`bool`, optional): Whether user_data should be saved by this
             persistence class. Default is :obj:`True`.
         store_bot_data (:obj:`bool`, optional): Whether bot_data should be saved by this
-            persistence class. Default is :obj:`True` .
-        single_file (:obj:`bool`, optional): When :obj:`False` will store 3 separate files of
-            `filename_user_data`, `filename_chat_data` and `filename_conversations`. Default is
-            :obj:`True`.
+            persistence class. Default is :obj:`True`.
+        store_callback_data (:obj:`bool`, optional): Whether callback_data should be saved by this
+            persistence class. Default is :obj:`False`.
+
+            .. versionadded:: 13.6
+        single_file (:obj:`bool`, optional): When :obj:`False` will store 5 separate files of
+            `filename_user_data`, `filename_chat_data`, `filename_bot_data`, `filename_chat_data`,
+            `filename_callback_data` and `filename_conversations`. Default is :obj:`True`.
         on_flush (:obj:`bool`, optional): When :obj:`True` will only save to file when
             :meth:`flush` is called and keep data in memory until that happens. When
             :obj:`False` will store data on any transaction *and* on call to :meth:`flush`.
@@ -79,9 +82,13 @@ class PicklePersistence(BasePersistence[UD, CD, BD]):
             persistence class.
         store_bot_data (:obj:`bool`): Optional. Whether bot_data should be saved by this
             persistence class.
-        single_file (:obj:`bool`): Optional. When :obj:`False` will store 3 separate files of
-            `filename_user_data`, `filename_chat_data` and `filename_conversations`. Default is
-            :obj:`True`.
+        store_callback_data (:obj:`bool`): Optional. Whether callback_data be saved by this
+            persistence class.
+
+            .. versionadded:: 13.6
+        single_file (:obj:`bool`): Optional. When :obj:`False` will store 5 separate files of
+            `filename_user_data`, `filename_chat_data`, `filename_bot_data`, `filename_chat_data`,
+            `filename_callback_data` and `filename_conversations`. Default is :obj:`True`.
         on_flush (:obj:`bool`, optional): When :obj:`True` will only save to file when
             :meth:`flush` is called and keep data in memory until that happens. When
             :obj:`False` will store data on any transaction *and* on call to :meth:`flush`.
@@ -99,6 +106,7 @@ class PicklePersistence(BasePersistence[UD, CD, BD]):
         'user_data',
         'chat_data',
         'bot_data',
+        'callback_data',
         'conversations',
         'context_types',
     )
@@ -112,6 +120,7 @@ class PicklePersistence(BasePersistence[UD, CD, BD]):
         store_bot_data: bool = True,
         single_file: bool = True,
         on_flush: bool = False,
+        store_callback_data: bool = False,
     ):
         ...
 
@@ -124,6 +133,7 @@ class PicklePersistence(BasePersistence[UD, CD, BD]):
         store_bot_data: bool = True,
         single_file: bool = True,
         on_flush: bool = False,
+        store_callback_data: bool = False,
         context_types: ContextTypes[Any, UD, CD, BD] = None,
     ):
         ...
@@ -136,12 +146,14 @@ class PicklePersistence(BasePersistence[UD, CD, BD]):
         store_bot_data: bool = True,
         single_file: bool = True,
         on_flush: bool = False,
+        store_callback_data: bool = False,
         context_types: ContextTypes[Any, UD, CD, BD] = None,
     ):
         super().__init__(
             store_user_data=store_user_data,
             store_chat_data=store_chat_data,
             store_bot_data=store_bot_data,
+            store_callback_data=store_callback_data,
         )
         self.filename = filename
         self.single_file = single_file
@@ -149,6 +161,7 @@ class PicklePersistence(BasePersistence[UD, CD, BD]):
         self.user_data: Optional[DefaultDict[int, UD]] = None
         self.chat_data: Optional[DefaultDict[int, CD]] = None
         self.bot_data: Optional[BD] = None
+        self.callback_data: Optional[CDCData] = None
         self.conversations: Optional[Dict[str, Dict[Tuple, object]]] = None
         self.context_types = cast(ContextTypes[Any, UD, CD, BD], context_types or ContextTypes())
 
@@ -161,12 +174,14 @@ class PicklePersistence(BasePersistence[UD, CD, BD]):
                 self.chat_data = defaultdict(self.context_types.chat_data, data['chat_data'])
                 # For backwards compatibility with files not containing bot data
                 self.bot_data = data.get('bot_data', self.context_types.bot_data())
+                self.callback_data = data.get('callback_data', {})
                 self.conversations = data['conversations']
         except OSError:
             self.conversations = {}
             self.user_data = defaultdict(self.context_types.user_data)
             self.chat_data = defaultdict(self.context_types.chat_data)
             self.bot_data = self.context_types.bot_data()
+            self.callback_data = None
         except pickle.UnpicklingError as exc:
             raise TypeError(f"File {filename} does not contain valid pickle data") from exc
         except Exception as exc:
@@ -191,6 +206,7 @@ class PicklePersistence(BasePersistence[UD, CD, BD]):
                 'user_data': self.user_data,
                 'chat_data': self.chat_data,
                 'bot_data': self.bot_data,
+                'callback_data': self.callback_data,
             }
             pickle.dump(data, file)
 
@@ -258,6 +274,29 @@ class PicklePersistence(BasePersistence[UD, CD, BD]):
             self._load_singlefile()
         return self.bot_data  # type: ignore[return-value]
 
+    def get_callback_data(self) -> Optional[CDCData]:
+        """Returns the callback data from the pickle file if it exists or :obj:`None`.
+
+        .. versionadded:: 13.6
+
+        Returns:
+            Optional[:class:`telegram.ext.utils.types.CDCData`]: The restored meta data or
+                :obj:`None`, if no data was stored.
+        """
+        if self.callback_data:
+            pass
+        elif not self.single_file:
+            filename = f"{self.filename}_callback_data"
+            data = self._load_file(filename)
+            if not data:
+                data = None
+            self.callback_data = data
+        else:
+            self._load_singlefile()
+        if self.callback_data is None:
+            return None
+        return self.callback_data[0], self.callback_data[1].copy()
+
     def get_conversations(self, name: str) -> ConversationDict:
         """Returns the conversations from the pickle file if it exsists or an empty dict.
 
@@ -359,6 +398,26 @@ class PicklePersistence(BasePersistence[UD, CD, BD]):
             else:
                 self._dump_singlefile()
 
+    def update_callback_data(self, data: CDCData) -> None:
+        """Will update the callback_data (if changed) and depending on :attr:`on_flush` save the
+        pickle file.
+
+        .. versionadded:: 13.6
+
+        Args:
+            data (:class:`telegram.ext.utils.types.CDCData`:): The relevant data to restore
+                :attr:`telegram.ext.dispatcher.bot.callback_data`.
+        """
+        if self.callback_data == data:
+            return
+        self.callback_data = (data[0], data[1].copy())
+        if not self.on_flush:
+            if not self.single_file:
+                filename = f"{self.filename}_callback_data"
+                self._dump_file(filename, self.callback_data)
+            else:
+                self._dump_singlefile()
+
     def refresh_user_data(self, user_id: int, user_data: UD) -> None:
         """Does nothing.
 
@@ -383,7 +442,13 @@ class PicklePersistence(BasePersistence[UD, CD, BD]):
     def flush(self) -> None:
         """Will save all data in memory to pickle file(s)."""
         if self.single_file:
-            if self.user_data or self.chat_data or self.bot_data or self.conversations:
+            if (
+                self.user_data
+                or self.chat_data
+                or self.bot_data
+                or self.callback_data
+                or self.conversations
+            ):
                 self._dump_singlefile()
         else:
             if self.user_data:
@@ -392,5 +457,7 @@ class PicklePersistence(BasePersistence[UD, CD, BD]):
                 self._dump_file(f"{self.filename}_chat_data", self.chat_data)
             if self.bot_data:
                 self._dump_file(f"{self.filename}_bot_data", self.bot_data)
+            if self.callback_data:
+                self._dump_file(f"{self.filename}_callback_data", self.callback_data)
             if self.conversations:
                 self._dump_file(f"{self.filename}_conversations", self.conversations)

telegram/ext/updater.py

@@ -41,9 +41,9 @@ from typing import (
 
 from telegram import Bot, TelegramError
 from telegram.error import InvalidToken, RetryAfter, TimedOut, Unauthorized
-from telegram.ext import Dispatcher, JobQueue, ContextTypes
+from telegram.ext import Dispatcher, JobQueue, ContextTypes, ExtBot
 from telegram.utils.deprecate import TelegramDeprecationWarning, set_new_attribute_deprecated
-from telegram.utils.helpers import get_signal_name
+from telegram.utils.helpers import get_signal_name, DEFAULT_FALSE, DefaultValue
 from telegram.utils.request import Request
 from telegram.ext.utils.types import CCT, UD, CD, BD
 from telegram.ext.utils.webhookhandler import WebhookAppClass, WebhookServer
@@ -65,8 +65,11 @@ class Updater(Generic[CCT, UD, CD, BD]):
 
     Note:
         * You must supply either a :attr:`bot` or a :attr:`token` argument.
-        * If you supply a :attr:`bot`, you will need to pass :attr:`defaults` to *both* the bot and
-          the :class:`telegram.ext.Updater`.
+        * If you supply a :attr:`bot`, you will need to pass :attr:`arbitrary_callback_data`,
+          and :attr:`defaults` to the bot instead of the :class:`telegram.ext.Updater`. In this
+          case, you'll have to use the class :class:`telegram.ext.ExtBot`.
+
+          .. versionchanged:: 13.6
 
     Args:
         token (:obj:`str`, optional): The bot's token given by the @BotFather.
@@ -98,6 +101,12 @@ class Updater(Generic[CCT, UD, CD, BD]):
             used).
         defaults (:class:`telegram.ext.Defaults`, optional): An object containing default values to
             be used if not set explicitly in the bot methods.
+        arbitrary_callback_data (:obj:`bool` | :obj:`int` | :obj:`None`, optional): Whether to
+            allow arbitrary objects as callback data for :class:`telegram.InlineKeyboardButton`.
+            Pass an integer to specify the maximum number of cached objects. For more details,
+            please see our wiki. Defaults to :obj:`False`.
+
+            .. versionadded:: 13.6
         context_types (:class:`telegram.ext.ContextTypes`, optional): Pass an instance
             of :class:`telegram.ext.ContextTypes` to customize the types used in the
             ``context`` interface. If not passed, the defaults documented in
@@ -158,6 +167,7 @@ class Updater(Generic[CCT, UD, CD, BD]):
         defaults: 'Defaults' = None,
         use_context: bool = True,
         base_file_url: str = None,
+        arbitrary_callback_data: Union[DefaultValue, bool, int, None] = DEFAULT_FALSE,
     ):
         ...
 
@@ -176,6 +186,7 @@ class Updater(Generic[CCT, UD, CD, BD]):
         defaults: 'Defaults' = None,
         use_context: bool = True,
         base_file_url: str = None,
+        arbitrary_callback_data: Union[DefaultValue, bool, int, None] = DEFAULT_FALSE,
         context_types: ContextTypes[CCT, UD, CD, BD] = None,
     ):
         ...
@@ -203,6 +214,7 @@ class Updater(Generic[CCT, UD, CD, BD]):
         use_context: bool = True,
         dispatcher=None,
         base_file_url: str = None,
+        arbitrary_callback_data: Union[DefaultValue, bool, int, None] = DEFAULT_FALSE,
         context_types: ContextTypes[CCT, UD, CD, BD] = None,
     ):
 
@@ -213,6 +225,12 @@ class Updater(Generic[CCT, UD, CD, BD]):
                 TelegramDeprecationWarning,
                 stacklevel=2,
             )
+        if arbitrary_callback_data is not DEFAULT_FALSE and bot:
+            warnings.warn(
+                'Passing arbitrary_callback_data to an Updater has no '
+                'effect when a Bot is passed as well. Pass them to the Bot instead.',
+                stacklevel=2,
+            )
 
         if dispatcher is None:
             if (token is None) and (bot is None):
@@ -258,7 +276,7 @@ class Updater(Generic[CCT, UD, CD, BD]):
                 if 'con_pool_size' not in request_kwargs:
                     request_kwargs['con_pool_size'] = con_pool_size
                 self._request = Request(**request_kwargs)
-                self.bot = Bot(
+                self.bot = ExtBot(
                     token,  # type: ignore[arg-type]
                     base_url,
                     base_file_url=base_file_url,
@@ -266,6 +284,11 @@ class Updater(Generic[CCT, UD, CD, BD]):
                     private_key=private_key,
                     private_key_password=private_key_password,
                     defaults=defaults,
+                    arbitrary_callback_data=(
+                        False  # type: ignore[arg-type]
+                        if arbitrary_callback_data is DEFAULT_FALSE
+                        else arbitrary_callback_data
+                    ),
                 )
             self.update_queue: Queue = Queue()
             self.job_queue = JobQueue()

telegram/ext/utils/types.py

@@ -20,11 +20,26 @@
 
 .. versionadded:: 13.6
 """
-from typing import TypeVar, TYPE_CHECKING
+from typing import TypeVar, TYPE_CHECKING, Tuple, List, Dict, Any, Optional
 
 if TYPE_CHECKING:
     from telegram.ext import CallbackContext  # noqa: F401
 
+
+ConversationDict = Dict[Tuple[int, ...], Optional[object]]
+"""Dicts as maintained by the :class:`telegram.ext.ConversationHandler`.
+
+    .. versionadded:: 13.6
+"""
+
+CDCData = Tuple[List[Tuple[str, float, Dict[str, Any]]], Dict[str, str]]
+"""Tuple[List[Tuple[:obj:`str`, :obj:`float`, Dict[:obj:`str`, :obj:`any`]]], \
+    Dict[:obj:`str`, :obj:`str`]]: Data returned by
+    :attr:`telegram.ext.CallbackDataCache.persistence_data`.
+
+    .. versionadded:: 13.6
+"""
+
 CCT = TypeVar('CCT', bound='CallbackContext')
 """An instance of :class:`telegram.ext.CallbackContext` or a custom subclass.
 

telegram/ext/utils/webhookhandler.py

@@ -30,6 +30,7 @@ from tornado.httpserver import HTTPServer
 from tornado.ioloop import IOLoop
 
 from telegram import Update
+from telegram.ext import ExtBot
 from telegram.utils.deprecate import set_new_attribute_deprecated
 from telegram.utils.types import JSONDict
 
@@ -143,6 +144,9 @@ class WebhookHandler(tornado.web.RequestHandler):
         update = Update.de_json(data, self.bot)
         if update:
             self.logger.debug('Received Update with ID %d on Webhook', update.update_id)
+            # handle arbitrary callback data, if necessary
+            if isinstance(self.bot, ExtBot):
+                self.bot.insert_callback_data(update)
             self.update_queue.put(update)
 
     def _validate_post(self) -> None:

telegram/inline/inlinekeyboardbutton.py

@@ -35,16 +35,31 @@ class InlineKeyboardButton(TelegramObject):
     and :attr:`pay` are equal.
 
     Note:
-        You must use exactly one of the optional fields. Mind that :attr:`callback_game` is not
-        working as expected. Putting a game short name in it might, but is not guaranteed to work.
+        * You must use exactly one of the optional fields. Mind that :attr:`callback_game` is not
+          working as expected. Putting a game short name in it might, but is not guaranteed to
+          work.
+        * If your bot allows for arbitrary callback data, in keyboards returned in a response
+          from telegram, :attr:`callback_data` maybe be an instance of
+          :class:`telegram.ext.InvalidCallbackData`. This will be the case, if the data
+          associated with the button was already deleted.
+
+          .. versionadded:: 13.6
+
+    Warning:
+        If your bot allows your arbitrary callback data, buttons whose callback data is a
+        non-hashable object will be come unhashable. Trying to evaluate ``hash(button)`` will
+        result in a :class:`TypeError`.
+
+        .. versionchanged:: 13.6
 
     Args:
         text (:obj:`str`): Label text on the button.
         url (:obj:`str`): HTTP or tg:// url to be opened when button is pressed.
         login_url (:class:`telegram.LoginUrl`, optional): An HTTP URL used to automatically
             authorize the user. Can be used as a replacement for the Telegram Login Widget.
-        callback_data (:obj:`str`, optional): Data to be sent in a callback query to the bot when
-            button is pressed, UTF-8 1-64 bytes.
+        callback_data (:obj:`str` | :obj:`Any`, optional): Data to be sent in a callback query to
+            the bot when button is pressed, UTF-8 1-64 bytes. If the bot instance allows arbitrary
+            callback data, anything can be passed.
         switch_inline_query (:obj:`str`, optional): If set, pressing the button will prompt the
             user to select one of their chats, open that chat and insert the bot's username and the
             specified inline query in the input field. Can be empty, in which case just the bot's
@@ -69,8 +84,8 @@ class InlineKeyboardButton(TelegramObject):
         url (:obj:`str`): Optional. HTTP or tg:// url to be opened when button is pressed.
         login_url (:class:`telegram.LoginUrl`): Optional. An HTTP URL used to automatically
             authorize the user. Can be used as a replacement for the Telegram Login Widget.
-        callback_data (:obj:`str`): Optional. Data to be sent in a callback query to the bot when
-            button is pressed, UTF-8 1-64 bytes.
+        callback_data (:obj:`str` | :obj:`object`): Optional. Data to be sent in a callback query
+            to the bot when button is pressed, UTF-8 1-64 bytes.
         switch_inline_query (:obj:`str`): Optional. Will prompt the user to select one of their
             chats, open that chat and insert the bot's username and the specified inline query in
             the input field. Can be empty, in which case just the bot’s username will be inserted.
@@ -99,7 +114,7 @@ class InlineKeyboardButton(TelegramObject):
         self,
         text: str,
         url: str = None,
-        callback_data: str = None,
+        callback_data: object = None,
         switch_inline_query: str = None,
         switch_inline_query_current_chat: str = None,
         callback_game: 'CallbackGame' = None,
@@ -118,7 +133,10 @@ class InlineKeyboardButton(TelegramObject):
         self.switch_inline_query_current_chat = switch_inline_query_current_chat
         self.callback_game = callback_game
         self.pay = pay
+        self._id_attrs = ()
+        self._set_id_attrs()
 
+    def _set_id_attrs(self) -> None:
         self._id_attrs = (
             self.text,
             self.url,
@@ -129,3 +147,16 @@ class InlineKeyboardButton(TelegramObject):
             self.callback_game,
             self.pay,
         )
+
+    def update_callback_data(self, callback_data: object) -> None:
+        """
+        Sets :attr:`callback_data` to the passed object. Intended to be used by
+        :class:`telegram.ext.CallbackDataCache`.
+
+        .. versionadded:: 13.6
+
+        Args:
+            callback_data (:obj:`obj`): The new callback data.
+        """
+        self.callback_data = callback_data
+        self._set_id_attrs()

telegram/utils/types.py

@@ -44,9 +44,6 @@ a local file path as string, :class:`pathlib.Path` or the file contents as :obj:
 JSONDict = Dict[str, Any]
 """Dictionary containing response from Telegram or data to send to the API."""
 
-ConversationDict = Dict[Tuple[int, ...], Optional[object]]
-"""Dicts as maintained by the :class:`telegram.ext.ConversationHandler`."""
-
 DVType = TypeVar('DVType')
 ODVInput = Optional[Union['DefaultValue[DVType]', DVType]]
 """Generic type for bot method parameters which can have defaults. ``ODVInput[type]`` is the same

tests/conftest.py

@@ -32,7 +32,6 @@ import pytest
 import pytz
 
 from telegram import (
-    Bot,
     Message,
     User,
     Chat,
@@ -46,7 +45,15 @@ from telegram import (
     File,
     ChatPermissions,
 )
-from telegram.ext import Dispatcher, JobQueue, Updater, MessageFilter, Defaults, UpdateFilter
+from telegram.ext import (
+    Dispatcher,
+    JobQueue,
+    Updater,
+    MessageFilter,
+    Defaults,
+    UpdateFilter,
+    ExtBot,
+)
 from telegram.error import BadRequest
 from telegram.utils.helpers import DefaultValue, DEFAULT_NONE
 from tests.bots import get_bot
@@ -84,10 +91,12 @@ def bot_info():
 
 @pytest.fixture(scope='session')
 def bot(bot_info):
-    class DictBot(Bot):  # Subclass Bot to allow monkey patching of attributes and functions, would
+    class DictExtBot(
+        ExtBot
+    ):  # Subclass Bot to allow monkey patching of attributes and functions, would
         pass  # come into effect when we __dict__ is dropped from slots
 
-    return DictBot(bot_info['token'], private_key=PRIVATE_KEY)
+    return DictExtBot(bot_info['token'], private_key=PRIVATE_KEY)
 
 
 DEFAULT_BOTS = {}
@@ -218,7 +227,10 @@ def pytest_configure(config):
 
 
 def make_bot(bot_info, **kwargs):
-    return Bot(bot_info['token'], private_key=PRIVATE_KEY, **kwargs)
+    """
+    Tests are executed on tg.ext.ExtBot, as that class only extends the functionality of tg.bot
+    """
+    return ExtBot(bot_info['token'], private_key=PRIVATE_KEY, **kwargs)
 
 
 CMD_PATTERN = re.compile(r'/[\da-z_]{1,32}(?:@\w{1,32})?')
@@ -444,7 +456,7 @@ def check_shortcut_signature(
 
 def check_shortcut_call(
     shortcut_method: Callable,
-    bot: Bot,
+    bot: ExtBot,
     bot_method_name: str,
     skip_params: Iterable[str] = None,
     shortcut_kwargs: Iterable[str] = None,
@@ -513,7 +525,7 @@ def check_shortcut_call(
 
 def check_defaults_handling(
     method: Callable,
-    bot: Bot,
+    bot: ExtBot,
     return_value=None,
 ) -> bool:
     """

tests/test_bot.py

@@ -19,6 +19,7 @@
 import inspect
 import time
 import datetime as dtm
+from collections import defaultdict
 from pathlib import Path
 from platform import python_implementation
 
@@ -45,10 +46,21 @@ from telegram import (
     Dice,
     MessageEntity,
     ParseMode,
+    CallbackQuery,
+    Message,
+    Chat,
+    InlineQueryResultVoice,
+    PollOption,
 )
 from telegram.constants import MAX_INLINE_QUERY_RESULTS
+from telegram.ext import ExtBot
 from telegram.error import BadRequest, InvalidToken, NetworkError, RetryAfter
-from telegram.utils.helpers import from_timestamp, escape_markdown, to_timestamp
+from telegram.ext.callbackdatacache import InvalidCallbackData
+from telegram.utils.helpers import (
+    from_timestamp,
+    escape_markdown,
+    to_timestamp,
+)
 from tests.conftest import expect_bad_request, check_defaults_handling, GITHUB_ACTION
 from tests.bots import FALLBACKS
 
@@ -109,6 +121,10 @@ def inst(request, bot_info, default_bot):
 
 
 class TestBot:
+    """
+    Most are executed on tg.ext.ExtBot, as that class only extends the functionality of tg.bot
+    """
+
     @pytest.mark.parametrize('inst', ['bot', "default_bot"], indirect=True)
     def test_slot_behaviour(self, inst, recwarn, mro_slots):
         for attr in inst.__slots__:
@@ -141,6 +157,15 @@ class TestBot:
         with pytest.raises(InvalidToken, match='Invalid token'):
             Bot(token)
 
+    @pytest.mark.parametrize(
+        'acd_in,maxsize,acd',
+        [(True, 1024, True), (False, 1024, False), (0, 0, True), (None, None, True)],
+    )
+    def test_callback_data_maxsize(self, bot, acd_in, maxsize, acd):
+        bot = ExtBot(bot.token, arbitrary_callback_data=acd_in)
+        assert bot.arbitrary_callback_data == acd
+        assert bot.callback_data_cache.maxsize == maxsize
+
     @flaky(3, 1)
     def test_invalid_token_server_response(self, monkeypatch):
         monkeypatch.setattr('telegram.Bot._validate_token', lambda x, y: True)
@@ -236,6 +261,40 @@ class TestBot:
         bot_method = getattr(bot, bot_method_name)
         assert check_defaults_handling(bot_method, bot)
 
+    def test_ext_bot_signature(self):
+        """
+        Here we make sure that all methods of ext.ExtBot have the same signature as the
+        corresponding methods of tg.Bot.
+        """
+        # Some methods of ext.ExtBot
+        global_extra_args = set()
+        extra_args_per_method = defaultdict(set, {'__init__': {'arbitrary_callback_data'}})
+        different_hints_per_method = defaultdict(set, {'__setattr__': {'ext_bot'}})
+
+        for name, method in inspect.getmembers(Bot, predicate=inspect.isfunction):
+            signature = inspect.signature(method)
+            ext_signature = inspect.signature(getattr(ExtBot, name))
+
+            assert (
+                ext_signature.return_annotation == signature.return_annotation
+            ), f'Wrong return annotation for method {name}'
+            assert (
+                set(signature.parameters)
+                == set(ext_signature.parameters) - global_extra_args - extra_args_per_method[name]
+            ), f'Wrong set of parameters for method {name}'
+            for param_name, param in signature.parameters.items():
+                if param_name in different_hints_per_method[name]:
+                    continue
+                assert (
+                    param.annotation == ext_signature.parameters[param_name].annotation
+                ), f'Wrong annotation for parameter {param_name} of method {name}'
+                assert (
+                    param.default == ext_signature.parameters[param_name].default
+                ), f'Wrong default value for parameter {param_name} of method {name}'
+                assert (
+                    param.kind == ext_signature.parameters[param_name].kind
+                ), f'Wrong parameter kind for parameter {param_name} of method {name}'
+
     @flaky(3, 1)
     def test_forward_message(self, bot, chat_id, message):
         forward_message = bot.forward_message(
@@ -1175,6 +1234,41 @@ class TestBot:
         if updates:
             assert isinstance(updates[0], Update)
 
+    def test_get_updates_invalid_callback_data(self, bot, monkeypatch):
+        def post(*args, **kwargs):
+            return [
+                Update(
+                    17,
+                    callback_query=CallbackQuery(
+                        id=1,
+                        from_user=None,
+                        chat_instance=123,
+                        data='invalid data',
+                        message=Message(
+                            1,
+                            from_user=User(1, '', False),
+                            date=None,
+                            chat=Chat(1, ''),
+                            text='Webhook',
+                        ),
+                    ),
+                ).to_dict()
+            ]
+
+        bot.arbitrary_callback_data = True
+        try:
+            monkeypatch.setattr(bot.request, 'post', post)
+            bot.delete_webhook()  # make sure there is no webhook set if webhook tests failed
+            updates = bot.get_updates(timeout=1)
+
+            assert isinstance(updates, list)
+            assert len(updates) == 1
+            assert isinstance(updates[0].callback_query.data, InvalidCallbackData)
+
+        finally:
+            # Reset b/c bots scope is session
+            bot.arbitrary_callback_data = False
+
     @flaky(3, 1)
     @pytest.mark.xfail
     def test_set_webhook_get_webhook_info_and_delete_webhook(self, bot):
@@ -1955,3 +2049,327 @@ class TestBot:
             assert len(message.caption_entities) == 1
         else:
             assert len(message.caption_entities) == 0
+
+    def test_replace_callback_data_send_message(self, bot, chat_id):
+        try:
+            bot.arbitrary_callback_data = True
+            replace_button = InlineKeyboardButton(text='replace', callback_data='replace_test')
+            no_replace_button = InlineKeyboardButton(
+                text='no_replace', url='http://python-telegram-bot.org/'
+            )
+            reply_markup = InlineKeyboardMarkup.from_row(
+                [
+                    replace_button,
+                    no_replace_button,
+                ]
+            )
+            message = bot.send_message(chat_id=chat_id, text='test', reply_markup=reply_markup)
+            inline_keyboard = message.reply_markup.inline_keyboard
+
+            assert inline_keyboard[0][1] == no_replace_button
+            assert inline_keyboard[0][0] == replace_button
+            keyboard = list(bot.callback_data_cache._keyboard_data)[0]
+            data = list(bot.callback_data_cache._keyboard_data[keyboard].button_data.values())[0]
+            assert data == 'replace_test'
+        finally:
+            bot.arbitrary_callback_data = False
+            bot.callback_data_cache.clear_callback_data()
+            bot.callback_data_cache.clear_callback_queries()
+
+    def test_replace_callback_data_stop_poll_and_repl_to_message(self, bot, chat_id):
+        poll_message = bot.send_poll(chat_id=chat_id, question='test', options=['1', '2'])
+        try:
+            bot.arbitrary_callback_data = True
+            replace_button = InlineKeyboardButton(text='replace', callback_data='replace_test')
+            no_replace_button = InlineKeyboardButton(
+                text='no_replace', url='http://python-telegram-bot.org/'
+            )
+            reply_markup = InlineKeyboardMarkup.from_row(
+                [
+                    replace_button,
+                    no_replace_button,
+                ]
+            )
+            poll_message.stop_poll(reply_markup=reply_markup)
+            helper_message = poll_message.reply_text('temp', quote=True)
+            message = helper_message.reply_to_message
+            inline_keyboard = message.reply_markup.inline_keyboard
+
+            assert inline_keyboard[0][1] == no_replace_button
+            assert inline_keyboard[0][0] == replace_button
+            keyboard = list(bot.callback_data_cache._keyboard_data)[0]
+            data = list(bot.callback_data_cache._keyboard_data[keyboard].button_data.values())[0]
+            assert data == 'replace_test'
+        finally:
+            bot.arbitrary_callback_data = False
+            bot.callback_data_cache.clear_callback_data()
+            bot.callback_data_cache.clear_callback_queries()
+
+    def test_replace_callback_data_copy_message(self, bot, chat_id):
+        """This also tests that data is inserted into the buttons of message.reply_to_message
+        where message is the return value of a bot method"""
+        original_message = bot.send_message(chat_id=chat_id, text='original')
+        try:
+            bot.arbitrary_callback_data = True
+            replace_button = InlineKeyboardButton(text='replace', callback_data='replace_test')
+            no_replace_button = InlineKeyboardButton(
+                text='no_replace', url='http://python-telegram-bot.org/'
+            )
+            reply_markup = InlineKeyboardMarkup.from_row(
+                [
+                    replace_button,
+                    no_replace_button,
+                ]
+            )
+            message_id = original_message.copy(chat_id=chat_id, reply_markup=reply_markup)
+            helper_message = bot.send_message(
+                chat_id=chat_id, reply_to_message_id=message_id.message_id, text='temp'
+            )
+            message = helper_message.reply_to_message
+            inline_keyboard = message.reply_markup.inline_keyboard
+
+            assert inline_keyboard[0][1] == no_replace_button
+            assert inline_keyboard[0][0] == replace_button
+            keyboard = list(bot.callback_data_cache._keyboard_data)[0]
+            data = list(bot.callback_data_cache._keyboard_data[keyboard].button_data.values())[0]
+            assert data == 'replace_test'
+        finally:
+            bot.arbitrary_callback_data = False
+            bot.callback_data_cache.clear_callback_data()
+            bot.callback_data_cache.clear_callback_queries()
+
+    # TODO: Needs improvement. We need incoming inline query to test answer.
+    def test_replace_callback_data_answer_inline_query(self, monkeypatch, bot, chat_id):
+        # For now just test that our internals pass the correct data
+        def make_assertion(
+            endpoint,
+            data=None,
+            timeout=None,
+            api_kwargs=None,
+        ):
+            inline_keyboard = InlineKeyboardMarkup.de_json(
+                data['results'][0]['reply_markup'], bot
+            ).inline_keyboard
+            assertion_1 = inline_keyboard[0][1] == no_replace_button
+            assertion_2 = inline_keyboard[0][0] != replace_button
+            keyboard, button = (
+                inline_keyboard[0][0].callback_data[:32],
+                inline_keyboard[0][0].callback_data[32:],
+            )
+            assertion_3 = (
+                bot.callback_data_cache._keyboard_data[keyboard].button_data[button]
+                == 'replace_test'
+            )
+            assertion_4 = 'reply_markup' not in data['results'][1]
+            return assertion_1 and assertion_2 and assertion_3 and assertion_4
+
+        try:
+            bot.arbitrary_callback_data = True
+            replace_button = InlineKeyboardButton(text='replace', callback_data='replace_test')
+            no_replace_button = InlineKeyboardButton(
+                text='no_replace', url='http://python-telegram-bot.org/'
+            )
+            reply_markup = InlineKeyboardMarkup.from_row(
+                [
+                    replace_button,
+                    no_replace_button,
+                ]
+            )
+
+            bot.username  # call this here so `bot.get_me()` won't be called after mocking
+            monkeypatch.setattr(bot, '_post', make_assertion)
+            results = [
+                InlineQueryResultArticle(
+                    '11', 'first', InputTextMessageContent('first'), reply_markup=reply_markup
+                ),
+                InlineQueryResultVoice(
+                    '22',
+                    'https://python-telegram-bot.org/static/testfiles/telegram.ogg',
+                    title='second',
+                ),
+            ]
+
+            assert bot.answer_inline_query(chat_id, results=results)
+
+        finally:
+            bot.arbitrary_callback_data = False
+            bot.callback_data_cache.clear_callback_data()
+            bot.callback_data_cache.clear_callback_queries()
+
+    def test_get_chat_arbitrary_callback_data(self, super_group_id, bot):
+        try:
+            bot.arbitrary_callback_data = True
+            reply_markup = InlineKeyboardMarkup.from_button(
+                InlineKeyboardButton(text='text', callback_data='callback_data')
+            )
+
+            message = bot.send_message(
+                super_group_id, text='get_chat_arbitrary_callback_data', reply_markup=reply_markup
+            )
+            message.pin()
+
+            keyboard = list(bot.callback_data_cache._keyboard_data)[0]
+            data = list(bot.callback_data_cache._keyboard_data[keyboard].button_data.values())[0]
+            assert data == 'callback_data'
+
+            chat = bot.get_chat(super_group_id)
+            assert chat.pinned_message == message
+            assert chat.pinned_message.reply_markup == reply_markup
+        finally:
+            bot.arbitrary_callback_data = False
+            bot.callback_data_cache.clear_callback_data()
+            bot.callback_data_cache.clear_callback_queries()
+            bot.unpin_all_chat_messages(super_group_id)
+
+    # In the following tests we check that get_updates inserts callback data correctly if necessary
+    # The same must be done in the webhook updater. This is tested over at test_updater.py, but
+    # here we test more extensively.
+
+    def test_arbitrary_callback_data_no_insert(self, monkeypatch, bot):
+        """Updates that don't need insertion shouldn.t fail obviously"""
+
+        def post(*args, **kwargs):
+            update = Update(
+                17,
+                poll=Poll(
+                    '42',
+                    'question',
+                    options=[PollOption('option', 0)],
+                    total_voter_count=0,
+                    is_closed=False,
+                    is_anonymous=True,
+                    type=Poll.REGULAR,
+                    allows_multiple_answers=False,
+                ),
+            )
+            return [update.to_dict()]
+
+        try:
+            bot.arbitrary_callback_data = True
+            monkeypatch.setattr(bot.request, 'post', post)
+            bot.delete_webhook()  # make sure there is no webhook set if webhook tests failed
+            updates = bot.get_updates(timeout=1)
+
+            assert len(updates) == 1
+            assert updates[0].update_id == 17
+            assert updates[0].poll.id == '42'
+        finally:
+            bot.arbitrary_callback_data = False
+
+    @pytest.mark.parametrize(
+        'message_type', ['channel_post', 'edited_channel_post', 'message', 'edited_message']
+    )
+    def test_arbitrary_callback_data_pinned_message_reply_to_message(
+        self, super_group_id, bot, monkeypatch, message_type
+    ):
+        bot.arbitrary_callback_data = True
+        reply_markup = InlineKeyboardMarkup.from_button(
+            InlineKeyboardButton(text='text', callback_data='callback_data')
+        )
+
+        message = Message(
+            1, None, None, reply_markup=bot.callback_data_cache.process_keyboard(reply_markup)
+        )
+        # We do to_dict -> de_json to make sure those aren't the same objects
+        message.pinned_message = Message.de_json(message.to_dict(), bot)
+
+        def post(*args, **kwargs):
+            update = Update(
+                17,
+                **{
+                    message_type: Message(
+                        1,
+                        None,
+                        None,
+                        pinned_message=message,
+                        reply_to_message=Message.de_json(message.to_dict(), bot),
+                    )
+                },
+            )
+            return [update.to_dict()]
+
+        try:
+            monkeypatch.setattr(bot.request, 'post', post)
+            bot.delete_webhook()  # make sure there is no webhook set if webhook tests failed
+            updates = bot.get_updates(timeout=1)
+
+            assert isinstance(updates, list)
+            assert len(updates) == 1
+
+            effective_message = updates[0][message_type]
+            assert (
+                effective_message.reply_to_message.reply_markup.inline_keyboard[0][0].callback_data
+                == 'callback_data'
+            )
+            assert (
+                effective_message.pinned_message.reply_markup.inline_keyboard[0][0].callback_data
+                == 'callback_data'
+            )
+
+            pinned_message = effective_message.reply_to_message.pinned_message
+            assert (
+                pinned_message.reply_markup.inline_keyboard[0][0].callback_data == 'callback_data'
+            )
+
+        finally:
+            bot.arbitrary_callback_data = False
+            bot.callback_data_cache.clear_callback_data()
+            bot.callback_data_cache.clear_callback_queries()
+
+    def test_arbitrary_callback_data_get_chat_no_pinned_message(self, super_group_id, bot):
+        bot.arbitrary_callback_data = True
+        bot.unpin_all_chat_messages(super_group_id)
+
+        try:
+            chat = bot.get_chat(super_group_id)
+
+            assert isinstance(chat, Chat)
+            assert int(chat.id) == int(super_group_id)
+            assert chat.pinned_message is None
+        finally:
+            bot.arbitrary_callback_data = False
+
+    @pytest.mark.parametrize(
+        'message_type', ['channel_post', 'edited_channel_post', 'message', 'edited_message']
+    )
+    @pytest.mark.parametrize('self_sender', [True, False])
+    def test_arbitrary_callback_data_via_bot(
+        self, super_group_id, bot, monkeypatch, self_sender, message_type
+    ):
+        bot.arbitrary_callback_data = True
+        reply_markup = InlineKeyboardMarkup.from_button(
+            InlineKeyboardButton(text='text', callback_data='callback_data')
+        )
+
+        reply_markup = bot.callback_data_cache.process_keyboard(reply_markup)
+        message = Message(
+            1,
+            None,
+            None,
+            reply_markup=reply_markup,
+            via_bot=bot.bot if self_sender else User(1, 'first', False),
+        )
+
+        def post(*args, **kwargs):
+            return [Update(17, **{message_type: message}).to_dict()]
+
+        try:
+            monkeypatch.setattr(bot.request, 'post', post)
+            bot.delete_webhook()  # make sure there is no webhook set if webhook tests failed
+            updates = bot.get_updates(timeout=1)
+
+            assert isinstance(updates, list)
+            assert len(updates) == 1
+
+            message = updates[0][message_type]
+            if self_sender:
+                assert message.reply_markup.inline_keyboard[0][0].callback_data == 'callback_data'
+            else:
+                assert (
+                    message.reply_markup.inline_keyboard[0][0].callback_data
+                    == reply_markup.inline_keyboard[0][0].callback_data
+                )
+        finally:
+            bot.arbitrary_callback_data = False
+            bot.callback_data_cache.clear_callback_data()
+            bot.callback_data_cache.clear_callback_queries()

tests/test_callbackcontext.py

@@ -19,7 +19,17 @@
 
 import pytest
 
-from telegram import Update, Message, Chat, User, TelegramError
+from telegram import (
+    Update,
+    Message,
+    Chat,
+    User,
+    TelegramError,
+    Bot,
+    InlineKeyboardMarkup,
+    InlineKeyboardButton,
+    CallbackQuery,
+)
 from telegram.ext import CallbackContext
 
 """
@@ -166,3 +176,57 @@ class TestCallbackContext:
     def test_dispatcher_attribute(self, cdp):
         callback_context = CallbackContext(cdp)
         assert callback_context.dispatcher == cdp
+
+    def test_drop_callback_data_exception(self, bot, cdp):
+        non_ext_bot = Bot(bot.token)
+        update = Update(
+            0, message=Message(0, None, Chat(1, 'chat'), from_user=User(1, 'user', False))
+        )
+
+        callback_context = CallbackContext.from_update(update, cdp)
+
+        with pytest.raises(RuntimeError, match='This telegram.ext.ExtBot instance does not'):
+            callback_context.drop_callback_data(None)
+
+        try:
+            cdp.bot = non_ext_bot
+            with pytest.raises(RuntimeError, match='telegram.Bot does not allow for'):
+                callback_context.drop_callback_data(None)
+        finally:
+            cdp.bot = bot
+
+    def test_drop_callback_data(self, cdp, monkeypatch, chat_id):
+        monkeypatch.setattr(cdp.bot, 'arbitrary_callback_data', True)
+
+        update = Update(
+            0, message=Message(0, None, Chat(1, 'chat'), from_user=User(1, 'user', False))
+        )
+
+        callback_context = CallbackContext.from_update(update, cdp)
+        cdp.bot.send_message(
+            chat_id=chat_id,
+            text='test',
+            reply_markup=InlineKeyboardMarkup.from_button(
+                InlineKeyboardButton('test', callback_data='callback_data')
+            ),
+        )
+        keyboard_uuid = cdp.bot.callback_data_cache.persistence_data[0][0][0]
+        button_uuid = list(cdp.bot.callback_data_cache.persistence_data[0][0][2])[0]
+        callback_data = keyboard_uuid + button_uuid
+        callback_query = CallbackQuery(
+            id='1',
+            from_user=None,
+            chat_instance=None,
+            data=callback_data,
+        )
+        cdp.bot.callback_data_cache.process_callback_query(callback_query)
+
+        try:
+            assert len(cdp.bot.callback_data_cache.persistence_data[0]) == 1
+            assert list(cdp.bot.callback_data_cache.persistence_data[1]) == ['1']
+
+            callback_context.drop_callback_data(callback_query)
+            assert cdp.bot.callback_data_cache.persistence_data == ([], {})
+        finally:
+            cdp.bot.callback_data_cache.clear_callback_data()
+            cdp.bot.callback_data_cache.clear_callback_queries()

tests/test_callbackdatacache.py

@@ -0,0 +1,387 @@
+#!/usr/bin/env python
+#
+# A library that provides a Python interface to the Telegram Bot API
+# Copyright (C) 2015-2021
+# Leandro Toledo de Souza <devs@python-telegram-bot.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser Public License for more details.
+#
+# You should have received a copy of the GNU Lesser Public License
+# along with this program.  If not, see [http://www.gnu.org/licenses/].
+import time
+from copy import deepcopy
+from datetime import datetime
+from uuid import uuid4
+
+import pytest
+import pytz
+
+from telegram import InlineKeyboardButton, InlineKeyboardMarkup, CallbackQuery, Message, User
+from telegram.ext.callbackdatacache import (
+    CallbackDataCache,
+    _KeyboardData,
+    InvalidCallbackData,
+)
+
+
+@pytest.fixture(scope='function')
+def callback_data_cache(bot):
+    return CallbackDataCache(bot)
+
+
+class TestInvalidCallbackData:
+    def test_slot_behaviour(self, mro_slots, recwarn):
+        invalid_callback_data = InvalidCallbackData()
+        for attr in invalid_callback_data.__slots__:
+            assert getattr(invalid_callback_data, attr, 'err') != 'err', f"got extra slot '{attr}'"
+        assert len(mro_slots(invalid_callback_data)) == len(
+            set(mro_slots(invalid_callback_data))
+        ), "duplicate slot"
+        with pytest.raises(AttributeError):
+            invalid_callback_data.custom
+
+
+class TestKeyboardData:
+    def test_slot_behaviour(self, mro_slots):
+        keyboard_data = _KeyboardData('uuid')
+        for attr in keyboard_data.__slots__:
+            assert getattr(keyboard_data, attr, 'err') != 'err', f"got extra slot '{attr}'"
+        assert len(mro_slots(keyboard_data)) == len(
+            set(mro_slots(keyboard_data))
+        ), "duplicate slot"
+        with pytest.raises(AttributeError):
+            keyboard_data.custom = 42
+
+
+class TestCallbackDataCache:
+    def test_slot_behaviour(self, callback_data_cache, mro_slots):
+        for attr in callback_data_cache.__slots__:
+            attr = (
+                f"_CallbackDataCache{attr}"
+                if attr.startswith('__') and not attr.endswith('__')
+                else attr
+            )
+            assert getattr(callback_data_cache, attr, 'err') != 'err', f"got extra slot '{attr}'"
+        assert len(mro_slots(callback_data_cache)) == len(
+            set(mro_slots(callback_data_cache))
+        ), "duplicate slot"
+        with pytest.raises(AttributeError):
+            callback_data_cache.custom = 42
+
+    @pytest.mark.parametrize('maxsize', [1, 5, 2048])
+    def test_init_maxsize(self, maxsize, bot):
+        assert CallbackDataCache(bot).maxsize == 1024
+        cdc = CallbackDataCache(bot, maxsize=maxsize)
+        assert cdc.maxsize == maxsize
+        assert cdc.bot is bot
+
+    def test_init_and_access__persistent_data(self, bot):
+        keyboard_data = _KeyboardData('123', 456, {'button': 678})
+        persistent_data = ([keyboard_data.to_tuple()], {'id': '123'})
+        cdc = CallbackDataCache(bot, persistent_data=persistent_data)
+
+        assert cdc.maxsize == 1024
+        assert dict(cdc._callback_queries) == {'id': '123'}
+        assert list(cdc._keyboard_data.keys()) == ['123']
+        assert cdc._keyboard_data['123'].keyboard_uuid == '123'
+        assert cdc._keyboard_data['123'].access_time == 456
+        assert cdc._keyboard_data['123'].button_data == {'button': 678}
+
+        assert cdc.persistence_data == persistent_data
+
+    def test_process_keyboard(self, callback_data_cache):
+        changing_button_1 = InlineKeyboardButton('changing', callback_data='some data 1')
+        changing_button_2 = InlineKeyboardButton('changing', callback_data='some data 2')
+        non_changing_button = InlineKeyboardButton('non-changing', url='https://ptb.org')
+        reply_markup = InlineKeyboardMarkup.from_row(
+            [non_changing_button, changing_button_1, changing_button_2]
+        )
+
+        out = callback_data_cache.process_keyboard(reply_markup)
+        assert out.inline_keyboard[0][0] is non_changing_button
+        assert out.inline_keyboard[0][1] != changing_button_1
+        assert out.inline_keyboard[0][2] != changing_button_2
+
+        keyboard_1, button_1 = callback_data_cache.extract_uuids(
+            out.inline_keyboard[0][1].callback_data
+        )
+        keyboard_2, button_2 = callback_data_cache.extract_uuids(
+            out.inline_keyboard[0][2].callback_data
+        )
+        assert keyboard_1 == keyboard_2
+        assert (
+            callback_data_cache._keyboard_data[keyboard_1].button_data[button_1] == 'some data 1'
+        )
+        assert (
+            callback_data_cache._keyboard_data[keyboard_2].button_data[button_2] == 'some data 2'
+        )
+
+    def test_process_keyboard_no_changing_button(self, callback_data_cache):
+        reply_markup = InlineKeyboardMarkup.from_button(
+            InlineKeyboardButton('non-changing', url='https://ptb.org')
+        )
+        assert callback_data_cache.process_keyboard(reply_markup) is reply_markup
+
+    def test_process_keyboard_full(self, bot):
+        cdc = CallbackDataCache(bot, maxsize=1)
+        changing_button_1 = InlineKeyboardButton('changing', callback_data='some data 1')
+        changing_button_2 = InlineKeyboardButton('changing', callback_data='some data 2')
+        non_changing_button = InlineKeyboardButton('non-changing', url='https://ptb.org')
+        reply_markup = InlineKeyboardMarkup.from_row(
+            [non_changing_button, changing_button_1, changing_button_2]
+        )
+
+        out1 = cdc.process_keyboard(reply_markup)
+        assert len(cdc.persistence_data[0]) == 1
+        out2 = cdc.process_keyboard(reply_markup)
+        assert len(cdc.persistence_data[0]) == 1
+
+        keyboard_1, button_1 = cdc.extract_uuids(out1.inline_keyboard[0][1].callback_data)
+        keyboard_2, button_2 = cdc.extract_uuids(out2.inline_keyboard[0][2].callback_data)
+        assert cdc.persistence_data[0][0][0] != keyboard_1
+        assert cdc.persistence_data[0][0][0] == keyboard_2
+
+    @pytest.mark.parametrize('data', [True, False])
+    @pytest.mark.parametrize('message', [True, False])
+    @pytest.mark.parametrize('invalid', [True, False])
+    def test_process_callback_query(self, callback_data_cache, data, message, invalid):
+        """This also tests large parts of process_message"""
+        changing_button_1 = InlineKeyboardButton('changing', callback_data='some data 1')
+        changing_button_2 = InlineKeyboardButton('changing', callback_data='some data 2')
+        non_changing_button = InlineKeyboardButton('non-changing', url='https://ptb.org')
+        reply_markup = InlineKeyboardMarkup.from_row(
+            [non_changing_button, changing_button_1, changing_button_2]
+        )
+
+        out = callback_data_cache.process_keyboard(reply_markup)
+        if invalid:
+            callback_data_cache.clear_callback_data()
+
+        effective_message = Message(message_id=1, date=None, chat=None, reply_markup=out)
+        effective_message.reply_to_message = deepcopy(effective_message)
+        effective_message.pinned_message = deepcopy(effective_message)
+        cq_id = uuid4().hex
+        callback_query = CallbackQuery(
+            cq_id,
+            from_user=None,
+            chat_instance=None,
+            # not all CallbackQueries have callback_data
+            data=out.inline_keyboard[0][1].callback_data if data else None,
+            # CallbackQueries from inline messages don't have the message attached, so we test that
+            message=effective_message if message else None,
+        )
+        callback_data_cache.process_callback_query(callback_query)
+
+        if not invalid:
+            if data:
+                assert callback_query.data == 'some data 1'
+                # make sure that we stored the mapping CallbackQuery.id -> keyboard_uuid correctly
+                assert len(callback_data_cache._keyboard_data) == 1
+                assert (
+                    callback_data_cache._callback_queries[cq_id]
+                    == list(callback_data_cache._keyboard_data.keys())[0]
+                )
+            else:
+                assert callback_query.data is None
+            if message:
+                for msg in (
+                    callback_query.message,
+                    callback_query.message.reply_to_message,
+                    callback_query.message.pinned_message,
+                ):
+                    assert msg.reply_markup == reply_markup
+        else:
+            if data:
+                assert isinstance(callback_query.data, InvalidCallbackData)
+            else:
+                assert callback_query.data is None
+            if message:
+                for msg in (
+                    callback_query.message,
+                    callback_query.message.reply_to_message,
+                    callback_query.message.pinned_message,
+                ):
+                    assert isinstance(
+                        msg.reply_markup.inline_keyboard[0][1].callback_data,
+                        InvalidCallbackData,
+                    )
+                    assert isinstance(
+                        msg.reply_markup.inline_keyboard[0][2].callback_data,
+                        InvalidCallbackData,
+                    )
+
+    @pytest.mark.parametrize('pass_from_user', [True, False])
+    @pytest.mark.parametrize('pass_via_bot', [True, False])
+    def test_process_message_wrong_sender(self, pass_from_user, pass_via_bot, callback_data_cache):
+        reply_markup = InlineKeyboardMarkup.from_button(
+            InlineKeyboardButton('test', callback_data='callback_data')
+        )
+        user = User(1, 'first', False)
+        message = Message(
+            1,
+            None,
+            None,
+            from_user=user if pass_from_user else None,
+            via_bot=user if pass_via_bot else None,
+            reply_markup=reply_markup,
+        )
+        callback_data_cache.process_message(message)
+        if pass_from_user or pass_via_bot:
+            # Here we can determine that the message is not from our bot, so no replacing
+            assert message.reply_markup.inline_keyboard[0][0].callback_data == 'callback_data'
+        else:
+            # Here we have no chance to know, so InvalidCallbackData
+            assert isinstance(
+                message.reply_markup.inline_keyboard[0][0].callback_data, InvalidCallbackData
+            )
+
+    @pytest.mark.parametrize('pass_from_user', [True, False])
+    def test_process_message_inline_mode(self, pass_from_user, callback_data_cache):
+        """Check that via_bot tells us correctly that our bot sent the message, even if
+        from_user is not our bot."""
+        reply_markup = InlineKeyboardMarkup.from_button(
+            InlineKeyboardButton('test', callback_data='callback_data')
+        )
+        user = User(1, 'first', False)
+        message = Message(
+            1,
+            None,
+            None,
+            from_user=user if pass_from_user else None,
+            via_bot=callback_data_cache.bot.bot,
+            reply_markup=callback_data_cache.process_keyboard(reply_markup),
+        )
+        callback_data_cache.process_message(message)
+        # Here we can determine that the message is not from our bot, so no replacing
+        assert message.reply_markup.inline_keyboard[0][0].callback_data == 'callback_data'
+
+    def test_process_message_no_reply_markup(self, callback_data_cache):
+        message = Message(1, None, None)
+        callback_data_cache.process_message(message)
+        assert message.reply_markup is None
+
+    def test_drop_data(self, callback_data_cache):
+        changing_button_1 = InlineKeyboardButton('changing', callback_data='some data 1')
+        changing_button_2 = InlineKeyboardButton('changing', callback_data='some data 2')
+        reply_markup = InlineKeyboardMarkup.from_row([changing_button_1, changing_button_2])
+
+        out = callback_data_cache.process_keyboard(reply_markup)
+        callback_query = CallbackQuery(
+            '1',
+            from_user=None,
+            chat_instance=None,
+            data=out.inline_keyboard[0][1].callback_data,
+        )
+        callback_data_cache.process_callback_query(callback_query)
+
+        assert len(callback_data_cache.persistence_data[1]) == 1
+        assert len(callback_data_cache.persistence_data[0]) == 1
+
+        callback_data_cache.drop_data(callback_query)
+        assert len(callback_data_cache.persistence_data[1]) == 0
+        assert len(callback_data_cache.persistence_data[0]) == 0
+
+    def test_drop_data_missing_data(self, callback_data_cache):
+        changing_button_1 = InlineKeyboardButton('changing', callback_data='some data 1')
+        changing_button_2 = InlineKeyboardButton('changing', callback_data='some data 2')
+        reply_markup = InlineKeyboardMarkup.from_row([changing_button_1, changing_button_2])
+
+        out = callback_data_cache.process_keyboard(reply_markup)
+        callback_query = CallbackQuery(
+            '1',
+            from_user=None,
+            chat_instance=None,
+            data=out.inline_keyboard[0][1].callback_data,
+        )
+
+        with pytest.raises(KeyError, match='CallbackQuery was not found in cache.'):
+            callback_data_cache.drop_data(callback_query)
+
+        callback_data_cache.process_callback_query(callback_query)
+        callback_data_cache.clear_callback_data()
+        callback_data_cache.drop_data(callback_query)
+        assert callback_data_cache.persistence_data == ([], {})
+
+    @pytest.mark.parametrize('method', ('callback_data', 'callback_queries'))
+    def test_clear_all(self, callback_data_cache, method):
+        changing_button_1 = InlineKeyboardButton('changing', callback_data='some data 1')
+        changing_button_2 = InlineKeyboardButton('changing', callback_data='some data 2')
+        reply_markup = InlineKeyboardMarkup.from_row([changing_button_1, changing_button_2])
+
+        for i in range(100):
+            out = callback_data_cache.process_keyboard(reply_markup)
+            callback_query = CallbackQuery(
+                str(i),
+                from_user=None,
+                chat_instance=None,
+                data=out.inline_keyboard[0][1].callback_data,
+            )
+            callback_data_cache.process_callback_query(callback_query)
+
+        if method == 'callback_data':
+            callback_data_cache.clear_callback_data()
+            # callback_data was cleared, callback_queries weren't
+            assert len(callback_data_cache.persistence_data[0]) == 0
+            assert len(callback_data_cache.persistence_data[1]) == 100
+        else:
+            callback_data_cache.clear_callback_queries()
+            # callback_queries were cleared, callback_data wasn't
+            assert len(callback_data_cache.persistence_data[0]) == 100
+            assert len(callback_data_cache.persistence_data[1]) == 0
+
+    @pytest.mark.parametrize('time_method', ['time', 'datetime', 'defaults'])
+    def test_clear_cutoff(self, callback_data_cache, time_method, tz_bot):
+        # Fill the cache with some fake data
+        for i in range(50):
+            reply_markup = InlineKeyboardMarkup.from_button(
+                InlineKeyboardButton('changing', callback_data=str(i))
+            )
+            out = callback_data_cache.process_keyboard(reply_markup)
+            callback_query = CallbackQuery(
+                str(i),
+                from_user=None,
+                chat_instance=None,
+                data=out.inline_keyboard[0][0].callback_data,
+            )
+            callback_data_cache.process_callback_query(callback_query)
+
+        # sleep a bit before saving the time cutoff, to make test more reliable
+        time.sleep(0.1)
+        if time_method == 'time':
+            cutoff = time.time()
+        elif time_method == 'datetime':
+            cutoff = datetime.now(pytz.utc)
+        else:
+            cutoff = datetime.now(tz_bot.defaults.tzinfo).replace(tzinfo=None)
+            callback_data_cache.bot = tz_bot
+        time.sleep(0.1)
+
+        # more fake data after the time cutoff
+        for i in range(50, 100):
+            reply_markup = InlineKeyboardMarkup.from_button(
+                InlineKeyboardButton('changing', callback_data=str(i))
+            )
+            out = callback_data_cache.process_keyboard(reply_markup)
+            callback_query = CallbackQuery(
+                str(i),
+                from_user=None,
+                chat_instance=None,
+                data=out.inline_keyboard[0][0].callback_data,
+            )
+            callback_data_cache.process_callback_query(callback_query)
+
+        callback_data_cache.clear_callback_data(time_cutoff=cutoff)
+        assert len(callback_data_cache.persistence_data[0]) == 50
+        assert len(callback_data_cache.persistence_data[1]) == 100
+        callback_data = [
+            list(data[2].values())[0] for data in callback_data_cache.persistence_data[0]
+        ]
+        assert callback_data == list(str(i) for i in range(50, 100))

tests/test_callbackqueryhandler.py

@@ -144,10 +144,42 @@ class TestCallbackQueryHandler:
         callback_query.callback_query.data = 'nothing here'
         assert not handler.check_update(callback_query)
 
-        callback_query.callback_query.data = False
+        callback_query.callback_query.data = None
         callback_query.callback_query.game_short_name = "this is a short game name"
         assert not handler.check_update(callback_query)
 
+    def test_with_callable_pattern(self, callback_query):
+        class CallbackData:
+            pass
+
+        def pattern(callback_data):
+            return isinstance(callback_data, CallbackData)
+
+        handler = CallbackQueryHandler(self.callback_basic, pattern=pattern)
+
+        callback_query.callback_query.data = CallbackData()
+        assert handler.check_update(callback_query)
+        callback_query.callback_query.data = 'callback_data'
+        assert not handler.check_update(callback_query)
+
+    def test_with_type_pattern(self, callback_query):
+        class CallbackData:
+            pass
+
+        handler = CallbackQueryHandler(self.callback_basic, pattern=CallbackData)
+
+        callback_query.callback_query.data = CallbackData()
+        assert handler.check_update(callback_query)
+        callback_query.callback_query.data = 'callback_data'
+        assert not handler.check_update(callback_query)
+
+        handler = CallbackQueryHandler(self.callback_basic, pattern=bool)
+
+        callback_query.callback_query.data = False
+        assert handler.check_update(callback_query)
+        callback_query.callback_query.data = 'callback_data'
+        assert not handler.check_update(callback_query)
+
     def test_with_passing_group_dict(self, dp, callback_query):
         handler = CallbackQueryHandler(
             self.callback_group, pattern='(?P<begin>.*)est(?P<end>.*)', pass_groups=True
@@ -243,3 +275,18 @@ class TestCallbackQueryHandler:
 
         cdp.process_update(callback_query)
         assert self.test_flag
+
+    def test_context_callable_pattern(self, cdp, callback_query):
+        class CallbackData:
+            pass
+
+        def pattern(callback_data):
+            return isinstance(callback_data, CallbackData)
+
+        def callback(update, context):
+            assert context.matches is None
+
+        handler = CallbackQueryHandler(callback, pattern=pattern)
+        cdp.add_handler(handler)
+
+        cdp.process_update(callback_query)

tests/test_dispatcher.py

@@ -177,6 +177,7 @@ class TestDispatcher:
                 self.store_user_data = False
                 self.store_chat_data = False
                 self.store_bot_data = False
+                self.store_callback_data = False
 
         with pytest.raises(
             TypeError, match='persistence must be based on telegram.ext.BasePersistence'
@@ -599,6 +600,13 @@ class TestDispatcher:
                 self.store_user_data = True
                 self.store_chat_data = True
                 self.store_bot_data = True
+                self.store_callback_data = True
+
+            def get_callback_data(self):
+                return None
+
+            def update_callback_data(self, data):
+                raise Exception
 
             def get_bot_data(self):
                 return {}
@@ -652,7 +660,7 @@ class TestDispatcher:
         dp.add_handler(CommandHandler('start', start1))
         dp.add_error_handler(error)
         dp.process_update(update)
-        assert increment == ["error", "error", "error"]
+        assert increment == ["error", "error", "error", "error"]
 
     def test_flow_stop_in_error_handler(self, dp, bot):
         passed = []
@@ -724,10 +732,14 @@ class TestDispatcher:
                 self.store_user_data = True
                 self.store_chat_data = True
                 self.store_bot_data = True
+                self.store_callback_data = True
 
             def update(self, data):
                 raise Exception('PersistenceError')
 
+            def update_callback_data(self, data):
+                self.update(data)
+
             def update_bot_data(self, data):
                 self.update(data)
 
@@ -746,6 +758,9 @@ class TestDispatcher:
             def get_user_data(self):
                 pass
 
+            def get_callback_data(self):
+                pass
+
             def get_conversations(self, name):
                 pass
 

tests/test_error.py

@@ -32,6 +32,7 @@ from telegram.error import (
     RetryAfter,
     Conflict,
 )
+from telegram.ext.callbackdatacache import InvalidCallbackData
 
 
 class TestErrors:
@@ -112,6 +113,7 @@ class TestErrors:
             (RetryAfter(12), ["message", "retry_after"]),
             (Conflict("test message"), ["message"]),
             (TelegramDecryptionError("test message"), ["message"]),
+            (InvalidCallbackData('test data'), ['callback_data']),
         ],
     )
     def test_errors_pickling(self, exception, attributes):
@@ -146,6 +148,7 @@ class TestErrors:
                     RetryAfter,
                     Conflict,
                     TelegramDecryptionError,
+                    InvalidCallbackData,
                 },
                 NetworkError: {BadRequest, TimedOut},
             }

tests/test_inlinekeyboardbutton.py

@@ -134,3 +134,26 @@ class TestInlineKeyboardButton:
 
         assert a != f
         assert hash(a) != hash(f)
+
+    @pytest.mark.parametrize('callback_data', ['foo', 1, ('da', 'ta'), object()])
+    def test_update_callback_data(self, callback_data):
+        button = InlineKeyboardButton(text='test', callback_data='data')
+        button_b = InlineKeyboardButton(text='test', callback_data='data')
+
+        assert button == button_b
+        assert hash(button) == hash(button_b)
+
+        button.update_callback_data(callback_data)
+        assert button.callback_data is callback_data
+        assert button != button_b
+        assert hash(button) != hash(button_b)
+
+        button_b.update_callback_data(callback_data)
+        assert button_b.callback_data is callback_data
+        assert button == button_b
+        assert hash(button) == hash(button_b)
+
+        button.update_callback_data({})
+        assert button.callback_data == {}
+        with pytest.raises(TypeError, match='unhashable'):
+            hash(button)

tests/test_invoice.py

@@ -42,7 +42,7 @@ class TestInvoice:
     description = 'description'
     start_parameter = 'start_parameter'
     currency = 'EUR'
-    total_amount = sum([p.amount for p in prices])
+    total_amount = sum(p.amount for p in prices)
     max_tip_amount = 42
     suggested_tip_amounts = [13, 42]
 

tests/test_message.py

@@ -164,7 +164,6 @@ def message(bot):
                 ]
             },
         },
-        {'quote': True},
         {'dice': Dice(4, '🎲')},
         {'via_bot': User(9, 'A_Bot', True)},
         {
@@ -222,7 +221,6 @@ def message(bot):
         'passport_data',
         'poll',
         'reply_markup',
-        'default_quote',
         'dice',
         'via_bot',
         'proximity_alert_triggered',

tests/test_slots.py

@@ -16,9 +16,9 @@
 #
 # You should have received a copy of the GNU Lesser Public License
 # along with this program.  If not, see [http://www.gnu.org/licenses/].
-import os
 import importlib
 import importlib.util
+import os
 from glob import iglob
 
 import inspect
@@ -32,6 +32,9 @@ excluded = {
     'telegram.deprecate',
     'TelegramDecryptionError',
     'ContextTypes',
+    'CallbackDataCache',
+    'InvalidCallbackData',
+    '_KeyboardData',
 }  # These modules/classes intentionally don't have __dict__.
 
 

tests/test_updater.py

@@ -36,9 +36,25 @@ from urllib.error import HTTPError
 
 import pytest
 
-from telegram import TelegramError, Message, User, Chat, Update, Bot
+from telegram import (
+    TelegramError,
+    Message,
+    User,
+    Chat,
+    Update,
+    Bot,
+    InlineKeyboardMarkup,
+    InlineKeyboardButton,
+)
 from telegram.error import Unauthorized, InvalidToken, TimedOut, RetryAfter
-from telegram.ext import Updater, Dispatcher, DictPersistence, Defaults
+from telegram.ext import (
+    Updater,
+    Dispatcher,
+    DictPersistence,
+    Defaults,
+    InvalidCallbackData,
+    ExtBot,
+)
 from telegram.utils.deprecate import TelegramDeprecationWarning
 from telegram.ext.utils.webhookhandler import WebhookServer
 
@@ -110,6 +126,11 @@ class TestUpdater:
         self.received = update.message.text
         self.cb_handler_called.set()
 
+    def test_warn_arbitrary_callback_data(self, bot, recwarn):
+        Updater(bot=bot, arbitrary_callback_data=True)
+        assert len(recwarn) == 1
+        assert 'Passing arbitrary_callback_data to an Updater' in str(recwarn[0].message)
+
     @pytest.mark.parametrize(
         ('error',),
         argvalues=[(TelegramError('Test Error 2'),), (Unauthorized('Test Unauthorized'),)],
@@ -185,7 +206,15 @@ class TestUpdater:
         event.wait()
         assert self.err_handler_called.wait(0.5) is not True
 
-    def test_webhook(self, monkeypatch, updater):
+    @pytest.mark.parametrize('ext_bot', [True, False])
+    def test_webhook(self, monkeypatch, updater, ext_bot):
+        # Testing with both ExtBot and Bot to make sure any logic in WebhookHandler
+        # that depends on this distinction works
+        if ext_bot and not isinstance(updater.bot, ExtBot):
+            updater.bot = ExtBot(updater.bot.token)
+        if not ext_bot and not type(updater.bot) is Bot:
+            updater.bot = Bot(updater.bot.token)
+
         q = Queue()
         monkeypatch.setattr(updater.bot, 'set_webhook', lambda *args, **kwargs: True)
         monkeypatch.setattr(updater.bot, 'delete_webhook', lambda *args, **kwargs: True)
@@ -226,6 +255,59 @@ class TestUpdater:
             assert not updater.httpd.is_running
             updater.stop()
 
+    @pytest.mark.parametrize('invalid_data', [True, False])
+    def test_webhook_arbitrary_callback_data(self, monkeypatch, updater, invalid_data):
+        """Here we only test one simple setup. telegram.ext.ExtBot.insert_callback_data is tested
+        extensively in test_bot.py in conjunction with get_updates."""
+        updater.bot.arbitrary_callback_data = True
+        try:
+            q = Queue()
+            monkeypatch.setattr(updater.bot, 'set_webhook', lambda *args, **kwargs: True)
+            monkeypatch.setattr(updater.bot, 'delete_webhook', lambda *args, **kwargs: True)
+            monkeypatch.setattr('telegram.ext.Dispatcher.process_update', lambda _, u: q.put(u))
+
+            ip = '127.0.0.1'
+            port = randrange(1024, 49152)  # Select random port
+            updater.start_webhook(ip, port, url_path='TOKEN')
+            sleep(0.2)
+            try:
+                # Now, we send an update to the server via urlopen
+                reply_markup = InlineKeyboardMarkup.from_button(
+                    InlineKeyboardButton(text='text', callback_data='callback_data')
+                )
+                if not invalid_data:
+                    reply_markup = updater.bot.callback_data_cache.process_keyboard(reply_markup)
+
+                message = Message(
+                    1,
+                    None,
+                    None,
+                    reply_markup=reply_markup,
+                )
+                update = Update(1, message=message)
+                self._send_webhook_msg(ip, port, update.to_json(), 'TOKEN')
+                sleep(0.2)
+                received_update = q.get(False)
+                assert received_update == update
+
+                button = received_update.message.reply_markup.inline_keyboard[0][0]
+                if invalid_data:
+                    assert isinstance(button.callback_data, InvalidCallbackData)
+                else:
+                    assert button.callback_data == 'callback_data'
+
+                # Test multiple shutdown() calls
+                updater.httpd.shutdown()
+            finally:
+                updater.httpd.shutdown()
+                sleep(0.2)
+                assert not updater.httpd.is_running
+                updater.stop()
+        finally:
+            updater.bot.arbitrary_callback_data = False
+            updater.bot.callback_data_cache.clear_callback_data()
+            updater.bot.callback_data_cache.clear_callback_queries()
+
     def test_start_webhook_no_warning_or_error_logs(self, caplog, updater, monkeypatch):
         monkeypatch.setattr(updater.bot, 'set_webhook', lambda *args, **kwargs: True)
         monkeypatch.setattr(updater.bot, 'delete_webhook', lambda *args, **kwargs: True)
@@ -590,25 +672,25 @@ class TestUpdater:
         with pytest.raises(ValueError):
             Updater(bot=bot, private_key=b'key')
 
-    def test_mutual_exclude_bot_dispatcher(self):
-        dispatcher = Dispatcher(None, None)
+    def test_mutual_exclude_bot_dispatcher(self, bot):
+        dispatcher = Dispatcher(bot, None)
         bot = Bot('123:zyxw')
         with pytest.raises(ValueError):
             Updater(bot=bot, dispatcher=dispatcher)
 
-    def test_mutual_exclude_persistence_dispatcher(self):
-        dispatcher = Dispatcher(None, None)
+    def test_mutual_exclude_persistence_dispatcher(self, bot):
+        dispatcher = Dispatcher(bot, None)
         persistence = DictPersistence()
         with pytest.raises(ValueError):
             Updater(dispatcher=dispatcher, persistence=persistence)
 
-    def test_mutual_exclude_workers_dispatcher(self):
-        dispatcher = Dispatcher(None, None)
+    def test_mutual_exclude_workers_dispatcher(self, bot):
+        dispatcher = Dispatcher(bot, None)
         with pytest.raises(ValueError):
             Updater(dispatcher=dispatcher, workers=8)
 
-    def test_mutual_exclude_use_context_dispatcher(self):
-        dispatcher = Dispatcher(None, None)
+    def test_mutual_exclude_use_context_dispatcher(self, bot):
+        dispatcher = Dispatcher(bot, None)
         use_context = not dispatcher.use_context
         with pytest.raises(ValueError):
             Updater(dispatcher=dispatcher, use_context=use_context)