Add Python 3.12 Beta to the Test Matrix (#3751)

.github/workflows/test.yml

@@ -16,7 +16,7 @@ jobs:
     runs-on: ${{matrix.os}}
     strategy:
       matrix:
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12.0-beta.3']
         os: [ubuntu-latest, windows-latest, macos-latest]
       fail-fast: False
     steps:

.github/workflows/type_completeness.yml

@@ -22,7 +22,7 @@ jobs:
           cache-dependency-path: '**/requirements*.txt'
       - name: Install Pyright
         run: |
-          python -W ignore -m pip install pyright~=1.1.291
+          python -W ignore -m pip install pyright~=1.1.316
       - name: Get PR Completeness
         # Must run before base completeness, as base completeness will checkout the base branch
         # And we can't go back to the PR branch after that in case the PR is coming from a fork

.pre-commit-config.yaml

@@ -35,7 +35,7 @@ repos:
           - aiolimiter~=1.1.0
           - . # this basically does `pip install -e .`
 -   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.3.0
+    rev: v1.4.1
     hooks:
     -   id: mypy
         name: mypy-ptb

telegram/ext/_application.py

@@ -23,6 +23,7 @@ import inspect
 import itertools
 import platform
 import signal
+import sys
 from collections import defaultdict
 from copy import deepcopy
 from pathlib import Path
@@ -64,6 +65,7 @@ from telegram.ext._updater import Updater
 from telegram.ext._utils.stack import was_called_by
 from telegram.ext._utils.trackingdict import TrackingDict
 from telegram.ext._utils.types import BD, BT, CCT, CD, JQ, RT, UD, ConversationKey, HandlerCallback
+from telegram.warnings import PTBDeprecationWarning
 
 if TYPE_CHECKING:
     from telegram import Message
@@ -78,6 +80,17 @@ _AppType = TypeVar("_AppType", bound="Application")  # pylint: disable=invalid-n
 _STOP_SIGNAL = object()
 _DEFAULT_0 = DefaultValue(0)
 
+
+# Since python 3.12, the coroutine passed to create_task should not be an (async) generator. Remove
+# this check when we drop support for python 3.11.
+if sys.version_info >= (3, 12):
+    _CoroType = Awaitable[RT]
+else:
+    _CoroType = Union[Generator["asyncio.Future[object]", None, RT], Awaitable[RT]]
+
+_ErrorCoroType = Optional[_CoroType[RT]]
+
+
 _LOGGER = get_logger(__name__)
 
 
@@ -950,7 +963,7 @@ class Application(Generic[BT, CCT, UD, CD, BD, JQ], AsyncContextManager["Applica
 
     def create_task(
         self,
-        coroutine: Union[Generator[Optional["asyncio.Future[object]"], None, RT], Awaitable[RT]],
+        coroutine: _CoroType[RT],
         update: Optional[object] = None,
         *,
         name: Optional[str] = None,
@@ -971,6 +984,8 @@ class Application(Generic[BT, CCT, UD, CD, BD, JQ], AsyncContextManager["Applica
 
                 .. versionchanged:: 20.2
                     Accepts :class:`asyncio.Future` and generator-based coroutine functions.
+                .. deprecated:: NEXT.VERSION
+                    Since Python 3.12, generator-based coroutine functions are no longer accepted.
             update (:obj:`object`, optional): If set, will be passed to :meth:`process_error`
                 as additional information for the error handlers. Moreover, the corresponding
                 :attr:`chat_data` and :attr:`user_data` entries will be updated in the next run of
@@ -988,7 +1003,7 @@ class Application(Generic[BT, CCT, UD, CD, BD, JQ], AsyncContextManager["Applica
 
     def __create_task(
         self,
-        coroutine: Union[Generator[Optional["asyncio.Future[object]"], None, RT], Awaitable[RT]],
+        coroutine: _CoroType[RT],
         update: Optional[object] = None,
         is_error_handler: bool = False,
         name: Optional[str] = None,
@@ -1025,14 +1040,22 @@ class Application(Generic[BT, CCT, UD, CD, BD, JQ], AsyncContextManager["Applica
 
     async def __create_task_callback(
         self,
-        coroutine: Union[Generator[Optional["asyncio.Future[object]"], None, RT], Awaitable[RT]],
+        coroutine: _CoroType[RT],
         update: Optional[object] = None,
         is_error_handler: bool = False,
     ) -> RT:
         try:
-            if isinstance(coroutine, Generator):
+            # Generator-based coroutines are not supported in Python 3.12+
+            if sys.version_info < (3, 12) and isinstance(coroutine, Generator):
+                warn(
+                    "Generator-based coroutines are deprecated in create_task and will not work"
+                    " in Python 3.12+",
+                    category=PTBDeprecationWarning,
+                )
                 return await asyncio.create_task(coroutine)
-            return await coroutine
+            # If user uses generator in python 3.12+, Exception will happen and we cannot do
+            # anything about it. (hence the type ignore if mypy is run on python 3.12-)
+            return await coroutine  # type: ignore
         except Exception as exception:
             if isinstance(exception, ApplicationHandlerStop):
                 warn(
@@ -1648,9 +1671,7 @@ class Application(Generic[BT, CCT, UD, CD, BD, JQ], AsyncContextManager["Applica
         update: Optional[object],
         error: Exception,
         job: Optional["Job[CCT]"] = None,
-        coroutine: Optional[
-            Union[Generator[Optional["asyncio.Future[object]"], None, RT], Awaitable[RT]]
-        ] = None,
+        coroutine: _ErrorCoroType[RT] = None,  # noqa: RUF013
     ) -> bool:
         """Processes an error by passing it to all error handlers registered with
         :meth:`add_error_handler`. If one of the error handlers raises

tests/ext/test_application.py

@@ -24,6 +24,7 @@ import logging
 import os
 import platform
 import signal
+import sys
 import threading
 import time
 from collections import defaultdict
@@ -54,7 +55,7 @@ from telegram.ext import (
     Updater,
     filters,
 )
-from telegram.warnings import PTBUserWarning
+from telegram.warnings import PTBDeprecationWarning, PTBUserWarning
 from tests.auxil.asyncio_helpers import call_after
 from tests.auxil.build_messages import make_message_update
 from tests.auxil.files import PROJECT_ROOT_PATH
@@ -1319,7 +1320,8 @@ class TestApplication:
         out = await app.create_task(asyncio.gather(callback()))
         assert out == [42]
 
-    async def test_create_task_awaiting_generator(self, app):
+    @pytest.mark.skipif(sys.version_info >= (3, 12), reason="generator coroutines are deprecated")
+    async def test_create_task_awaiting_generator(self, app, recwarn):
         event = asyncio.Event()
 
         def gen():
@@ -1328,6 +1330,9 @@ class TestApplication:
 
         await app.create_task(gen())
         assert event.is_set()
+        assert len(recwarn) == 2  # 1st warning is: tasks not being awaited when app isn't running
+        assert recwarn[1].category is PTBDeprecationWarning
+        assert "Generator-based coroutines are deprecated" in str(recwarn[1].message)
 
     async def test_no_update_processor(self, app):
         queue = asyncio.Queue()

tests/ext/test_basepersistence.py

@@ -22,6 +22,7 @@ import copy
 import enum
 import functools
 import logging
+import sys
 import time
 from pathlib import Path
 from typing import NamedTuple, Optional
@@ -377,15 +378,13 @@ class TestBasePersistence:
         assert persistence.store_data.callback_data == callback_data
 
     def test_abstract_methods(self):
+        methods = list(BasePersistence.__abstractmethods__)
+        methods.sort()
         with pytest.raises(
             TypeError,
-            match=(
-                "drop_chat_data, drop_user_data, flush, get_bot_data, get_callback_data, "
-                "get_chat_data, get_conversations, "
-                "get_user_data, refresh_bot_data, refresh_chat_data, "
-                "refresh_user_data, update_bot_data, update_callback_data, "
-                "update_chat_data, update_conversation, update_user_data"
-            ),
+            match=", ".join(methods)
+            if sys.version_info < (3, 12)
+            else ", ".join(f"'{i}'" for i in methods),
         ):
             BasePersistence()
 

tests/ext/test_picklepersistence.py

@@ -20,6 +20,7 @@ import datetime
 import gzip
 import os
 import pickle
+import sys
 from pathlib import Path
 
 import pytest
@@ -872,7 +873,8 @@ class TestPicklePersistence:
             "function was specified."
         )
         with Path("pickletest_chat_data").open("rb") as f, pytest.raises(
-            pickle.UnpicklingError, match=err_msg
+            pickle.UnpicklingError,
+            match=err_msg if sys.version_info < (3, 12) else err_msg.replace("\n", " "),
         ):
             pickle.load(f)