Python 3.8 support (#1614)

* github workflow: Add python-3.8

* workaround for tornado issue on win with py>=3.8

* add workaround to webhookhandler

* Try making codecov and codacy happy

* Fix stupid mistake

Co-authored-by: Hinrich Mahler <hinrich.mahler@freenet.de>

.github/workflows/test.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ${{matrix.os}}
     strategy:
       matrix:
-        python-version: [3.5, 3.6, 3.7]
+        python-version: [3.5, 3.6, 3.7, 3.8]
         os: [ubuntu-latest, windows-latest]
         include:
           - os: ubuntu-latest
@@ -51,7 +51,7 @@ jobs:
           exit ${global_exit}
         env:
           JOB_INDEX: ${{ strategy.job-index }}
-          BOTS: W3sidG9rZW4iOiAiNjk2MTg4NzMyOkFBR1Z3RUtmSEhsTmpzY3hFRE5LQXdraEdzdFpfa28xbUMwIiwgInBheW1lbnRfcHJvdmlkZXJfdG9rZW4iOiAiMjg0Njg1MDYzOlRFU1Q6WldGaU1UUmxNbVF5TnpNeSIsICJib3RfbmFtZSI6ICJQVEIgdGVzdHMgb24gVHJhdmlzIHVzaW5nIENQeXRob24gMi43IiwgImJvdF91c2VybmFtZSI6ICJAcHRiX3RyYXZpc19jcHl0aG9uXzI3X2JvdCJ9LCB7InRva2VuIjogIjY3MTQ2ODg4NjpBQUdQR2ZjaVJJQlVORmU4MjR1SVZkcTdKZTNfWW5BVE5HdyIsICJwYXltZW50X3Byb3ZpZGVyX3Rva2VuIjogIjI4NDY4NTA2MzpURVNUOlpHWXdPVGxrTXpNeE4yWTIiLCAiYm90X25hbWUiOiAiUFRCIHRlc3RzIG9uIFRyYXZpcyB1c2luZyBDUHl0aG9uIDMuNCIsICJib3RfdXNlcm5hbWUiOiAiQHB0Yl90cmF2aXNfY3B5dGhvbl8zNF9ib3QifSwgeyJ0b2tlbiI6ICI2MjkzMjY1Mzg6QUFGUnJaSnJCN29CM211ekdzR0pYVXZHRTVDUXpNNUNVNG8iLCAicGF5bWVudF9wcm92aWRlcl90b2tlbiI6ICIyODQ2ODUwNjM6VEVTVDpNbU01WVdKaFl6a3hNMlUxIiwgImJvdF9uYW1lIjogIlBUQiB0ZXN0cyBvbiBUcmF2aXMgdXNpbmcgQ1B5dGhvbiAzLjUiLCAiYm90X3VzZXJuYW1lIjogIkBwdGJfdHJhdmlzX2NweXRob25fMzVfYm90In0sIHsidG9rZW4iOiAiNjQwMjA4OTQzOkFBRmhCalFwOXFtM1JUeFN6VXBZekJRakNsZS1Kano1aGNrIiwgInBheW1lbnRfcHJvdmlkZXJfdG9rZW4iOiAiMjg0Njg1MDYzOlRFU1Q6WXpoa1pUZzFOamMxWXpWbCIsICJib3RfbmFtZSI6ICJQVEIgdGVzdHMgb24gVHJhdmlzIHVzaW5nIENQeXRob24gMy42IiwgImJvdF91c2VybmFtZSI6ICJAcHRiX3RyYXZpc19jcHl0aG9uXzM2X2JvdCJ9LCB7InRva2VuIjogIjY5NTEwNDA4ODpBQUhmenlsSU9qU0lJUy1lT25JMjB5MkUyMEhvZEhzZnotMCIsICJwYXltZW50X3Byb3ZpZGVyX3Rva2VuIjogIjI4NDY4NTA2MzpURVNUOk9HUTFNRGd3WmpJd1pqRmwiLCAiYm90X25hbWUiOiAiUFRCIHRlc3RzIG9uIFRyYXZpcyB1c2luZyBDUHl0aG9uIDMuNyIsICJib3RfdXNlcm5hbWUiOiAiQHB0Yl90cmF2aXNfY3B5dGhvbl8zN19ib3QifSwgeyJ0b2tlbiI6ICI2OTE0MjM1NTQ6QUFGOFdrakNaYm5IcVBfaTZHaFRZaXJGRWxackdhWU9oWDAiLCAicGF5bWVudF9wcm92aWRlcl90b2tlbiI6ICIyODQ2ODUwNjM6VEVTVDpZamM1TlRoaU1tUXlNV1ZoIiwgImJvdF9uYW1lIjogIlBUQiB0ZXN0cyBvbiBUcmF2aXMgdXNpbmcgUHlQeSAyLjciLCAiYm90X3VzZXJuYW1lIjogIkBwdGJfdHJhdmlzX3B5cHlfMjdfYm90In0sIHsidG9rZW4iOiAiNjg0MzM5OTg0OkFBRk1nRUVqcDAxcjVyQjAwN3lDZFZOc2c4QWxOc2FVLWNjIiwgInBheW1lbnRfcHJvdmlkZXJfdG9rZW4iOiAiMjg0Njg1MDYzOlRFU1Q6TVRBek1UWTNNR1V5TmpnMCIsICJib3RfbmFtZSI6ICJQVEIgdGVzdHMgb24gVHJhdmlzIHVzaW5nIFB5UHkgMy41IiwgImJvdF91c2VybmFtZSI6ICJAcHRiX3RyYXZpc19weXB5XzM1X2JvdCJ9LCB7InRva2VuIjogIjY5MDA5MTM0NzpBQUZMbVI1cEFCNVljcGVfbU9oN3pNNEpGQk9oMHozVDBUbyIsICJwYXltZW50X3Byb3ZpZGVyX3Rva2VuIjogIjI4NDY4NTA2MzpURVNUOlpEaGxOekU1TURrd1lXSmkiLCAiYm90X25hbWUiOiAiUFRCIHRlc3RzIG9uIEFwcFZleW9yIHVzaW5nIENQeXRob24gMy40IiwgImJvdF91c2VybmFtZSI6ICJAcHRiX2FwcHZleW9yX2NweXRob25fMzRfYm90In0sIHsidG9rZW4iOiAiNjk0MzA4MDUyOkFBRUIyX3NvbkNrNTVMWTlCRzlBTy1IOGp4aVBTNTVvb0JBIiwgInBheW1lbnRfcHJvdmlkZXJfdG9rZW4iOiAiMjg0Njg1MDYzOlRFU1Q6WW1aaVlXWm1NakpoWkdNeSIsICJib3RfbmFtZSI6ICJQVEIgdGVzdHMgb24gQXBwVmV5b3IgdXNpbmcgQ1B5dGhvbiAyLjciLCAiYm90X3VzZXJuYW1lIjogIkBwdGJfYXBwdmV5b3JfY3B5dGhvbl8yN19ib3QifV0=
+          BOTS: W3sidG9rZW4iOiAiNjk2MTg4NzMyOkFBR1Z3RUtmSEhsTmpzY3hFRE5LQXdraEdzdFpfa28xbUMwIiwgInBheW1lbnRfcHJvdmlkZXJfdG9rZW4iOiAiMjg0Njg1MDYzOlRFU1Q6WldGaU1UUmxNbVF5TnpNeSIsICJib3RfbmFtZSI6ICJQVEIgdGVzdHMgb24gVHJhdmlzIHVzaW5nIENQeXRob24gMi43IiwgImJvdF91c2VybmFtZSI6ICJAcHRiX3RyYXZpc19jcHl0aG9uXzI3X2JvdCJ9LCB7InRva2VuIjogIjY3MTQ2ODg4NjpBQUdQR2ZjaVJJQlVORmU4MjR1SVZkcTdKZTNfWW5BVE5HdyIsICJwYXltZW50X3Byb3ZpZGVyX3Rva2VuIjogIjI4NDY4NTA2MzpURVNUOlpHWXdPVGxrTXpNeE4yWTIiLCAiYm90X25hbWUiOiAiUFRCIHRlc3RzIG9uIFRyYXZpcyB1c2luZyBDUHl0aG9uIDMuNCIsICJib3RfdXNlcm5hbWUiOiAiQHB0Yl90cmF2aXNfY3B5dGhvbl8zNF9ib3QifSwgeyJ0b2tlbiI6ICI2MjkzMjY1Mzg6QUFGUnJaSnJCN29CM211ekdzR0pYVXZHRTVDUXpNNUNVNG8iLCAicGF5bWVudF9wcm92aWRlcl90b2tlbiI6ICIyODQ2ODUwNjM6VEVTVDpNbU01WVdKaFl6a3hNMlUxIiwgImJvdF9uYW1lIjogIlBUQiB0ZXN0cyBvbiBUcmF2aXMgdXNpbmcgQ1B5dGhvbiAzLjUiLCAiYm90X3VzZXJuYW1lIjogIkBwdGJfdHJhdmlzX2NweXRob25fMzVfYm90In0sIHsidG9rZW4iOiAiNjQwMjA4OTQzOkFBRmhCalFwOXFtM1JUeFN6VXBZekJRakNsZS1Kano1aGNrIiwgInBheW1lbnRfcHJvdmlkZXJfdG9rZW4iOiAiMjg0Njg1MDYzOlRFU1Q6WXpoa1pUZzFOamMxWXpWbCIsICJib3RfbmFtZSI6ICJQVEIgdGVzdHMgb24gVHJhdmlzIHVzaW5nIENQeXRob24gMy42IiwgImJvdF91c2VybmFtZSI6ICJAcHRiX3RyYXZpc19jcHl0aG9uXzM2X2JvdCJ9LCB7InRva2VuIjogIjY5NTEwNDA4ODpBQUhmenlsSU9qU0lJUy1lT25JMjB5MkUyMEhvZEhzZnotMCIsICJwYXltZW50X3Byb3ZpZGVyX3Rva2VuIjogIjI4NDY4NTA2MzpURVNUOk9HUTFNRGd3WmpJd1pqRmwiLCAiYm90X25hbWUiOiAiUFRCIHRlc3RzIG9uIFRyYXZpcyB1c2luZyBDUHl0aG9uIDMuNyIsICJib3RfdXNlcm5hbWUiOiAiQHB0Yl90cmF2aXNfY3B5dGhvbl8zN19ib3QifSwgeyJ0b2tlbiI6ICI2OTE0MjM1NTQ6QUFGOFdrakNaYm5IcVBfaTZHaFRZaXJGRWxackdhWU9oWDAiLCAicGF5bWVudF9wcm92aWRlcl90b2tlbiI6ICIyODQ2ODUwNjM6VEVTVDpZamM1TlRoaU1tUXlNV1ZoIiwgImJvdF9uYW1lIjogIlBUQiB0ZXN0cyBvbiBUcmF2aXMgdXNpbmcgUHlQeSAyLjciLCAiYm90X3VzZXJuYW1lIjogIkBwdGJfdHJhdmlzX3B5cHlfMjdfYm90In0sIHsidG9rZW4iOiAiNjg0MzM5OTg0OkFBRk1nRUVqcDAxcjVyQjAwN3lDZFZOc2c4QWxOc2FVLWNjIiwgInBheW1lbnRfcHJvdmlkZXJfdG9rZW4iOiAiMjg0Njg1MDYzOlRFU1Q6TVRBek1UWTNNR1V5TmpnMCIsICJib3RfbmFtZSI6ICJQVEIgdGVzdHMgb24gVHJhdmlzIHVzaW5nIFB5UHkgMy41IiwgImJvdF91c2VybmFtZSI6ICJAcHRiX3RyYXZpc19weXB5XzM1X2JvdCJ9LCB7InRva2VuIjogIjY5MDA5MTM0NzpBQUZMbVI1cEFCNVljcGVfbU9oN3pNNEpGQk9oMHozVDBUbyIsICJwYXltZW50X3Byb3ZpZGVyX3Rva2VuIjogIjI4NDY4NTA2MzpURVNUOlpEaGxOekU1TURrd1lXSmkiLCAiYm90X25hbWUiOiAiUFRCIHRlc3RzIG9uIEFwcFZleW9yIHVzaW5nIENQeXRob24gMy40IiwgImJvdF91c2VybmFtZSI6ICJAcHRiX2FwcHZleW9yX2NweXRob25fMzRfYm90In0sIHsidG9rZW4iOiAiNjk0MzA4MDUyOkFBRUIyX3NvbkNrNTVMWTlCRzlBTy1IOGp4aVBTNTVvb0JBIiwgInBheW1lbnRfcHJvdmlkZXJfdG9rZW4iOiAiMjg0Njg1MDYzOlRFU1Q6WW1aaVlXWm1NakpoWkdNeSIsICJib3RfbmFtZSI6ICJQVEIgdGVzdHMgb24gQXBwVmV5b3IgdXNpbmcgQ1B5dGhvbiAyLjciLCAiYm90X3VzZXJuYW1lIjogIkBwdGJfYXBwdmV5b3JfY3B5dGhvbl8yN19ib3QifSwgeyJ0b2tlbiI6ICIxMDU1Mzk3NDcxOkFBRzE4bkJfUzJXQXd1SjNnN29oS0JWZ1hYY2VNbklPeVNjIiwgInBheW1lbnRfcHJvdmlkZXJfdG9rZW4iOiAiMjg0Njg1MDYzOlRFU1Q6TmpBd056QXpZalZpTkdOayIsICJuYW1lIjogIlBUQiB0ZXN0cyBbMF0iLCAidXNlcm5hbWUiOiAicHRiXzBfYm90In0sIHsidG9rZW4iOiAiMTA0NzMyNjc3MTpBQUY4bk90ODFGcFg4bGJidno4VWV3UVF2UmZUYkZmQnZ1SSIsICJwYXltZW50X3Byb3ZpZGVyX3Rva2VuIjogIjI4NDY4NTA2MzpURVNUOllUVTFOVEk0WkdSallqbGkiLCAibmFtZSI6ICJQVEIgdGVzdHMgWzFdIiwgInVzZXJuYW1lIjogInB0Yl8xX2JvdCJ9LCB7InRva2VuIjogIjk3MTk5Mjc0NTpBQUdPa09hVzBOSGpnSXY1LTlqUWJPajR2R3FkaFNGLVV1cyIsICJwYXltZW50X3Byb3ZpZGVyX3Rva2VuIjogIjI4NDY4NTA2MzpURVNUOk5XWmtNV1ZoWWpsallqVTUiLCAibmFtZSI6ICJQVEIgdGVzdHMgWzJdIiwgInVzZXJuYW1lIjogInB0Yl8yX2JvdCJ9XQ==
           TEST_BUILD: ${{ matrix.test-build }}
           TEST_PRE_COMMIT: ${{ matrix.test-pre-commit }}
         shell: bash --noprofile --norc {0}

telegram/utils/webhookhandler.py

@@ -16,6 +16,7 @@
 #
 # You should have received a copy of the GNU Lesser Public License
 # along with this program.  If not, see [http://www.gnu.org/licenses/].
+import sys
 import logging
 from telegram import Update
 from future.utils import bytes_to_native_str
@@ -27,7 +28,6 @@ except ImportError:
 from tornado.httpserver import HTTPServer
 from tornado.ioloop import IOLoop
 import tornado.web
-import tornado.iostream
 
 logging.getLogger(__name__).addHandler(logging.NullHandler())
 
@@ -91,6 +91,35 @@ class WebhookHandler(tornado.web.RequestHandler):
     def __init__(self, application, request, **kwargs):
         super(WebhookHandler, self).__init__(application, request, **kwargs)
         self.logger = logging.getLogger(__name__)
+        self._init_asyncio_patch()
+
+    def _init_asyncio_patch(self):
+        """set default asyncio policy to be compatible with tornado
+        Tornado 6 (at least) is not compatible with the default
+        asyncio implementation on Windows
+        Pick the older SelectorEventLoopPolicy on Windows
+        if the known-incompatible default policy is in use.
+        do this as early as possible to make it a low priority and overrideable
+        ref: https://github.com/tornadoweb/tornado/issues/2608
+        TODO: if/when tornado supports the defaults in asyncio,
+                remove and bump tornado requirement for py38
+        Copied from https://github.com/ipython/ipykernel/pull/456/
+        """
+        if sys.platform.startswith("win") and sys.version_info >= (3, 8):
+            import asyncio
+            try:
+                from asyncio import (
+                    WindowsProactorEventLoopPolicy,
+                    WindowsSelectorEventLoopPolicy,
+                )
+            except ImportError:
+                pass
+                # not affected
+            else:
+                if isinstance(asyncio.get_event_loop_policy(), WindowsProactorEventLoopPolicy):
+                    # WindowsProactorEventLoopPolicy is not compatible with tornado 6
+                    # fallback to the pre-3.8 default of Selector
+                    asyncio.set_event_loop_policy(WindowsSelectorEventLoopPolicy())
 
     def initialize(self, bot, update_queue, default_quote=None):
         self.bot = bot

tests/test_updater.py

@@ -20,6 +20,7 @@ import logging
 import os
 import signal
 import sys
+import asyncio
 from flaky import flaky
 from functools import partial
 from queue import Queue
@@ -47,6 +48,33 @@ signalskip = pytest.mark.skipif(sys.platform == 'win32',
                                        'whole process on windows')
 
 
+if sys.platform.startswith("win") and sys.version_info >= (3, 8):
+    """set default asyncio policy to be compatible with tornado
+    Tornado 6 (at least) is not compatible with the default
+    asyncio implementation on Windows
+    Pick the older SelectorEventLoopPolicy on Windows
+    if the known-incompatible default policy is in use.
+    do this as early as possible to make it a low priority and overrideable
+    ref: https://github.com/tornadoweb/tornado/issues/2608
+    TODO: if/when tornado supports the defaults in asyncio,
+            remove and bump tornado requirement for py38
+    Copied from https://github.com/ipython/ipykernel/pull/456/
+    """
+    try:
+        from asyncio import (
+            WindowsProactorEventLoopPolicy,
+            WindowsSelectorEventLoopPolicy,
+        )
+    except ImportError:
+        pass
+        # not affected
+    else:
+        if type(asyncio.get_event_loop_policy()) is WindowsProactorEventLoopPolicy:
+            # WindowsProactorEventLoopPolicy is not compatible with tornado 6
+            # fallback to the pre-3.8 default of Selector
+            asyncio.set_event_loop_policy(WindowsSelectorEventLoopPolicy())
+
+
 class TestUpdater(object):
     message_count = 0
     received = None
@@ -246,6 +274,33 @@ class TestUpdater(object):
         assert q.get(False).message.default_quote is True
         updater.stop()
 
+    @pytest.mark.skipif(not (sys.platform.startswith("win") and sys.version_info >= (3, 8)),
+                        reason="only relevant on win with py>=3.8")
+    def test_webhook_tornado_win_py38_workaround(self, updater, monkeypatch):
+        updater._default_quote = True
+        q = Queue()
+        monkeypatch.setattr(updater.bot, 'set_webhook', lambda *args, **kwargs: True)
+        monkeypatch.setattr(updater.bot, 'delete_webhook', lambda *args, **kwargs: True)
+        monkeypatch.setattr('telegram.ext.Dispatcher.process_update', lambda _, u: q.put(u))
+
+        ip = '127.0.0.1'
+        port = randrange(1024, 49152)  # Select random port
+        updater.start_webhook(
+            ip,
+            port,
+            url_path='TOKEN')
+        sleep(.2)
+
+        try:
+            from asyncio import (WindowsSelectorEventLoopPolicy)
+        except ImportError:
+            pass
+            # not affected
+        else:
+            assert isinstance(asyncio.get_event_loop_policy(), WindowsSelectorEventLoopPolicy)
+
+        updater.stop()
+
     @pytest.mark.parametrize(('error',),
                              argvalues=[(TelegramError(''),)],
                              ids=('TelegramError',))