Fix Server response could not be decoded using UTF-8 (#1623)

Co-authored-by: Noam Meltzer <tsnoam@gmail.com>

AUTHORS.rst

@@ -68,6 +68,7 @@ The following wonderful people contributed directly or indirectly to this projec
 - `Pieter Schutz <https://github.com/eldinnie>`_
 - `Poolitzer <https://github.com/Poolitzer>`_
 - `Rahiel Kasim <https://github.com/rahiel>`_
+- `Riko Naka <https://github.com/rikonaka>`_
 - `Rizlas <https://github.com/rizlas>`_
 - `Sahil Sharma <https://github.com/sahilsharma811>`_
 - `Sascha <https://github.com/saschalalala>`_

telegram/utils/request.py

@@ -178,13 +178,9 @@ class Request(object):
 
         """
 
+        decoded_s = json_data.decode('utf-8', 'replace')
         try:
-            decoded_s = json_data.decode('utf-8')
             data = json.loads(decoded_s)
-        except UnicodeDecodeError:
-            logging.getLogger(__name__).debug(
-                'Logging raw invalid UTF-8 response:\n%r', json_data)
-            raise TelegramError('Server response could not be decoded using UTF-8')
         except ValueError:
             raise TelegramError('Invalid server response')
 

tests/test_request.py

@@ -23,12 +23,22 @@ from telegram import TelegramError
 from telegram.utils.request import Request
 
 
-def test_parse_illegal_callback_data():
+def test_replaced_unprintable_char():
     """
     Clients can send arbitrary bytes in callback data.
     Make sure the correct error is raised in this case.
     """
-    server_response = b'{"invalid utf-8": "\x80"}'
+    server_response = b'{"invalid utf-8": "\x80", "result": "KUKU"}'
 
-    with pytest.raises(TelegramError, match='Server response could not be decoded using UTF-8'):
+    assert Request._parse(server_response) == 'KUKU'
+
+
+def test_parse_illegal_json():
+    """
+    Clients can send arbitrary bytes in callback data.
+    Make sure the correct error is raised in this case.
+    """
+    server_response = b'{"invalid utf-8": "\x80", result: "KUKU"}'
+
+    with pytest.raises(TelegramError, match='Invalid server response'):
         Request._parse(server_response)