mirror of
https://github.com/MarshalX/telegram-crawler.git
synced 2025-03-08 10:24:43 +01:00
handle timeout and retry
This commit is contained in:
parent
cd5653605f
commit
5f29beda73
1 changed files with 2 additions and 1 deletions
|
@ -2,6 +2,7 @@ import asyncio
|
|||
import logging
|
||||
import os
|
||||
import re
|
||||
from asyncio.exceptions import TimeoutError
|
||||
from html import unescape
|
||||
from time import time
|
||||
from urllib.parse import unquote
|
||||
|
@ -215,7 +216,7 @@ async def crawl(url: str, session: aiohttp.ClientSession):
|
|||
logger.info(f'Unhandled type: {content_type}')
|
||||
except UnicodeDecodeError:
|
||||
logger.warning('Codec can\'t decode byte. So its was a tgs file')
|
||||
except ClientConnectorError:
|
||||
except (TimeoutError, ClientConnectorError):
|
||||
await asyncio.gather(crawl(url, session))
|
||||
|
||||
|
||||
|
|
Loading…
Add table
Reference in a new issue