handle timeout and retry

This commit is contained in:
Il'ya (Marshal) 2021-04-24 11:38:33 +02:00
parent cd5653605f
commit 5f29beda73

View file

@ -2,6 +2,7 @@ import asyncio
import logging
import os
import re
from asyncio.exceptions import TimeoutError
from html import unescape
from time import time
from urllib.parse import unquote
@ -215,7 +216,7 @@ async def crawl(url: str, session: aiohttp.ClientSession):
logger.info(f'Unhandled type: {content_type}')
except UnicodeDecodeError:
logger.warning('Codec can\'t decode byte. So its was a tgs file')
except ClientConnectorError:
except (TimeoutError, ClientConnectorError):
await asyncio.gather(crawl(url, session))