OpenSource/telegram-crawler
OpenSource/telegram-crawler
1
0
Fork 0
mirror of https://github.com/MarshalX/telegram-crawler.git synced 2025-01-07 11:15:42 +01:00
Code Issues Projects Releases Packages Wiki Activity

resend requests with server error response

Browse source
This commit is contained in:
Il'ya (Marshal) 2021-06-21 18:39:20 +02:00
parent 0b86273326
commit 12672aba45
2 changed files with 10 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
make_files_tree.py
View file

@ -38,10 +38,11 @@ async def crawl(url: str, session: aiohttp.ClientSession):
try:
logger.info(f'Process {url}')
async with session.get(f'{PROTOCOL}{url}', allow_redirects=False) as response:
if response.status == 302:
return
if response.status == 500:
return await asyncio.gather(crawl(url, session))
if response.status != 200:
if response.status != 302:
content = await response.text()
logger.debug(f'Skip {url} because status code == {response.status}. Content: {content}')
return

5
make_tracked_links_list.py
View file

@ -198,10 +198,11 @@ async def crawl(url: str, session: aiohttp.ClientSession):
async with session.get(f'{PROTOCOL}{url}', allow_redirects=False, timeout=TIMEOUT) as response:
content_type = response.headers.get('content-type')
if response.status == 302:
return
if response.status == 500:
return await asyncio.gather(crawl(url, session))
if response.status != 200:
if response.status != 302:
content = await response.text()
logger.debug(f'Skip {url} because status code == {response.status}. Content: {content}')
return