From 12672aba456d3a85f368bedb3b175358c1a58909 Mon Sep 17 00:00:00 2001 From: "Il'ya (Marshal)" Date: Mon, 21 Jun 2021 18:39:20 +0200 Subject: [PATCH] resend requests with server error response --- make_files_tree.py | 9 +++++---- make_tracked_links_list.py | 9 +++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/make_files_tree.py b/make_files_tree.py index 6e38aabb77..db31a13b16 100644 --- a/make_files_tree.py +++ b/make_files_tree.py @@ -38,12 +38,13 @@ async def crawl(url: str, session: aiohttp.ClientSession): try: logger.info(f'Process {url}') async with session.get(f'{PROTOCOL}{url}', allow_redirects=False) as response: - if response.status == 302: - return + if response.status == 500: + return await asyncio.gather(crawl(url, session)) if response.status != 200: - content = await response.text() - logger.debug(f'Skip {url} because status code == {response.status}. Content: {content}') + if response.status != 302: + content = await response.text() + logger.debug(f'Skip {url} because status code == {response.status}. Content: {content}') return # bypass external slashes and so on diff --git a/make_tracked_links_list.py b/make_tracked_links_list.py index 368923df9b..4892bb0f52 100644 --- a/make_tracked_links_list.py +++ b/make_tracked_links_list.py @@ -198,12 +198,13 @@ async def crawl(url: str, session: aiohttp.ClientSession): async with session.get(f'{PROTOCOL}{url}', allow_redirects=False, timeout=TIMEOUT) as response: content_type = response.headers.get('content-type') - if response.status == 302: - return + if response.status == 500: + return await asyncio.gather(crawl(url, session)) if response.status != 200: - content = await response.text() - logger.debug(f'Skip {url} because status code == {response.status}. Content: {content}') + if response.status != 302: + content = await response.text() + logger.debug(f'Skip {url} because status code == {response.status}. Content: {content}') return if 'text/html' in content_type: