OpenSource/telegram-crawler
OpenSource/telegram-crawler
1
0
Fork 0
mirror of https://github.com/MarshalX/telegram-crawler.git synced 2025-01-21 00:17:06 +01:00
Code Issues Projects Releases Packages Wiki Activity

add sha256 ext; fix ext

Browse source
This commit is contained in:
Ilya (Marshal) 2023-10-28 21:41:27 +02:00
parent 1001a9729e
commit 95d645abfa
No known key found for this signature in database
GPG key ID: F8FB1A006AD3156D
1 changed files with 8 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
make_files_tree.py
View file

@ -710,10 +710,16 @@ async def _crawl(url: str, session: aiohttp.ClientSession, output_dir: str):
# handle pure domains and html pages without ext in url as html do enable syntax highlighting
page_type, _ = mimetypes.guess_type(url)
ext = ''
if page_type:
ext = mimetypes.guess_extension(page_type) or ''
if ext != '' and url.endswith(ext):
ext = ''
if url.endswith('.tl'):
page_type = 'text/plain'
ext = ''
if page_type is None or len(url_parts) == 1:
ext = '.html'
content_type = 'text/html'
@ -735,7 +741,7 @@ async def _crawl(url: str, session: aiohttp.ClientSession, output_dir: str):
# Some servers do not return a correct content type.
# Some servers do...
if is_hashable_only or is_sucking_file:
ext = ''
ext = '.sha256'
filename = os.path.join(output_dir, *url_parts) + ext
os.makedirs(os.path.dirname(filename), exist_ok=True)