OpenSource/telegram-crawler
OpenSource/telegram-crawler
1
0
Fork 0
mirror of https://github.com/MarshalX/telegram-crawler.git synced 2024-11-28 19:23:11 +01:00
Code Issues Projects Releases Packages Wiki Activity

update apktool

Browse source
This commit is contained in:
Ilya (Marshal) 2023-10-28 20:21:49 +02:00
parent 9ba3442ca1
commit 7e6a4c05a1
No known key found for this signature in database
GPG key ID: F8FB1A006AD3156D
1 changed files with 7 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
make_files_tree.py
View file

@ -142,7 +142,6 @@ async def track_additional_files(
content = get_hash(content)
else:
content = re.sub(r'id=".*"', 'id="tgcrawl"', content)
content = re.sub(r'name="APKTOOL_DUMMY_.*" id', 'name="tgcrawl" id', content)
filename = os.path.join(output_dir_name, file)
os.makedirs(os.path.dirname(filename), exist_ok=True)
@ -362,7 +361,7 @@ async def download_telegram_android_beta_and_extract_resources(session: aiohttp.
return
await asyncio.gather(
download_file('https://bitbucket.org/iBotPeaches/apktool/downloads/apktool_2.6.1.jar', 'tool.apk', session),
download_file('https://bitbucket.org/iBotPeaches/apktool/downloads/apktool_2.9.0.jar', 'tool.apk', session),
download_file(download_url, 'android.apk', session),
)
@ -709,8 +708,9 @@ async def _crawl(url: str, session: aiohttp.ClientSession, output_dir: str):
is_hashable_only = is_hashable_only_content_type(response.content_type)
# amazing dirt for media files like
# telegram.org/file/811140591/1/q7zZHjgES6s/9d121a89ffb0015837
# with response content type HTML instead of image. Shame on you
# sometimes it returns correct type. noice load balancing
# with response content type HTML instead of image.
# shame on you.
# sometimes it returns a correct type. noice load balancing
is_sucking_file = '/file/' in url and 'text' in response.content_type
# handle pure domains and html pages without ext in url as html do enable syntax highlighting
@ -720,8 +720,9 @@ async def _crawl(url: str, session: aiohttp.ClientSession, output_dir: str):
ext = '.html' if page_type is None or len(url_parts) == 1 else ''
# I don't add ext by content type for images and so on cuz TG servers sucks.
# Some servers do not return correct content type. Some servers do...
# I don't add ext by content type for images, and so on cuz TG servers suck.
# Some servers do not return a correct content type.
# Some servers do...
if is_hashable_only or is_sucking_file:
ext = ''