OpenSource/telegram-crawler
OpenSource/telegram-crawler
1
0
Fork 0
mirror of https://github.com/MarshalX/telegram-crawler.git synced 2025-03-08 10:24:43 +01:00
Code Issues Projects Releases Packages Wiki Activity

fix tracking of ad panel

Browse source
This commit is contained in:
Il'ya (Marshal) 2021-10-26 17:55:08 +02:00
parent fd4c92dba0
commit 45c7211f67
1 changed files with 7 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
make_tracked_links_list.py
View file

@ -29,12 +29,13 @@ HIDDEN_URLS = {
'core.telegram.org/resources/cidr.txt',
'core.telegram.org/apple_privacy',
'core.telegram.org/getProxyConfig',
'promote.telegram.org',
'contest.com',
}
ADDITIONAL_URLS = {
'raw.githubusercontent.com/telegramdesktop/tdesktop/dev/Telegram/Resources/tl/api.tl',
'raw.githubusercontent.com/tdlib/td/master/td/generate/scheme/telegram_api.tl',
'promote.telegram.org',
'contest.com',
}
BASE_URL_REGEX = r'telegram.org'
@ -160,6 +161,10 @@ def find_absolute_links(html: str) -> set[str]:
def find_relative_links(html: str, cur_link: str) -> set[str]:
matches = re.findall(DIRECT_LINK_REGEX, cur_link)
if not matches:
return set()
direct_cur_link = re.findall(DIRECT_LINK_REGEX, cur_link)[0]
# optimization. when we want to exclude domain
if should_exclude(cur_link):