bellingcat/snscrape
1
0
Fork 0
mirror of https://github.com/bellingcat/snscrape.git synced 2026-06-08 02:28:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

fixed issue where Telegram scraper terminated early because some pages didn't have a next page link (added reasonable default)

Browse Source
This commit is contained in:
Tristan Lee
2022-04-17 04:33:22 -05:00
parent babcddda19
commit 1e4e0c278d
1 changed files with 6 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
snscrape/modules/telegram.py
View File

@@ -214,8 +214,13 @@ class TelegramChannelScraper(snscrape.base.Scraper):
yield from self._soup_to_items(soup, r.url) yield from self._soup_to_items(soup, r.url)
pageLink = soup.find('a', attrs = {'class': 'tme_messages_more', 'data-before': True}) pageLink = soup.find('a', attrs = {'class': 'tme_messages_more', 'data-before': True})
if not pageLink: if not pageLink:
break nextPostIndex = int(nextPageUrl.split('=')[-1]) - 20
if nextPostIndex > 20:
pageLink = {'href': nextPageUrl.split('=')[0] + f'={nextPostIndex}'}
else:
break
nextPageUrl = urllib.parse.urljoin(r.url, pageLink['href']) nextPageUrl = urllib.parse.urljoin(r.url, pageLink['href'])
print(f'nextPageUrl: {nextPageUrl}')
r = self._get(nextPageUrl, headers = self._headers, responseOkCallback = telegramResponseOkCallback) r = self._get(nextPageUrl, headers = self._headers, responseOkCallback = telegramResponseOkCallback)
if r.status_code != 200: if r.status_code != 200:
raise snscrape.base.ScraperException(f'Got status code {r.status_code}') raise snscrape.base.ScraperException(f'Got status code {r.status_code}')