bellingcat/snscrape
1
0
Fork 0
mirror of https://github.com/bellingcat/snscrape.git synced 2026-06-11 03:48:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

Fix crash when Twitter search produces no results (fixes #41)

Browse Source
This commit is contained in:
JustAnotherArchivist
2019-05-24 11:51:50 +00:00
parent 8ada279b57
commit 907a003a59
1 changed files with 11 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
snscrape/modules/twitter.py
View File

@@ -37,8 +37,14 @@ class TwitterSearchScraper(snscrape.base.Scraper):
if withMinPosition: if withMinPosition:
streamContainer = soup.find('div', 'stream-container') streamContainer = soup.find('div', 'stream-container')
if not streamContainer or not streamContainer.has_attr('data-min-position'): if not streamContainer or not streamContainer.has_attr('data-min-position'):
raise RuntimeError('Unable to find min-position') if soup.find('div', 'SearchEmptyTimeline'):
minPosition = streamContainer['data-min-position'] # No results found
minPosition = None
else:
# Unknown error condition
raise RuntimeError('Unable to find min-position')
else:
minPosition = streamContainer['data-min-position']
else: else:
minPosition = None minPosition = None
return feed, minPosition return feed, minPosition
@@ -104,6 +110,9 @@ class TwitterSearchScraper(snscrape.base.Scraper):
else: else:
maxPosition = self._maxPosition maxPosition = self._maxPosition
if not maxPosition:
return
while True: while True:
logger.info(f'Retrieving scroll page {maxPosition}') logger.info(f'Retrieving scroll page {maxPosition}')
r = self._get('https://twitter.com/i/search/timeline', r = self._get('https://twitter.com/i/search/timeline',