Fix crash when Twitter search produces no results (fixes #41)

This commit is contained in:
JustAnotherArchivist
2019-05-24 11:51:50 +00:00
parent 8ada279b57
commit 907a003a59

View File

@@ -37,8 +37,14 @@ class TwitterSearchScraper(snscrape.base.Scraper):
if withMinPosition:
streamContainer = soup.find('div', 'stream-container')
if not streamContainer or not streamContainer.has_attr('data-min-position'):
raise RuntimeError('Unable to find min-position')
minPosition = streamContainer['data-min-position']
if soup.find('div', 'SearchEmptyTimeline'):
# No results found
minPosition = None
else:
# Unknown error condition
raise RuntimeError('Unable to find min-position')
else:
minPosition = streamContainer['data-min-position']
else:
minPosition = None
return feed, minPosition
@@ -104,6 +110,9 @@ class TwitterSearchScraper(snscrape.base.Scraper):
else:
maxPosition = self._maxPosition
if not maxPosition:
return
while True:
logger.info(f'Retrieving scroll page {maxPosition}')
r = self._get('https://twitter.com/i/search/timeline',