From 907a003a593f39edc60fcbf44df875cbd1a507b6 Mon Sep 17 00:00:00 2001 From: JustAnotherArchivist Date: Fri, 24 May 2019 11:51:50 +0000 Subject: [PATCH] Fix crash when Twitter search produces no results (fixes #41) --- snscrape/modules/twitter.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/snscrape/modules/twitter.py b/snscrape/modules/twitter.py index c34d49c..4c2679a 100644 --- a/snscrape/modules/twitter.py +++ b/snscrape/modules/twitter.py @@ -37,8 +37,14 @@ class TwitterSearchScraper(snscrape.base.Scraper): if withMinPosition: streamContainer = soup.find('div', 'stream-container') if not streamContainer or not streamContainer.has_attr('data-min-position'): - raise RuntimeError('Unable to find min-position') - minPosition = streamContainer['data-min-position'] + if soup.find('div', 'SearchEmptyTimeline'): + # No results found + minPosition = None + else: + # Unknown error condition + raise RuntimeError('Unable to find min-position') + else: + minPosition = streamContainer['data-min-position'] else: minPosition = None return feed, minPosition @@ -104,6 +110,9 @@ class TwitterSearchScraper(snscrape.base.Scraper): else: maxPosition = self._maxPosition + if not maxPosition: + return + while True: logger.info(f'Retrieving scroll page {maxPosition}') r = self._get('https://twitter.com/i/search/timeline',