diff --git a/snscrape/modules/twitter.py b/snscrape/modules/twitter.py index c34d49c..4c2679a 100644 --- a/snscrape/modules/twitter.py +++ b/snscrape/modules/twitter.py @@ -37,8 +37,14 @@ class TwitterSearchScraper(snscrape.base.Scraper): if withMinPosition: streamContainer = soup.find('div', 'stream-container') if not streamContainer or not streamContainer.has_attr('data-min-position'): - raise RuntimeError('Unable to find min-position') - minPosition = streamContainer['data-min-position'] + if soup.find('div', 'SearchEmptyTimeline'): + # No results found + minPosition = None + else: + # Unknown error condition + raise RuntimeError('Unable to find min-position') + else: + minPosition = streamContainer['data-min-position'] else: minPosition = None return feed, minPosition @@ -104,6 +110,9 @@ class TwitterSearchScraper(snscrape.base.Scraper): else: maxPosition = self._maxPosition + if not maxPosition: + return + while True: logger.info(f'Retrieving scroll page {maxPosition}') r = self._get('https://twitter.com/i/search/timeline',