From 7d1916292c8b2b63cebe0ee81a7fbb930185c96c Mon Sep 17 00:00:00 2001 From: JustAnotherArchivist Date: Mon, 10 Jun 2019 14:38:25 +0000 Subject: [PATCH] Twitter: stop recursion based on whether the server returns the same position instead of detecting an empty feed Fixes #37 --- snscrape/modules/twitter.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/snscrape/modules/twitter.py b/snscrape/modules/twitter.py index 4c2679a..e086d6e 100644 --- a/snscrape/modules/twitter.py +++ b/snscrape/modules/twitter.py @@ -133,9 +133,10 @@ class TwitterSearchScraper(snscrape.base.Scraper): obj = json.loads(r.text) feed, _ = self._get_feed_from_html(obj['items_html'], False) - if not feed: + if feed: + yield from self._feed_to_items(feed) + if obj['min_position'] == maxPosition: return - yield from self._feed_to_items(feed) maxPosition = obj['min_position'] @classmethod