bellingcat/snscrape
1
0
Fork 0
mirror of https://github.com/bellingcat/snscrape.git synced 2026-06-10 19:38:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

Twitter: stop recursion based on whether the server returns the same position instead of detecting an empty feed

Browse Source 
Fixes #37
This commit is contained in:
JustAnotherArchivist
2019-06-10 14:38:25 +00:00
parent 0d509c4ba0
commit 7d1916292c
1 changed files with 3 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
snscrape/modules/twitter.py
View File

@@ -133,9 +133,10 @@ class TwitterSearchScraper(snscrape.base.Scraper):
obj = json.loads(r.text)
feed, _ = self._get_feed_from_html(obj['items_html'], False)
if not feed:
if feed:
yield from self._feed_to_items(feed)
if obj['min_position'] == maxPosition:
return
yield from self._feed_to_items(feed)
maxPosition = obj['min_position']
@classmethod