bellingcat/snscrape
1
0
Fork 0
mirror of https://github.com/bellingcat/snscrape.git synced 2026-06-12 12:28:28 +03:00
Code Issues Packages Projects Releases Wiki Activity

Fix order of processing in Twitter module for more useful locals dump output

Browse Source
This commit is contained in:
JustAnotherArchivist
2019-05-16 22:22:53 +00:00
parent c6a11298ac
commit 9c8bbf051c
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
snscrape/modules/twitter.py
View File

@@ -75,9 +75,9 @@ class TwitterSearchScraper(snscrape.base.Scraper):
feed = self._get_feed_from_html(r.text) feed = self._get_feed_from_html(r.text)
if not feed: if not feed:
return return
yield from self._feed_to_items(feed)
newestID = feed[0]['data-item-id'] newestID = feed[0]['data-item-id']
maxPosition = f'TWEET-{feed[-1]["data-item-id"]}-{newestID}' maxPosition = f'TWEET-{feed[-1]["data-item-id"]}-{newestID}'
yield from self._feed_to_items(feed)
else: else:
_, _, newestID = self._maxPosition.split('-') _, _, newestID = self._maxPosition.split('-')
maxPosition = self._maxPosition maxPosition = self._maxPosition
@@ -103,8 +103,8 @@ class TwitterSearchScraper(snscrape.base.Scraper):
feed = self._get_feed_from_html(json.loads(r.text)['items_html']) feed = self._get_feed_from_html(json.loads(r.text)['items_html'])
if not feed: if not feed:
return return
maxPosition = f'TWEET-{feed[-1]["data-item-id"]}-{newestID}'
yield from self._feed_to_items(feed) yield from self._feed_to_items(feed)
maxPosition = f'TWEET-{feed[-1]["data-item-id"]}-{newestID}'
@classmethod @classmethod
def setup_parser(cls, subparser): def setup_parser(cls, subparser):