bellingcat/snscrape
1
0
Fork 0
mirror of https://github.com/bellingcat/snscrape.git synced 2026-06-12 20:38:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

Reduce warnings about duplicate users on cards

Browse Source
This commit is contained in:
JustAnotherArchivist
2023-01-13 08:28:52 +00:00
parent e846a6a4cd
commit 238bdcd560
1 changed files with 7 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
snscrape/modules/twitter.py
View File

@@ -1023,13 +1023,15 @@ class _TwitterAPIScraper(snscrape.base.Scraper):
for o in card['legacy'].get('user_refs_results', []): for o in card['legacy'].get('user_refs_results', []):
o = o['result'] o = o['result']
userId = int(o['rest_id']) userId = int(o['rest_id'])
if userId in userRefs:
_logger.warning(f'Duplicate user {userId} in card on tweet {tweetId}')
continue
if 'legacy' in o: if 'legacy' in o:
userRefs[userId] = self._user_to_user(o['legacy'], id_ = userId) user = self._user_to_user(o['legacy'], id_ = userId)
else: else:
userRefs[userId] = UserRef(id = userId) user = UserRef(id = userId)
if userId in userRefs:
if userRefs[userId] != user:
_logger.warning(f'Duplicate user {userId} with differing data in card on tweet {tweetId}')
continue
userRefs[userId] = user
if apiType is _TwitterAPIType.V2: if apiType is _TwitterAPIType.V2:
messyBindingValues = card['binding_values'].items() messyBindingValues = card['binding_values'].items()