From 238bdcd56069cc8ee63a3523fca3437d531a7a04 Mon Sep 17 00:00:00 2001 From: JustAnotherArchivist Date: Fri, 13 Jan 2023 08:28:52 +0000 Subject: [PATCH] Reduce warnings about duplicate users on cards --- snscrape/modules/twitter.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/snscrape/modules/twitter.py b/snscrape/modules/twitter.py index b147248..62a5a01 100644 --- a/snscrape/modules/twitter.py +++ b/snscrape/modules/twitter.py @@ -1023,13 +1023,15 @@ class _TwitterAPIScraper(snscrape.base.Scraper): for o in card['legacy'].get('user_refs_results', []): o = o['result'] userId = int(o['rest_id']) - if userId in userRefs: - _logger.warning(f'Duplicate user {userId} in card on tweet {tweetId}') - continue if 'legacy' in o: - userRefs[userId] = self._user_to_user(o['legacy'], id_ = userId) + user = self._user_to_user(o['legacy'], id_ = userId) else: - userRefs[userId] = UserRef(id = userId) + user = UserRef(id = userId) + if userId in userRefs: + if userRefs[userId] != user: + _logger.warning(f'Duplicate user {userId} with differing data in card on tweet {tweetId}') + continue + userRefs[userId] = user if apiType is _TwitterAPIType.V2: messyBindingValues = card['binding_values'].items()