bellingcat/snscrape
1
0
Fork 0
mirror of https://github.com/bellingcat/snscrape.git synced 2026-06-08 02:28:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

Fix KeyErrors on display_url and expanded_url for certain users with broken profile links

Browse Source 
Fixes #480
This commit is contained in:
JustAnotherArchivist
2022-05-29 17:23:38 +00:00
parent dc6bc9bf9d
commit 530f4fa122
1 changed files with 4 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
snscrape/modules/twitter.py
View File

@@ -1349,8 +1349,8 @@ class _TwitterAPIScraper(snscrape.base.Scraper):
if not entity or entity['url'] != user['url']: if not entity or entity['url'] != user['url']:
self.logger.warning(f'Link inconsistency on user {kwargs["id"]}') self.logger.warning(f'Link inconsistency on user {kwargs["id"]}')
if not entity: if not entity:
entity = {'display_url': None, 'expanded_url': user['url'], 'indices': (0, len(user['url']))} entity = {'indices': (0, len(user['url']))}
kwargs['link'] = TextLink(text = entity['display_url'], url = entity['expanded_url'], tcourl = user['url'], indices = tuple(entity['indices'])) kwargs['link'] = TextLink(text = entity.get('display_url'), url = entity.get('expanded_url', user['url']), tcourl = user['url'], indices = tuple(entity['indices']))
kwargs['profileImageUrl'] = user['profile_image_url_https'] kwargs['profileImageUrl'] = user['profile_image_url_https']
kwargs['profileBannerUrl'] = user.get('profile_banner_url') kwargs['profileBannerUrl'] = user.get('profile_banner_url')
if 'ext' in user and (label := user['ext']['highlightedLabel']['r']['ok'].get('label')): if 'ext' in user and (label := user['ext']['highlightedLabel']['r']['ok'].get('label')):
@@ -1482,8 +1482,8 @@ class TwitterUserScraper(TwitterSearchScraper):
if not entity or entity['url'] != user['legacy']['url']: if not entity or entity['url'] != user['legacy']['url']:
self.logger.warning(f'Link inconsistency on user') self.logger.warning(f'Link inconsistency on user')
if not entity: if not entity:
entity = {'display_url': None, 'expanded_url': user['legacy']['url'], 'indices': (0, len(user['legacy']['url']))} entity = {'indices': (0, len(user['legacy']['url']))}
link = TextLink(text = entity['display_url'], url = entity['expanded_url'], tcourl = user['legacy']['url'], indices = tuple(entity['indices'])) link = TextLink(text = entity.get('display_url'), url = entity.get('expanded_url', user['legacy']['url']), tcourl = user['legacy']['url'], indices = tuple(entity['indices']))
label = None label = None
if (labelO := user['affiliates_highlighted_label'].get('label')): if (labelO := user['affiliates_highlighted_label'].get('label')):
label = self._user_label_to_user_label(labelO) label = self._user_label_to_user_label(labelO)