bellingcat/snscrape
1
0
Fork 0
mirror of https://github.com/bellingcat/snscrape.git synced 2026-06-12 12:28:28 +03:00
Code Issues Packages Projects Releases Wiki Activity

Fix type of description URL objects

Browse Source
This commit is contained in:
JustAnotherArchivist
2022-05-29 05:06:51 +00:00
parent ef7c4fad3e
commit 01cf6a09b3
1 changed files with 12 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
snscrape/modules/twitter.py
View File

@@ -1303,7 +1303,12 @@ class _TwitterAPIScraper(snscrape.base.Scraper):
kwargs['description'] = self._render_text_with_urls(user['description'], user['entities']['description'].get('urls')) kwargs['description'] = self._render_text_with_urls(user['description'], user['entities']['description'].get('urls'))
kwargs['rawDescription'] = user['description'] kwargs['rawDescription'] = user['description']
if user['entities']['description'].get('urls'): if user['entities']['description'].get('urls'):
kwargs['descriptionUrls'] = [{'text': x.get('display_url'), 'url': x['expanded_url'], 'tcourl': x['url'], 'indices': tuple(x['indices'])} for x in user['entities']['description']['urls']] kwargs['descriptionUrls'] = [DescriptionURL(
text = x.get('display_url'),
url = x['expanded_url'],
tcourl = x['url'],
indices = tuple(x['indices']),
) for x in user['entities']['description']['urls']]
kwargs['verified'] = user.get('verified') kwargs['verified'] = user.get('verified')
kwargs['created'] = email.utils.parsedate_to_datetime(user['created_at']) kwargs['created'] = email.utils.parsedate_to_datetime(user['created_at'])
kwargs['followersCount'] = user['followers_count'] kwargs['followersCount'] = user['followers_count']
@@ -1451,7 +1456,12 @@ class TwitterUserScraper(TwitterSearchScraper):
displayname = user['legacy']['name'], displayname = user['legacy']['name'],
description = description, description = description,
rawDescription = rawDescription, rawDescription = rawDescription,
descriptionUrls = [{'text': x.get('display_url'), 'url': x['expanded_url'], 'tcourl': x['url'], 'indices': tuple(x['indices'])} for x in user['legacy']['entities']['description']['urls']], descriptionUrls = [DescriptionURL(
text = x.get('display_url'),
url = x['expanded_url'],
tcourl = x['url'],
indices = tuple(x['indices']),
) for x in user['legacy']['entities']['description']['urls']],
verified = user['legacy']['verified'], verified = user['legacy']['verified'],
created = email.utils.parsedate_to_datetime(user['legacy']['created_at']), created = email.utils.parsedate_to_datetime(user['legacy']['created_at']),
followersCount = user['legacy']['followers_count'], followersCount = user['legacy']['followers_count'],