bellingcat/snscrape
1
0
Fork 0
mirror of https://github.com/bellingcat/snscrape.git synced 2026-06-11 11:58:28 +03:00
Code Issues Packages Projects Releases Wiki Activity

Fix crash on missing source label data

Browse Source 
This data had been announced in mid-November to disappear but was still always returned by the API until very recently.
This commit is contained in:
JustAnotherArchivist
2023-01-13 08:32:02 +00:00
parent 238bdcd560
commit 27374285a2
1 changed files with 7 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
snscrape/modules/twitter.py
View File

@@ -69,7 +69,7 @@ class Tweet(snscrape.base.Item):
quoteCount: int
conversationId: int
lang: str
source: str
source: typing.Optional[str] = None
sourceUrl: typing.Optional[str] = None
sourceLabel: typing.Optional[str] = None
links: typing.Optional[typing.List['TextLink']] = None
@@ -899,11 +899,12 @@ class _TwitterAPIScraper(snscrape.base.Scraper):
kwargs['quoteCount'] = tweet['quote_count']
kwargs['conversationId'] = tweet['conversation_id'] if 'conversation_id' in tweet else int(tweet['conversation_id_str'])
kwargs['lang'] = tweet['lang']
kwargs['source'] = tweet['source']
if (match := re.search(r'href=[\'"]?([^\'" >]+)', tweet['source'])):
kwargs['sourceUrl'] = match.group(1)
if (match := re.search(r'>([^<]*)<', tweet['source'])):
kwargs['sourceLabel'] = match.group(1)
if 'source' in tweet:
kwargs['source'] = tweet['source']
if (match := re.search(r'href=[\'"]?([^\'" >]+)', tweet['source'])):
kwargs['sourceUrl'] = match.group(1)
if (match := re.search(r'>([^<]*)<', tweet['source'])):
kwargs['sourceLabel'] = match.group(1)
if 'extended_entities' in tweet and 'media' in tweet['extended_entities']:
media = []
for medium in tweet['extended_entities']['media']: