bellingcat/snscrape
1
0
Fork 0
mirror of https://github.com/bellingcat/snscrape.git synced 2026-06-14 21:28:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #141 from gitshrl/twitter/split-source-url-label

Browse Source 
Split tweet source into URL and label
This commit is contained in:
JustAnotherArchivist
2020-10-27 18:44:10 +00:00
committed by GitHub
parent 7499384110 d2dce37fa0
commit 61a1ecffc5
1 changed files with 6 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
snscrape/modules/twitter.py
View File

@@ -35,6 +35,8 @@ class Tweet(snscrape.base.Item):
conversationId: int
lang: str
source: str
sourceUrl: typing.Optional[str] = None
sourceLabel: typing.Optional[str] = None
media: typing.Optional[typing.List['Medium']] = None
retweetedTweet: typing.Optional['Tweet'] = None
quotedTweet: typing.Optional['Tweet'] = None
@@ -315,6 +317,10 @@ class TwitterAPIScraper(snscrape.base.Scraper):
kwargs['conversationId'] = tweet['conversation_id'] if 'conversation_id' in tweet else int(tweet['conversation_id_str'])
kwargs['lang'] = tweet['lang']
kwargs['source'] = tweet['source']
if (match := re.search(r'href=[\'"]?([^\'" >]+)', tweet['source'])):
kwargs['sourceUrl'] = match.group(1)
if (match := re.search(r'>([^<]*)<', tweet['source'])):
kwargs['sourceLabel'] = match.group(1)
if 'extended_entities' in tweet and 'media' in tweet['extended_entities']:
media = []
for medium in tweet['extended_entities']['media']: