diff --git a/snscrape/modules/twitter.py b/snscrape/modules/twitter.py index d6ac2f1..b77388e 100644 --- a/snscrape/modules/twitter.py +++ b/snscrape/modules/twitter.py @@ -34,7 +34,8 @@ class Tweet(snscrape.base.Item): quoteCount: int conversationId: int lang: str - source: str + sourceUrl: str + sourceLabel: str media: typing.Optional[typing.List['Medium']] = None retweetedTweet: typing.Optional['Tweet'] = None quotedTweet: typing.Optional['Tweet'] = None @@ -314,7 +315,10 @@ class TwitterAPIScraper(snscrape.base.Scraper): kwargs['quoteCount'] = tweet['quote_count'] kwargs['conversationId'] = tweet['conversation_id'] if 'conversation_id' in tweet else int(tweet['conversation_id_str']) kwargs['lang'] = tweet['lang'] - kwargs['source'] = tweet['source'] + if (match := re.search(r'href=[\'"]?([^\'" >]+)', tweet['source'])): + kwargs['sourceUrl'] = match.group(1) + if (match := re.search(r'>([^<]*)<', tweet['source'])): + kwargs['sourceLabel'] = match.group(1) if 'extended_entities' in tweet and 'media' in tweet['extended_entities']: media = [] for medium in tweet['extended_entities']['media']: