bellingcat/snscrape
1
0
Fork 0
mirror of https://github.com/bellingcat/snscrape.git synced 2026-06-13 12:58:27 +03:00
Code Issues Packages Projects Releases Wiki Activity

Add support for image_collection_website unified cards

Browse Source
This commit is contained in:
JustAnotherArchivist
2023-01-13 07:36:53 +00:00
parent 28f5a45825
commit 3e19f8f84b
1 changed files with 15 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
snscrape/modules/twitter.py
View File

@@ -324,11 +324,12 @@ class UnifiedCard(Card):
apps: typing.Optional[typing.Dict[UnifiedCardAppKey, typing.List['UnifiedCardApp']]] = None apps: typing.Optional[typing.Dict[UnifiedCardAppKey, typing.List['UnifiedCardApp']]] = None
components: typing.Optional[typing.List[UnifiedCardComponentKey]] = None components: typing.Optional[typing.List[UnifiedCardComponentKey]] = None
swipeableLayoutSlides: typing.Optional[typing.List['UnifiedCardSwipeableLayoutSlide']] = None swipeableLayoutSlides: typing.Optional[typing.List['UnifiedCardSwipeableLayoutSlide']] = None
collectionLayoutSlides: typing.Optional[typing.List['UnifiedCardCollectionLayoutSlide']] = None
type: typing.Optional[str] = None type: typing.Optional[str] = None
def __post_init__(self): def __post_init__(self):
if (self.components is None) == (self.swipeableLayoutSlides is None): if (self.components is not None) + (self.swipeableLayoutSlides is not None) + (self.collectionLayoutSlides is not None) != 1:
raise ValueError('did not get exactly one of components or swipeableLayoutSlides') raise ValueError('did not get exactly one of components, swipeableLayoutSlides, and collectionLayoutSlides')
if self.components and not all(k in self.componentObjects for k in self.components): if self.components and not all(k in self.componentObjects for k in self.components):
raise ValueError('missing components') raise ValueError('missing components')
if self.swipeableLayoutSlides and not all(s.mediumComponentKey in self.componentObjects and s.componentKey in self.componentObjects for s in self.swipeableLayoutSlides): if self.swipeableLayoutSlides and not all(s.mediumComponentKey in self.componentObjects and s.componentKey in self.componentObjects for s in self.swipeableLayoutSlides):
@@ -452,6 +453,12 @@ class UnifiedCardSwipeableLayoutSlide:
componentKey: UnifiedCardComponentKey componentKey: UnifiedCardComponentKey
@dataclasses.dataclass
class UnifiedCardCollectionLayoutSlide:
detailsComponentKey: UnifiedCardComponentKey
mediumComponentKey: UnifiedCardComponentKey
@dataclasses.dataclass @dataclasses.dataclass
class Vibe: class Vibe:
text: str text: str
@@ -1190,6 +1197,7 @@ class _TwitterAPIScraper(snscrape.base.Scraper):
'image_app', 'image_app',
'image_carousel_app', 'image_carousel_app',
'image_carousel_website', 'image_carousel_website',
'image_collection_website',
'image_multi_dest_carousel_website', 'image_multi_dest_carousel_website',
'image_website', 'image_website',
'mixed_media_multi_dest_carousel_website', 'mixed_media_multi_dest_carousel_website',
@@ -1300,10 +1308,13 @@ class _TwitterAPIScraper(snscrape.base.Scraper):
kwargs['components'] = o['components'] kwargs['components'] = o['components']
if 'layout' in o: if 'layout' in o:
if o['layout']['type'] != 'swipeable': if o['layout']['type'] == 'swipeable':
kwargs['swipeableLayoutSlides'] = [UnifiedCardSwipeableLayoutSlide(mediumComponentKey = v[0], componentKey = v[1]) for v in o['layout']['data']['slides']]
elif o['layout']['type'] == 'collection':
kwargs['collectionLayoutSlides'] = [UnifiedCardCollectionLayoutSlide(detailsComponentKey = v[0], mediumComponentKey = v[1]) for v in o['layout']['data']['slides']]
else:
_logger.warning(f'Unsupported unified_card layout type on tweet {tweetId}: {o["layout"]["type"]!r}') _logger.warning(f'Unsupported unified_card layout type on tweet {tweetId}: {o["layout"]["type"]!r}')
return return
kwargs['swipeableLayoutSlides'] = [UnifiedCardSwipeableLayoutSlide(mediumComponentKey = v[0], componentKey = v[1]) for v in o['layout']['data']['slides']]
return UnifiedCard(**kwargs) return UnifiedCard(**kwargs)