bellingcat/snscrape
1
0
Fork 0
mirror of https://github.com/bellingcat/snscrape.git synced 2026-06-08 02:28:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

Refactor class instantiation to remove the need to repeat 'retries' everywhere

Browse Source
This commit is contained in:
JustAnotherArchivist
2021-09-30 09:58:10 +00:00
parent d4567da23c
commit 5e829e2541
8 changed files with 19 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
snscrape/base.py
View File

@@ -207,11 +207,13 @@ class Scraper:
return self._request('POST', *args, **kwargs)
@classmethod
@abc.abstractmethod
def setup_parser(cls, subparser):
pass
@classmethod
@abc.abstractmethod
def from_args(cls, args):
pass
return cls._construct(args)
@classmethod
def _construct(cls, argparseArgs, *args, **kwargs):
return cls(*args, **kwargs, retries = argparseArgs.retries)

4
snscrape/modules/facebook.py
View File

@@ -207,7 +207,7 @@ class FacebookUserAndCommunityScraper(FacebookCommonScraper):
@classmethod
def from_args(cls, args):
return cls(args.username, retries = args.retries)
return cls._construct(args, args.username)
class FacebookUserScraper(FacebookUserAndCommunityScraper):
@@ -358,4 +358,4 @@ class FacebookGroupScraper(FacebookCommonScraper):
@classmethod
def from_args(cls, args):
return cls(args.group, retries = args.retries)
return cls._construct(args, args.group)

6
snscrape/modules/instagram.py
View File

@@ -178,7 +178,7 @@ class InstagramUserScraper(InstagramCommonScraper):
@classmethod
def from_args(cls, args):
return cls('User', args.username, retries = args.retries)
return cls._construct(args, 'User', args.username)
def _get_entity(self):
r = self._initial_page()
@@ -223,7 +223,7 @@ class InstagramHashtagScraper(InstagramCommonScraper):
@classmethod
def from_args(cls, args):
return cls('Hashtag', args.hashtag, retries = args.retries)
return cls._construct(args, 'Hashtag', args.hashtag)
class InstagramLocationScraper(InstagramCommonScraper):
@@ -235,4 +235,4 @@ class InstagramLocationScraper(InstagramCommonScraper):
@classmethod
def from_args(cls, args):
return cls('Location', args.locationid, retries = args.retries)
return cls._construct(args, 'Location', args.locationid)

2
snscrape/modules/reddit.py
View File

@@ -222,7 +222,7 @@ def _make_scraper(name_, validationFunc, apiField):
@classmethod
def from_args(cls, args):
return cls(getattr(args, name_), submissions = not args.noSubmissions, comments = not args.noComments, before = args.before, after = args.after, retries = args.retries)
return cls._construct(args, getattr(args, name_), submissions = not args.noSubmissions, comments = not args.noComments, before = args.before, after = args.after)
Scraper.__name__ = f'Reddit{name_.capitalize()}Scraper'
Scraper.__qualname__ = Scraper.__name__

2
snscrape/modules/telegram.py
View File

@@ -197,4 +197,4 @@ class TelegramChannelScraper(snscrape.base.Scraper):
@classmethod
def from_args(cls, args):
return cls(args.channel, retries = args.retries)
return cls._construct(args, args.channel)

18
snscrape/modules/twitter.py
View File

@@ -560,7 +560,7 @@ class TwitterSearchScraper(TwitterAPIScraper):
@classmethod
def from_args(cls, args):
return cls(args.query, cursor = args.cursor, top = args.top, retries = args.retries)
return cls._construct(args, args.query, cursor = args.cursor, top = args.top)
class TwitterUserScraper(TwitterSearchScraper):
@@ -640,7 +640,7 @@ class TwitterUserScraper(TwitterSearchScraper):
@classmethod
def from_args(cls, args):
return cls(args.username, args.isUserId, retries = args.retries)
return cls._construct(args, args.username, args.isUserId)
class TwitterProfileScraper(TwitterUserScraper):
@@ -699,7 +699,7 @@ class TwitterHashtagScraper(TwitterSearchScraper):
@classmethod
def from_args(cls, args):
return cls(args.hashtag, retries = args.retries)
return cls._construct(args, args.hashtag)
class TwitterTweetScraperMode(enum.Enum):
@@ -783,7 +783,7 @@ class TwitterTweetScraper(TwitterAPIScraper):
@classmethod
def from_args(cls, args):
return cls(args.tweetId, TwitterTweetScraperMode.from_args(args), retries = args.retries)
return cls._construct(args, args.tweetId, TwitterTweetScraperMode.from_args(args))
class TwitterListPostsScraper(TwitterSearchScraper):
@@ -799,7 +799,7 @@ class TwitterListPostsScraper(TwitterSearchScraper):
@classmethod
def from_args(cls, args):
return cls(args.list, retries = args.retries)
return cls._construct(args, args.list)
class TwitterTrendsScraper(TwitterAPIScraper):
@@ -847,11 +847,3 @@ class TwitterTrendsScraper(TwitterAPIScraper):
for item in entry['content']['timelineModule']['items']:
trend = item['item']['content']['trend']
yield Trend(name = trend['name'], metaDescription = trend['trendMetadata'].get('metaDescription'), domainContext = trend['trendMetadata']['domainContext'])
@classmethod
def setup_parser(cls, subparser):
pass
@classmethod
def from_args(cls, args):
return cls(retries = args.retries)

2
snscrape/modules/vkontakte.py
View File

@@ -370,5 +370,5 @@ class VKontakteUserScraper(snscrape.base.Scraper):
@classmethod
def from_args(cls, args):
return cls(args.username, retries = args.retries)
return cls._construct(args, args.username)

2
snscrape/modules/weibo.py
View File

@@ -149,4 +149,4 @@ class WeiboUserScraper(snscrape.base.Scraper):
else:
uid = None
name = args.user
return cls(name = name, uid = uid, retries = args.retries)
return cls._construct(args, name = name, uid = uid)