From 5e829e2541fe47c0f107636d8abd1f9129558d29 Mon Sep 17 00:00:00 2001 From: JustAnotherArchivist Date: Thu, 30 Sep 2021 09:58:10 +0000 Subject: [PATCH] Refactor class instantiation to remove the need to repeat 'retries' everywhere --- snscrape/base.py | 8 +++++--- snscrape/modules/facebook.py | 4 ++-- snscrape/modules/instagram.py | 6 +++--- snscrape/modules/reddit.py | 2 +- snscrape/modules/telegram.py | 2 +- snscrape/modules/twitter.py | 18 +++++------------- snscrape/modules/vkontakte.py | 2 +- snscrape/modules/weibo.py | 2 +- 8 files changed, 19 insertions(+), 25 deletions(-) diff --git a/snscrape/base.py b/snscrape/base.py index 02dd2e1..6860845 100644 --- a/snscrape/base.py +++ b/snscrape/base.py @@ -207,11 +207,13 @@ class Scraper: return self._request('POST', *args, **kwargs) @classmethod - @abc.abstractmethod def setup_parser(cls, subparser): pass @classmethod - @abc.abstractmethod def from_args(cls, args): - pass + return cls._construct(args) + + @classmethod + def _construct(cls, argparseArgs, *args, **kwargs): + return cls(*args, **kwargs, retries = argparseArgs.retries) diff --git a/snscrape/modules/facebook.py b/snscrape/modules/facebook.py index 8285489..db205f5 100644 --- a/snscrape/modules/facebook.py +++ b/snscrape/modules/facebook.py @@ -207,7 +207,7 @@ class FacebookUserAndCommunityScraper(FacebookCommonScraper): @classmethod def from_args(cls, args): - return cls(args.username, retries = args.retries) + return cls._construct(args, args.username) class FacebookUserScraper(FacebookUserAndCommunityScraper): @@ -358,4 +358,4 @@ class FacebookGroupScraper(FacebookCommonScraper): @classmethod def from_args(cls, args): - return cls(args.group, retries = args.retries) + return cls._construct(args, args.group) diff --git a/snscrape/modules/instagram.py b/snscrape/modules/instagram.py index 5c1b8d6..cb4f5d3 100644 --- a/snscrape/modules/instagram.py +++ b/snscrape/modules/instagram.py @@ -178,7 +178,7 @@ class InstagramUserScraper(InstagramCommonScraper): @classmethod def from_args(cls, args): - return cls('User', args.username, retries = args.retries) + return cls._construct(args, 'User', args.username) def _get_entity(self): r = self._initial_page() @@ -223,7 +223,7 @@ class InstagramHashtagScraper(InstagramCommonScraper): @classmethod def from_args(cls, args): - return cls('Hashtag', args.hashtag, retries = args.retries) + return cls._construct(args, 'Hashtag', args.hashtag) class InstagramLocationScraper(InstagramCommonScraper): @@ -235,4 +235,4 @@ class InstagramLocationScraper(InstagramCommonScraper): @classmethod def from_args(cls, args): - return cls('Location', args.locationid, retries = args.retries) + return cls._construct(args, 'Location', args.locationid) diff --git a/snscrape/modules/reddit.py b/snscrape/modules/reddit.py index 892cbbf..bc4c721 100644 --- a/snscrape/modules/reddit.py +++ b/snscrape/modules/reddit.py @@ -222,7 +222,7 @@ def _make_scraper(name_, validationFunc, apiField): @classmethod def from_args(cls, args): - return cls(getattr(args, name_), submissions = not args.noSubmissions, comments = not args.noComments, before = args.before, after = args.after, retries = args.retries) + return cls._construct(args, getattr(args, name_), submissions = not args.noSubmissions, comments = not args.noComments, before = args.before, after = args.after) Scraper.__name__ = f'Reddit{name_.capitalize()}Scraper' Scraper.__qualname__ = Scraper.__name__ diff --git a/snscrape/modules/telegram.py b/snscrape/modules/telegram.py index d8819c9..8b64ca3 100644 --- a/snscrape/modules/telegram.py +++ b/snscrape/modules/telegram.py @@ -197,4 +197,4 @@ class TelegramChannelScraper(snscrape.base.Scraper): @classmethod def from_args(cls, args): - return cls(args.channel, retries = args.retries) + return cls._construct(args, args.channel) diff --git a/snscrape/modules/twitter.py b/snscrape/modules/twitter.py index 154ea1d..3f08efe 100644 --- a/snscrape/modules/twitter.py +++ b/snscrape/modules/twitter.py @@ -560,7 +560,7 @@ class TwitterSearchScraper(TwitterAPIScraper): @classmethod def from_args(cls, args): - return cls(args.query, cursor = args.cursor, top = args.top, retries = args.retries) + return cls._construct(args, args.query, cursor = args.cursor, top = args.top) class TwitterUserScraper(TwitterSearchScraper): @@ -640,7 +640,7 @@ class TwitterUserScraper(TwitterSearchScraper): @classmethod def from_args(cls, args): - return cls(args.username, args.isUserId, retries = args.retries) + return cls._construct(args, args.username, args.isUserId) class TwitterProfileScraper(TwitterUserScraper): @@ -699,7 +699,7 @@ class TwitterHashtagScraper(TwitterSearchScraper): @classmethod def from_args(cls, args): - return cls(args.hashtag, retries = args.retries) + return cls._construct(args, args.hashtag) class TwitterTweetScraperMode(enum.Enum): @@ -783,7 +783,7 @@ class TwitterTweetScraper(TwitterAPIScraper): @classmethod def from_args(cls, args): - return cls(args.tweetId, TwitterTweetScraperMode.from_args(args), retries = args.retries) + return cls._construct(args, args.tweetId, TwitterTweetScraperMode.from_args(args)) class TwitterListPostsScraper(TwitterSearchScraper): @@ -799,7 +799,7 @@ class TwitterListPostsScraper(TwitterSearchScraper): @classmethod def from_args(cls, args): - return cls(args.list, retries = args.retries) + return cls._construct(args, args.list) class TwitterTrendsScraper(TwitterAPIScraper): @@ -847,11 +847,3 @@ class TwitterTrendsScraper(TwitterAPIScraper): for item in entry['content']['timelineModule']['items']: trend = item['item']['content']['trend'] yield Trend(name = trend['name'], metaDescription = trend['trendMetadata'].get('metaDescription'), domainContext = trend['trendMetadata']['domainContext']) - - @classmethod - def setup_parser(cls, subparser): - pass - - @classmethod - def from_args(cls, args): - return cls(retries = args.retries) diff --git a/snscrape/modules/vkontakte.py b/snscrape/modules/vkontakte.py index 7b0d9ba..1703baf 100644 --- a/snscrape/modules/vkontakte.py +++ b/snscrape/modules/vkontakte.py @@ -370,5 +370,5 @@ class VKontakteUserScraper(snscrape.base.Scraper): @classmethod def from_args(cls, args): - return cls(args.username, retries = args.retries) + return cls._construct(args, args.username) diff --git a/snscrape/modules/weibo.py b/snscrape/modules/weibo.py index 2823c53..d972461 100644 --- a/snscrape/modules/weibo.py +++ b/snscrape/modules/weibo.py @@ -149,4 +149,4 @@ class WeiboUserScraper(snscrape.base.Scraper): else: uid = None name = args.user - return cls(name = name, uid = uid, retries = args.retries) + return cls._construct(args, name = name, uid = uid)