bellingcat/snscrape
1
0
Fork 0
mirror of https://github.com/bellingcat/snscrape.git synced 2026-06-11 03:48:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

Throw an error on empty arguments

Browse Source 
Fixes #290
This commit is contained in:
JustAnotherArchivist
2021-10-10 17:43:27 +00:00
parent 5e829e2541
commit a6b6f3faaa
8 changed files with 25 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
snscrape/base.py
View File

@@ -217,3 +217,13 @@ class Scraper:
@classmethod
def _construct(cls, argparseArgs, *args, **kwargs):
return cls(*args, **kwargs, retries = argparseArgs.retries)
def nonempty_string(name):
def f(s):
s = s.strip()
if s:
return s
raise ValueError('must not be an empty string')
f.__name__ = name
return f

4
snscrape/modules/facebook.py
View File

@@ -203,7 +203,7 @@ class FacebookUserAndCommunityScraper(FacebookCommonScraper):
@classmethod
def setup_parser(cls, subparser):
subparser.add_argument('username', help = 'A Facebook username or user ID')
subparser.add_argument('username', type = snscrape.base.nonempty_string('username'), help = 'A Facebook username or user ID')
@classmethod
def from_args(cls, args):
@@ -354,7 +354,7 @@ class FacebookGroupScraper(FacebookCommonScraper):
@classmethod
def setup_parser(cls, subparser):
subparser.add_argument('group', help = 'A group name or ID')
subparser.add_argument('group', type = snscrape.base.nonempty_string('group'), help = 'A group name or ID')
@classmethod
def from_args(cls, args):

4
snscrape/modules/instagram.py
View File

@@ -174,7 +174,7 @@ class InstagramUserScraper(InstagramCommonScraper):
@classmethod
def setup_parser(cls, subparser):
subparser.add_argument('username', help = 'An Instagram username (no leading @)')
subparser.add_argument('username', type = snscrape.base.nonempty_string('username'), help = 'An Instagram username (no leading @)')
@classmethod
def from_args(cls, args):
@@ -219,7 +219,7 @@ class InstagramHashtagScraper(InstagramCommonScraper):
@classmethod
def setup_parser(cls, subparser):
subparser.add_argument('hashtag', help = 'An Instagram hashtag (no leading #)')
subparser.add_argument('hashtag', type = snscrape.base.nonempty_string('hashtag'), help = 'An Instagram hashtag (no leading #)')
@classmethod
def from_args(cls, args):

2
snscrape/modules/reddit.py
View File

@@ -218,7 +218,7 @@ def _make_scraper(name_, validationFunc, apiField):
@classmethod
def setup_parser(cls, subparser):
super()._setup_parser_opts(subparser)
subparser.add_argument(name_)
subparser.add_argument(name_, type = snscrape.base.nonempty_string(name_))
@classmethod
def from_args(cls, args):

2
snscrape/modules/telegram.py
View File

@@ -193,7 +193,7 @@ class TelegramChannelScraper(snscrape.base.Scraper):
@classmethod
def setup_parser(cls, subparser):
subparser.add_argument('channel', help = 'A channel name')
subparser.add_argument('channel', type = snscrape.base.nonempty_string('channel'), help = 'A channel name')
@classmethod
def from_args(cls, args):

10
snscrape/modules/twitter.py
View File

@@ -495,6 +495,8 @@ class TwitterSearchScraper(TwitterAPIScraper):
name = 'twitter-search'
def __init__(self, query, cursor = None, top = False, **kwargs):
if not query.strip():
raise ValueError('empty query')
super().__init__(baseUrl = 'https://twitter.com/search?' + urllib.parse.urlencode({'f': 'live', 'lang': 'en', 'q': query, 'src': 'spelling_expansion_revert_click'}), **kwargs)
self._query = query # Note: may get replaced by subclasses when using user ID resolution
self._cursor = cursor
@@ -511,6 +513,8 @@ class TwitterSearchScraper(TwitterAPIScraper):
return True, None
def get_items(self):
if not self._query.strip():
raise ValueError('empty query')
paginationParams = {
'include_profile_interstitial_type': '1',
'include_blocking': '1',
@@ -556,7 +560,7 @@ class TwitterSearchScraper(TwitterAPIScraper):
def setup_parser(cls, subparser):
subparser.add_argument('--cursor', metavar = 'CURSOR')
subparser.add_argument('--top', action = 'store_true', default = False, help = 'Enable fetching top tweets instead of live/chronological')
subparser.add_argument('query', help = 'A Twitter search string')
subparser.add_argument('query', type = snscrape.base.nonempty_string('query'), help = 'A Twitter search string')
@classmethod
def from_args(cls, args):
@@ -695,7 +699,7 @@ class TwitterHashtagScraper(TwitterSearchScraper):
@classmethod
def setup_parser(cls, subparser):
subparser.add_argument('hashtag', help = 'A Twitter hashtag (without #)')
subparser.add_argument('hashtag', type = snscrape.base.nonempty_string('hashtag'), help = 'A Twitter hashtag (without #)')
@classmethod
def from_args(cls, args):
@@ -795,7 +799,7 @@ class TwitterListPostsScraper(TwitterSearchScraper):
@classmethod
def setup_parser(cls, subparser):
subparser.add_argument('list', help = 'A Twitter list ID or a string of the form "username/listname" (replace spaces with dashes)')
subparser.add_argument('list', type = snscrape.base.nonempty_string('list'), help = 'A Twitter list ID or a string of the form "username/listname" (replace spaces with dashes)')
@classmethod
def from_args(cls, args):

2
snscrape/modules/vkontakte.py
View File

@@ -366,7 +366,7 @@ class VKontakteUserScraper(snscrape.base.Scraper):
@classmethod
def setup_parser(cls, subparser):
subparser.add_argument('username', help = 'A VK username')
subparser.add_argument('username', type = snscrape.base.nonempty_string('username'), help = 'A VK username')
@classmethod
def from_args(cls, args):

2
snscrape/modules/weibo.py
View File

@@ -139,7 +139,7 @@ class WeiboUserScraper(snscrape.base.Scraper):
@classmethod
def setup_parser(cls, subparser):
subparser.add_argument('user', help = 'A user name or ID')
subparser.add_argument('user', type = snscrape.base.nonempty_string('user'), help = 'A user name or ID')
@classmethod
def from_args(cls, args):