bellingcat/snscrape
1
0
Fork 0
mirror of https://github.com/bellingcat/snscrape.git synced 2026-06-08 02:28:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

Add support for POST requests

Browse Source
This commit is contained in:
JustAnotherArchivist
2018-05-18 16:24:14 +02:00
parent e28a2cdb4b
commit 897f5bebe6
1 changed files with 10 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
socialmediascraper/base.py
View File

@@ -49,12 +49,14 @@ class Scraper:
'''Iterator yielding Items.'''
pass
def _get(self, url, params = None, headers = None, timeout = 10, responseOkCallback = None):
def _request(self, method, url, params = None, data = None, headers = None, timeout = 10, responseOkCallback = None):
for attempt in range(self._retries + 1):
# The request is newly prepared on each retry because of potential cookie updates.
req = self._session.prepare_request(requests.Request('GET', url, params = params, headers = headers))
req = self._session.prepare_request(requests.Request(method, url, params = params, data = data, headers = headers))
logger.info(f'Retrieving {req.url}')
logger.debug(f'... with headers: {headers!r}')
if data:
logger.debug(f'... with data: {data!r}')
try:
r = self._session.send(req, timeout = timeout)
if responseOkCallback is None or responseOkCallback(r):
@@ -72,6 +74,12 @@ class Scraper:
raise ScraperException(msg)
raise RuntimeError('Reached unreachable code')
def _get(self, *args, **kwargs):
return self._request('GET', *args, **kwargs)
def _post(self, *args, **kwargs):
return self._request('POST', *args, **kwargs)
@classmethod
@abc.abstractmethod
def setup_parser(cls, subparser):