From 897f5bebe6c384e3c5a9f153751d669ac57f6a38 Mon Sep 17 00:00:00 2001 From: JustAnotherArchivist Date: Fri, 18 May 2018 16:24:14 +0200 Subject: [PATCH] Add support for POST requests --- socialmediascraper/base.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/socialmediascraper/base.py b/socialmediascraper/base.py index 62213a5..a4a6f21 100644 --- a/socialmediascraper/base.py +++ b/socialmediascraper/base.py @@ -49,12 +49,14 @@ class Scraper: '''Iterator yielding Items.''' pass - def _get(self, url, params = None, headers = None, timeout = 10, responseOkCallback = None): + def _request(self, method, url, params = None, data = None, headers = None, timeout = 10, responseOkCallback = None): for attempt in range(self._retries + 1): # The request is newly prepared on each retry because of potential cookie updates. - req = self._session.prepare_request(requests.Request('GET', url, params = params, headers = headers)) + req = self._session.prepare_request(requests.Request(method, url, params = params, data = data, headers = headers)) logger.info(f'Retrieving {req.url}') logger.debug(f'... with headers: {headers!r}') + if data: + logger.debug(f'... with data: {data!r}') try: r = self._session.send(req, timeout = timeout) if responseOkCallback is None or responseOkCallback(r): @@ -72,6 +74,12 @@ class Scraper: raise ScraperException(msg) raise RuntimeError('Reached unreachable code') + def _get(self, *args, **kwargs): + return self._request('GET', *args, **kwargs) + + def _post(self, *args, **kwargs): + return self._request('POST', *args, **kwargs) + @classmethod @abc.abstractmethod def setup_parser(cls, subparser):