bellingcat/snscrape
1
0
Fork 0
mirror of https://github.com/bellingcat/snscrape.git synced 2026-06-13 12:58:27 +03:00
Code Issues Packages Projects Releases Wiki Activity

Add support for disabling following redirects

Browse Source
This commit is contained in:
JustAnotherArchivist
2020-09-13 00:52:26 +00:00
parent f296f9d21d
commit 800cfd5be0
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
snscrape/base.py
View File

@@ -76,7 +76,7 @@ class Scraper:
def entity(self): def entity(self):
return self._get_entity() return self._get_entity()
def _request(self, method, url, params = None, data = None, headers = None, timeout = 10, responseOkCallback = None): def _request(self, method, url, params = None, data = None, headers = None, timeout = 10, responseOkCallback = None, allowRedirects = True):
for attempt in range(self._retries + 1): for attempt in range(self._retries + 1):
# The request is newly prepared on each retry because of potential cookie updates. # The request is newly prepared on each retry because of potential cookie updates.
req = self._session.prepare_request(requests.Request(method, url, params = params, data = data, headers = headers)) req = self._session.prepare_request(requests.Request(method, url, params = params, data = data, headers = headers))
@@ -85,7 +85,7 @@ class Scraper:
if data: if data:
logger.debug(f'... with data: {data!r}') logger.debug(f'... with data: {data!r}')
try: try:
r = self._session.send(req, timeout = timeout) r = self._session.send(req, allow_redirects = allowRedirects, timeout = timeout)
except requests.exceptions.RequestException as exc: except requests.exceptions.RequestException as exc:
if attempt < self._retries: if attempt < self._retries:
retrying = ', retrying' retrying = ', retrying'