From 6c50eee31b667b057739c1b26a64f572550b960e Mon Sep 17 00:00:00 2001 From: JustAnotherArchivist Date: Fri, 15 Apr 2022 19:23:54 +0000 Subject: [PATCH] Fix proxies not being applied correctly due to missing merge with environment settings Fixes #447 --- snscrape/base.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/snscrape/base.py b/snscrape/base.py index bda493f..71ab649 100644 --- a/snscrape/base.py +++ b/snscrape/base.py @@ -163,16 +163,19 @@ class Scraper: return self._get_entity() def _request(self, method, url, params = None, data = None, headers = None, timeout = 10, responseOkCallback = None, allowRedirects = True, proxies = None): - proxies = proxies or self._proxies + proxies = proxies or self._proxies or {} for attempt in range(self._retries + 1): # The request is newly prepared on each retry because of potential cookie updates. req = self._session.prepare_request(requests.Request(method, url, params = params, data = data, headers = headers)) + environmentSettings = self._session.merge_environment_settings(req.url, proxies, None, None, None) logger.info(f'Retrieving {req.url}') logger.debug(f'... with headers: {headers!r}') if data: logger.debug(f'... with data: {data!r}') + if environmentSettings: + logger.debug(f'... with environmentSettings: {environmentSettings!r}') try: - r = self._session.send(req, allow_redirects = allowRedirects, timeout = timeout, proxies = proxies) + r = self._session.send(req, allow_redirects = allowRedirects, timeout = timeout, **environmentSettings) except requests.exceptions.RequestException as exc: if attempt < self._retries: retrying = ', retrying'