bellingcat/snscrape
1
0
Fork 0
mirror of https://github.com/bellingcat/snscrape.git synced 2026-06-11 20:08:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

Better log output for temporary failures (fixes #2)

Browse Source
This commit is contained in:
JustAnotherArchivist
2018-10-01 03:24:29 +02:00
parent 4d0350e541
commit d0d3c8b2a6
2 changed files with 27 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
snscrape/base.py
View File

@@ -59,11 +59,32 @@ class Scraper:
logger.debug(f'... with data: {data!r}')
try:
r = self._session.send(req, timeout = timeout)
if responseOkCallback is None or responseOkCallback(r):
logger.debug(f'{req.url} retrieved successfully')
return r
except requests.exceptions.RequestException as exc:
logger.error(f'Error retrieving {url}: {exc!r}')
if attempt < self._retries:
retrying = ', retrying'
level = logging.WARNING
else:
retrying = ''
level = logging.ERROR
logger.log(level, f'Error retrieving {req.url}: {exc!r}{retrying}')
else:
if responseOkCallback is not None:
success, msg = responseOkCallback(r)
else:
success, msg = (True, None)
msg = f': {msg}' if msg else ''
if success:
logger.debug(f'{req.url} retrieved successfully{msg}')
return r
else:
if attempt < self._retries:
retrying = ', retrying'
level = logging.WARNING
else:
retrying = ''
level = logging.ERROR
logger.log(level, f'Error retrieving {req.url}{msg}{retrying}')
if attempt < self._retries:
sleepTime = 1.0 * 2**attempt # exponential backoff: sleep 1 second after first attempt, 2 after second, 4 after third, etc.
logger.info(f'Waiting {sleepTime:.0f} seconds')