From ed117e8891d6bf81d97df9db1379e578ae59e234 Mon Sep 17 00:00:00 2001 From: JustAnotherArchivist Date: Sun, 29 Aug 2021 18:26:00 +0000 Subject: [PATCH] Log response status code and redirects --- snscrape/base.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/snscrape/base.py b/snscrape/base.py index 829bb69..9817f1b 100644 --- a/snscrape/base.py +++ b/snscrape/base.py @@ -168,6 +168,11 @@ class Scraper: level = logging.ERROR logger.log(level, f'Error retrieving {req.url}: {exc!r}{retrying}') else: + redirected = f' (redirected to {r.url})' if r.history else '' + logger.info(f'Retrieved {req.url}{redirected}: {r.status_code}') + if r.history: + for i, redirect in enumerate(r.history): + logger.debug('... request {i}: {redirect.request.url}: {r.status_code} (Location: {r.headers.get("Location")})') if responseOkCallback is not None: success, msg = responseOkCallback(r) else: