bellingcat/snscrape
1
0
Fork 0
mirror of https://github.com/bellingcat/snscrape.git synced 2026-06-12 20:38:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

Log response headers for debugging

Browse Source
This commit is contained in:
JustAnotherArchivist
2023-01-16 03:48:21 +00:00
parent 49270f6d3a
commit 36e85c54c1
1 changed files with 2 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
snscrape/base.py
View File

@@ -219,9 +219,11 @@ class Scraper:
else: else:
redirected = f' (redirected to {r.url})' if r.history else '' redirected = f' (redirected to {r.url})' if r.history else ''
logger.info(f'Retrieved {req.url}{redirected}: {r.status_code}') logger.info(f'Retrieved {req.url}{redirected}: {r.status_code}')
logger.debug(f'... with response headers: {r.headers!r}')
if r.history: if r.history:
for i, redirect in enumerate(r.history): for i, redirect in enumerate(r.history):
logger.debug(f'... request {i}: {redirect.request.url}: {redirect.status_code} (Location: {redirect.headers.get("Location")})') logger.debug(f'... request {i}: {redirect.request.url}: {redirect.status_code} (Location: {redirect.headers.get("Location")})')
logger.debug(f'... ... with response headers: {redirect.headers!r}')
if responseOkCallback is not None: if responseOkCallback is not None:
success, msg = responseOkCallback(r) success, msg = responseOkCallback(r)
errors.append(msg) errors.append(msg)