From adac052723f34060743a4e81f9e005b3838c93c4 Mon Sep 17 00:00:00 2001 From: JustAnotherArchivist Date: Mon, 20 Mar 2023 19:15:11 +0000 Subject: [PATCH] Treat 404 responses from Twitter as a block --- snscrape/modules/twitter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/snscrape/modules/twitter.py b/snscrape/modules/twitter.py index 593eb42..33df8eb 100644 --- a/snscrape/modules/twitter.py +++ b/snscrape/modules/twitter.py @@ -678,7 +678,7 @@ class _TwitterAPIScraper(snscrape.base.Scraper): def _check_guest_token_response(self, r): if r.status_code != 200: self._set_random_user_agent() - return False, f'non-200 response ({r.status_code})' + return False, ('non-200 response' if r.status_code != 404 else 'blocked') + f' ({r.status_code})' return True, None def _ensure_guest_token(self, url = None): @@ -710,7 +710,7 @@ class _TwitterAPIScraper(snscrape.base.Scraper): del self._apiHeaders['x-guest-token'] def _check_api_response(self, r): - if r.status_code in (403, 429): + if r.status_code in (403, 404, 429): self._unset_guest_token() self._ensure_guest_token() return False, f'blocked ({r.status_code})'