bellingcat/snscrape
1
0
Fork 0
mirror of https://github.com/bellingcat/snscrape.git synced 2026-06-08 02:28:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

Adjust to Twitter's new method of handing out guest tokens

Browse Source 
Fixes #64
This commit is contained in:
JustAnotherArchivist
2020-06-24 21:22:58 +00:00
parent 7b3c7deb28
commit 90f9598ecc
1 changed files with 4 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
snscrape/modules/twitter.py
View File

@@ -98,10 +98,9 @@ class TwitterSearchScraper(TwitterCommonScraper):
def _get_guest_token(self):
logger.info(f'Retrieving guest token from search page')
r = self._get(self._baseUrl, headers = {'User-Agent': self._userAgent})
match = re.search(r'document\.cookie = decodeURIComponent\("gt=(\d+);', r.text)
if not match:
raise snscrape.base.ScraperException('Unable to find guest token')
return match.group(1)
if 'gt' not in r.cookies:
raise snscrape.base.ScraperException("Twitter didn't set the cookie")
return r.cookies['gt']
def _check_scroll_response(self, r):
if r.status_code == 429:
@@ -162,6 +161,7 @@ class TwitterSearchScraper(TwitterCommonScraper):
r = self._get('https://api.twitter.com/2/search/adaptive.json', params = params, headers = headers, responseOkCallback = self._check_scroll_response)
if r.status_code == 429:
guestToken = None
del self._session.cookies['gt']
continue
try:
obj = r.json()