bellingcat/snscrape
1
0
Fork 0
mirror of https://github.com/bellingcat/snscrape.git synced 2026-06-12 20:38:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

Override TLS ciphers to get past Twitter's new fingerprinting

Browse Source 
Fixes #647
This commit is contained in:
JustAnotherArchivist
2023-01-13 02:25:39 +00:00
parent ceb06664f0
commit 7de8d734e9
1 changed files with 13 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
snscrape/modules/twitter.py
View File

@@ -26,11 +26,13 @@ import random
import logging import logging
import os import os
import re import re
import requests.adapters
import snscrape.base import snscrape.base
import string import string
import time import time
import typing import typing
import urllib.parse import urllib.parse
import urllib3.util.ssl_
import warnings import warnings
@@ -49,6 +51,7 @@ _logger = logging.getLogger(__name__)
_API_AUTHORIZATION_HEADER = 'Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs=1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA' _API_AUTHORIZATION_HEADER = 'Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs=1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA'
_globalGuestTokenManager = None _globalGuestTokenManager = None
_GUEST_TOKEN_VALIDITY = 10800 _GUEST_TOKEN_VALIDITY = 10800
_CIPHERS_CHROME = 'TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-RSA-AES128-SHA:ECDHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA:AES256-SHA'
@dataclasses.dataclass @dataclasses.dataclass
@@ -612,6 +615,13 @@ class _CLIGuestTokenManager(GuestTokenManager):
pass pass
class _TwitterTLSAdapter(requests.adapters.HTTPAdapter):
def init_poolmanager(self, *args, **kwargs):
#FIXME: When urllib3 2.0.0 is out and can be required, this should use urllib3.util.create_urllib3_context instead of the private, undocumented ssl_ module.
kwargs['ssl_context'] = urllib3.util.ssl_.create_urllib3_context(ciphers = _CIPHERS_CHROME)
return super().init_poolmanager(*args, **kwargs)
class _TwitterAPIType(enum.Enum): class _TwitterAPIType(enum.Enum):
V2 = 0 # Introduced with the redesign V2 = 0 # Introduced with the redesign
GRAPHQL = 1 GRAPHQL = 1
@@ -633,6 +643,9 @@ class _TwitterAPIScraper(snscrape.base.Scraper):
'Referer': self._baseUrl, 'Referer': self._baseUrl,
'Accept-Language': 'en-US,en;q=0.5', 'Accept-Language': 'en-US,en;q=0.5',
} }
adapter = _TwitterTLSAdapter()
self._session.mount('https://twitter.com', adapter)
self._session.mount('https://api.twitter.com', adapter)
self._set_random_user_agent() self._set_random_user_agent()
def _set_random_user_agent(self): def _set_random_user_agent(self):