bellingcat/snscrape
1
0
Fork 0
mirror of https://github.com/bellingcat/snscrape.git synced 2026-06-12 20:38:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #26 from ludios/avoid-twitter-bans

Browse Source 
twitter: randomize user agent to avoid Twitter's (IP, UA)-keyed bans
This commit is contained in:
JustAnotherArchivist
2018-12-25 02:19:17 +01:00
committed by GitHub
parent c5a5dcb92c 1984110f78
commit f5ee1f7ac5
1 changed files with 2 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
snscrape/modules/twitter.py
View File

@@ -1,5 +1,6 @@
import bs4 import bs4
import json import json
import random
import logging import logging
import snscrape.base import snscrape.base
@@ -31,7 +32,7 @@ class TwitterSearchScraper(snscrape.base.Scraper):
return True, None return True, None
def get_items(self): def get_items(self):
headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'} headers = {'User-Agent': f'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.{random.randint(1, 3500)}.{random.randint(1, 160)} Safari/537.36'}
# First page # First page
logger.info(f'Retrieving search page for {self._query}') logger.info(f'Retrieving search page for {self._query}')