bellingcat/snscrape
1
0
Fork 0
mirror of https://github.com/bellingcat/snscrape.git synced 2026-06-12 20:38:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

Check that Instagram returned valid JSON, take 2

Browse Source 
Fixes #22
This commit is contained in:
JustAnotherArchivist
2019-06-10 15:03:15 +00:00
parent 7d1916292c
commit 731a2e8c8b
1 changed files with 13 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
snscrape/modules/instagram.py
View File

@@ -60,6 +60,17 @@ class InstagramCommonScraper(snscrape.base.Scraper):
displayUrl = node['node']['display_url'], displayUrl = node['node']['display_url'],
) )
def _check_initial_page_callback(self, r):
if r.status_code != 200:
return True, None
jsonData = r.text.split('<script type="text/javascript">window._sharedData = ')[1].split(';</script>')[0] # May throw an IndexError if Instagram changes something again; we just let that bubble.
try:
obj = json.loads(jsonData)
except json.JSONDecodeError:
return False, 'invalid JSON'
r._snscrape_json_obj = obj
return True, None
def _check_json_callback(self, r): def _check_json_callback(self, r):
try: try:
obj = json.loads(r.text) obj = json.loads(r.text)
@@ -72,15 +83,14 @@ class InstagramCommonScraper(snscrape.base.Scraper):
headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'} headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'}
logger.info('Retrieving initial data') logger.info('Retrieving initial data')
r = self._get(self._initialUrl, headers = headers) r = self._get(self._initialUrl, headers = headers, responseOkCallback = self._check_initial_page_callback)
if r.status_code == 404: if r.status_code == 404:
logger.warning(f'{self._mode} does not exist') logger.warning(f'{self._mode} does not exist')
return return
elif r.status_code != 200: elif r.status_code != 200:
logger.error(f'Got status code {r.status_code}') logger.error(f'Got status code {r.status_code}')
return return
jsonData = r.text.split('<script type="text/javascript">window._sharedData = ')[1].split(';</script>')[0] # May throw an IndexError if Instagram changes something again; we just let that bubble. response = r._snscrape_json_obj
response = json.loads(jsonData)
rhxGis = response['rhx_gis'] if 'rhx_gis' in response else '' rhxGis = response['rhx_gis'] if 'rhx_gis' in response else ''
if response['entry_data'][self._pageName][0]['graphql'][self._responseContainer][self._edgeXToMedia]['count'] == 0: if response['entry_data'][self._pageName][0]['graphql'][self._responseContainer][self._edgeXToMedia]['count'] == 0:
logger.info(f'{self._mode} has no posts') logger.info(f'{self._mode} has no posts')