bellingcat/snscrape
1
0
Fork 0
mirror of https://github.com/bellingcat/snscrape.git synced 2026-06-12 12:28:28 +03:00
Code Issues Packages Projects Releases Wiki Activity

Fix Instagram again

Browse Source 
- __a=1 is no longer supported, so we need to extract the JSON from the HTML page instead.
- There is now a X-Instagram-GIS header that needs to be set correctly.
This commit is contained in:
JustAnotherArchivist
2018-04-26 15:06:28 +02:00
parent 5a084af85c
commit e28a2cdb4b
1 changed files with 16 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
socialmediascraper/modules/instagram.py
View File

@@ -1,3 +1,4 @@
import hashlib
import json
import logging
import socialmediascraper.base
@@ -22,31 +23,35 @@ class InstagramUserScraper(socialmediascraper.base.Scraper):
headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'}
logger.info('Retrieving initial data')
r = self._get(f'https://www.instagram.com/{self._username}/?__a=1', headers = headers)
r = self._get(f'https://www.instagram.com/{self._username}/', headers = headers)
if r.status_code == 404:
logger.warning('User does not exist')
return
elif r.status_code != 200:
logger.error(f'Got status code {r.status_code}')
return
response = json.loads(r.text)
if response['graphql']['user']['edge_owner_to_timeline_media']['count'] == 0:
jsonData = r.text.split('<script type="text/javascript">window._sharedData = ')[1].split(';</script>')[0] # May throw an IndexError if Instagram changes something again; we just let that bubble.
response = json.loads(jsonData)
rhxGis = response['rhx_gis']
if response['entry_data']['ProfilePage'][0]['graphql']['user']['edge_owner_to_timeline_media']['count'] == 0:
logger.info('User has no posts')
return
if not response['graphql']['user']['edge_owner_to_timeline_media']['edges']:
if not response['entry_data']['ProfilePage'][0]['graphql']['user']['edge_owner_to_timeline_media']['edges']:
logger.warning('Private account')
return
userID = response['graphql']['user']['id']
username = response['graphql']['user']['username'] # Might have different capitalisation than self._username
yield from self._response_to_items(response['graphql'], username)
if not response['graphql']['user']['edge_owner_to_timeline_media']['page_info']['has_next_page']:
userID = response['entry_data']['ProfilePage'][0]['graphql']['user']['id']
username = response['entry_data']['ProfilePage'][0]['graphql']['user']['username'] # Might have different capitalisation than self._username
yield from self._response_to_items(response['entry_data']['ProfilePage'][0]['graphql'], username)
if not response['entry_data']['ProfilePage'][0]['graphql']['user']['edge_owner_to_timeline_media']['page_info']['has_next_page']:
return
endCursor = response['graphql']['user']['edge_owner_to_timeline_media']['page_info']['end_cursor']
endCursor = response['entry_data']['ProfilePage'][0]['graphql']['user']['edge_owner_to_timeline_media']['page_info']['end_cursor']
# Cf. https://stackoverflow.com/questions/49265339/instagram-a-1-url-doesnt-allow-max-id and https://github.com/rarcega/instagram-scraper
while True:
logger.info(f'Retrieving endCursor = {endCursor!r}')
r = self._get(f'https://www.instagram.com/graphql/query/?query_hash=42323d64886122307be10013ad2dcc44&variables={{"id":"{userID}","first":12,"after":"{endCursor}"}}', headers = headers)
variables = f'{{"id":"{userID}","first":50,"after":"{endCursor}"}}'
headers['X-Requested-With'] = 'XMLHttpRequest'
headers['X-Instagram-GIS'] = hashlib.md5(f'{rhxGis}:{variables}'.encode('utf-8')).hexdigest()
r = self._get(f'https://www.instagram.com/graphql/query/?query_hash=42323d64886122307be10013ad2dcc44&variables={variables}', headers = headers)
if r.status_code != 200:
logger.error(f'Got status code {r.status_code}')