diff --git a/socialmediascraper/modules/instagram.py b/socialmediascraper/modules/instagram.py index 6d8186a..8266929 100644 --- a/socialmediascraper/modules/instagram.py +++ b/socialmediascraper/modules/instagram.py @@ -1,3 +1,4 @@ +import hashlib import json import logging import socialmediascraper.base @@ -22,31 +23,35 @@ class InstagramUserScraper(socialmediascraper.base.Scraper): headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'} logger.info('Retrieving initial data') - r = self._get(f'https://www.instagram.com/{self._username}/?__a=1', headers = headers) + r = self._get(f'https://www.instagram.com/{self._username}/', headers = headers) if r.status_code == 404: logger.warning('User does not exist') return elif r.status_code != 200: logger.error(f'Got status code {r.status_code}') return - response = json.loads(r.text) - if response['graphql']['user']['edge_owner_to_timeline_media']['count'] == 0: + jsonData = r.text.split('')[0] # May throw an IndexError if Instagram changes something again; we just let that bubble. + response = json.loads(jsonData) + rhxGis = response['rhx_gis'] + if response['entry_data']['ProfilePage'][0]['graphql']['user']['edge_owner_to_timeline_media']['count'] == 0: logger.info('User has no posts') return - if not response['graphql']['user']['edge_owner_to_timeline_media']['edges']: + if not response['entry_data']['ProfilePage'][0]['graphql']['user']['edge_owner_to_timeline_media']['edges']: logger.warning('Private account') return - userID = response['graphql']['user']['id'] - username = response['graphql']['user']['username'] # Might have different capitalisation than self._username - yield from self._response_to_items(response['graphql'], username) - if not response['graphql']['user']['edge_owner_to_timeline_media']['page_info']['has_next_page']: + userID = response['entry_data']['ProfilePage'][0]['graphql']['user']['id'] + username = response['entry_data']['ProfilePage'][0]['graphql']['user']['username'] # Might have different capitalisation than self._username + yield from self._response_to_items(response['entry_data']['ProfilePage'][0]['graphql'], username) + if not response['entry_data']['ProfilePage'][0]['graphql']['user']['edge_owner_to_timeline_media']['page_info']['has_next_page']: return - endCursor = response['graphql']['user']['edge_owner_to_timeline_media']['page_info']['end_cursor'] + endCursor = response['entry_data']['ProfilePage'][0]['graphql']['user']['edge_owner_to_timeline_media']['page_info']['end_cursor'] - # Cf. https://stackoverflow.com/questions/49265339/instagram-a-1-url-doesnt-allow-max-id and https://github.com/rarcega/instagram-scraper while True: logger.info(f'Retrieving endCursor = {endCursor!r}') - r = self._get(f'https://www.instagram.com/graphql/query/?query_hash=42323d64886122307be10013ad2dcc44&variables={{"id":"{userID}","first":12,"after":"{endCursor}"}}', headers = headers) + variables = f'{{"id":"{userID}","first":50,"after":"{endCursor}"}}' + headers['X-Requested-With'] = 'XMLHttpRequest' + headers['X-Instagram-GIS'] = hashlib.md5(f'{rhxGis}:{variables}'.encode('utf-8')).hexdigest() + r = self._get(f'https://www.instagram.com/graphql/query/?query_hash=42323d64886122307be10013ad2dcc44&variables={variables}', headers = headers) if r.status_code != 200: logger.error(f'Got status code {r.status_code}')