From 731a2e8c8bbee23dc8833e2d11e8c9a94abfeb6c Mon Sep 17 00:00:00 2001 From: JustAnotherArchivist Date: Mon, 10 Jun 2019 15:03:15 +0000 Subject: [PATCH] Check that Instagram returned valid JSON, take 2 Fixes #22 --- snscrape/modules/instagram.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/snscrape/modules/instagram.py b/snscrape/modules/instagram.py index f042adf..ef96d3d 100644 --- a/snscrape/modules/instagram.py +++ b/snscrape/modules/instagram.py @@ -60,6 +60,17 @@ class InstagramCommonScraper(snscrape.base.Scraper): displayUrl = node['node']['display_url'], ) + def _check_initial_page_callback(self, r): + if r.status_code != 200: + return True, None + jsonData = r.text.split('')[0] # May throw an IndexError if Instagram changes something again; we just let that bubble. + try: + obj = json.loads(jsonData) + except json.JSONDecodeError: + return False, 'invalid JSON' + r._snscrape_json_obj = obj + return True, None + def _check_json_callback(self, r): try: obj = json.loads(r.text) @@ -72,15 +83,14 @@ class InstagramCommonScraper(snscrape.base.Scraper): headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'} logger.info('Retrieving initial data') - r = self._get(self._initialUrl, headers = headers) + r = self._get(self._initialUrl, headers = headers, responseOkCallback = self._check_initial_page_callback) if r.status_code == 404: logger.warning(f'{self._mode} does not exist') return elif r.status_code != 200: logger.error(f'Got status code {r.status_code}') return - jsonData = r.text.split('')[0] # May throw an IndexError if Instagram changes something again; we just let that bubble. - response = json.loads(jsonData) + response = r._snscrape_json_obj rhxGis = response['rhx_gis'] if 'rhx_gis' in response else '' if response['entry_data'][self._pageName][0]['graphql'][self._responseContainer][self._edgeXToMedia]['count'] == 0: logger.info(f'{self._mode} has no posts')