bellingcat/snscrape
1
0
Fork 0
mirror of https://github.com/bellingcat/snscrape.git synced 2026-06-12 20:38:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

Remove dirtyUrl which does not appear to be used anymore by Instagram

Browse Source 
#234
This commit is contained in:
JustAnotherArchivist
2021-05-27 00:32:03 +00:00
parent 157e4d4265
commit 57d0aaafc1
1 changed files with 4 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
snscrape/modules/instagram.py
View File

@@ -13,8 +13,7 @@ logger = logging.getLogger(__name__)
@dataclasses.dataclass @dataclasses.dataclass
class InstagramPost(snscrape.base.Item): class InstagramPost(snscrape.base.Item):
cleanUrl: str url: str
dirtyUrl: str
date: datetime.datetime date: datetime.datetime
content: str content: str
thumbnailUrl: str thumbnailUrl: str
@@ -26,7 +25,7 @@ class InstagramPost(snscrape.base.Item):
isVideo: bool isVideo: bool
def __str__(self): def __str__(self):
return self.cleanUrl return self.url
@dataclasses.dataclass @dataclasses.dataclass
@@ -85,11 +84,9 @@ class InstagramCommonScraper(snscrape.base.Scraper):
for node in response[self._responseContainer][self._edgeXToMedia]['edges']: for node in response[self._responseContainer][self._edgeXToMedia]['edges']:
code = node['node']['shortcode'] code = node['node']['shortcode']
username = node['node']['owner']['username'] if 'username' in node['node']['owner'] else None username = node['node']['owner']['username'] if 'username' in node['node']['owner'] else None
usernameQuery = '?taken-by=' + (username or '') url = f'https://www.instagram.com/p/{code}/'
cleanUrl = f'https://www.instagram.com/p/{code}/'
yield InstagramPost( yield InstagramPost(
cleanUrl = cleanUrl, url = url,
dirtyUrl = f'{cleanUrl}{usernameQuery}',
date = datetime.datetime.fromtimestamp(node['node']['taken_at_timestamp'], datetime.timezone.utc), date = datetime.datetime.fromtimestamp(node['node']['taken_at_timestamp'], datetime.timezone.utc),
content = node['node']['edge_media_to_caption']['edges'][0]['node']['text'] if len(node['node']['edge_media_to_caption']['edges']) else None, content = node['node']['edge_media_to_caption']['edges'][0]['node']['text'] if len(node['node']['edge_media_to_caption']['edges']) else None,
thumbnailUrl = node['node']['thumbnail_src'], thumbnailUrl = node['node']['thumbnail_src'],