bellingcat/snscrape
1
0
Fork 0
mirror of https://github.com/bellingcat/snscrape.git synced 2026-06-11 11:58:28 +03:00
Code Issues Packages Projects Releases Wiki Activity

Add support for photo.php URLs

Browse Source 
Fixes #42
This commit is contained in:
JustAnotherArchivist
2019-06-30 18:36:39 +00:00
parent 6d6e3fa16c
commit 180f4dfeb7
1 changed files with 5 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
snscrape/modules/facebook.py
View File

@@ -30,6 +30,10 @@ class FacebookCommonScraper(snscrape.base.Scraper):
# Retain only story_fbid and id parameters
q = urllib.parse.parse_qs(u.query)
clean = (u.scheme, u.netloc, u.path, urllib.parse.urlencode((('story_fbid', q['story_fbid'][0]), ('id', q['id'][0]))), '')
elif u.path == '/photo.php':
# Retain only the fbid parameter
q = urllib.parse.parse_qs(u.query)
clean = (u.scheme, u.netloc, u.path, urllib.parse.urlencode((('fbid', q['fbid'][0]),)), '')
elif u.path.split('/')[2] == 'posts' or u.path.startswith('/events/') or u.path.startswith('/notes/'):
# No manipulation of the path needed, but strip the query string
clean = (u.scheme, u.netloc, u.path, '', '')
@@ -53,7 +57,7 @@ class FacebookCommonScraper(snscrape.base.Scraper):
def _is_odd_link(self, href, entryText, mode):
# Returns (isOddLink: bool, warn: bool|None)
if mode == 'user':
if not any(x in href for x in ('/posts/', '/photos/', '/videos/', '/permalink.php?', '/events/', '/notes/')):
if not any(x in href for x in ('/posts/', '/photos/', '/videos/', '/permalink.php?', '/events/', '/notes/', '/photo.php?')):
if href == '#' and 'new photo' in entryText and 'to the album' in entryText:
# Don't print a warning if it's a "User added 5 new photos to the album"-type entry, which doesn't have a permalink.
return True, False