bellingcat/snscrape
1
0
Fork 0
mirror of https://github.com/bellingcat/snscrape.git synced 2026-06-12 12:28:28 +03:00
Code Issues Packages Projects Releases Wiki Activity

Include permalink.php, events, and notes (fixes #32)

Browse Source
This commit is contained in:
JustAnotherArchivist
2019-04-18 04:22:47 +02:00
parent c2cebd9166
commit f8d812f799
1 changed files with 4 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
snscrape/modules/facebook.py
View File

@@ -22,8 +22,10 @@ class FacebookUserScraper(snscrape.base.Scraper):
for entry in soup.find_all('div', class_ = '_5pcr'): # also class 'fbUserContent' in 2017 and 'userContentWrapper' in 2019 for entry in soup.find_all('div', class_ = '_5pcr'): # also class 'fbUserContent' in 2017 and 'userContentWrapper' in 2019
entryA = entry.find('a', class_ = '_5pcq') # There can be more than one, e.g. when a post is shared by another user, but the first one is always the one of this entry. entryA = entry.find('a', class_ = '_5pcq') # There can be more than one, e.g. when a post is shared by another user, but the first one is always the one of this entry.
href = entryA.get('href') href = entryA.get('href')
if not ('/posts/' in href or '/photos/' in href or '/videos/' in href): if not any(x in href for x in ('/posts/', '/photos/', '/videos/', '/permalink.php?', '/events/', '/notes/')):
logger.debug(f'Ignoring odd link: {href}') if href != '#' or 'new photo' not in entry.text or 'to the album' not in entry.text:
# Don't print a warning if it's a "User added 5 new photos to the album"-type entry, which doesn't have a permalink.
logger.warning(f'Ignoring odd link: {href}')
continue continue
link = urllib.parse.urljoin(baseUrl, href) link = urllib.parse.urljoin(baseUrl, href)
if link not in yielded: if link not in yielded: