bellingcat/snscrape
1
0
Fork 0
mirror of https://github.com/bellingcat/snscrape.git synced 2026-06-11 11:58:28 +03:00
Code Issues Packages Projects Releases Wiki Activity

Rewrite link extraction on Facebook (fixes #17)

Browse Source 
Facebook's returned HTML has a large number of inconsistencies; some (most) pages include a <link rel="canonical" /> but some don't, for example. This was at the root of the failing post extraction for some Facebook pages (#17). The previous link extraction technique was also quite poor for other reasons though. The new method uses the relevant CSS classes instead. Despite probably being the result of a CSS minimiser or similar, these seem to be quite stable: they haven't changed in the past two years (but the more readable ones have!).
This commit is contained in:
JustAnotherArchivist
2019-04-18 02:09:42 +02:00
parent b59c7e8d8f
commit 8458c12218
1 changed files with 14 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
snscrape/modules/facebook.py
View File

@@ -16,15 +16,19 @@ class FacebookUserScraper(snscrape.base.Scraper):
super().__init__(**kwargs)
self._username = username
def _soup_to_items(self, soup, username, baseUrl):
def _soup_to_items(self, soup, baseUrl):
yielded = set()
for a in soup.find_all('a', href = re.compile(r'^/[^/]+/(posts|photos|videos)/[^/]*\d')):
href = a.get('href')
if href.startswith(f'/{username}/'):
link = urllib.parse.urljoin(baseUrl, href)
if link not in yielded:
yield snscrape.base.URLItem(link)
yielded.add(link)
for entry in soup.find_all('div', class_ = '_5pcr'): # also class 'fbUserContent' in 2017 and 'userContentWrapper' in 2019
entryA = entry.find('a', class_ = '_5pcq') # There can be more than one, e.g. when a post is shared by another user, but the first one is always the one of this entry.
href = entryA.get('href')
if not ('/posts/' in href or '/photos/' in href or '/videos/' in href):
logger.debug(f'Ignoring odd link: {href}')
continue
link = urllib.parse.urljoin(baseUrl, href)
if link not in yielded:
yield snscrape.base.URLItem(link)
yielded.add(link)
def get_items(self):
headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36'}
@@ -42,9 +46,7 @@ class FacebookUserScraper(snscrape.base.Scraper):
logger.error('Got status code {r.status_code}')
return
soup = bs4.BeautifulSoup(r.text, 'lxml')
username = re.sub(r'^https://www\.facebook\.com/([^/]+)/$', r'\1', soup.find('link').get('href')) # Canonical capitalisation
baseUrl = f'https://www.facebook.com/{username}/'
yield from self._soup_to_items(soup, username, baseUrl)
yield from self._soup_to_items(soup, baseUrl)
nextPageLink = soup.find('a', ajaxify = nextPageLinkPattern)
while nextPageLink:
@@ -65,7 +67,7 @@ class FacebookUserScraper(snscrape.base.Scraper):
assert response['domops'][0][2] == False
assert '__html' in response['domops'][0][3]
soup = bs4.BeautifulSoup(response['domops'][0][3]['__html'], 'lxml')
yield from self._soup_to_items(soup, username, baseUrl)
yield from self._soup_to_items(soup, baseUrl)
nextPageLink = soup.find('a', ajaxify = nextPageLinkPattern)
@classmethod