bellingcat/snscrape
1
0
Fork 0
mirror of https://github.com/bellingcat/snscrape.git synced 2026-06-12 12:28:28 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #518 from hgrsd/fix/vkontakte-photo-scrape

Browse Source 
fix(vkontakte): update photo detection
This commit is contained in:
JustAnotherArchivist
2022-08-21 01:49:59 +00:00
committed by GitHub
parent 2c7a85a620 279d1cf4a1
commit d6bce5b1d6
1 changed files with 4 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
snscrape/modules/vkontakte.py
View File

@@ -117,6 +117,9 @@ class VKontakteUserScraper(snscrape.base.Scraper):
return urllib.parse.unquote(a['href'][13 : end]) return urllib.parse.unquote(a['href'][13 : end])
return None return None
def is_photo(self, a):
return 'aria-label' in a.attrs and a.attrs['aria-label'].startswith('photo')
def _date_span_to_date(self, dateSpan): def _date_span_to_date(self, dateSpan):
if not dateSpan: if not dateSpan:
return None return None
@@ -172,7 +175,7 @@ class VKontakteUserScraper(snscrape.base.Scraper):
not (not isCopy and thumbsDiv.parent.name == 'div' and 'class' in thumbsDiv.parent.attrs and 'copy_quote' in thumbsDiv.parent.attrs['class']): # Skip post quotes not (not isCopy and thumbsDiv.parent.name == 'div' and 'class' in thumbsDiv.parent.attrs and 'copy_quote' in thumbsDiv.parent.attrs['class']): # Skip post quotes
photos = [] photos = []
for a in thumbsDiv.find_all('a', class_ = 'page_post_thumb_wrap'): for a in thumbsDiv.find_all('a', class_ = 'page_post_thumb_wrap'):
if 'data-photo-id' not in a.attrs and 'data-video' not in a.attrs: if not self.is_photo(a) and 'data-video' not in a.attrs:
_logger.warning(f'Skipping non-photo and non-video thumb wrap on {url}') _logger.warning(f'Skipping non-photo and non-video thumb wrap on {url}')
continue continue
if 'data-video' in a.attrs: if 'data-video' in a.attrs: