bellingcat/snscrape
1
0
Fork 0
mirror of https://github.com/bellingcat/snscrape.git synced 2026-06-10 03:18:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

Fix vkontakte-user pagination

Browse Source
This commit is contained in:
Jody Leonard
2019-10-30 20:39:27 -04:00
parent e6aae35304
commit 20ea117a2c
1 changed files with 6 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
snscrape/modules/vkontakte.py
View File

@@ -80,14 +80,15 @@ class VKontakteUserScraper(snscrape.base.Scraper):
if r.status_code != 200:
logger.error(f'Got status code {r.status_code}')
return
fields = r.content.split(b'<!>')
if fields[5].startswith(b'<div class="page_block no_posts">'):
# Convert to JSON and read the HTML payload. Note that this implicitly converts the data to a Python string (i.e., Unicode), away from a windows-1251-encoded bytes.
posts = r.json()['payload'][1][0]
if posts.startswith('<div class="page_block no_posts">'):
# Reached the end
break
if not fields[5].startswith(b'<div id="post'):
logger.error(f'Got an unknown response: {fields[5][:200]!r}...')
if not posts.startswith('<div id="post'):
logger.error(f'Got an unknown response: {posts[:200]!r}...')
break
soup = bs4.BeautifulSoup(fields[5], 'lxml', from_encoding = r.encoding)
soup = bs4.BeautifulSoup(posts, 'lxml')
yield from self._soup_to_items(soup, baseUrl)
@classmethod