bellingcat/cisticola
1
0
Fork 0
mirror of https://github.com/bellingcat/cisticola.git synced 2026-06-08 03:18:34 +03:00
Code Issues Packages Projects Releases Wiki Activity

made get_videos_user use request_from_bitchute requests wrapper to catch errors

Browse Source
This commit is contained in:
Tristan Lee
2022-04-06 11:40:43 -05:00
parent 36c81c8e17
commit 1f667d532e
1 changed files with 2 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
cisticola/scraper/bitchute.py
View File

@@ -184,7 +184,7 @@ def request_from_bitchute(session, method, url, headers=None, data=None):
raise NotImplemented()
if request.status_code >= 300:
raise ValueError("Response %i from BitChut for URL %s, need to retry" % (request.status_code, url))
raise ValueError("Response %i from BitChute for URL %s, need to retry" % (request.status_code, url))
response = request.json()
return response
@@ -421,14 +421,8 @@ def get_videos_user(session, user, csrftoken, detail):
post_data = {"csrfmiddlewaretoken": csrftoken, "name": "", "offset": str(offset)}
try:
request = session.post(url, data=post_data, headers=headers)
if request.status_code != 200:
raise ConnectionError()
response = request.json()
response = request_from_bitchute(session, "POST", url, headers=headers, data=post_data)
except (json.JSONDecodeError, requests.RequestException, ConnectionError) as e:
raise ValueError('FALSE')
soup = BeautifulSoup(response["html"], 'html.parser')
videos = soup.select(".channel-videos-container")
comments = []