bellingcat/cisticola
1
0
Fork 0
mirror of https://github.com/bellingcat/cisticola.git synced 2026-06-11 12:58:33 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #44 from bellingcat/bitchute-error

Browse Source 
Catch errors while retrieving Bitchute videos
This commit is contained in:
Logan Williams
2022-04-12 11:13:52 +02:00
committed by GitHub
parent e05f69bbee 1f667d532e
commit 1f7f957e62
1 changed files with 2 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
cisticola/scraper/bitchute.py
View File

@@ -184,7 +184,7 @@ def request_from_bitchute(session, method, url, headers=None, data=None):
raise NotImplemented()
if request.status_code >= 300:
raise ValueError("Response %i from BitChut for URL %s, need to retry" % (request.status_code, url))
raise ValueError("Response %i from BitChute for URL %s, need to retry" % (request.status_code, url))
response = request.json()
return response
@@ -421,14 +421,8 @@ def get_videos_user(session, user, csrftoken, detail):
post_data = {"csrfmiddlewaretoken": csrftoken, "name": "", "offset": str(offset)}
try:
request = session.post(url, data=post_data, headers=headers)
if request.status_code != 200:
raise ConnectionError()
response = request.json()
response = request_from_bitchute(session, "POST", url, headers=headers, data=post_data)
except (json.JSONDecodeError, requests.RequestException, ConnectionError) as e:
raise ValueError('FALSE')
soup = BeautifulSoup(response["html"], 'html.parser')
videos = soup.select(".channel-videos-container")
comments = []