From 1f667d532eeb2b38f8b0e1a6df7681a4ee9a6bac Mon Sep 17 00:00:00 2001 From: Tristan Lee Date: Wed, 6 Apr 2022 11:40:43 -0500 Subject: [PATCH] made get_videos_user use request_from_bitchute requests wrapper to catch errors --- cisticola/scraper/bitchute.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/cisticola/scraper/bitchute.py b/cisticola/scraper/bitchute.py index 643d013..0682e98 100644 --- a/cisticola/scraper/bitchute.py +++ b/cisticola/scraper/bitchute.py @@ -184,7 +184,7 @@ def request_from_bitchute(session, method, url, headers=None, data=None): raise NotImplemented() if request.status_code >= 300: - raise ValueError("Response %i from BitChut for URL %s, need to retry" % (request.status_code, url)) + raise ValueError("Response %i from BitChute for URL %s, need to retry" % (request.status_code, url)) response = request.json() return response @@ -421,14 +421,8 @@ def get_videos_user(session, user, csrftoken, detail): post_data = {"csrfmiddlewaretoken": csrftoken, "name": "", "offset": str(offset)} - try: - request = session.post(url, data=post_data, headers=headers) - if request.status_code != 200: - raise ConnectionError() - response = request.json() + response = request_from_bitchute(session, "POST", url, headers=headers, data=post_data) - except (json.JSONDecodeError, requests.RequestException, ConnectionError) as e: - raise ValueError('FALSE') soup = BeautifulSoup(response["html"], 'html.parser') videos = soup.select(".channel-videos-container") comments = []