bellingcat/cisticola
1
0
Fork 0
mirror of https://github.com/bellingcat/cisticola.git synced 2026-06-13 05:48:33 +03:00
Code Issues Packages Projects Releases Wiki Activity

converted bitchute to yield, got video archiving working on bitchute and gettr, added url_to_blob method that downloads media bytes blob from url and converted archive_media to take in the media bytes blob instead of the media url.

Browse Source
This commit is contained in:
Tristan Lee
2022-02-25 13:43:30 -06:00
parent 8ab56ff5ba
commit ef83cc4b0a
8 changed files with 101 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
cisticola/scraper/base.py
View File

@@ -1,4 +1,4 @@
from typing import Generator
from typing import Generator, Tuple
import cisticola.base
import requests
import os
@@ -24,7 +24,8 @@ class Scraper:
def __str__(self):
return self.__version__
def archive_media(self, url: str, key: str = None) -> str:
def url_to_blob(self, url: str, key: str = None) -> Tuple[bytes, str, str]:
n_retries = 0
r = requests.get(url)
@@ -38,13 +39,16 @@ class Scraper:
return url
blob = r.content
content_type = r.headers.get('Content-Type')
if key is None:
key = url.split('/')[-1]
key = key.split('?')[0]
return blob, content_type, key
def archive_media(self, blob: bytes, content_type: str, key: str) -> str:
filename = self.__version__.replace(' ', '_') + '/' + key
self.s3_client.upload_fileobj(BytesIO(blob), Bucket=os.getenv(