diff --git a/.gitignore b/.gitignore index 2885782..62a5815 100644 --- a/.gitignore +++ b/.gitignore @@ -16,4 +16,6 @@ config.yaml config-*.yaml logs/* local_archive/ -vk_config*.json \ No newline at end of file +vk_config*.json + +secrets/* \ No newline at end of file diff --git a/archivers/base_archiver.py b/archivers/base_archiver.py index 815d31e..8951115 100644 --- a/archivers/base_archiver.py +++ b/archivers/base_archiver.py @@ -31,9 +31,10 @@ class Archiver(ABC): name = "default" retry_regex = r"retrying at (\d+)$" - def __init__(self, storage: Storage, driver): + def __init__(self, storage: Storage, driver, hash_algorithm): self.storage = storage self.driver = driver + self.hash_algorithm = hash_algorithm def __str__(self): return self.__class__.__name__ @@ -163,10 +164,13 @@ class Archiver(ABC): def get_hash(self, filename): with open(filename, "rb") as f: bytes = f.read() # read entire file as bytes - # TODO: customizable hash - hash = hashlib.sha256(bytes) - # option to use SHA3_512 instead - # hash = hashlib.sha3_512(bytes) + ha = self.hash_algorithm + logger.debug(f'Hash algorithm is {ha}') + + if ha == "SHA3_512": hash = hashlib.sha3_512(bytes) + elif ha == "SHA256": hash = hashlib.sha256(bytes) + else: raise Exception("Unknown Hash Algorithm of {ha}") + return hash.hexdigest() def get_screenshot(self, url): diff --git a/archivers/telegram_archiver.py b/archivers/telegram_archiver.py index 0b6e777..c38dd30 100644 --- a/archivers/telegram_archiver.py +++ b/archivers/telegram_archiver.py @@ -11,6 +11,9 @@ from storages import Storage class TelegramArchiver(Archiver): name = "telegram" + def __init__(self, storage: Storage, driver, hash_algorithm): + super().__init__(storage, driver, hash_algorithm) + def download(self, url, check_if_exists=False): # detect URLs that we definitely cannot handle if 't.me' != self.get_netloc(url): diff --git a/archivers/telethon_archiver.py b/archivers/telethon_archiver.py index f35e323..bce34d2 100644 --- a/archivers/telethon_archiver.py +++ b/archivers/telethon_archiver.py @@ -15,8 +15,8 @@ class TelethonArchiver(Archiver): name = "telethon" link_pattern = re.compile(r"https:\/\/t\.me(\/c){0,1}\/(.+)\/(\d+)") - def __init__(self, storage: Storage, driver, config: TelethonConfig): - super().__init__(storage, driver) + def __init__(self, storage: Storage, driver, config: TelethonConfig, hash_algorithm): + super().__init__(storage, driver, hash_algorithm) if config: self.client = TelegramClient("./anon", config.api_id, config.api_hash) self.bot_token = config.bot_token diff --git a/archivers/tiktok_archiver.py b/archivers/tiktok_archiver.py index 8100bb1..30b8c7c 100644 --- a/archivers/tiktok_archiver.py +++ b/archivers/tiktok_archiver.py @@ -15,6 +15,9 @@ class TiktokArchiver(Archiver): status = 'success' + def __init__(self, storage: Storage, driver, hash_algorithm): + super().__init__(storage, driver, hash_algorithm) + try: info = tiktok_downloader.info_post(url) key = self.get_key(f'{info.id}.mp4') diff --git a/archivers/twitter_api_archiver.py b/archivers/twitter_api_archiver.py index ef2bf40..99fb8f1 100644 --- a/archivers/twitter_api_archiver.py +++ b/archivers/twitter_api_archiver.py @@ -13,8 +13,8 @@ from .twitter_archiver import TwitterArchiver class TwitterApiArchiver(TwitterArchiver): name = "twitter_api" - def __init__(self, storage: Storage, driver, config: TwitterApiConfig): - super().__init__(storage, driver) + def __init__(self, storage: Storage, driver, config: TwitterApiConfig, hash_algorithm): + super().__init__(storage, driver, hash_algorithm) if config.bearer_token: self.api = Api(bearer_token=config.bearer_token) @@ -54,7 +54,7 @@ class TwitterApiArchiver(TwitterArchiver): for u in urls: if u is None: - logger.error(f"Should not have gotten None url for {tweet.includes.media=}") + logger.debug(f"Should not have gotten None url for {tweet.includes.media=} so going to download_alternative in twitter_archiver") return self.download_alternative(url, tweet_id) logger.debug(f"found {urls=}") diff --git a/archivers/twitter_archiver.py b/archivers/twitter_archiver.py index 1c1b173..750d2c4 100644 --- a/archivers/twitter_archiver.py +++ b/archivers/twitter_archiver.py @@ -5,12 +5,16 @@ from snscrape.modules.twitter import TwitterTweetScraper, Video, Gif, Photo from .base_archiver import Archiver, ArchiveResult +from storages import Storage class TwitterArchiver(Archiver): """ This Twitter Archiver uses unofficial scraping methods, and it works as an alternative to TwitterApiArchiver when no API credentials are provided. """ + def __init__(self, storage: Storage, driver, hash_algorithm): + super().__init__(storage, driver, hash_algorithm) + name = "twitter" link_pattern = re.compile(r"twitter.com\/(?:\#!\/)?(\w+)\/status(?:es)?\/(\d+)") diff --git a/archivers/vk_archiver.py b/archivers/vk_archiver.py index c448367..6ddba10 100644 --- a/archivers/vk_archiver.py +++ b/archivers/vk_archiver.py @@ -17,8 +17,8 @@ class VkArchiver(Archiver): wall_pattern = re.compile(r"(wall.{0,1}\d+_\d+)") photo_pattern = re.compile(r"(photo.{0,1}\d+_\d+)") - def __init__(self, storage: Storage, driver, config: VkConfig): - super().__init__(storage, driver) + def __init__(self, storage: Storage, driver, config: VkConfig, hash_algorithm): + super().__init__(storage, driver, hash_algorithm) if config != None: self.vks = VkScraper(config.username, config.password) diff --git a/archivers/wayback_archiver.py b/archivers/wayback_archiver.py index f46d1cb..c19ca4f 100644 --- a/archivers/wayback_archiver.py +++ b/archivers/wayback_archiver.py @@ -15,8 +15,8 @@ class WaybackArchiver(Archiver): """ name = "wayback" - def __init__(self, storage: Storage, driver, config: WaybackConfig): - super(WaybackArchiver, self).__init__(storage, driver) + def __init__(self, storage: Storage, driver, config: WaybackConfig, hash_algorithm): + super(WaybackArchiver, self).__init__(storage, driver, hash_algorithm) self.config = config self.seen_urls = {} diff --git a/archivers/youtubedl_archiver.py b/archivers/youtubedl_archiver.py index 7990131..a41b6c6 100644 --- a/archivers/youtubedl_archiver.py +++ b/archivers/youtubedl_archiver.py @@ -12,8 +12,8 @@ class YoutubeDLArchiver(Archiver): name = "youtube_dl" ydl_opts = {'outtmpl': f'{Storage.TMP_FOLDER}%(id)s.%(ext)s', 'quiet': False} - def __init__(self, storage: Storage, driver, fb_cookie): - super().__init__(storage, driver) + def __init__(self, storage: Storage, driver, fb_cookie, hash_algorithm): + super().__init__(storage, driver, hash_algorithm) self.fb_cookie = fb_cookie def download(self, url, check_if_exists=False): diff --git a/auto_archive.py b/auto_archive.py index f12b9c4..72cb748 100644 --- a/auto_archive.py +++ b/auto_archive.py @@ -104,14 +104,14 @@ def process_sheet(c: Config): # order matters, first to succeed excludes remaining active_archivers = [ - TelethonArchiver(storage, c.webdriver, c.telegram_config), - TiktokArchiver(storage, c.webdriver), - TwitterApiArchiver(storage, c.webdriver, c.twitter_config), - YoutubeDLArchiver(storage, c.webdriver, c.facebook_cookie), - TelegramArchiver(storage, c.webdriver), - TwitterArchiver(storage, c.webdriver), - VkArchiver(storage, c.webdriver, c.vk_config), - WaybackArchiver(storage, c.webdriver, c.wayback_config) + TelethonArchiver(storage, c.webdriver, c.telegram_config, c.hash_algorithm), + TiktokArchiver(storage, c.webdriver, c.hash_algorithm), + TwitterApiArchiver(storage, c.webdriver, c.twitter_config, c.hash_algorithm), + YoutubeDLArchiver(storage, c.webdriver, c.facebook_cookie,c.hash_algorithm), + TelegramArchiver(storage, c.webdriver, c.hash_algorithm), + TwitterArchiver(storage, c.webdriver, c.hash_algorithm), + VkArchiver(storage, c.webdriver, c.vk_config, c.hash_algorithm), + WaybackArchiver(storage, c.webdriver, c.wayback_config, c.hash_algorithm) ] for archiver in active_archivers: diff --git a/configs/config.py b/configs/config.py index 41b531a..2d134da 100644 --- a/configs/config.py +++ b/configs/config.py @@ -81,6 +81,8 @@ class Config: ) self.webdriver = "not initialized" + self.hash_algorithm = execution.get("hash_algorithm") + # ---------------------- SECRETS - APIs and service configurations secrets = self.config.get("secrets", {}) diff --git a/example.config.yaml b/example.config.yaml index 3092efc..f823c47 100644 --- a/example.config.yaml +++ b/example.config.yaml @@ -104,3 +104,8 @@ execution: duration: duration screenshot: screenshot hash: hash + + # Must be either SHA256 or SHA3_512 + hash_algorithm: SHA3_512 + # hash_algorithm: SHA256 +