Catch exceptions in archive_files so that archiver continues to run

This commit is contained in:
Logan Williams
2022-04-03 14:12:23 +02:00
parent 9c838aae39
commit ecae1aad05
6 changed files with 6 additions and 0 deletions

View File

@@ -236,6 +236,7 @@ class Scraper:
return archived_url
@logger.catch
def archive_files(self, result: ScraperResult) -> ScraperResult:
"""Archive files corresponding to ``archived_url`` dict keys, if the
files have not previously been archived.

View File

@@ -79,6 +79,7 @@ class OdyseeScraper(Scraper):
archived_urls={},
media_archived=datetime.now(timezone.utc))
@logger.catch
def archive_files(self, result: ScraperResult) -> ScraperResult:
for url in result.archived_urls:
if result.archived_urls[url] is None:

View File

@@ -50,6 +50,7 @@ class RumbleScraper(Scraper):
key = urlparse(url).path.split('/')[-2] + ext
return key
@logger.catch
def archive_files(self, result: ScraperResult) -> ScraperResult:
for url in result.archived_urls:
if result.archived_urls[url] is None:

View File

@@ -26,6 +26,7 @@ class TelegramTelethonScraper(Scraper):
username = username.split('s/')[1]
return username
@logger.catch
def archive_files(self, result: ScraperResult, client : TelegramClient = None) -> ScraperResult:
if len(result.archived_urls.keys()) == 0:
return result

View File

@@ -73,6 +73,7 @@ class VkontakteScraper(Scraper):
archived_urls=archived_urls,
media_archived=datetime.now(timezone.utc) if archive_media else None)
@logger.catch
def archive_files(self, result: ScraperResult) -> ScraperResult:
for url in result.archived_urls:
if result.archived_urls[url] is None:

View File

@@ -83,6 +83,7 @@ class YoutubeScraper(Scraper):
if channel.platform == "Youtube" and channel.url:
return True
@logger.catch
def archive_files(self, result: ScraperResult) -> ScraperResult:
for url in result.archived_urls:
if result.archived_urls[url] is None: