mirror of
https://github.com/bellingcat/cisticola.git
synced 2026-06-08 03:18:34 +03:00
Catch exceptions in archive_files so that archiver continues to run
This commit is contained in:
@@ -236,6 +236,7 @@ class Scraper:
|
||||
|
||||
return archived_url
|
||||
|
||||
@logger.catch
|
||||
def archive_files(self, result: ScraperResult) -> ScraperResult:
|
||||
"""Archive files corresponding to ``archived_url`` dict keys, if the
|
||||
files have not previously been archived.
|
||||
|
||||
@@ -79,6 +79,7 @@ class OdyseeScraper(Scraper):
|
||||
archived_urls={},
|
||||
media_archived=datetime.now(timezone.utc))
|
||||
|
||||
@logger.catch
|
||||
def archive_files(self, result: ScraperResult) -> ScraperResult:
|
||||
for url in result.archived_urls:
|
||||
if result.archived_urls[url] is None:
|
||||
|
||||
@@ -50,6 +50,7 @@ class RumbleScraper(Scraper):
|
||||
key = urlparse(url).path.split('/')[-2] + ext
|
||||
return key
|
||||
|
||||
@logger.catch
|
||||
def archive_files(self, result: ScraperResult) -> ScraperResult:
|
||||
for url in result.archived_urls:
|
||||
if result.archived_urls[url] is None:
|
||||
|
||||
@@ -26,6 +26,7 @@ class TelegramTelethonScraper(Scraper):
|
||||
username = username.split('s/')[1]
|
||||
return username
|
||||
|
||||
@logger.catch
|
||||
def archive_files(self, result: ScraperResult, client : TelegramClient = None) -> ScraperResult:
|
||||
if len(result.archived_urls.keys()) == 0:
|
||||
return result
|
||||
|
||||
@@ -73,6 +73,7 @@ class VkontakteScraper(Scraper):
|
||||
archived_urls=archived_urls,
|
||||
media_archived=datetime.now(timezone.utc) if archive_media else None)
|
||||
|
||||
@logger.catch
|
||||
def archive_files(self, result: ScraperResult) -> ScraperResult:
|
||||
for url in result.archived_urls:
|
||||
if result.archived_urls[url] is None:
|
||||
|
||||
@@ -83,6 +83,7 @@ class YoutubeScraper(Scraper):
|
||||
if channel.platform == "Youtube" and channel.url:
|
||||
return True
|
||||
|
||||
@logger.catch
|
||||
def archive_files(self, result: ScraperResult) -> ScraperResult:
|
||||
for url in result.archived_urls:
|
||||
if result.archived_urls[url] is None:
|
||||
|
||||
Reference in New Issue
Block a user