python auto_archive.py --helptitle: '{{ title }}'
+| about | +preview | +
|---|---|
+
|
+
+ {% if 'image' in m.mimetype %}
+ |
+
| key | +value | +
|---|---|
| {{ key }} | +{{ metadata[key] }} | +
made with bellingcat/auto-archiver, add suggestions and report issues on the project's github page
\ No newline at end of file diff --git a/src/media.py b/src/media.py index 58eae27..3c416be 100644 --- a/src/media.py +++ b/src/media.py @@ -12,10 +12,13 @@ class Media: key: str = None cdn_url: str = None mimetype: str = None # eg: image/jpeg - # id: str = None + id: str = None # in case this type of media needs a special id, eg: screenshot # hash: str = None # TODO: added by enrichers def set_mimetype(self) -> Media: if not self.mimetype: self.mimetype = mimetypes.guess_type(self.filename)[0] return self + + def is_video(self) -> bool: + return self.mimetype.startswith("video") diff --git a/src/metadata.py b/src/metadata.py index ceece8d..7af923c 100644 --- a/src/metadata.py +++ b/src/metadata.py @@ -28,9 +28,12 @@ class Metadata: """ merges two Metadata instances, will overwrite according to overwrite_left flag """ + if right is None: return self if overwrite_left: - self.status = right.status + if right.status and len(right.status): + self.status = right.status self.rearchivable |= right.rearchivable + self.tmp_keys |= right.tmp_keys for k, v in right.metadata.items(): assert k not in self.metadata or type(v) == type(self.get(k)) if type(v) not in [dict, list, set] or k not in self.metadata: @@ -76,6 +79,12 @@ class Metadata: def get_title(self) -> str: return self.get("title") + def set_tmp_dir(self, tmp_dir: str) -> Metadata: + return self.set("tmp_dir", tmp_dir, True) + + def get_tmp_dir(self) -> str: + return self.get("tmp_dir") + def set_timestamp(self, timestamp: datetime.datetime) -> Metadata: assert type(timestamp) == datetime.datetime, "set_timestamp expects a datetime instance" return self.set("timestamp", timestamp) @@ -88,9 +97,15 @@ class Metadata: return ts def add_media(self, media: Media) -> Metadata: + if media is None: return media.set_mimetype() return self.media.append(media) + def get_media_by_id(self, id:str) -> Media: + for m in self.media: + if m.id == id: return m + return None + def set_final_media(self, final: Media) -> Metadata: if final: if self.final_media: @@ -100,6 +115,7 @@ class Metadata: return self def get_single_media(self) -> Media: + #TODO: could be refactored to use a custom media.id if self.final_media: return self.final_media return self.media[0] diff --git a/src/orchestrator.py b/src/orchestrator.py index 5a8ff31..3d554e0 100644 --- a/src/orchestrator.py +++ b/src/orchestrator.py @@ -52,74 +52,6 @@ Cisticola considerations: 2. So the auto-archiver becomes like a puzzle and fixes to Cisticola scrapers can immediately benefit it, and contributions are focused on a single source or scraping """ -# @dataclass -# class Metadata: -# # does not handle files, only primitives -# # the only piece of logic to handle files is the archiver, enricher, and storage -# status: str -# # title: str -# # url: str -# # hash: str -# main_file: Metadata -# metadata: Dict[str, Metadata] - -# @staticmethod -# def merge(left, right : Metadata, overwrite_left=True) -> Metadata: -# # should return a merged version of the Metadata -# # will work for archived() and enriched() -# # what if 2 metadatas contain the same keys? only one can remain! : overwrite_left -# pass - -# def get(self, key) -> Union[Metadata, str]: -# # goes through metadata and returns the Metadata available -# pass - -# def as_json(self) -> str: -# # converts all metadata and data into JSON -# pass - - -""" -@dataclass -class ArchiveResult: - # maybe metadata can have status as well, eg: screenshot fails. should that be registered in the databases? likely yes - status: str - url: str - metadata: Metadata - # title, url, hash, other={} - # cdn_url: str = None - # thumbnail: str = None - # thumbnail_index: str = None - # duration: float = None - # title: str = None - # timestamp: datetime.datetime = None - # screenshot: str = None - # wacz: str = None - # hash: str = None - # media: list = field(default_factory=list) - - def __init__(self) -> None: pass - - def update(self, metadata) -> None: - # receive a Metadata instance and update itself with it! - pass - - def as_json(self) -> str: - # converts all metadata and data into JSON - pass -""" - -""" -There is a Superclass for: - * Database (should_process) - -How can GSheets work? it needs to feed from a READER (GSheets Feeder) - -Once an archiver returns a link to a local file (for eg to a storage), how do we then delete the produced local files? -The context metadata should include a temporary folder (maybe a LocalStorage instance?) -""" - - class ArchivingOrchestrator: def __init__(self, config) -> None: # in config.py we should test that the archivers exist and log mismatches (blocking execution) @@ -128,7 +60,7 @@ class ArchivingOrchestrator: # Is it possible to overwrite config.yaml values? it could be useful: share config file and modify gsheet_feeder.sheet via CLI # where does that update/processing happen? in config.py - # reflection for Archiver to know wihch child classes it has? use Archiver.__subclasses__ + # reflection for Archiver to know which child classes it has? use Archiver.__subclasses__ # self.archivers = [ # Archiver.init(a, config) # for a in config.archivers @@ -166,7 +98,7 @@ class ArchivingOrchestrator: print("ARCHIVING", item) try: with tempfile.TemporaryDirectory(dir="./") as tmp_dir: - item.set("tmp_dir", tmp_dir, True) + item.set_tmp_dir(tmp_dir) result = self.archive(item) print(result) except KeyboardInterrupt: @@ -226,6 +158,7 @@ class ArchivingOrchestrator: # do they need to be refreshed with every execution? # this is where the Hashes come from, the place with access to all content # the archiver does not have access to storage + # a.download(result) # TODO: refactor so there's not merge here result.merge(a.download(result)) # TODO: fix logic if True or result.is_success(): break @@ -237,7 +170,7 @@ class ArchivingOrchestrator: # maybe as a PDF? or a Markdown file # side captures: screenshot, wacz, webarchive, thumbnails, HTMLgenerator for e in self.enrichers: - result.merge(e.enrich(result)) + e.enrich(result) # store media unstored_media = result.media[::] From 6ca46417feeda7f6ac586214cbf40917f9d9b50f Mon Sep 17 00:00:00 2001 From: msramalho <19508417+msramalho@users.noreply.github.com> Date: Thu, 12 Jan 2023 02:09:39 +0000 Subject: [PATCH 034/190] local storage + multiple storage support --- src/archivers/telethon_archiverv2.py | 4 +- src/databases/gsheet_db.py | 4 +- src/enrichers/wayback_enricher.py | 2 +- src/formatters/templates/html_template.html | 19 +++++---- src/media.py | 28 +++++++++---- src/metadata.py | 2 - src/orchestrator.py | 14 +++---- src/storages/__init__.py | 3 +- src/storages/local.py | 46 +++++++++++++++++++++ src/storages/s3.py | 25 +++++------ src/storages/storage.py | 24 ++++++----- 11 files changed, 117 insertions(+), 54 deletions(-) create mode 100644 src/storages/local.py diff --git a/src/archivers/telethon_archiverv2.py b/src/archivers/telethon_archiverv2.py index 66ecd74..6851cb5 100644 --- a/src/archivers/telethon_archiverv2.py +++ b/src/archivers/telethon_archiverv2.py @@ -132,11 +132,11 @@ class TelethonArchiver(Archiverv2): if mp.entities: other_media_urls = [e.url for e in mp.entities if hasattr(e, "url") and e.url and self._guess_file_type(e.url) in ["video", "image", "audio"]] if len(other_media_urls): - logger.debug(f"Got {len(other_media_urls)} other medial urls from {mp.id=}: {other_media_urls}") + logger.debug(f"Got {len(other_media_urls)} other media urls from {mp.id=}: {other_media_urls}") for i, om_url in enumerate(other_media_urls): filename = os.path.join(tmp_dir, f'{chat}_{group_id}_{i}') self.download_from_url(om_url, filename) - result.add_media(Media(filename)) + result.add_media(Media(filename=filename, id=f"{group_id}_{i}")) filename_dest = os.path.join(tmp_dir, f'{chat}_{group_id}', str(mp.id)) filename = self.client.download_media(mp.media, filename_dest) diff --git a/src/databases/gsheet_db.py b/src/databases/gsheet_db.py index 26aae68..0cf65ed 100644 --- a/src/databases/gsheet_db.py +++ b/src/databases/gsheet_db.py @@ -63,13 +63,13 @@ class GsheetsDb(Database): media: Media = item.get_single_media() - batch_if_valid('archive', media.cdn_url) + batch_if_valid('archive', "\n".join(media.urls)) batch_if_valid('date', True, datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc).isoformat()) batch_if_valid('title', item.get_title()) batch_if_valid('text', item.get("content", "")[:500]) batch_if_valid('timestamp', item.get_timestamp()) if (screenshot := item.get_media_by_id("screenshot")): - batch_if_valid('screenshot', screenshot.cdn_url) + batch_if_valid('screenshot', "\n".join(screenshot.urls)) # batch_if_valid('status', item.status) # TODO: AFTER ENRICHMENTS diff --git a/src/enrichers/wayback_enricher.py b/src/enrichers/wayback_enricher.py index 09a43e0..bf55923 100644 --- a/src/enrichers/wayback_enricher.py +++ b/src/enrichers/wayback_enricher.py @@ -21,7 +21,7 @@ class WaybackEnricher(Enricher): @staticmethod def configs() -> dict: return { - "timeout": {"default": 5, "help": "number of seconds to wait for a response from webarchive's wayback machine, after that only job_id is saved but page will still be processed."}, + "timeout": {"default": 15, "help": "seconds to wait for successful archive confirmation from wayback, if more than this passes the result contains the job_id so the status can later be checked manually."}, "key": {"default": None, "help": "wayback API key. to get credentials visit https://archive.org/account/s3.php"}, "secret": {"default": None, "help": "wayback API secret. to get credentials visit https://archive.org/account/s3.php"} } diff --git a/src/formatters/templates/html_template.html b/src/formatters/templates/html_template.html index fc986f0..f488a5f 100644 --- a/src/formatters/templates/html_template.html +++ b/src/formatters/templates/html_template.html @@ -26,6 +26,7 @@ table, th, td { + margin: auto; border: 1px solid; border-collapse: collapse; } @@ -43,18 +44,17 @@title: '{{ title }}'
+title: '{{ title }}'
| about | -preview | +preview(s) |
|---|---|---|
|
+ {% for url in m.urls %}
{% if 'image' in m.mimetype %}
- |
| key | @@ -100,7 +103,7 @@
|---|
made with bellingcat/auto-archiver, add suggestions and report issues on the project's github page
+Made with bellingcat/auto-archiver