Merge branch 'more_mainifests' into load_modules

src/auto_archiver/modules/api_db/__init__.py

@@ -0,0 +1 @@
+from api_db import AAApiDb

src/auto_archiver/modules/api_db/__manifest__.py

@@ -0,0 +1,33 @@
+{
+    "name": "Auto-Archiver API Database",
+    "type": ["database"],
+    "entry_point": "api_db:AAApiDb",
+    "requires_setup": True,
+    "external_dependencies": {
+        "python": ["requests",
+                   "loguru"],
+    },
+    "configs": {
+            "api_endpoint": {"default": None, "help": "API endpoint where calls are made to"},
+            "api_token": {"default": None, "help": "API Bearer token."},
+            "public": {"default": False, "help": "whether the URL should be publicly available via the API"},
+            "author_id": {"default": None, "help": "which email to assign as author"},
+            "group_id": {"default": None, "help": "which group of users have access to the archive in case public=false as author"},
+            "allow_rearchive": {"default": True, "help": "if False then the API database will be queried prior to any archiving operations and stop if the link has already been archived", "type": "bool",},
+            "store_results": {"default": True, "help": "when set, will send the results to the API database.", "type": "bool",},
+            "tags": {"default": [], "help": "what tags to add to the archived URL",}
+        },
+    "description": """
+     Provides integration with the Auto-Archiver API for querying and storing archival data.
+
+### Features
+- **API Integration**: Supports querying for existing archives and submitting results.
+- **Duplicate Prevention**: Avoids redundant archiving when `allow_rearchive` is disabled.
+- **Configurable**: Supports settings like API endpoint, authentication token, tags, and permissions.
+- **Tagging and Metadata**: Adds tags and manages metadata for archives.
+- **Optional Storage**: Archives results conditionally based on configuration.
+
+### Setup
+Requires access to an Auto-Archiver API instance and a valid API token.
+     """,
+}

src/auto_archiver/modules/api_db/api_db.py

@@ -0,0 +1,59 @@
+from typing import Union
+import requests, os
+from loguru import logger
+
+from auto_archiver.base_processors import Database
+from auto_archiver.core import Metadata
+
+
+class AAApiDb(Database):
+    """
+        Connects to auto-archiver-api instance
+    """
+    name = "auto_archiver_api_db"
+
+    def __init__(self, config: dict) -> None:
+        # without this STEP.__init__ is not called
+        super().__init__(config)
+        self.allow_rearchive = bool(self.allow_rearchive)
+        self.store_results = bool(self.store_results)
+        self.assert_valid_string("api_endpoint")
+
+
+    def fetch(self, item: Metadata) -> Union[Metadata, bool]:
+        """ query the database for the existence of this item.
+            Helps avoid re-archiving the same URL multiple times.
+        """
+        if not self.allow_rearchive: return
+        
+        params = {"url": item.get_url(), "limit": 15}
+        headers = {"Authorization": f"Bearer {self.api_token}", "accept": "application/json"}
+        response = requests.get(os.path.join(self.api_endpoint, "tasks/search-url"), params=params, headers=headers)
+
+        if response.status_code == 200:
+            if len(response.json()):
+                logger.success(f"API returned {len(response.json())} previously archived instance(s)")
+                fetched_metadata = [Metadata.from_dict(r["result"]) for r in response.json()]
+                return Metadata.choose_most_complete(fetched_metadata)
+        else:
+            logger.error(f"AA API FAIL ({response.status_code}): {response.json()}")
+        return False
+
+
+    def done(self, item: Metadata, cached: bool=False) -> None:
+        """archival result ready - should be saved to DB"""
+        if not self.store_results: return
+        if cached: 
+            logger.debug(f"skipping saving archive of {item.get_url()} to the AA API because it was cached")
+            return
+        logger.debug(f"saving archive of {item.get_url()} to the AA API.")
+
+        payload = {'result': item.to_json(), 'public': self.public, 'author_id': self.author_id, 'group_id': self.group_id, 'tags': list(self.tags)}
+        headers = {"Authorization": f"Bearer {self.api_token}"}
+        response = requests.post(os.path.join(self.api_endpoint, "submit-archive"), json=payload, headers=headers)
+
+        if response.status_code == 200:
+            logger.success(f"AA API: {response.json()}")
+        else:
+            logger.error(f"AA API FAIL ({response.status_code}): {response.json()}")
+

src/auto_archiver/modules/atlos/__init__.py

@@ -0,0 +1 @@
+from .atlos import AtlosStorage

src/auto_archiver/modules/atlos/__manifest__.py

@@ -0,0 +1,40 @@
+{
+    "name": "atlos_storage",
+    "type": ["storage"],
+    "requires_setup": True,
+    "external_dependencies": {"python": ["loguru", "requests"], "bin": [""]},
+    "configs": {
+        "path_generator": {
+            "default": "url",
+            "help": "how to store the file in terms of directory structure: 'flat' sets to root; 'url' creates a directory based on the provided URL; 'random' creates a random directory.",
+        },
+        "filename_generator": {
+            "default": "random",
+            "help": "how to name stored files: 'random' creates a random string; 'static' uses a replicable strategy such as a hash.",
+        },
+        "api_token": {
+            "default": None,
+            "help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/",
+            "type": "str",
+        },
+        "atlos_url": {
+            "default": "https://platform.atlos.org",
+            "help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.",
+            "type": "str",
+        },
+    },
+    "description": """
+    AtlosStorage: A storage module for saving media files to the Atlos platform.
+
+    ### Features
+    - Uploads media files to Atlos using Atlos-specific APIs.
+    - Automatically calculates SHA-256 hashes of media files for integrity verification.
+    - Skips uploads for files that already exist on Atlos with the same hash.
+    - Supports attaching metadata, such as `atlos_id`, to the uploaded files.
+    - Provides CDN-like URLs for accessing uploaded media.
+
+    ### Notes
+    - Requires Atlos API configuration, including `atlos_url` and `api_token`.
+    - Files are linked to an `atlos_id` in the metadata, ensuring proper association with Atlos source materials.
+    """,
+}

src/auto_archiver/modules/atlos/atlos.py

@@ -0,0 +1,70 @@
+import os
+from typing import IO, List, Optional
+from loguru import logger
+import requests
+import hashlib
+
+from auto_archiver.core import Media, Metadata
+from auto_archiver.base_processors import Storage
+from auto_archiver.utils import get_atlos_config_options
+
+
+class AtlosStorage(Storage):
+    name = "atlos_storage"
+
+    def __init__(self, config: dict) -> None:
+        super().__init__(config)
+
+    def get_cdn_url(self, _media: Media) -> str:
+        # It's not always possible to provide an exact URL, because it's
+        # possible that the media once uploaded could have been copied to
+        # another project.
+        return self.atlos_url
+    
+    def _hash(self, media: Media) -> str:
+        # Hash the media file using sha-256. We don't use the existing auto archiver
+        # hash because there's no guarantee that the configuerer is using sha-256, which
+        # is how Atlos hashes files.
+
+        sha256 = hashlib.sha256()
+        with open(media.filename, "rb") as f:
+            while True:
+                buf = f.read(4096)
+                if not buf: break
+                sha256.update(buf)
+        return sha256.hexdigest()
+
+    def upload(self, media: Media, metadata: Optional[Metadata]=None, **_kwargs) -> bool:
+        atlos_id = metadata.get("atlos_id")
+        if atlos_id is None:
+            logger.error(f"No Atlos ID found in metadata; can't store {media.filename} on Atlos")
+            return False
+        
+        media_hash = self._hash(media)
+        
+        # Check whether the media has already been uploaded
+        source_material = requests.get(
+            f"{self.atlos_url}/api/v2/source_material/{atlos_id}",
+            headers={"Authorization": f"Bearer {self.api_token}"},
+        ).json()["result"]
+        existing_media = [x["file_hash_sha256"] for x in source_material.get("artifacts", [])]
+        if media_hash in existing_media:
+            logger.info(f"{media.filename} with SHA256 {media_hash} already uploaded to Atlos")
+            return True
+        
+        # Upload the media to the Atlos API
+        requests.post(
+            f"{self.atlos_url}/api/v2/source_material/upload/{atlos_id}",
+            headers={"Authorization": f"Bearer {self.api_token}"},
+            params={
+                "title": media.properties
+            },
+            files={"file": (os.path.basename(media.filename), open(media.filename, "rb"))},
+        ).raise_for_status()
+
+        logger.info(f"Uploaded {media.filename} to Atlos with ID {atlos_id} and title {media.key}")
+        
+        return True
+
+    # must be implemented even if unused
+    def uploadf(self, file: IO[bytes], key: str, **kwargs: dict) -> bool: pass

src/auto_archiver/modules/atlos_db/__init__.py

@@ -0,0 +1 @@
+from atlos_db import AtlosDb

src/auto_archiver/modules/atlos_db/__manifest__.py

@@ -0,0 +1,36 @@
+{
+    "name": "Atlos Database",
+    "type": ["database"],
+    "entry_point": "atlos_db:AtlosDb",
+    "requires_setup": True,
+    "external_dependencies":
+        {"python": ["loguru",
+                    ""],
+         "bin": [""]},
+    "configs": {
+        "api_token": {
+            "default": None,
+            "help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/",
+        },
+        "atlos_url": {
+            "default": "https://platform.atlos.org",
+            "help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.",
+            "type": "str"
+        },
+    },
+    "description": """
+Handles integration with the Atlos platform for managing archival results.
+
+### Features
+- Outputs archival results to the Atlos API for storage and tracking.
+- Updates failure status with error details when archiving fails.
+- Processes and formats metadata, including ISO formatting for datetime fields.
+- Skips processing for items without an Atlos ID.
+
+### Setup
+Required configs:
+- atlos_url: Base URL for the Atlos API.
+- api_token: Authentication token for API access.
+"""
+,
+}

src/auto_archiver/modules/atlos_db/atlos_db.py

@@ -0,0 +1,76 @@
+import os
+
+from typing import Union
+from loguru import logger
+from csv import DictWriter
+from dataclasses import asdict
+import requests
+
+from auto_archiver.base_processors import Database
+from auto_archiver.core import Metadata
+from auto_archiver.utils import get_atlos_config_options
+
+
+class AtlosDb(Database):
+    """
+    Outputs results to Atlos
+    """
+
+    name = "atlos_db"
+
+    def __init__(self, config: dict) -> None:
+        # without this STEP.__init__ is not called
+        super().__init__(config)
+
+    def failed(self, item: Metadata, reason: str) -> None:
+        """Update DB accordingly for failure"""
+        # If the item has no Atlos ID, there's nothing for us to do
+        if not item.metadata.get("atlos_id"):
+            logger.info(f"Item {item.get_url()} has no Atlos ID, skipping")
+            return
+
+        requests.post(
+            f"{self.atlos_url}/api/v2/source_material/metadata/{item.metadata['atlos_id']}/auto_archiver",
+            headers={"Authorization": f"Bearer {self.api_token}"},
+            json={"metadata": {"processed": True, "status": "error", "error": reason}},
+        ).raise_for_status()
+        logger.info(
+            f"Stored failure for {item.get_url()} (ID {item.metadata['atlos_id']}) on Atlos: {reason}"
+        )
+
+    def fetch(self, item: Metadata) -> Union[Metadata, bool]:
+        """check and fetch if the given item has been archived already, each
+        database should handle its own caching, and configuration mechanisms"""
+        return False
+
+    def _process_metadata(self, item: Metadata) -> dict:
+        """Process metadata for storage on Atlos. Will convert any datetime
+        objects to ISO format."""
+
+        return {
+            k: v.isoformat() if hasattr(v, "isoformat") else v
+            for k, v in item.metadata.items()
+        }
+
+    def done(self, item: Metadata, cached: bool = False) -> None:
+        """archival result ready - should be saved to DB"""
+
+        if not item.metadata.get("atlos_id"):
+            logger.info(f"Item {item.get_url()} has no Atlos ID, skipping")
+            return
+
+        requests.post(
+            f"{self.atlos_url}/api/v2/source_material/metadata/{item.metadata['atlos_id']}/auto_archiver",
+            headers={"Authorization": f"Bearer {self.api_token}"},
+            json={
+                "metadata": dict(
+                    processed=True,
+                    status="success",
+                    results=self._process_metadata(item),
+                )
+            },
+        ).raise_for_status()
+
+        logger.info(
+            f"Stored success for {item.get_url()} (ID {item.metadata['atlos_id']}) on Atlos"
+        )

src/auto_archiver/modules/atlos_db/base_configs.py

@@ -0,0 +1,13 @@
+def get_atlos_config_options():
+    return {
+        "api_token": {
+            "default": None,
+            "help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/",
+            "type": str
+        },
+        "atlos_url": {
+            "default": "https://platform.atlos.org",
+            "help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.",
+            "type": str
+        },
+    }

src/auto_archiver/modules/atlos_feeder/__init__.py

@@ -0,0 +1 @@
+from .atlos_feeder import AtlosFeeder

src/auto_archiver/modules/atlos_feeder/__manifest__.py

@@ -0,0 +1,34 @@
+{
+    "name": "Atlos Feeder",
+    "type": ["feeder"],
+    "requires_setup": True,
+    "external_dependencies": {
+        "python": ["loguru", "requests"],
+    },
+    "configs": {
+        "api_token": {
+            "default": None,
+            "help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/",
+            "type": "str"
+        },
+        "atlos_url": {
+            "default": "https://platform.atlos.org",
+            "help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.",
+            "type": "str"
+        },
+    },
+    "description": """
+    AtlosFeeder: A feeder module that integrates with the Atlos API to fetch source material URLs for archival.
+
+    ### Features
+    - Connects to the Atlos API to retrieve a list of source material URLs.
+    - Filters source materials based on visibility, processing status, and metadata.
+    - Converts filtered source materials into `Metadata` objects with the relevant `atlos_id` and URL.
+    - Iterates through paginated results using a cursor for efficient API interaction.
+
+    ### Notes
+    - Requires an Atlos API endpoint and a valid API token for authentication.
+    - Ensures only unprocessed, visible, and ready-to-archive URLs are returned.
+    - Handles pagination transparently when retrieving data from the Atlos API.
+    """
+}

src/auto_archiver/modules/atlos_feeder/atlos_feeder.py

@@ -0,0 +1,52 @@
+from loguru import logger
+import requests
+
+from auto_archiver.base_processors import Feeder
+from auto_archiver.core import Metadata, ArchivingContext
+from auto_archiver.utils import get_atlos_config_options
+
+
+class AtlosFeeder(Feeder):
+    name = "atlos_feeder"
+
+    def __init__(self, config: dict) -> None:
+        # without this STEP.__init__ is not called
+        super().__init__(config)
+        if type(self.api_token) != str:
+            raise Exception("Atlos Feeder did not receive an Atlos API token")
+
+    def __iter__(self) -> Metadata:
+        # Get all the urls from the Atlos API
+        count = 0
+        cursor = None
+        while True:
+            response = requests.get(
+                f"{self.atlos_url}/api/v2/source_material",
+                headers={"Authorization": f"Bearer {self.api_token}"},
+                params={"cursor": cursor},
+            )
+            data = response.json()
+            response.raise_for_status()
+            cursor = data["next"]
+
+            for item in data["results"]:
+                if (
+                    item["source_url"] not in [None, ""]
+                    and (
+                        item["metadata"]
+                        .get("auto_archiver", {})
+                        .get("processed", False)
+                        != True
+                    )
+                    and item["visibility"] == "visible"
+                    and item["status"] not in ["processing", "pending"]
+                ):
+                    yield Metadata().set_url(item["source_url"]).set(
+                        "atlos_id", item["id"]
+                    )
+                    count += 1
+
+            if len(data["results"]) == 0 or cursor is None:
+                break
+
+        logger.success(f"Processed {count} URL(s)")

src/auto_archiver/modules/cli_feeder/__init__.py

@@ -0,0 +1 @@
+from .cli_feeder import CLIFeeder

src/auto_archiver/modules/cli_feeder/__manifest__.py

@@ -0,0 +1,23 @@
+{
+    "name": "CLI Feeder",
+    "type": ["feeder"],
+    "requires_setup": False,
+    "external_dependencies": {
+        "python": ["loguru"],
+    },
+    "configs": {
+        "urls": {
+            "default": None,
+            "help": "URL(s) to archive, either a single URL or a list of urls, should not come from config.yaml",
+        },
+    },
+    "description": """
+    Processes URLs to archive passed via the command line and feeds them into the archiving pipeline.
+
+    ### Features
+    - Takes a single URL or a list of URLs provided via the command line.
+    - Converts each URL into a `Metadata` object and yields it for processing.
+    - Ensures URLs are processed only if they are explicitly provided.
+
+    """
+}

src/auto_archiver/modules/cli_feeder/cli_feeder.py

@@ -0,0 +1,22 @@
+from loguru import logger
+
+from auto_archiver.base_processors import Feeder
+from auto_archiver.core import Metadata, ArchivingContext
+
+
+class CLIFeeder(Feeder):
+    name = "cli_feeder"
+
+    def __init__(self, config: dict) -> None:
+        # without this STEP.__init__ is not called
+        super().__init__(config)
+        if type(self.urls) != list or len(self.urls) == 0:
+            raise Exception("CLI Feeder did not receive any URL to process")
+
+    def __iter__(self) -> Metadata:
+        for url in self.urls:
+            logger.debug(f"Processing {url}")
+            yield Metadata().set_url(url)
+            ArchivingContext.set("folder", "cli")
+
+        logger.success(f"Processed {len(self.urls)} URL(s)")

src/auto_archiver/modules/console_db/__init__.py

@@ -0,0 +1 @@
+from .console_db import ConsoleDb

src/auto_archiver/modules/console_db/__manifest__.py

@@ -0,0 +1,22 @@
+{
+    "name": "Console Database",
+    "type": ["database"],
+    "requires_setup": False,
+    "external_dependencies": {
+        "python": ["loguru"],
+    },
+    "description": """
+Provides a simple database implementation that outputs archival results and status updates to the console.
+
+### Features
+- Logs the status of archival tasks directly to the console, including:
+  - started
+  - failed (with error details)
+  - aborted
+  - done (with optional caching status)
+- Useful for debugging or lightweight setups where no external database is required.
+
+### Setup
+No additional configuration is required.
+""",
+}

src/auto_archiver/modules/console_db/console_db.py

@@ -0,0 +1,28 @@
+from loguru import logger
+
+from auto_archiver.base_processors import Database
+from auto_archiver.core import Metadata
+
+
+class ConsoleDb(Database):
+    """
+        Outputs results to the console
+    """
+    name = "console_db"
+
+    def __init__(self, config: dict) -> None:
+        # without this STEP.__init__ is not called
+        super().__init__(config)
+
+    def started(self, item: Metadata) -> None:
+        logger.warning(f"STARTED {item}")
+
+    def failed(self, item: Metadata, reason:str) -> None:
+        logger.error(f"FAILED {item}: {reason}")
+
+    def aborted(self, item: Metadata) -> None:
+        logger.warning(f"ABORTED {item}")
+
+    def done(self, item: Metadata, cached: bool=False) -> None:
+        """archival result ready - should be saved to DB"""
+        logger.success(f"DONE {item}")

src/auto_archiver/modules/csv_db/__init__.py

@@ -0,0 +1 @@
+from .csv_db import CSVDb

src/auto_archiver/modules/csv_db/__manifest__.py

@@ -0,0 +1,22 @@
+{
+    "name": "csv_db",
+    "type": ["database"],
+    "requires_setup": False,
+    "external_dependencies": {"python": ["loguru"]
+                              },
+    "configs": {
+            "csv_file": {"default": "db.csv", "help": "CSV file name"}
+        },
+    "description": """
+Handles exporting archival results to a CSV file.
+
+### Features
+- Saves archival metadata as rows in a CSV file.
+- Automatically creates the CSV file with a header if it does not exist.
+- Appends new metadata entries to the existing file.
+
+### Setup
+Required config:
+- csv_file: Path to the CSV file where results will be stored (default: "db.csv").
+""",
+}

src/auto_archiver/modules/csv_db/csv_db.py

@@ -0,0 +1,29 @@
+import os
+from loguru import logger
+from csv import DictWriter
+from dataclasses import asdict
+
+from auto_archiver.base_processors import Database
+from auto_archiver.core import Metadata
+
+
+class CSVDb(Database):
+    """
+        Outputs results to a CSV file
+    """
+    name = "csv_db"
+
+    def __init__(self, config: dict) -> None:
+        # without this STEP.__init__ is not called
+        super().__init__(config)
+        self.assert_valid_string("csv_file")
+
+
+    def done(self, item: Metadata, cached: bool=False) -> None:
+        """archival result ready - should be saved to DB"""
+        logger.success(f"DONE {item}")
+        is_empty = not os.path.isfile(self.csv_file) or os.path.getsize(self.csv_file) == 0
+        with open(self.csv_file, "a", encoding="utf-8") as outf:
+            writer = DictWriter(outf, fieldnames=asdict(Metadata()))
+            if is_empty: writer.writeheader()
+            writer.writerow(asdict(item))

src/auto_archiver/modules/csv_feeder/__init__.py

@@ -0,0 +1 @@
+from .csv_feeder import CSVFeeder

src/auto_archiver/modules/csv_feeder/__manifest__.py

@@ -0,0 +1,32 @@
+{
+    "name": "CSV Feeder",
+    "type": ["feeder"],
+    "requires_setup": False,
+    "external_dependencies": {
+        "python": ["loguru"],
+        "bin": [""]
+    },
+    "configs": {
+            "files": {
+                "default": None,
+                "help": "Path to the input file(s) to read the URLs from, comma separated. \
+                        Input files should be formatted with one URL per line",
+            },
+            "column": {
+                "default": None,
+                "help": "Column number or name to read the URLs from, 0-indexed",
+            }
+        },
+    "description": """
+    Reads URLs from CSV files and feeds them into the archiving process.
+
+    ### Features
+    - Supports reading URLs from multiple input files, specified as a comma-separated list.
+    - Allows specifying the column number or name to extract URLs from.
+    - Skips header rows if the first value is not a valid URL.
+    - Integrates with the `ArchivingContext` to manage URL feeding.
+
+    ### Setu N
+    - Input files should be formatted with one URL per line.
+    """
+}

src/auto_archiver/modules/csv_feeder/csv_feeder.py

@@ -0,0 +1,27 @@
+from loguru import logger
+import csv
+
+from auto_archiver.base_processors import Feeder
+from auto_archiver.core import Metadata, ArchivingContext
+from auto_archiver.utils import url_or_none
+
+class CSVFeeder(Feeder):
+
+    name = "csv_feeder"
+
+    def __iter__(self) -> Metadata:
+        url_column = self.column or 0
+        for file in self.files:
+            with open(file, "r") as f:
+                reader = csv.reader(f)
+                first_row = next(reader)
+                if not(url_or_none(first_row[url_column])):
+                    # it's a header row, skip it
+                    logger.debug(f"Skipping header row: {first_row}")
+                for row in reader:
+                    url = row[0]
+                    logger.debug(f"Processing {url}")
+                    yield Metadata().set_url(url)
+            ArchivingContext.set("folder", "cli")
+
+        logger.success(f"Processed {len(self.urls)} URL(s)")

src/auto_archiver/modules/gdrive_storage/__init__.py

@@ -0,0 +1 @@
+from .gdrive_storage import GDriveStorage

src/auto_archiver/modules/gdrive_storage/__manifest__.py

@@ -0,0 +1,43 @@
+{
+    "name": "Google Drive Storage",
+    "type": ["storage"],
+    "requires_setup": True,
+    "external_dependencies": {
+        "python": [
+            "loguru",
+            "google-api-python-client",
+            "google-auth",
+            "google-auth-oauthlib",
+            "google-auth-httplib2"
+        ],
+    },
+    "configs": {
+        "path_generator": {
+            "default": "url",
+            "help": "how to store the file in terms of directory structure: 'flat' sets to root; 'url' creates a directory based on the provided URL; 'random' creates a random directory.",
+            "choices": ["flat", "url", "random"],
+        },
+        "filename_generator": {
+            "default": "random",
+            "help": "how to name stored files: 'random' creates a random string; 'static' uses a replicable strategy such as a hash.",
+            "choices": ["random", "static"],
+        },
+        "root_folder_id": {"default": None, "help": "root google drive folder ID to use as storage, found in URL: 'https://drive.google.com/drive/folders/FOLDER_ID'"},
+        "oauth_token": {"default": None, "help": "JSON filename with Google Drive OAuth token: check auto-archiver repository scripts folder for create_update_gdrive_oauth_token.py. NOTE: storage used will count towards owner of GDrive folder, therefore it is best to use oauth_token_filename over service_account."},
+        "service_account": {"default": "secrets/service_account.json", "help": "service account JSON file path, same as used for Google Sheets. NOTE: storage used will count towards the developer account."},
+    },
+    "description": """
+    GDriveStorage: A storage module for saving archived content to Google Drive.
+
+    ### Features
+    - Saves media files to Google Drive, organizing them into folders based on the provided path structure.
+    - Supports OAuth token-based authentication or service account credentials for API access.
+    - Automatically creates folders in Google Drive if they don't exist.
+    - Retrieves CDN URLs for stored files, enabling easy sharing and access.
+
+    ### Notes
+    - Requires setup with either a Google OAuth token or a service account JSON file.
+    - Files are uploaded to the specified `root_folder_id` and organized by the `media.key` structure.
+    - Automatically handles Google Drive API token refreshes for long-running jobs.
+    """
+}

src/auto_archiver/modules/gdrive_storage/gdrive_storage.py

@@ -0,0 +1,176 @@
+
+import shutil, os, time, json
+from typing import IO
+from loguru import logger
+
+from googleapiclient.discovery import build
+from googleapiclient.http import MediaFileUpload
+from google.oauth2 import service_account
+from google.oauth2.credentials import Credentials
+from google.auth.transport.requests import Request
+
+from auto_archiver.core import Media
+from auto_archiver.base_processors import Storage
+
+
+class GDriveStorage(Storage):
+    name = "gdrive_storage"
+
+    def __init__(self, config: dict) -> None:
+        super().__init__(config)
+
+        SCOPES = ['https://www.googleapis.com/auth/drive']
+
+        if self.oauth_token is not None:
+            """
+            Tokens are refreshed after 1 hour 
+            however keep working for 7 days (tbc)
+            so as long as the job doesn't last for 7 days
+            then this method of refreshing only once per run will work
+            see this link for details on the token
+            https://davemateer.com/2022/04/28/google-drive-with-python#tokens
+            """
+            logger.debug(f'Using GD OAuth token {self.oauth_token}')
+            # workaround for missing 'refresh_token' in from_authorized_user_file
+            with open(self.oauth_token, 'r') as stream:
+                creds_json = json.load(stream)
+                creds_json['refresh_token'] = creds_json.get("refresh_token", "")
+            creds = Credentials.from_authorized_user_info(creds_json, SCOPES)
+            # creds = Credentials.from_authorized_user_file(self.oauth_token, SCOPES)
+
+            if not creds or not creds.valid:
+                if creds and creds.expired and creds.refresh_token:
+                    logger.debug('Requesting new GD OAuth token')
+                    creds.refresh(Request())
+                else:
+                    raise Exception("Problem with creds - create the token again")
+
+                # Save the credentials for the next run
+                with open(self.oauth_token, 'w') as token:
+                    logger.debug('Saving new GD OAuth token')
+                    token.write(creds.to_json())
+            else:
+                logger.debug('GD OAuth Token valid')
+        else:
+            gd_service_account = self.service_account
+            logger.debug(f'Using GD Service Account {gd_service_account}')
+            creds = service_account.Credentials.from_service_account_file(gd_service_account, scopes=SCOPES)
+
+        self.service = build('drive', 'v3', credentials=creds)
+
+    def get_cdn_url(self, media: Media) -> str:
+        """
+        only support files saved in a folder for GD
+        S3 supports folder and all stored in the root
+        """
+
+        # full_name = os.path.join(self.folder, media.key)
+        parent_id, folder_id = self.root_folder_id, None
+        path_parts = media.key.split(os.path.sep)
+        filename = path_parts[-1]
+        logger.info(f"looking for folders for {path_parts[0:-1]} before getting url for {filename=}")
+        for folder in path_parts[0:-1]:
+            folder_id = self._get_id_from_parent_and_name(parent_id, folder, use_mime_type=True, raise_on_missing=True)
+            parent_id = folder_id
+
+        # get id of file inside folder (or sub folder)
+        file_id = self._get_id_from_parent_and_name(folder_id, filename)
+        return f"https://drive.google.com/file/d/{file_id}/view?usp=sharing"
+
+    def upload(self, media: Media, **kwargs) -> bool:
+        logger.debug(f'[{self.__class__.name}] storing file {media.filename} with key {media.key}')
+        """
+        1. for each sub-folder in the path check if exists or create
+        2. upload file to root_id/other_paths.../filename
+        """
+        parent_id, upload_to = self.root_folder_id, None
+        path_parts = media.key.split(os.path.sep)
+        filename = path_parts[-1]
+        logger.info(f"checking folders {path_parts[0:-1]} exist (or creating) before uploading {filename=}")
+        for folder in path_parts[0:-1]:
+            upload_to = self._get_id_from_parent_and_name(parent_id, folder, use_mime_type=True, raise_on_missing=False)
+            if upload_to is None:
+                upload_to = self._mkdir(folder, parent_id)
+            parent_id = upload_to
+
+        # upload file to gd
+        logger.debug(f'uploading {filename=} to folder id {upload_to}')
+        file_metadata = {
+            'name': [filename],
+            'parents': [upload_to]
+        }
+        media = MediaFileUpload(media.filename, resumable=True)
+        gd_file = self.service.files().create(supportsAllDrives=True, body=file_metadata, media_body=media, fields='id').execute()
+        logger.debug(f'uploadf: uploaded file {gd_file["id"]} successfully in folder={upload_to}')
+
+    # must be implemented even if unused
+    def uploadf(self, file: IO[bytes], key: str, **kwargs: dict) -> bool: pass
+
+    def _get_id_from_parent_and_name(self, parent_id: str, name: str, retries: int = 1, sleep_seconds: int = 10, use_mime_type: bool = False, raise_on_missing: bool = True, use_cache=False):
+        """
+        Retrieves the id of a folder or file from its @name and the @parent_id folder
+        Optionally does multiple @retries and sleeps @sleep_seconds between them
+        If @use_mime_type will restrict search to "mimeType='application/vnd.google-apps.folder'"
+        If @raise_on_missing will throw error when not found, or returns None
+        Will remember previous calls to avoid duplication if @use_cache - might not have all edge cases tested, so use at own risk
+        Returns the id of the file or folder from its name as a string
+        """
+        # cache logic
+        if use_cache:
+            self.api_cache = getattr(self, "api_cache", {})
+            cache_key = f"{parent_id}_{name}_{use_mime_type}"
+            if cache_key in self.api_cache:
+                logger.debug(f"cache hit for {cache_key=}")
+                return self.api_cache[cache_key]
+
+        # API logic
+        debug_header: str = f"[searching {name=} in {parent_id=}]"
+        query_string = f"'{parent_id}' in parents and name = '{name}' and trashed = false "
+        if use_mime_type:
+            query_string += f" and mimeType='application/vnd.google-apps.folder' "
+
+        for attempt in range(retries):
+            results = self.service.files().list(
+                # both below for Google Shared Drives
+                supportsAllDrives=True,
+                includeItemsFromAllDrives=True,
+                q=query_string,
+                spaces='drive',  # ie not appDataFolder or photos
+                fields='files(id, name)'
+            ).execute()
+            items = results.get('files', [])
+
+            if len(items) > 0:
+                logger.debug(f"{debug_header} found {len(items)} matches, returning last of {','.join([i['id'] for i in items])}")
+                _id = items[-1]['id']
+                if use_cache: self.api_cache[cache_key] = _id
+                return _id
+            else:
+                logger.debug(f'{debug_header} not found, attempt {attempt+1}/{retries}.')
+                if attempt < retries - 1:
+                    logger.debug(f'sleeping for {sleep_seconds} second(s)')
+                    time.sleep(sleep_seconds)
+
+        if raise_on_missing:
+            raise ValueError(f'{debug_header} not found after {retries} attempt(s)')
+        return None
+
+    def _mkdir(self, name: str, parent_id: str):
+        """
+        Creates a new GDrive folder @name inside folder @parent_id
+        Returns id of the created folder
+        """
+        logger.debug(f'Creating new folder with {name=} inside {parent_id=}')
+        file_metadata = {
+            'name': [name],
+            'mimeType': 'application/vnd.google-apps.folder',
+            'parents': [parent_id]
+        }
+        gd_folder = self.service.files().create(supportsAllDrives=True, body=file_metadata, fields='id').execute()
+        return gd_folder.get('id')
+
+    # def exists(self, key):
+    #     try:
+    #         self.get_cdn_url(key)
+    #         return True
+    #     except: return False

src/auto_archiver/modules/generic_extractor/bluesky.py

@@ -1,17 +1,12 @@
-import os
-import mimetypes
-
-import requests
 from loguru import logger
 
-from auto_archiver.core.context import ArchivingContext
-from auto_archiver.archivers.archiver import Archiver
+from auto_archiver.base_processors.extractor import Extractor
 from auto_archiver.core.metadata import Metadata, Media
 from .dropin import GenericDropin, InfoExtractor
 
 class Bluesky(GenericDropin):
 
-    def create_metadata(self, post: dict, ie_instance: InfoExtractor, archiver: Archiver, url: str) -> Metadata:
+    def create_metadata(self, post: dict, ie_instance: InfoExtractor, archiver: Extractor, url: str) -> Metadata:
         result = Metadata()
         result.set_url(url)
         result.set_title(post["record"]["text"])
@@ -42,7 +37,7 @@ class Bluesky(GenericDropin):
 
 
 
-    def _download_bsky_embeds(self, post: dict, archiver: Archiver) -> list[Media]:
+    def _download_bsky_embeds(self, post: dict, archiver: Extractor) -> list[Media]:
         """
         Iterates over image(s) or video in a Bluesky post and downloads them        
         """

src/auto_archiver/modules/generic_extractor/dropin.py

@@ -1,6 +1,6 @@
 from yt_dlp.extractor.common import InfoExtractor
 from auto_archiver.core.metadata import Metadata
-from auto_archiver.archivers.archiver import Archiver
+from auto_archiver.base_processors.extractor import Extractor
 
 class GenericDropin:
     """Base class for dropins for the generic extractor.
@@ -30,7 +30,7 @@ class GenericDropin:
         raise NotImplementedError("This method should be implemented in the subclass")
     
 
-    def create_metadata(self, post: dict, ie_instance: InfoExtractor, archiver: Archiver, url: str) -> Metadata:
+    def create_metadata(self, post: dict, ie_instance: InfoExtractor, archiver: Extractor, url: str) -> Metadata:
         """
         This method should create a Metadata object from the post data.
         """

src/auto_archiver/modules/generic_extractor/generic_extractor.py

@@ -5,10 +5,10 @@ from yt_dlp.extractor.common import InfoExtractor
 
 from loguru import logger
 
-from auto_archiver.archivers.archiver import Archiver
+from auto_archiver.base_processors.extractor import Extractor
 from ...core import Metadata, Media, ArchivingContext
 
-class GenericExtractor(Archiver):
+class GenericExtractor(Extractor):
     name = "youtubedl_archiver" #left as is for backwards compat
     _dropins = {}
 

src/auto_archiver/modules/generic_extractor/truth.py

@@ -2,7 +2,7 @@ from typing import Type
 
 from auto_archiver.utils import traverse_obj
 from auto_archiver.core.metadata import Metadata, Media
-from auto_archiver.archivers.archiver import Archiver
+from auto_archiver.base_processors.extractor import Extractor
 from yt_dlp.extractor.common import InfoExtractor
 
 from dateutil.parser import parse as parse_dt
@@ -19,7 +19,7 @@ class Truth(GenericDropin):
     def skip_ytdlp_download(self, url, ie_instance: Type[InfoExtractor]) -> bool:
         return True
 
-    def create_metadata(self, post: dict, ie_instance: InfoExtractor, archiver: Archiver, url: str) -> Metadata:
+    def create_metadata(self, post: dict, ie_instance: InfoExtractor, archiver: Extractor, url: str) -> Metadata:
         """
         Creates metadata from a truth social post
         

src/auto_archiver/modules/generic_extractor/twitter.py

@@ -6,7 +6,7 @@ from slugify import slugify
 
 from auto_archiver.core.metadata import Metadata, Media
 from auto_archiver.utils import UrlUtil
-from auto_archiver.archivers.archiver import Archiver
+from auto_archiver.base_processors.extractor import Extractor
 
 from .dropin import GenericDropin, InfoExtractor
 
@@ -32,7 +32,7 @@ class Twitter(GenericDropin):
         twid = ie_instance._match_valid_url(url).group('id')
         return ie_instance._extract_status(twid=twid)
 
-    def create_metadata(self, tweet: dict, ie_instance: InfoExtractor, archiver: Archiver, url: str) -> Metadata:
+    def create_metadata(self, tweet: dict, ie_instance: InfoExtractor, archiver: Extractor, url: str) -> Metadata:
         result = Metadata()
         try:
             if not tweet.get("user") or not tweet.get("created_at"):

src/auto_archiver/modules/gsheet_db/__init__.py

@@ -0,0 +1 @@
+from .gsheet_db import GsheetsDb

src/auto_archiver/modules/gsheet_db/__manifest__.py

@@ -0,0 +1,36 @@
+{
+    "name": "Google Sheets Database",
+    "type": ["database"],
+    "requires_setup": True,
+    "external_dependencies": {
+        "python": ["loguru", "gspread", "python-slugify"],
+    },
+    "configs": {
+        "allow_worksheets": {
+            "default": set(),
+            "help": "(CSV) only worksheets whose name is included in allow are included (overrides worksheet_block), leave empty so all are allowed",
+        },
+        "block_worksheets": {
+            "default": set(),
+            "help": "(CSV) explicitly block some worksheets from being processed",
+        },
+        "use_sheet_names_in_stored_paths": {
+            "default": True,
+            "help": "if True the stored files path will include 'workbook_name/worksheet_name/...'",
+        }
+    },
+    "description": """
+    GsheetsDatabase:
+    Handles integration with Google Sheets for tracking archival tasks.
+
+### Features
+- Updates a Google Sheet with the status of the archived URLs, including in progress, success or failure, and method used.
+- Saves metadata such as title, text, timestamp, hashes, screenshots, and media URLs to designated columns.
+- Formats media-specific metadata, such as thumbnails and PDQ hashes for the sheet.
+- Skips redundant updates for empty or invalid data fields.
+
+### Notes
+- Currently works only with metadata provided by GsheetFeeder. 
+- Requires configuration of a linked Google Sheet and appropriate API credentials.
+    """
+}

src/auto_archiver/modules/gsheet_db/gsheet_db.py

@@ -0,0 +1,108 @@
+from typing import Union, Tuple
+
+import datetime
+from urllib.parse import quote
+
+from loguru import logger
+
+from auto_archiver.base_processors import Database
+from auto_archiver.core import Metadata, Media, ArchivingContext
+from auto_archiver.modules.gsheet_feeder import GWorksheet
+
+
+class GsheetsDb(Database):
+    """
+        NB: only works if GsheetFeeder is used. 
+        could be updated in the future to support non-GsheetFeeder metadata 
+    """
+    name = "gsheet_db"
+
+    def __init__(self, config: dict) -> None:
+        # without this STEP.__init__ is not called
+        super().__init__(config)
+
+    def started(self, item: Metadata) -> None:
+        logger.warning(f"STARTED {item}")
+        gw, row = self._retrieve_gsheet(item)
+        gw.set_cell(row, 'status', 'Archive in progress')
+
+    def failed(self, item: Metadata, reason:str) -> None:
+        logger.error(f"FAILED {item}")
+        self._safe_status_update(item, f'Archive failed {reason}')
+
+    def aborted(self, item: Metadata) -> None:
+        logger.warning(f"ABORTED {item}")
+        self._safe_status_update(item, '')
+
+    def fetch(self, item: Metadata) -> Union[Metadata, bool]:
+        """check if the given item has been archived already"""
+        return False
+
+    def done(self, item: Metadata, cached: bool=False) -> None:
+        """archival result ready - should be saved to DB"""
+        logger.success(f"DONE {item.get_url()}")
+        gw, row = self._retrieve_gsheet(item)
+        # self._safe_status_update(item, 'done')
+
+        cell_updates = []
+        row_values = gw.get_row(row)
+
+        def batch_if_valid(col, val, final_value=None):
+            final_value = final_value or val
+            try:
+                if val and gw.col_exists(col) and gw.get_cell(row_values, col) == '':
+                    cell_updates.append((row, col, final_value))
+            except Exception as e:
+                logger.error(f"Unable to batch {col}={final_value} due to {e}")
+        status_message = item.status
+        if cached:
+            status_message = f"[cached] {status_message}"
+        cell_updates.append((row, 'status', status_message))
+
+        media: Media = item.get_final_media()
+        if hasattr(media, "urls"):
+            batch_if_valid('archive', "\n".join(media.urls))
+        batch_if_valid('date', True, datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=datetime.timezone.utc).isoformat())
+        batch_if_valid('title', item.get_title())
+        batch_if_valid('text', item.get("content", ""))
+        batch_if_valid('timestamp', item.get_timestamp())
+        if media: batch_if_valid('hash', media.get("hash", "not-calculated"))
+
+        # merge all pdq hashes into a single string, if present
+        pdq_hashes = []
+        all_media = item.get_all_media()
+        for m in all_media:
+            if pdq := m.get("pdq_hash"):
+                pdq_hashes.append(pdq)
+        if len(pdq_hashes):
+            batch_if_valid('pdq_hash', ",".join(pdq_hashes))
+
+        if (screenshot := item.get_media_by_id("screenshot")) and hasattr(screenshot, "urls"):
+            batch_if_valid('screenshot', "\n".join(screenshot.urls))
+
+        if (thumbnail := item.get_first_image("thumbnail")):
+            if hasattr(thumbnail, "urls"):
+                batch_if_valid('thumbnail', f'=IMAGE("{thumbnail.urls[0]}")')
+
+        if (browsertrix := item.get_media_by_id("browsertrix")):
+            batch_if_valid('wacz', "\n".join(browsertrix.urls))
+            batch_if_valid('replaywebpage', "\n".join([f'https://replayweb.page/?source={quote(wacz)}#view=pages&url={quote(item.get_url())}' for wacz in browsertrix.urls]))
+
+        gw.batch_set_cell(cell_updates)
+
+    def _safe_status_update(self, item: Metadata, new_status: str) -> None:
+        try:
+            gw, row = self._retrieve_gsheet(item)
+            gw.set_cell(row, 'status', new_status)
+        except Exception as e:
+            logger.debug(f"Unable to update sheet: {e}")
+
+    def _retrieve_gsheet(self, item: Metadata) -> Tuple[GWorksheet, int]:
+        # TODO: to make gsheet_db less coupled with gsheet_feeder's "gsheet" parameter, this method could 1st try to fetch "gsheet" from ArchivingContext and, if missing, manage its own singleton - not needed for now
+        if gsheet := ArchivingContext.get("gsheet"):
+            gw: GWorksheet = gsheet.get("worksheet")
+            row: int = gsheet.get("row")
+        elif self.sheet_id:
+            print(self.sheet_id)
+
+        return gw, row

src/auto_archiver/modules/gsheet_feeder/__init__.py

@@ -0,0 +1,2 @@
+from .gworksheet import GWorksheet
+from .gsheet_feeder import GsheetsFeeder

src/auto_archiver/modules/gsheet_feeder/__manifest__.py

@@ -0,0 +1,65 @@
+{
+    "name": "Google Sheets Feeder",
+    "type": ["feeder"],
+    "entry_point": "GsheetsFeeder",
+    "requires_setup": True,
+    "external_dependencies": {
+        "python": ["loguru", "gspread", "python-slugify"],
+    },
+    "configs": {
+            "sheet": {"default": None, "help": "name of the sheet to archive"},
+            "sheet_id": {"default": None, "help": "(alternative to sheet name) the id of the sheet to archive"},
+            "header": {"default": 1, "help": "index of the header row (starts at 1)"},
+            "service_account": {"default": "secrets/service_account.json", "help": "service account JSON file path"},
+            "columns": {
+                "default": {
+                    'url': 'link',
+                    'status': 'archive status',
+                    'folder': 'destination folder',
+                    'archive': 'archive location',
+                    'date': 'archive date',
+                    'thumbnail': 'thumbnail',
+                    'timestamp': 'upload timestamp',
+                    'title': 'upload title',
+                    'text': 'text content',
+                    'screenshot': 'screenshot',
+                    'hash': 'hash',
+                    'pdq_hash': 'perceptual hashes',
+                    'wacz': 'wacz',
+                    'replaywebpage': 'replaywebpage',
+                },
+                "help": "names of columns in the google sheet (stringified JSON object)",
+                "type": "auto_archiver.utils.json_loader",
+            },
+        "allow_worksheets": {
+            "default": set(),
+            "help": "(CSV) only worksheets whose name is included in allow are included (overrides worksheet_block), leave empty so all are allowed",
+        },
+        "block_worksheets": {
+            "default": set(),
+            "help": "(CSV) explicitly block some worksheets from being processed",
+        },
+        "use_sheet_names_in_stored_paths": {
+            "default": True,
+            "help": "if True the stored files path will include 'workbook_name/worksheet_name/...'",
+            "type": "bool",
+        }
+    },
+    "description": """
+    GsheetsFeeder 
+    A Google Sheets-based feeder for the Auto Archiver.
+
+    This reads data from Google Sheets and filters rows based on user-defined rules.
+    The filtered rows are processed into `Metadata` objects.
+
+    ### Features
+    - Validates the sheet structure and filters rows based on input configurations.
+    - Processes only worksheets allowed by the `allow_worksheets` and `block_worksheets` configurations.
+    - Ensures only rows with valid URLs and unprocessed statuses are included for archival.
+    - Supports organizing stored files into folder paths based on sheet and worksheet names.
+
+    ### Notes
+    - Requires a Google Service Account JSON file for authentication. Suggested location is `secrets/gsheets_service_account.json`.
+    - Create the sheet using the template provided in the docs.
+    """
+}

src/auto_archiver/modules/gsheet_feeder/gsheet_feeder.py

@@ -0,0 +1,122 @@
+"""
+GsheetsFeeder: A Google Sheets-based feeder for the Auto Archiver.
+
+This reads data from Google Sheets and filters rows based on user-defined rules.
+The filtered rows are processed into `Metadata` objects.
+
+### Key properties
+- validates the sheet's structure and filters rows based on input configurations.
+- Ensures only rows with valid URLs and unprocessed statuses are included.
+"""
+import os
+import gspread
+
+from loguru import logger
+from slugify import slugify
+
+from auto_archiver.base_processors import Feeder
+from auto_archiver.core import Metadata, ArchivingContext
+from . import GWorksheet
+
+
+class GsheetsFeeder(Feeder):
+    name = "gsheet_feeder"
+
+    def __init__(self) -> None:
+        """
+        Initializes the GsheetsFeeder with preloaded configurations.
+        """
+        super().__init__()
+        # Initialize the gspread client with the provided service account file
+        # self.gsheets_client = gspread.service_account(filename=self.config["service_account"])
+    #
+    #     # Set up feeder-specific configurations from the config
+    #     self.sheet_name = config.get("sheet")
+    #     self.sheet_id = config.get("sheet_id")
+    #     self.header = config.get("header", 1)
+    #     self.columns = config.get("columns", {})
+    #     assert self.sheet_name or self.sheet_id, (
+    #         "You need to define either a 'sheet' name or a 'sheet_id' in your manifest."
+    #     )
+
+
+        # # Configuration attributes
+        # self.sheet = config.get("sheet")
+        # self.sheet_id = config.get("sheet_id")
+        # self.header = config.get("header", 1)
+        # self.columns = config.get("columns", {})
+        # self.allow_worksheets = config.get("allow_worksheets", set())
+        # self.block_worksheets = config.get("block_worksheets", set())
+        # self.use_sheet_names_in_stored_paths = config.get("use_sheet_names_in_stored_paths", True)
+
+        # Ensure the header is an integer
+    #     try:
+    #         self.header = int(self.header)
+    #     except ValueError:
+    #         pass
+    #     assert isinstance(self.header, int), f"Header must be an integer, got {type(self.header)}"
+    #     assert self.sheet or self.sheet_id, "Either 'sheet' or 'sheet_id' must be defined."
+    #
+
+    def open_sheet(self):
+        if self.sheet:
+            return self.gsheets_client.open(self.sheet)
+        else:  # self.sheet_id
+            return self.gsheets_client.open_by_key(self.sheet_id)
+
+
+    def __iter__(self) -> Metadata:
+        sh = self.open_sheet()
+        for ii, wks in enumerate(sh.worksheets()):
+            if not self.should_process_sheet(wks.title):
+                logger.debug(f"SKIPPED worksheet '{wks.title}' due to allow/block rules")
+                continue
+
+            logger.info(f'Opening worksheet {ii=}: {wks.title=} header={self.header}')
+            gw = GWorksheet(wks, header_row=self.header, columns=self.columns)
+
+            if len(missing_cols := self.missing_required_columns(gw)):
+                logger.warning(f"SKIPPED worksheet '{wks.title}' due to missing required column(s) for {missing_cols}")
+                continue
+
+            for row in range(1 + self.header, gw.count_rows() + 1):
+                url = gw.get_cell(row, 'url').strip()
+                if not len(url): continue
+
+                original_status = gw.get_cell(row, 'status')
+                status = gw.get_cell(row, 'status', fresh=original_status in ['', None])
+                # TODO: custom status parser(?) aka should_retry_from_status
+                if status not in ['', None]: continue
+
+                # All checks done - archival process starts here
+                m = Metadata().set_url(url)
+                ArchivingContext.set("gsheet", {"row": row, "worksheet": gw}, keep_on_reset=True)
+                if gw.get_cell_or_default(row, 'folder', "") is None:
+                    folder = ''
+                else:
+                    folder = slugify(gw.get_cell_or_default(row, 'folder', "").strip())
+                if len(folder):
+                    if self.use_sheet_names_in_stored_paths:
+                        ArchivingContext.set("folder", os.path.join(folder, slugify(self.sheet), slugify(wks.title)), True)
+                    else:
+                        ArchivingContext.set("folder", folder, True)
+
+                yield m
+
+            logger.success(f'Finished worksheet {wks.title}')
+
+    def should_process_sheet(self, sheet_name: str) -> bool:
+        if len(self.allow_worksheets) and sheet_name not in self.allow_worksheets:
+            # ALLOW rules exist AND sheet name not explicitly allowed
+            return False
+        if len(self.block_worksheets) and sheet_name in self.block_worksheets:
+            # BLOCK rules exist AND sheet name is blocked
+            return False
+        return True
+
+    def missing_required_columns(self, gw: GWorksheet) -> list:
+        missing = []
+        for required_col in ['url', 'status']:
+            if not gw.col_exists(required_col):
+                missing.append(required_col)
+        return missing

src/auto_archiver/modules/gsheet_feeder/gworksheet.py

@@ -0,0 +1,108 @@
+from gspread import utils
+
+
+class GWorksheet:
+    """
+    This class makes read/write operations to the a worksheet easier.
+    It can read the headers from a custom row number, but the row references
+    should always include the offset of the header. 
+    eg: if header=4, row 5 will be the first with data. 
+    """
+    COLUMN_NAMES = {
+        'url': 'link',
+        'status': 'archive status',
+        'folder': 'destination folder',
+        'archive': 'archive location',
+        'date': 'archive date',
+        'thumbnail': 'thumbnail',
+        'timestamp': 'upload timestamp',
+        'title': 'upload title',
+        'screenshot': 'screenshot',
+        'hash': 'hash',
+        'pdq_hash': 'perceptual hashes',
+        'wacz': 'wacz',
+        'replaywebpage': 'replaywebpage',
+    }
+
+    def __init__(self, worksheet, columns=COLUMN_NAMES, header_row=1):
+        self.wks = worksheet
+        self.columns = columns
+        self.values = self.wks.get_values()
+        if len(self.values) > 0:
+            self.headers = [v.lower() for v in self.values[header_row - 1]]
+        else:
+            self.headers = []
+
+    def _check_col_exists(self, col: str):
+        if col not in self.columns:
+            raise Exception(f'Column {col} is not in the configured column names: {self.columns.keys()}')
+
+    def _col_index(self, col: str):
+        self._check_col_exists(col)
+        return self.headers.index(self.columns[col].lower())
+
+    def col_exists(self, col: str):
+        self._check_col_exists(col)
+        return self.columns[col].lower() in self.headers
+
+    def count_rows(self):
+        return len(self.values)
+
+    def get_row(self, row: int):
+        # row is 1-based
+        return self.values[row - 1]
+
+    def get_values(self):
+        return self.values
+
+    def get_cell(self, row, col: str, fresh=False):
+        """
+        returns the cell value from (row, col), 
+        where row can be an index (1-based) OR list of values
+        as received from self.get_row(row)
+        if fresh=True, the sheet is queried again for this cell
+        """
+        col_index = self._col_index(col)
+
+        if fresh:
+            return self.wks.cell(row, col_index + 1).value
+        if type(row) == int:
+            row = self.get_row(row)
+
+        if col_index >= len(row):
+            return ''
+        return row[col_index]
+
+    def get_cell_or_default(self, row, col: str, default: str = None, fresh=False, when_empty_use_default=True):
+        """
+        return self.get_cell or default value on error (eg: column is missing)
+        """
+        try:
+            val = self.get_cell(row, col, fresh)
+            if when_empty_use_default and val.strip() == "":
+                return default
+            return val
+        except:
+            return default
+
+    def set_cell(self, row: int, col: str, val):
+        # row is 1-based
+        col_index = self._col_index(col) + 1
+        self.wks.update_cell(row, col_index, val)
+
+    def batch_set_cell(self, cell_updates):
+        """
+        receives a list of [(row:int, col:str, val)] and batch updates it, the parameters are the same as in the self.set_cell() method
+        """
+        cell_updates = [
+            {
+                'range': self.to_a1(row, col),
+                'values': [[str(val)[0:49999]]]
+            }
+            for row, col, val in cell_updates
+        ]
+        self.wks.batch_update(cell_updates, value_input_option='USER_ENTERED')
+
+    def to_a1(self, row: int, col: str):
+        # row is 1-based
+        return utils.rowcol_to_a1(row, self._col_index(col) + 1)

src/auto_archiver/modules/hash_enricher/__init__.py

@@ -0,0 +1 @@
+from .hash_enricher import HashEnricher

src/auto_archiver/modules/hash_enricher/__manifest__.py

@@ -0,0 +1,28 @@
+{
+    "name": "Hash Enricher",
+    "type": ["enricher"],
+    "requires_setup": False,
+    "external_dependencies": {
+                          "python": ["loguru"],
+    },
+    "configs": {
+            "algorithm": {"default": "SHA-256", "help": "hash algorithm to use", "choices": ["SHA-256", "SHA3-512"]},
+            # TODO add non-negative requirement to match previous implementation?
+            "chunksize": {"default": 1.6e7, "help": "number of bytes to use when reading files in chunks (if this value is too large you will run out of RAM), default is 16MB"},
+        },
+    "description": """
+Generates cryptographic hashes for media files to ensure data integrity and authenticity.
+
+### Features
+- Calculates cryptographic hashes (SHA-256 or SHA3-512) for media files stored in `Metadata` objects.
+- Ensures content authenticity, integrity validation, and duplicate identification.
+- Efficiently processes large files by reading file bytes in configurable chunk sizes.
+- Supports dynamic configuration of hash algorithms and chunk sizes.
+- Updates media metadata with the computed hash value in the format `<algorithm>:<hash>`.
+
+### Notes
+- Default hash algorithm is SHA-256, but SHA3-512 is also supported.
+- Chunk size defaults to 16 MB but can be adjusted based on memory requirements.
+- Useful for workflows requiring hash-based content validation or deduplication.
+""",
+}

src/auto_archiver/modules/hash_enricher/hash_enricher.py

@@ -0,0 +1,72 @@
+""" Hash Enricher for generating cryptographic hashes of media files.
+
+The `HashEnricher` calculates cryptographic hashes (e.g., SHA-256, SHA3-512)
+for media files stored in `Metadata` objects. These hashes are used for
+validating content integrity, ensuring data authenticity, and identifying
+exact duplicates. The hash is computed by reading the file's bytes in chunks,
+making it suitable for handling large files efficiently.
+
+"""
+import hashlib
+from loguru import logger
+
+from auto_archiver.base_processors import Enricher
+from auto_archiver.core import Metadata, ArchivingContext
+
+
+class HashEnricher(Enricher):
+    """
+    Calculates hashes for Media instances
+    """
+    name = "hash_enricher"
+
+    def __init__(self, config: dict) -> None:
+        # without this STEP.__init__ is not called
+        super().__init__(config)
+        algos = self.configs()["algorithm"]
+        algo_choices = algos["choices"]
+        if not getattr(self, 'algorithm', None):
+            if not config.get('algorithm'):
+                logger.warning(f"No hash algorithm selected, defaulting to {algos['default']}")
+                self.algorithm = algos["default"]
+            else:
+                self.algorithm = config["algorithm"]
+
+        assert self.algorithm in algo_choices, f"Invalid hash algorithm selected, must be one of {algo_choices} (you selected {self.algorithm})."
+
+        if not getattr(self, 'chunksize', None):
+            if config.get('chunksize'):
+                self.chunksize = config["chunksize"]
+            else:
+                self.chunksize = self.configs()["chunksize"]["default"]
+
+        try:
+            self.chunksize = int(self.chunksize)
+        except ValueError:
+            raise ValueError(f"Invalid chunksize value: {self.chunksize}. Must be an integer.")
+
+        assert self.chunksize >= -1, "read length must be non-negative or -1"
+
+        ArchivingContext.set("hash_enricher.algorithm", self.algorithm, keep_on_reset=True)
+
+    def enrich(self, to_enrich: Metadata) -> None:
+        url = to_enrich.get_url()
+        logger.debug(f"calculating media hashes for {url=} (using {self.algorithm})")
+
+        for i, m in enumerate(to_enrich.media):
+            if len(hd := self.calculate_hash(m.filename)):
+                to_enrich.media[i].set("hash", f"{self.algorithm}:{hd}")
+
+    def calculate_hash(self, filename) -> str:
+        hash = None
+        if self.algorithm == "SHA-256":
+            hash = hashlib.sha256()
+        elif self.algorithm == "SHA3-512":
+            hash = hashlib.sha3_512()
+        else: return ""
+        with open(filename, "rb") as f:
+            while True:
+                buf = f.read(self.chunksize)
+                if not buf: break
+                hash.update(buf)
+        return hash.hexdigest()

src/auto_archiver/modules/html_formatter/__init__.py

@@ -0,0 +1 @@
+from .html_formatter import HtmlFormatter

src/auto_archiver/modules/html_formatter/__manifest__.py

@@ -0,0 +1,13 @@
+{
+    "name": "HTML Formatter",
+    "type": ["formatter"],
+    "requires_setup": False,
+    "external_dependencies": {
+                          "python": ["loguru", "jinja2"],
+                          "bin": [""]
+    },
+    "configs": {
+            "detect_thumbnails": {"default": True, "help": "if true will group by thumbnails generated by thumbnail enricher by id 'thumbnail_00'"}
+        },
+    "description": """ """,
+}

src/auto_archiver/modules/html_formatter/html_formatter.py

@@ -0,0 +1,93 @@
+from __future__ import annotations
+from dataclasses import dataclass
+import mimetypes, os, pathlib
+from jinja2 import Environment, FileSystemLoader
+from urllib.parse import quote
+from loguru import logger
+import json
+import base64
+
+from auto_archiver.version import __version__
+from auto_archiver.core import Metadata, Media, ArchivingContext
+from auto_archiver.base_processors import Formatter
+from auto_archiver.modules.hash_enricher import HashEnricher
+from auto_archiver.utils.misc import random_str
+
+
+@dataclass
+class HtmlFormatter(Formatter):
+    name = "html_formatter"
+
+    def __init__(self, config: dict) -> None:
+        # without this STEP.__init__ is not called
+        super().__init__(config)
+        self.environment = Environment(loader=FileSystemLoader(os.path.join(pathlib.Path(__file__).parent.resolve(), "templates/")), autoescape=True)
+        # JinjaHelper class static methods are added as filters
+        self.environment.filters.update({
+            k: v.__func__ for k, v in JinjaHelpers.__dict__.items() if isinstance(v, staticmethod)
+        })
+        self.template = self.environment.get_template("html_template.html")
+
+    def format(self, item: Metadata) -> Media:
+        url = item.get_url()
+        if item.is_empty():
+            logger.debug(f"[SKIP] FORMAT there is no media or metadata to format: {url=}")
+            return
+
+        content = self.template.render(
+            url=url,
+            title=item.get_title(),
+            media=item.media,
+            metadata=item.metadata,
+            version=__version__
+        )
+
+        html_path = os.path.join(ArchivingContext.get_tmp_dir(), f"formatted{random_str(24)}.html")
+        with open(html_path, mode="w", encoding="utf-8") as outf:
+            outf.write(content)
+        final_media = Media(filename=html_path, _mimetype="text/html")
+
+        he = HashEnricher({"hash_enricher": {"algorithm": ArchivingContext.get("hash_enricher.algorithm"), "chunksize": 1.6e7}})
+        if len(hd := he.calculate_hash(final_media.filename)):
+            final_media.set("hash", f"{he.algorithm}:{hd}")
+
+        return final_media
+
+
+# JINJA helper filters
+class JinjaHelpers:
+    @staticmethod
+    def is_list(v) -> bool:
+        return isinstance(v, list)
+
+    @staticmethod
+    def is_video(s: str) -> bool:
+        m = mimetypes.guess_type(s)[0]
+        return "video" in (m or "")
+
+    @staticmethod
+    def is_image(s: str) -> bool:
+        m = mimetypes.guess_type(s)[0]
+        return "image" in (m or "")
+
+    @staticmethod
+    def is_audio(s: str) -> bool:
+        m = mimetypes.guess_type(s)[0]
+        return "audio" in (m or "")
+
+    @staticmethod
+    def is_media(v) -> bool:
+        return isinstance(v, Media)
+
+    @staticmethod
+    def get_extension(filename: str) -> str:
+        return os.path.splitext(filename)[1]
+
+    @staticmethod
+    def quote(s: str) -> str:
+        return quote(s)
+
+    @staticmethod
+    def json_dump_b64(d: dict) -> str:
+        j = json.dumps(d, indent=4, default=str)
+        return base64.b64encode(j.encode()).decode()

src/auto_archiver/modules/html_formatter/templates/html_template.html

@@ -0,0 +1,332 @@
+{# templates/results.html #}
+{% import 'macros.html' as macros %}
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+    <meta charset="utf-8">
+    <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300italic,700,700italic">
+    <title>{{ url }}</title>
+    <style>
+        html {
+            font-family: 'Roboto', sans-serif;
+        }
+
+        table {
+            table-layout: fixed;
+            width: 90%;
+        }
+
+        table td {
+            word-wrap: break-word;
+            overflow-wrap: break-word;
+            padding: 5px;
+        }
+
+        table,
+        th,
+        td {
+            margin: auto;
+            border: 1px solid;
+            border-collapse: collapse;
+            vertical-align: top;
+        }
+
+        table.metadata td:first-child {
+            text-align: center;
+        }
+
+        table.content td:nth-child(2),
+        .center {
+            text-align: center;
+        }
+
+        .copy:hover {
+            background: aliceblue;
+            cursor: copy;
+        }
+
+        #notification {
+            position: fixed;
+            right: 20px;
+            top: 20px;
+            background: aquamarine;
+            box-shadow: 6px 8px 5px 0px #000000;
+            padding: 10px;
+            font-size: large;
+            display: none;
+        }
+
+        img,
+        video {
+            filter: gray;
+            -webkit-filter: grayscale(1);
+            filter: grayscale(1);
+        }
+
+        /* Disable grayscale on hover */
+        /* img:hover,
+        video:hover {
+            -webkit-filter: grayscale(0);
+            filter: none;
+        } */
+
+
+        .collapsible {
+            background-color: #777;
+            color: white;
+            cursor: pointer;
+            padding: 5px;
+            margin: 10px;
+            width: 100%;
+            border: none;
+            text-align: left;
+            outline: none;
+            font-size: 15px;
+        }
+
+        .active,
+        .collapsible:hover {
+            background-color: #555;
+        }
+
+        .collapsible-content {
+            padding: 0 18px;
+            display: none;
+            overflow: hidden;
+            background-color: #f1f1f1;
+        }
+
+        .pem-certificate, .text-preview {
+            text-align: left;
+            font-size: small;
+        }
+        .text-preview{
+            padding-left: 10px;
+            padding-right: 10px;
+            white-space: pre-wrap;
+        }
+    </style>
+</head>
+
+<body>
+    <div id="notification"></div>
+    <h2>Archived media for <span class="copy">{{ url }}</span> - <a href="{{ url }}">open</a></h2>
+    {% if title | string | length > 0 %}
+    <p><b>title:</b> '<span class="copy">{{ title }}</span>'</p>
+    {% endif %}
+    <h2 class="center">content {{ media | length }} item(s)</h2>
+    <form class="center">
+        <label>
+            <input type="checkbox" id="safe-media-view" checked>
+            Safe Media View
+        </label>
+    </form>
+    <table class="content">
+        <tr>
+            <th>about</th>
+            <th>files and preview</th>
+        </tr>
+        <tbody>
+            {% for m in media %}
+            <tr>
+                <td>
+                    {{ macros.display_recursive(m, true) }}
+                </td>
+                <td>
+                    {{ macros.display_media(m, true, url) }}
+                </td>
+            </tr>
+            {% endfor %}
+        </tbody>
+    </table>
+    <h2 class="center">metadata</h2>
+    <table class="metadata">
+        <tr>
+            <th>key</th>
+            <th>value</th>
+        </tr>
+        {% for key in metadata %}
+        <tr>
+            <td>{{ key }}</td>
+            <td>
+                {% if metadata[key] is mapping %}
+                <div class="center copy" copy-value64='{{metadata[key] | json_dump_b64}}'>Copy as JSON</div>
+                {% endif %}
+                {{ macros.copy_urlize(metadata[key]) }}
+            </td>
+        </tr>
+        {% endfor %}
+    </table>
+
+    <p class="center">Made with <a href="https://github.com/bellingcat/auto-archiver">bellingcat/auto-archiver</a>
+        v{{ version }}</p>
+</body>
+<script src="https://cdnjs.cloudflare.com/ajax/libs/forge/0.10.0/forge.min.js"></script>
+<script defer>
+    // partial decode of SSL certificates
+    function decodeCertificate(sslCert) {
+        var cert = forge.pki.certificateFromPem(sslCert);
+        return `SSL CERTIFICATE PREVIEW:<br/><ul>
+            <li><b>Subject:</b> <span class="copy">${cert.subject.attributes.map(attr => `${attr.shortName}: ${attr.value}`).join(", ")}</span></li>
+            <li><b>Issuer:</b> <span class="copy">${cert.issuer.attributes.map(attr => `${attr.shortName}: ${attr.value}`).join(", ")}</span></li>
+            <li><b>Valid From:</b> <span class="copy">${cert.validity.notBefore}</span></li>
+            <li><b>Valid To:</b> <span class="copy">${cert.validity.notAfter}</span></li>
+            <li><b>Serial Number:</b> <span class="copy">${cert.serialNumber}</span></li>
+        </ul>`;
+    }
+
+    async function run() {
+        let setupFunctions = [
+            previewCertificates,
+            previewText,
+            enableCopyLogic,
+            enableCollapsibleLogic,
+            setupSafeView
+        ];
+        setupFunctions.forEach(async f => {
+            try {
+                await f();
+            } catch (e) {
+                console.error(`Error in ${f.name}: ${e}`);
+            }
+        });
+    }
+
+    async function previewCertificates() {
+        await Promise.all(
+            Array.from(document.querySelectorAll(".pem-certificate")).map(async el => {
+                let certificate = await (await fetch(el.getAttribute("pem"))).text();
+                el.innerHTML = decodeCertificate(certificate);
+
+                let cyberChefUrl =
+                    `https://gchq.github.io/CyberChef/#recipe=Parse_X.509_certificate('PEM')&input=${btoa(certificate)}`;
+                // create a new anchor with this url and append after the code
+                let a = document.createElement("a");
+                a.href = cyberChefUrl;
+                a.textContent = "Full certificate details";
+                el.parentElement.appendChild(a);
+            })
+        );
+        console.log("certificate preview done");
+    }
+
+    async function previewText() {
+        await Promise.all(
+            Array.from(document.querySelectorAll(".text-preview")).map(async el => {
+                let textContent = await (await fetch(el.getAttribute("url"))).text();
+                el.textContent = textContent;
+            })
+        );
+        console.log("text preview done");
+    }
+
+    // notification logic
+    const notification = document.getElementById("notification");
+
+    function showNotification(message, miliseconds) {
+        notification.style.display = "block";
+        notification.innerText = message;
+        setTimeout(() => {
+            notification.style.display = "none";
+            notification.innerText = "";
+        }, miliseconds || 1000)
+    }
+
+    // copy logic
+    async function enableCopyLogic() {
+        await Promise.all(
+            Array.from(document.querySelectorAll(".copy")).map(el => {
+                el.onclick = () => {
+                    document.execCommand("copy");
+                }
+                el.addEventListener("copy", (e) => {
+                    e.preventDefault();
+                    if (e.clipboardData) {
+                        if (el.hasAttribute("copy-value")) {
+                            e.clipboardData.setData("text/plain", el.getAttribute("copy-value"));
+                        } else if (el.hasAttribute("copy-value64")) {
+                            // TODO: figure out how to decode unicode chars into utf-8
+                            e.clipboardData.setData("text/plain", new String(atob(el.getAttribute(
+                                "copy-value64"))));
+                        } else {
+                            e.clipboardData.setData("text/plain", el.textContent);
+                        }
+                        console.log(e.clipboardData.getData("text"))
+                        showNotification("copied!")
+                    }
+                })
+            })
+        )
+        console.log("copy logic enabled");
+    }
+
+    // collapsibles
+    async function enableCollapsibleLogic() {
+        let coll = document.getElementsByClassName("collapsible");
+        for (let i = 0; i < coll.length; i++) {
+            await new Promise(resolve => {
+                coll[i].addEventListener("click", function () {
+                    this.classList.toggle("active");
+                    // let content = this.nextElementSibling;
+                    let content = this.parentElement.querySelector(".collapsible-content");
+                    if (content.style.display === "block") {
+                        content.style.display = "none";
+                    } else {
+                        content.style.display = "block";
+                    }
+                });
+                resolve();
+            })
+        }
+        console.log("collapsible logic enabled");
+    }
+
+    async function setupSafeView() {
+        // logic for enabled/disabled greyscale
+        // Get references to the checkboxes and images/videos
+        const safeImageViewCheckbox = document.getElementById('safe-media-view');
+        const visualPreviews = document.querySelectorAll('img, video,embed');
+
+        // Function to toggle grayscale effect
+        function toggleGrayscale() {
+            visualPreviews.forEach(element => {
+                if (safeImageViewCheckbox.checked) {
+                    // Enable grayscale effect
+                    element.style.filter = 'grayscale(1)';
+                    element.style.webkitFilter = 'grayscale(1)';
+                } else {
+                    // Disable grayscale effect
+                    element.style.filter = 'none';
+                    element.style.webkitFilter = 'none';
+                }
+            });
+        }
+
+        // Add event listener to the checkbox to trigger the toggleGrayscale function
+        safeImageViewCheckbox.addEventListener('change', toggleGrayscale);
+
+        // Handle the hover effect using JavaScript
+        visualPreviews.forEach(element => {
+            element.addEventListener('mouseenter', () => {
+                // Disable grayscale effect on hover
+                element.style.filter = 'none';
+                element.style.webkitFilter = 'none';
+            });
+
+            element.addEventListener('mouseleave', () => {
+                // Re-enable grayscale effect if checkbox is checked
+                if (safeImageViewCheckbox.checked) {
+                    element.style.filter = 'grayscale(1)';
+                    element.style.webkitFilter = 'grayscale(1)';
+                }
+            });
+        });
+        toggleGrayscale();
+        console.log("grayscale logic enabled");
+    }
+
+    run();
+</script>
+
+</html>

src/auto_archiver/modules/html_formatter/templates/macros.html

@@ -0,0 +1,151 @@
+{% macro display_media(m, links, main_url) -%}
+
+{% for url in m.urls %}
+{% if url | length == 0 %}
+No URL available for {{ m.key }}.
+{% elif 'http://' in url or 'https://' in url or url.startswith('/') %}
+{% if 'image' in m.mimetype %}
+<div>
+    <a href="{{ url }}">
+        <img src="{{ url }}" style="max-height:400px;max-width:400px;"></img>
+    </a>
+
+    <div>
+        Reverse Image Search:&nbsp;
+        <a href="https://www.google.com/searchbyimage?sbisrc=4chanx&image_url={{ url | quote }}&safe=off">Google</a>,&nbsp;
+        <a href="https://lens.google.com/uploadbyurl?url={{ url | quote }}">Google Lens</a>,&nbsp;
+        <a href="https://yandex.ru/images/touch/search?rpt=imageview&url={{ url | quote }}">Yandex</a>,&nbsp;
+        <a href="https://www.bing.com/images/search?view=detailv2&iss=sbi&form=SBIVSP&sbisrc=UrlPaste&q=imgurl:{{ url | quote }}">Bing</a>,&nbsp;
+        <a href="https://www.tineye.com/search/?url={{ url | quote }}">Tineye</a>
+    </div>
+
+    <div>
+        Image Forensics:&nbsp;
+        <a href="https://fotoforensics.com/?url={{ url | quote }}">FotoForensics</a>,&nbsp;
+        <a href="https://mever.iti.gr/forensics/?image={{ url }}">Media Verification Assistant</a>
+    </div>
+    <p></p>
+</div>
+{% elif 'video' in m.mimetype %}
+<div>
+    <video src="{{ url }}" controls style="max-height:400px;max-width:600px;">
+        Your browser does not support the video element.
+    </video>
+</div>
+{% elif 'application/pdf' in m.mimetype %}
+<div>
+    <embed src="{{ url }}" width="100%" height="400px"/>
+</div>
+{% elif 'audio' in m.mimetype %}
+<div>
+    <audio controls>
+        <source src="{{ url }}" type="{{ m.mimetype }}">
+        Your browser does not support the audio element.
+    </audio>
+</div>
+{% elif m.filename | get_extension == ".wacz" %}
+<a href="https://replayweb.page/?source={{ url | quote }}#view=pages&url={{ main_url }}">replayweb</a>
+
+{% elif m.filename | get_extension == ".pem" %}
+<code class="pem-certificate" pem="{{url}}"></code>
+
+{% elif 'text' in m.mimetype %}
+<div>PREVIEW:<br/><code><pre class="text-preview" url="{{url}}"></pre></code></div>
+
+{% else %}
+No preview available for <code>{{ m.key }}</code>.
+{% endif %}
+{% else %}
+{{ m.url | urlize }}
+{% endif %}
+{% if links %}
+<a href="{{ url }}">open</a> or
+<a href="{{ url }}" download="">download</a> or
+{{ copy_urlize(url, "copy") }}
+
+<br>
+{% endif %}
+{% endfor %}
+
+{%- endmacro -%}
+
+{% macro copy_urlize(val, href_text) -%}
+
+{% if val | is_list %}
+    {% for item in val %}
+    {{ copy_urlize(item) }}
+    {% endfor %}
+
+{% elif val is mapping %}
+<ul>
+    {% for key in val %}
+    <li>
+        <b>{{ key }}:</b> {{ copy_urlize(val[key]) }}
+    </li>
+    {% endfor %}
+</ul>
+
+{% else %}
+{% if href_text | length == 0 %}
+<span class="copy">{{ val | string | urlize }}</span>
+{% else %}
+<span class="copy" copy-value="{{val}}">{{ href_text | string | urlize }}</span>
+{% endif %}
+{% endif %}
+
+{%- endmacro -%}
+
+
+{% macro display_recursive(prop, skip_display) -%}
+    {% if prop is mapping %}
+    <div class="center copy" copy-value64='{{prop | json_dump_b64}}'>Copy as JSON</div>
+    <ul>
+        {% for subprop in prop %}
+            <li>
+                <b>{{ subprop }}:</b>
+                {{ display_recursive(prop[subprop]) }}
+            </li>
+        {% endfor %}
+    </ul>
+
+    {% elif prop | is_list %}
+        {% for item in prop %}
+            <li>
+                {{ display_recursive(item) }}
+            </li>
+        {% endfor %}
+
+
+    {% elif prop | is_media %}
+        {% if not skip_display %}
+            {{ display_media(prop, true) }}
+        {% endif %}
+        <ul>
+            <li><b>key:</b> <span class="copy">{{ prop.key }}</span></li>
+            <li><b>type:</b> <span class="copy">{{ prop.mimetype }}</span></li>
+            {% for subprop in prop.properties %}
+                
+
+                {% if prop.properties[subprop] | is_list %}
+                <p></p>
+                <div>
+                    <b class="collapsible" title="expand">{{ subprop }} ({{ prop.properties[subprop] | length }}):</b>
+                    <p></p>
+                    <div class="collapsible-content">
+                        {% for subsubprop in prop.properties[subprop] %}
+                            {{ display_recursive(subsubprop) }}
+                        {% endfor %}
+                    </div>
+                </div>
+                <p></p>
+                {% elif prop.properties[subprop] | string | length > 1 %}
+                <li><b>{{ subprop }}:</b> {{ copy_urlize(prop.properties[subprop]) }}</li>
+                {% endif %}
+
+            {% endfor %}
+
+        </ul>
+    {% else %}
+        {{ copy_urlize(prop) }}
+    {% endif %}
+{%- endmacro -%}

src/auto_archiver/modules/instagram_api_extractor/__init__.py

@@ -0,0 +1 @@
+from .instagram_api_extractor import InstagramAPIExtractor

src/auto_archiver/modules/instagram_api_extractor/__manifest__.py

@@ -1,15 +1,13 @@
 {
-    "name": "Instagram API Archiver",
+    "name": "Instagram API Extractor",
     "type": ["extractor"],
-    "entry_point": "instagram_api_archiver:InstagramApiArchiver",
-    "depends": ["core"],
     "external_dependencies":
         {"python": ["requests",
                     "loguru",
                     "retrying",
                     "tqdm",],
          },
-    "no_setup_required": False,
+    "requires_setup": True,
     "configs": {
         "access_token": {"default": None, "help": "a valid instagrapi-api token"},
         "api_endpoint": {"default": None, "help": "API endpoint to use"},
@@ -26,5 +24,22 @@
             "help": "if true, will remove empty values from the json output",
         },
     },
-    "description": "",
+    "description": """
+Archives various types of Instagram content using the Instagrapi API.
+
+### Features
+- Connects to an Instagrapi API deployment to fetch Instagram profiles, posts, stories, highlights, reels, and tagged content.
+- Supports advanced configuration options, including:
+  - Full profile download (all posts, stories, highlights, and tagged content).
+  - Limiting the number of posts to fetch for large profiles.
+  - Minimising JSON output to remove empty fields and redundant data.
+- Provides robust error handling and retries for API calls.
+- Ensures efficient media scraping, including handling nested or carousel media items.
+- Adds downloaded media and metadata to the result for further processing.
+
+### Notes
+- Requires a valid Instagrapi API token (`access_token`) and API endpoint (`api_endpoint`).
+- Full-profile downloads can be limited by setting `full_profile_max_posts`.
+- Designed to fetch content in batches for large profiles, minimising API load.
+""",
 }

src/auto_archiver/modules/instagram_api_extractor/instagram_api_extractor.py

@@ -1,5 +1,5 @@
 """
-The `instagram_api_archiver` module provides tools for archiving various types of Instagram content
+The `instagram_api_extractor` module provides tools for archiving various types of Instagram content
 using the [Instagrapi API](https://github.com/subzeroid/instagrapi).
 
 Connects to an Instagrapi API deployment and allows for downloading Instagram user profiles,
@@ -16,19 +16,19 @@ from loguru import logger
 from retrying import retry
 from tqdm import tqdm
 
-from auto_archiver.archivers import Archiver
+from auto_archiver.base_processors import Extractor
 from auto_archiver.core import Media
 from auto_archiver.core import Metadata
 
 
-class InstagramAPIArchiver(Archiver):
+class InstagramAPIExtractor(Extractor):
     """
     Uses an https://github.com/subzeroid/instagrapi API deployment to fetch instagram posts data
 
     # TODO: improvement collect aggregates of locations[0].location and mentions for all posts
     """
 
-    name = "instagram_api_archiver"
+    name = "instagram_api_extractor"
 
     global_pattern = re.compile(
         r"(?:(?:http|https):\/\/)?(?:www.)?(?:instagram.com)\/(stories(?:\/highlights)?|p|reel)?\/?([^\/\?]*)\/?(\d+)?"
@@ -45,25 +45,6 @@ class InstagramAPIArchiver(Archiver):
         self.full_profile = bool(self.full_profile)
         self.minimize_json_output = bool(self.minimize_json_output)
 
-    @staticmethod
-    def configs() -> dict:
-        return {
-            "access_token": {"default": None, "help": "a valid instagrapi-api token"},
-            "api_endpoint": {"default": None, "help": "API endpoint to use"},
-            "full_profile": {
-                "default": False,
-                "help": "if true, will download all posts, tagged posts, stories, and highlights for a profile, if false, will only download the profile pic and information.",
-            },
-            "full_profile_max_posts": {
-                "default": 0,
-                "help": "Use to limit the number of posts to download when full_profile is true. 0 means no limit. limit is applied softly since posts are fetched in batch, once to: posts, tagged posts, and highlights",
-            },
-            "minimize_json_output": {
-                "default": True,
-                "help": "if true, will remove empty values from the json output",
-            },
-        }
-
     def download(self, item: Metadata) -> Metadata:
         url = item.get_url()
 

src/auto_archiver/modules/instagram_extractor/__init__.py

@@ -0,0 +1 @@
+from .instagram_extractor import InstagramExtractor

src/auto_archiver/modules/instagram_extractor/__manifest__.py

@@ -1,13 +1,13 @@
 {
-    "name": "Instagram Archiver",
+    "name": "Instagram Extractor",
     "type": ["extractor"],
-    "entry_point": "instagram_archiver:InstagramArchiver",
-    "depends": ["core"],
     "external_dependencies": {
-        "python": ["instaloader",
-                   "loguru",],
+        "python": [
+            "instaloader",
+            "loguru",
+        ],
     },
-    "no_setup_required": False,
+    "requires_setup": True,
     "configs": {
         "username": {"default": None, "help": "a valid Instagram username"},
         "password": {

src/auto_archiver/modules/instagram_extractor/instagram_extractor.py

@@ -7,15 +7,15 @@ import re, os, shutil, traceback
 import instaloader  # https://instaloader.github.io/as-module.html
 from loguru import logger
 
-from auto_archiver.archivers import Archiver
+from auto_archiver.base_processors import Extractor
 from auto_archiver.core import Metadata
 from auto_archiver.core import Media
 
-class InstagramArchiver(Archiver):
+class InstagramExtractor(Extractor):
     """
     Uses Instaloader to download either a post (inc images, videos, text) or as much as possible from a profile (posts, stories, highlights, ...)
     """
-    name = "instagram_archiver"
+    name = "instagram_extractor"
 
     # NB: post regex should be tested before profile
     # https://regex101.com/r/MGPquX/1
@@ -45,16 +45,7 @@ class InstagramArchiver(Archiver):
             except Exception as e2:
                 logger.error(f"Unable to finish login (retrying from file): {e2}\n{traceback.format_exc()}")
 
-    @staticmethod
-    def configs() -> dict:
-        return {
-            "username": {"default": None, "help": "a valid Instagram username"},
-            "password": {"default": None, "help": "the corresponding Instagram account password"},
-            "download_folder": {"default": "instaloader", "help": "name of a folder to temporarily download content to"},
-            "session_file": {"default": "secrets/instaloader.session", "help": "path to the instagram session which saves session credentials"},
-            #TODO: fine-grain
-            # "download_stories": {"default": True, "help": "if the link is to a user profile: whether to get stories information"},
-        }
+
 
     def download(self, item: Metadata) -> Metadata:
         url = item.get_url()
@@ -76,7 +67,7 @@ class InstagramArchiver(Archiver):
             elif len(profile_matches):
                 result = self.download_profile(url, profile_matches[0])
         except Exception as e:
-            logger.error(f"Failed to download with instagram archiver due to: {e}, make sure your account credentials are valid.")
+            logger.error(f"Failed to download with instagram extractor due to: {e}, make sure your account credentials are valid.")
         finally:
             shutil.rmtree(self.download_folder, ignore_errors=True)
         return result

src/auto_archiver/modules/instagram_tbot_extractor/__init__.py

@@ -0,0 +1 @@
+from .instagram_tbot_extractor import InstagramTbotExtractor

src/auto_archiver/modules/instagram_tbot_extractor/__manifest__.py

@@ -1,8 +1,6 @@
 {
-    "name": "Instagram Telegram Bot Archiver",
+    "name": "Instagram Telegram Bot Extractor",
     "type": ["extractor"],
-    "entry_point": "instagram_tbot_archiver:InstagramTbotArchiver",
-    "depends": ["core", "utils"],
     "external_dependencies": {"python": ["loguru",
                                          "telethon",],
                               },
@@ -14,7 +12,7 @@
             "timeout": {"default": 45, "help": "timeout to fetch the instagram content in seconds."},
     },
     "description": """
-The `InstagramTbotArchiver` module uses a Telegram bot (`instagram_load_bot`) to fetch and archive Instagram content,
+The `InstagramTbotExtractor` module uses a Telegram bot (`instagram_load_bot`) to fetch and archive Instagram content,
 such as posts and stories. It leverages the Telethon library to interact with the Telegram API, sending Instagram URLs
 to the bot and downloading the resulting media and metadata. The downloaded content is stored as `Media` objects and
 returned as part of a `Metadata` object.
@@ -27,7 +25,7 @@ returned as part of a `Metadata` object.
 
 ### Setup
 
-To use the `InstagramTbotArchiver`, you need to provide the following configuration settings:
+To use the `InstagramTbotExtractor`, you need to provide the following configuration settings:
 - **API ID and Hash**: Telegram API credentials obtained from [my.telegram.org/apps](https://my.telegram.org/apps).
 - **Session File**: Optional path to store the Telegram session file for future use.
 

src/auto_archiver/modules/instagram_tbot_extractor/instagram_tbot_extractor.py

@@ -1,5 +1,5 @@
 """
-InstagramTbotArchiver Module
+InstagramTbotExtractor Module
 
 This module provides functionality to archive Instagram content (posts, stories, etc.) using a Telegram bot (`instagram_load_bot`).
 It interacts with the Telegram API via the Telethon library to send Instagram URLs to the bot, which retrieves the
@@ -15,18 +15,18 @@ from sqlite3 import OperationalError
 from loguru import logger
 from telethon.sync import TelegramClient
 
-from auto_archiver.archivers import Archiver
+from auto_archiver.base_processors import Extractor
 from auto_archiver.core import Metadata, Media, ArchivingContext
 from auto_archiver.utils import random_str
 
 
-class InstagramTbotArchiver(Archiver):
+class InstagramTbotExtractor(Extractor):
     """
     calls a telegram bot to fetch instagram posts/stories... and gets available media from it
     https://github.com/adw0rd/instagrapi
     https://t.me/instagram_load_bot
     """
-    name = "instagram_tbot_archiver"
+    name = "instagram_tbot_extractor"
 
     def __init__(self, config: dict) -> None:
         super().__init__(config)
@@ -34,15 +34,6 @@ class InstagramTbotArchiver(Archiver):
         self.assert_valid_string("api_hash")
         self.timeout = int(self.timeout)
 
-    @staticmethod
-    def configs() -> dict:
-        return {
-            "api_id": {"default": None, "help": "telegram API_ID value, go to https://my.telegram.org/apps"},
-            "api_hash": {"default": None, "help": "telegram API_HASH value, go to https://my.telegram.org/apps"},
-            "session_file": {"default": "secrets/anon-insta", "help": "optional, records the telegram login session for future usage, '.session' will be appended to the provided value."},
-            "timeout": {"default": 45, "help": "timeout to fetch the instagram content in seconds."},
-        }
-
     def setup(self) -> None:
         """
         1. makes a copy of session_file that is removed in cleanup
@@ -58,7 +49,7 @@ class InstagramTbotArchiver(Archiver):
         try:
             self.client = TelegramClient(self.session_file, self.api_id, self.api_hash)
         except OperationalError as e:
-            logger.error(f"Unable to access the {self.session_file} session, please make sure you don't use the same session file here and in telethon_archiver. if you do then disable at least one of the archivers for the 1st time you setup telethon session: {e}")
+            logger.error(f"Unable to access the {self.session_file} session, please make sure you don't use the same session file here and in telethon_extractor. if you do then disable at least one of the archivers for the 1st time you setup telethon session: {e}")
 
         with self.client.start():
             logger.success(f"SETUP {self.name} login works.")

src/auto_archiver/modules/local_storage/__init__.py

@@ -0,0 +1 @@
+from .local import LocalStorage

src/auto_archiver/modules/local_storage/__manifest__.py

@@ -0,0 +1,35 @@
+{
+    "name": "Local Storage",
+    "type": ["storage"],
+    "requires_setup": False,
+    "external_dependencies": {
+        "python": ["loguru"],
+    },
+    "configs": {
+        "path_generator": {
+            "default": "url",
+            "help": "how to store the file in terms of directory structure: 'flat' sets to root; 'url' creates a directory based on the provided URL; 'random' creates a random directory.",
+            "choices": ["flat", "url", "random"],
+        },
+        "filename_generator": {
+            "default": "random",
+            "help": "how to name stored files: 'random' creates a random string; 'static' uses a replicable strategy such as a hash.",
+            "choices": ["random", "static"],
+        },
+        "save_to": {"default": "./archived", "help": "folder where to save archived content"},
+        "save_absolute": {"default": False, "help": "whether the path to the stored file is absolute or relative in the output result inc. formatters (WARN: leaks the file structure)"},
+    },
+    "description": """
+    LocalStorage: A storage module for saving archived content locally on the filesystem.
+
+    ### Features
+    - Saves archived media files to a specified folder on the local filesystem.
+    - Maintains file metadata during storage using `shutil.copy2`.
+    - Supports both absolute and relative paths for stored files, configurable via `save_absolute`.
+    - Automatically creates directories as needed for storing files.
+
+    ### Notes
+    - Default storage folder is `./archived`, but this can be changed via the `save_to` configuration.
+    - The `save_absolute` option can reveal the file structure in output formats; use with caution.
+    """
+}

src/auto_archiver/modules/local_storage/local.py

@@ -0,0 +1,35 @@
+
+import shutil
+from typing import IO
+import os
+from loguru import logger
+
+from auto_archiver.core import Media
+from auto_archiver.base_processors import Storage
+
+
+class LocalStorage(Storage):
+    name = "local_storage"
+
+    def __init__(self, config: dict) -> None:
+        super().__init__(config)
+        os.makedirs(self.save_to, exist_ok=True)
+
+    def get_cdn_url(self, media: Media) -> str:
+        # TODO: is this viable with Storage.configs on path/filename?
+        dest = os.path.join(self.save_to, media.key)
+        if self.save_absolute:
+            dest = os.path.abspath(dest)
+        return dest
+
+    def upload(self, media: Media, **kwargs) -> bool:
+        # override parent so that we can use shutil.copy2 and keep metadata
+        dest = os.path.join(self.save_to, media.key)
+        os.makedirs(os.path.dirname(dest), exist_ok=True)
+        logger.debug(f'[{self.__class__.name}] storing file {media.filename} with key {media.key} to {dest}')
+        res = shutil.copy2(media.filename, dest)
+        logger.info(res)
+        return True
+
+    # must be implemented even if unused
+    def uploadf(self, file: IO[bytes], key: str, **kwargs: dict) -> bool: pass

src/auto_archiver/modules/meta_enricher/__init__.py

@@ -0,0 +1 @@
+from .meta_enricher import MetaEnricher

src/auto_archiver/modules/meta_enricher/__manifest__.py

@@ -0,0 +1,22 @@
+{
+    "name": "Archive Metadata Enricher",
+    "type": ["enricher"],
+    "requires_setup": False,
+    "external_dependencies": {
+                          "python": ["loguru"],
+    },
+    "description": """ 
+    Adds metadata information about the archive operations, Adds metadata about archive operations, including file sizes and archive duration./
+    To be included at the end of all enrichments.
+    
+    ### Features
+- Calculates the total size of all archived media files, storing the result in human-readable and byte formats.
+- Computes the duration of the archival process, storing the elapsed time in seconds.
+- Ensures all enrichments are performed only if the `Metadata` object contains valid data.
+- Adds detailed metadata to provide insights into file sizes and archival performance.
+
+### Notes
+- Skips enrichment if no media or metadata is available in the `Metadata` object.
+- File sizes are calculated using the `os.stat` module, ensuring accurate byte-level reporting.
+""",
+}

src/auto_archiver/modules/meta_enricher/meta_enricher.py

@@ -0,0 +1,55 @@
+import datetime
+import os
+from loguru import logger
+
+from auto_archiver.base_processors import Enricher
+from auto_archiver.core import Metadata
+
+
+class MetaEnricher(Enricher):
+    """
+    Adds metadata information about the archive operations, to be included at the end of all enrichments
+    """
+    name = "meta_enricher"
+
+
+    def __init__(self, config: dict) -> None:
+        # without this STEP.__init__ is not called
+        super().__init__(config)
+
+    def enrich(self, to_enrich: Metadata) -> None:
+        url = to_enrich.get_url()
+        if to_enrich.is_empty():
+            logger.debug(f"[SKIP] META_ENRICHER there is no media or metadata to enrich: {url=}")
+            return
+
+        logger.debug(f"calculating archive metadata information for {url=}")
+
+        self.enrich_file_sizes(to_enrich)
+        self.enrich_archive_duration(to_enrich)
+
+    def enrich_file_sizes(self, to_enrich: Metadata):
+        logger.debug(f"calculating archive file sizes for url={to_enrich.get_url()} ({len(to_enrich.media)} media files)")
+        total_size = 0
+        for media in to_enrich.get_all_media():
+            file_stats = os.stat(media.filename)
+            media.set("bytes", file_stats.st_size)
+            media.set("size", self.human_readable_bytes(file_stats.st_size))
+            total_size += file_stats.st_size
+
+        to_enrich.set("total_bytes", total_size)
+        to_enrich.set("total_size", self.human_readable_bytes(total_size))
+
+
+    def human_readable_bytes(self, size: int) -> str:
+        # receives number of bytes and returns human readble size
+        for unit in ["bytes", "KB", "MB", "GB", "TB"]:
+            if size < 1024:
+                return f"{size:.1f} {unit}"
+            size /= 1024
+
+    def enrich_archive_duration(self, to_enrich):
+        logger.debug(f"calculating archive duration for url={to_enrich.get_url()} ")
+
+        archive_duration = datetime.datetime.now(datetime.timezone.utc) - to_enrich.get("_processed_at")
+        to_enrich.set("archive_duration_seconds", archive_duration.seconds)

src/auto_archiver/modules/metadata_enricher/__init__.py

@@ -0,0 +1 @@
+from .metadata_enricher import MetadataEnricher

src/auto_archiver/modules/metadata_enricher/__manifest__.py

@@ -0,0 +1,22 @@
+{
+    "name": "Media Metadata Enricher",
+    "type": ["enricher"],
+    "requires_setup": False,
+    "external_dependencies": {
+        "python": ["loguru"], 
+        "bin": ["exiftool"]
+
+    },
+    "description": """
+    Extracts metadata information from files using ExifTool.
+
+    ### Features
+    - Uses ExifTool to extract detailed metadata from media files.
+    - Processes file-specific data like camera settings, geolocation, timestamps, and other embedded metadata.
+    - Adds extracted metadata to the corresponding `Media` object within the `Metadata`.
+
+    ### Notes
+    - Requires ExifTool to be installed and accessible via the system's PATH.
+    - Skips enrichment for files where metadata extraction fails.
+    """
+}

src/auto_archiver/modules/metadata_enricher/metadata_enricher.py

@@ -0,0 +1,44 @@
+import subprocess
+import traceback
+from loguru import logger
+
+from auto_archiver.base_processors import Enricher
+from auto_archiver.core import Metadata
+
+
+class MetadataEnricher(Enricher):
+    """
+    Extracts metadata information from files using exiftool.
+    """
+    name = "metadata_enricher"
+
+    def __init__(self, config: dict) -> None:
+        # without this STEP.__init__ is not called
+        super().__init__(config)
+
+
+    def enrich(self, to_enrich: Metadata) -> None:
+        url = to_enrich.get_url()
+        logger.debug(f"extracting EXIF metadata for {url=}")
+
+        for i, m in enumerate(to_enrich.media):
+            if len(md := self.get_metadata(m.filename)):
+                to_enrich.media[i].set("metadata", md)
+
+    def get_metadata(self, filename: str) -> dict:
+        try:
+            # Run ExifTool command to extract metadata from the file
+            cmd = ['exiftool', filename]
+            result = subprocess.run(cmd, capture_output=True, text=True)
+
+            # Process the output to extract individual metadata fields
+            metadata = {}
+            for line in result.stdout.splitlines():
+                field, value = line.strip().split(':', 1)
+                metadata[field.strip()] = value.strip()
+            return metadata
+        except FileNotFoundError:
+            logger.error("[exif_enricher] ExifTool not found. Make sure ExifTool is installed and added to PATH.")
+        except Exception as e:
+            logger.error(f"Error occurred: {e}: {traceback.format_exc()}")
+        return {}

src/auto_archiver/modules/mute_formatter/__init__.py

@@ -0,0 +1 @@
+from .mute_formatter import MuteFormatter

src/auto_archiver/modules/mute_formatter/__manifest__.py

@@ -0,0 +1,9 @@
+m = {
+    "name": "Mute Formatter",
+    "type": ["formatter"],
+    "requires_setup": False,
+    "external_dependencies": {
+    },
+    "description": """ Default formatter.
+    """,
+}

src/auto_archiver/modules/mute_formatter/mute_formatter.py

@@ -0,0 +1,16 @@
+from __future__ import annotations
+from dataclasses import dataclass
+
+from ..core import Metadata, Media
+from . import Formatter
+
+
+@dataclass
+class MuteFormatter(Formatter):
+    name = "mute_formatter"
+
+    def __init__(self, config: dict) -> None:
+        # without this STEP.__init__ is not called
+        super().__init__(config)
+
+    def format(self, item: Metadata) -> Media: return None

src/auto_archiver/modules/pdq_hash_enricher/__init__.py

@@ -0,0 +1 @@
+from .pdq_hash_enricher import PdqHashEnricher

src/auto_archiver/modules/pdq_hash_enricher/__manifest__.py

@@ -0,0 +1,21 @@
+{
+    "name": "PDQ Hash Enricher",
+    "type": ["enricher"],
+    "requires_setup": False,
+    "external_dependencies": {
+        "python": ["loguru", "pdqhash", "numpy", "Pillow"],
+    },
+    "description": """
+    PDQ Hash Enricher for generating perceptual hashes of media files.
+
+    ### Features
+    - Calculates perceptual hashes for image files using the PDQ hashing algorithm.
+    - Enables detection of duplicate or near-duplicate visual content.
+    - Processes images stored in `Metadata` objects, adding computed hashes to the corresponding `Media` entries.
+    - Skips non-image media or files unsuitable for hashing (e.g., corrupted or unsupported formats).
+
+    ### Notes
+    - Best used after enrichers like `thumbnail_enricher` or `screenshot_enricher` to ensure images are available.
+    - Uses the `pdqhash` library to compute 256-bit perceptual hashes, which are stored as hexadecimal strings.
+    """
+}

src/auto_archiver/modules/pdq_hash_enricher/pdq_hash_enricher.py

@@ -0,0 +1,60 @@
+"""
+PDQ Hash Enricher for generating perceptual hashes of media files.
+
+The `PdqHashEnricher` processes media files (e.g., images) in `Metadata`
+objects and calculates perceptual hashes using the PDQ hashing algorithm.
+These hashes are designed specifically for images and can be used
+for detecting duplicate or near-duplicate visual content.
+
+This enricher is typically used after thumbnail or screenshot enrichers
+to ensure images are available for hashing.
+
+"""
+import traceback
+import pdqhash
+import numpy as np
+from PIL import Image, UnidentifiedImageError
+from loguru import logger
+
+from auto_archiver.base_processors import Enricher
+from auto_archiver.core import Metadata
+
+
+class PdqHashEnricher(Enricher):
+    """
+    Calculates perceptual hashes for Media instances using PDQ, allowing for (near-)duplicate detection.
+    Ideally this enrichment is orchestrated to run after the thumbnail_enricher.
+    """
+    name = "pdq_hash_enricher"
+
+    def __init__(self, config: dict) -> None:
+        # Without this STEP.__init__ is not called
+        super().__init__(config)
+
+    def enrich(self, to_enrich: Metadata) -> None:
+        url = to_enrich.get_url()
+        logger.debug(f"calculating perceptual hashes for {url=}")
+        media_with_hashes = []
+
+        for m in to_enrich.media:
+            for media in m.all_inner_media(True):
+                media_id = media.get("id", "")
+                if media.is_image() and "screenshot" not in media_id and "warc-file-" not in media_id and len(hd := self.calculate_pdq_hash(media.filename)):
+                    media.set("pdq_hash", hd)
+                    media_with_hashes.append(media.filename)
+
+        logger.debug(f"calculated '{len(media_with_hashes)}' perceptual hashes for {url=}: {media_with_hashes}")
+
+    def calculate_pdq_hash(self, filename):
+        # returns a hexadecimal string with the perceptual hash for the given filename
+        try:
+            with Image.open(filename) as img:
+                # convert the image to RGB
+                image_rgb = np.array(img.convert("RGB"))
+                # compute the 256-bit PDQ hash (we do not store the quality score)
+                hash_array, _ = pdqhash.compute(image_rgb)
+                hash = "".join(str(b) for b in hash_array)
+                return hex(int(hash, 2))[2:]
+        except UnidentifiedImageError as e:
+            logger.error(f"Image {filename=} is likely corrupted or in unsupported format {e}: {traceback.format_exc()}")
+        return ""

src/auto_archiver/modules/s3_storage/__init__.py

@@ -0,0 +1 @@
+from .s3 import S3Storage

src/auto_archiver/modules/s3_storage/__manifest__.py

@@ -0,0 +1,49 @@
+{
+    "name": "S3 Storage",
+    "type": ["storage"],
+    "requires_setup": True,
+    "external_dependencies": {
+        "python": ["boto3", "loguru"],
+    },
+    "configs": {
+        "path_generator": {
+            "default": "url",
+            "help": "how to store the file in terms of directory structure: 'flat' sets to root; 'url' creates a directory based on the provided URL; 'random' creates a random directory.",
+            "choices": ["flat", "url", "random"],
+        },
+        "filename_generator": {
+            "default": "random",
+            "help": "how to name stored files: 'random' creates a random string; 'static' uses a replicable strategy such as a hash.",
+            "choices": ["random", "static"],
+        },
+        "bucket": {"default": None, "help": "S3 bucket name"},
+        "region": {"default": None, "help": "S3 region name"},
+        "key": {"default": None, "help": "S3 API key"},
+        "secret": {"default": None, "help": "S3 API secret"},
+        "random_no_duplicate": {"default": False, "help": f"if set, it will override `path_generator`, `filename_generator` and `folder`. It will check if the file already exists and if so it will not upload it again. Creates a new root folder path `{NO_DUPLICATES_FOLDER}`"},
+        "endpoint_url": {
+            "default": 'https://{region}.digitaloceanspaces.com',
+            "help": "S3 bucket endpoint, {region} are inserted at runtime"
+        },
+        "cdn_url": {
+            "default": 'https://{bucket}.{region}.cdn.digitaloceanspaces.com/{key}',
+            "help": "S3 CDN url, {bucket}, {region} and {key} are inserted at runtime"
+        },
+        "private": {"default": False, "help": "if true S3 files will not be readable online"},
+    },
+    "description": """
+    S3Storage: A storage module for saving media files to an S3-compatible object storage.
+
+    ### Features
+    - Uploads media files to an S3 bucket with customizable configurations.
+    - Supports `random_no_duplicate` mode to avoid duplicate uploads by checking existing files based on SHA-256 hashes.
+    - Automatically generates unique paths for files when duplicates are found.
+    - Configurable endpoint and CDN URL for different S3-compatible providers.
+    - Supports both private and public file storage, with public files being readable online.
+
+    ### Notes
+    - Requires S3 credentials (API key and secret) and a bucket name to function.
+    - The `random_no_duplicate` option ensures no duplicate uploads by leveraging hash-based folder structures.
+    - Uses `boto3` for interaction with the S3 API.
+    """
+}

src/auto_archiver/modules/s3_storage/s3.py

@@ -0,0 +1,75 @@
+
+from typing import IO
+import boto3, os
+
+from auto_archiver.utils.misc import random_str
+from auto_archiver.core import Media
+from auto_archiver.base_processors import Storage
+# TODO
+from auto_archiver.modules.hash_enricher import HashEnricher
+from loguru import logger
+
+NO_DUPLICATES_FOLDER = "no-dups/"
+class S3Storage(Storage):
+    name = "s3_storage"
+
+    def __init__(self, config: dict) -> None:
+        super().__init__(config)
+        self.s3 = boto3.client(
+            's3',
+            region_name=self.region,
+            endpoint_url=self.endpoint_url.format(region=self.region),
+            aws_access_key_id=self.key,
+            aws_secret_access_key=self.secret
+        )
+        self.random_no_duplicate = bool(self.random_no_duplicate)
+        if self.random_no_duplicate:
+            logger.warning("random_no_duplicate is set to True, this will override `path_generator`, `filename_generator` and `folder`.")
+
+    def get_cdn_url(self, media: Media) -> str:
+        return self.cdn_url.format(bucket=self.bucket, region=self.region, key=media.key)
+
+    def uploadf(self, file: IO[bytes], media: Media, **kwargs: dict) -> None:
+        if not self.is_upload_needed(media): return True
+
+        extra_args = kwargs.get("extra_args", {})
+        if not self.private and 'ACL' not in extra_args:
+            extra_args['ACL'] = 'public-read'
+
+        if 'ContentType' not in extra_args:
+            try:
+                if media.mimetype:
+                    extra_args['ContentType'] = media.mimetype
+            except Exception as e:
+                logger.warning(f"Unable to get mimetype for {media.key=}, error: {e}")
+
+        self.s3.upload_fileobj(file, Bucket=self.bucket, Key=media.key, ExtraArgs=extra_args)
+        return True
+    
+    def is_upload_needed(self, media: Media) -> bool:
+        if self.random_no_duplicate:
+            # checks if a folder with the hash already exists, if so it skips the upload
+            he = HashEnricher({"hash_enricher": {"algorithm": "SHA-256", "chunksize": 1.6e7}})
+            hd = he.calculate_hash(media.filename)
+            path = os.path.join(NO_DUPLICATES_FOLDER, hd[:24])
+
+            if existing_key:=self.file_in_folder(path):
+                media.key = existing_key
+                media.set("previously archived", True)
+                logger.debug(f"skipping upload of {media.filename} because it already exists in {media.key}")
+                return False
+            
+            _, ext = os.path.splitext(media.key)
+            media.key = os.path.join(path, f"{random_str(24)}{ext}")
+        return True
+    
+    
+    def file_in_folder(self, path:str) -> str:
+        # checks if path exists and is not an empty folder
+        if not path.endswith('/'):
+            path = path + '/' 
+        resp = self.s3.list_objects(Bucket=self.bucket, Prefix=path, Delimiter='/', MaxKeys=1)
+        if 'Contents' in resp:
+            return resp['Contents'][0]['Key']
+        return False
+

src/auto_archiver/modules/screenshot_enricher/__init__.py

@@ -0,0 +1 @@
+from .screenshot_enricher import ScreenshotEnricher

src/auto_archiver/modules/screenshot_enricher/__manifest__.py

@@ -0,0 +1,30 @@
+{
+    "name": "Screenshot Enricher",
+    "type": ["enricher"],
+    "requires_setup": True,
+    "external_dependencies": {
+        "python": ["loguru", "selenium"],
+        "bin": ["chromedriver"]
+    },
+    "configs": {
+            "width": {"default": 1280, "help": "width of the screenshots"},
+            "height": {"default": 720, "help": "height of the screenshots"},
+            "timeout": {"default": 60, "help": "timeout for taking the screenshot"},
+            "sleep_before_screenshot": {"default": 4, "help": "seconds to wait for the pages to load before taking screenshot"},
+            "http_proxy": {"default": "", "help": "http proxy to use for the webdriver, eg http://proxy-user:password@proxy-ip:port"},
+            "save_to_pdf": {"default": False, "help": "save the page as pdf along with the screenshot. PDF saving options can be adjusted with the 'print_options' parameter"},
+            "print_options": {"default": {}, "help": "options to pass to the pdf printer"}
+        },
+    "description": """
+    Captures screenshots and optionally saves web pages as PDFs using a WebDriver.
+
+    ### Features
+    - Takes screenshots of web pages, with configurable width, height, and timeout settings.
+    - Optionally saves pages as PDFs, with additional configuration for PDF printing options.
+    - Bypasses URLs detected as authentication walls.
+    - Integrates seamlessly with the metadata enrichment pipeline, adding screenshots and PDFs as media.
+
+    ### Notes
+    - Requires a WebDriver (e.g., ChromeDriver) installed and accessible via the system's PATH.
+    """
+}

src/auto_archiver/modules/screenshot_enricher/screenshot_enricher.py

@@ -0,0 +1,42 @@
+from loguru import logger
+import time, os
+import base64
+
+from selenium.common.exceptions import TimeoutException
+
+
+from auto_archiver.base_processors import Enricher
+from auto_archiver.utils import Webdriver, UrlUtil, random_str
+from auto_archiver.core import Media, Metadata, ArchivingContext
+
+class ScreenshotEnricher(Enricher):
+    name = "screenshot_enricher"
+
+    def __init__(self, config: dict) -> None:
+        super().__init__(config)
+
+    def enrich(self, to_enrich: Metadata) -> None:
+        url = to_enrich.get_url()
+
+        if UrlUtil.is_auth_wall(url):
+            logger.debug(f"[SKIP] SCREENSHOT since url is behind AUTH WALL: {url=}")
+            return
+
+        logger.debug(f"Enriching screenshot for {url=}")
+        with Webdriver(self.width, self.height, self.timeout, 'facebook.com' in url, http_proxy=self.http_proxy, print_options=self.print_options) as driver:
+            try:
+                driver.get(url)
+                time.sleep(int(self.sleep_before_screenshot))
+                screenshot_file = os.path.join(ArchivingContext.get_tmp_dir(), f"screenshot_{random_str(8)}.png")
+                driver.save_screenshot(screenshot_file)
+                to_enrich.add_media(Media(filename=screenshot_file), id="screenshot")
+                if self.save_to_pdf:
+                    pdf_file = os.path.join(ArchivingContext.get_tmp_dir(), f"pdf_{random_str(8)}.pdf")
+                    pdf = driver.print_page(driver.print_options)
+                    with open(pdf_file, "wb") as f:
+                        f.write(base64.b64decode(pdf))
+                    to_enrich.add_media(Media(filename=pdf_file), id="pdf")
+            except TimeoutException:
+                logger.info("TimeoutException loading page for screenshot")
+            except Exception as e:
+                logger.error(f"Got error while loading webdriver for screenshot enricher: {e}")

src/auto_archiver/modules/ssl_enricher/__init__.py

@@ -0,0 +1 @@
+from .ssl_enricher import SSLEnricher

src/auto_archiver/modules/ssl_enricher/__manifest__.py

@@ -0,0 +1,22 @@
+{
+    "name": "SSL Certificate Enricher",
+    "type": ["enricher"],
+    "requires_setup": False,
+    "external_dependencies": {
+        "python": ["loguru", "python-slugify"],
+    },
+    "configs": {
+        "skip_when_nothing_archived": {"default": True, "help": "if true, will skip enriching when no media is archived"},
+    },
+    "description": """
+    Retrieves SSL certificate information for a domain and stores it as a file.
+
+    ### Features
+    - Fetches SSL certificates for domains using the HTTPS protocol.
+    - Stores certificates in PEM format and adds them as media to the metadata.
+    - Skips enrichment if no media has been archived, based on the `skip_when_nothing_archived` configuration.
+
+    ### Notes
+    - Requires the target URL to use the HTTPS scheme; other schemes are not supported.
+    """
+}

src/auto_archiver/modules/ssl_enricher/ssl_enricher.py

@@ -0,0 +1,33 @@
+import ssl, os
+from slugify import slugify
+from urllib.parse import urlparse
+from loguru import logger
+
+from auto_archiver.base_processors import Enricher
+from auto_archiver.core import Metadata, ArchivingContext, Media
+
+
+class SSLEnricher(Enricher):
+    """
+    Retrieves SSL certificate information for a domain, as a file
+    """
+    name = "ssl_enricher"
+
+    def __init__(self, config: dict) -> None:
+        super().__init__(config)
+        self.skip_when_nothing_archived = bool(self.skip_when_nothing_archived)
+
+    def enrich(self, to_enrich: Metadata) -> None:
+        if not to_enrich.media and self.skip_when_nothing_archived: return
+        
+        url = to_enrich.get_url()
+        parsed = urlparse(url)
+        assert parsed.scheme in ["https"], f"Invalid URL scheme {url=}"
+        
+        domain = parsed.netloc
+        logger.debug(f"fetching SSL certificate for {domain=} in {url=}")
+
+        cert = ssl.get_server_certificate((domain, 443))
+        cert_fn = os.path.join(ArchivingContext.get_tmp_dir(), f"{slugify(domain)}.pem")
+        with open(cert_fn, "w") as f: f.write(cert)
+        to_enrich.add_media(Media(filename=cert_fn), id="ssl_certificate")

src/auto_archiver/modules/telegram_extractor/__init__.py

@@ -0,0 +1 @@
+from .telegram_extractor import TelegramExtractor

src/auto_archiver/modules/telegram_extractor/__manifest__.py

@@ -1,9 +1,7 @@
 {
-    "name": "Telegram Archiver",
+    "name": "Telegram Extractor",
     "type": ["extractor"],
-    "entry_point": "telegram_archiver:TelegramArchiver",
     "requires_setup": False,
-    "depends": ["core"],
     "external_dependencies": {
         "python": [
             "requests",
@@ -12,7 +10,7 @@
         ],
     },
     "description": """ 
-        The `TelegramArchiver` retrieves publicly available media content from Telegram message links without requiring login credentials. 
+        The `TelegramExtractor` retrieves publicly available media content from Telegram message links without requiring login credentials. 
         It processes URLs to fetch images and videos embedded in Telegram messages, ensuring a structured output using `Metadata` 
         and `Media` objects. Recommended for scenarios where login-based archiving is not viable, although `telethon_archiver` 
         is advised for more comprehensive functionality.

src/auto_archiver/modules/telegram_extractor/telegram_extractor.py

@@ -2,23 +2,20 @@ import requests, re, html
 from bs4 import BeautifulSoup
 from loguru import logger
 
-from auto_archiver.archivers import Archiver
+from auto_archiver.base_processors import Extractor
 from auto_archiver.core import Metadata, Media
 
 
-class TelegramArchiver(Archiver):
+class TelegramExtractor(Extractor):
     """
-    Archiver for telegram that does not require login, but the telethon_archiver is much more advised,
+    Extractor for telegram that does not require login, but the telethon_extractor is much more advised,
     will only return if at least one image or one video is found
     """
-    name = "telegram_archiver"
+    name = "telegram_extractor"
 
     def __init__(self, config: dict) -> None:
         super().__init__(config)
 
-    @staticmethod
-    def configs() -> dict:
-        return {}
 
     def download(self, item: Metadata) -> Metadata:
         url = item.get_url()

src/auto_archiver/modules/telethon_extractor/__init__.py

@@ -0,0 +1 @@
+from .telethon_extractor import TelethonArchiver

src/auto_archiver/modules/telethon_extractor/__manifest__.py

@@ -1,10 +1,8 @@
-# TODO rm dependency on json
+import json
 {
-    "name": "telethon_archiver",
+    "name": "telethon_extractor",
     "type": ["extractor"],
-    "entry_point": "telethon_archiver:TelethonArchiver",
     "requires_setup": True,
-    "depends": [""],
     "external_dependencies": {
         "python": ["telethon",
                    "loguru",
@@ -21,12 +19,11 @@
             "channel_invites": {
                 "default": {},
                 "help": "(JSON string) private channel invite links (format: t.me/joinchat/HASH OR t.me/+HASH) and (optional but important to avoid hanging for minutes on startup) channel id (format: CHANNEL_ID taken from a post url like https://t.me/c/CHANNEL_ID/1), the telegram account will join any new channels on setup",
-                # TODO
-                #"cli_set": lambda cli_val, cur_val: dict(cur_val, **json.loads(cli_val))
+                "type": "auto_archiver.utils.json_loader",
             }
         },
     "description": """
-The `TelethonArchiver` uses the Telethon library to archive posts and media from Telegram channels and groups. 
+The `TelethonExtractor` uses the Telethon library to archive posts and media from Telegram channels and groups. 
 It supports private and public channels, downloading grouped posts with media, and can join channels using invite links 
 if provided in the configuration. 
 
@@ -38,7 +35,7 @@ if provided in the configuration.
 - Outputs structured metadata and media using `Metadata` and `Media` objects.
 
 ### Setup
-To use the `TelethonArchiver`, you must configure the following:
+To use the `TelethonExtractor`, you must configure the following:
 - **API ID and API Hash**: Obtain these from [my.telegram.org](https://my.telegram.org/apps).
 - **Session File**: Optional, but records login sessions for future use (default: `secrets/anon.session`).
 - **Bot Token**: Optional, allows access to additional content (e.g., large videos) but limits private channel archiving.

src/auto_archiver/modules/telethon_extractor/telethon_extractor.py

@@ -8,13 +8,13 @@ from loguru import logger
 from tqdm import tqdm
 import re, time, json, os
 
-from auto_archiver.archivers import Archiver
+from auto_archiver.base_processors import Extractor
 from auto_archiver.core import Metadata, Media, ArchivingContext
 from auto_archiver.utils import random_str
 
 
-class TelethonArchiver(Archiver):
-    name = "telethon_archiver"
+class TelethonArchiver(Extractor):
+    name = "telethon_extractor"
     link_pattern = re.compile(r"https:\/\/t\.me(\/c){0,1}\/(.+)\/(\d+)")
     invite_pattern = re.compile(r"t.me(\/joinchat){0,1}\/\+?(.+)")
 
@@ -23,20 +23,6 @@ class TelethonArchiver(Archiver):
         self.assert_valid_string("api_id")
         self.assert_valid_string("api_hash")
 
-    @staticmethod
-    def configs() -> dict:
-        return {
-            "api_id": {"default": None, "help": "telegram API_ID value, go to https://my.telegram.org/apps"},
-            "api_hash": {"default": None, "help": "telegram API_HASH value, go to https://my.telegram.org/apps"},
-            "bot_token": {"default": None, "help": "optional, but allows access to more content such as large videos, talk to @botfather"},
-            "session_file": {"default": "secrets/anon", "help": "optional, records the telegram login session for future usage, '.session' will be appended to the provided value."},
-            "join_channels": {"default": True, "help": "disables the initial setup with channel_invites config, useful if you have a lot and get stuck"},
-            "channel_invites": {
-                "default": {},
-                "help": "(JSON string) private channel invite links (format: t.me/joinchat/HASH OR t.me/+HASH) and (optional but important to avoid hanging for minutes on startup) channel id (format: CHANNEL_ID taken from a post url like https://t.me/c/CHANNEL_ID/1), the telegram account will join any new channels on setup",
-                "cli_set": lambda cli_val, cur_val: dict(cur_val, **json.loads(cli_val))
-            }
-        }
 
     def setup(self) -> None:
         """

src/auto_archiver/modules/thumbnail_enricher/__init__.py

@@ -0,0 +1 @@
+from .thumbnail_enricher import ThumbnailEnricher

src/auto_archiver/modules/thumbnail_enricher/__manifest__.py

@@ -0,0 +1,27 @@
+{
+    "name": "Thumbnail Enricher",
+    "type": ["enricher"],
+    "requires_setup": False,
+    "external_dependencies": {
+        "python": ["loguru", "ffmpeg-python"],
+        "bin": ["ffmpeg"]
+    },
+    "configs": {
+            "thumbnails_per_minute": {"default": 60, "help": "how many thumbnails to generate per minute of video, can be limited by max_thumbnails"},
+            "max_thumbnails": {"default": 16, "help": "limit the number of thumbnails to generate per video, 0 means no limit"},
+        },
+    "description": """
+    Generates thumbnails for video files to provide visual previews.
+
+    ### Features
+    - Processes video files and generates evenly distributed thumbnails.
+    - Calculates the number of thumbnails based on video duration, `thumbnails_per_minute`, and `max_thumbnails`.
+    - Distributes thumbnails equally across the video's duration and stores them as media objects.
+    - Adds metadata for each thumbnail, including timestamps and IDs.
+
+    ### Notes
+    - Requires `ffmpeg` to be installed and accessible via the system's PATH.
+    - Handles videos without pre-existing duration metadata by probing with `ffmpeg`.
+    - Skips enrichment for non-video media files.
+    """
+}

src/auto_archiver/modules/thumbnail_enricher/thumbnail_enricher.py

@@ -0,0 +1,69 @@
+"""Thumbnail Enricher for generating visual previews of video files.
+
+The `ThumbnailEnricher` processes video files in `Metadata` objects and
+creates evenly distributed thumbnail images. These thumbnails provide
+visual snapshots of the video's keyframes, helping users preview content
+and identify important moments without watching the entire video.
+
+"""
+import ffmpeg, os
+from loguru import logger
+
+from auto_archiver.base_processors import Enricher
+from auto_archiver.core import Media, Metadata, ArchivingContext
+from auto_archiver.utils.misc import random_str
+
+
+class ThumbnailEnricher(Enricher):
+    """
+    Generates thumbnails for all the media
+    """
+    name = "thumbnail_enricher"
+
+    def __init__(self, config: dict) -> None:
+        # without this STEP.__init__ is not called
+        super().__init__(config)
+        self.thumbnails_per_second = int(self.thumbnails_per_minute) / 60
+        self.max_thumbnails = int(self.max_thumbnails)
+    
+    def enrich(self, to_enrich: Metadata) -> None:
+        """
+        Uses or reads the video duration to generate thumbnails
+        Calculates how many thumbnails to generate and at which timestamps based on the video duration, the number of thumbnails per minute and the max number of thumbnails.
+        Thumbnails are equally distributed across the video duration.
+        """
+        logger.debug(f"generating thumbnails for {to_enrich.get_url()}")
+        for m_id, m in enumerate(to_enrich.media[::]):
+            if m.is_video():
+                folder = os.path.join(ArchivingContext.get_tmp_dir(), random_str(24))
+                os.makedirs(folder, exist_ok=True)
+                logger.debug(f"generating thumbnails for {m.filename}")
+                duration = m.get("duration")
+
+                if duration is None:
+                    try:
+                        probe = ffmpeg.probe(m.filename)
+                        duration = float(next(stream for stream in probe['streams'] if stream['codec_type'] == 'video')['duration'])
+                        to_enrich.media[m_id].set("duration", duration)
+                    except Exception as e:
+                        logger.error(f"error getting duration of video {m.filename}: {e}")
+                        return
+
+                num_thumbs = int(min(max(1, duration * self.thumbnails_per_second), self.max_thumbnails))
+                timestamps = [duration / (num_thumbs + 1) * i for i in range(1, num_thumbs + 1)]
+
+                thumbnails_media = []
+                for index, timestamp in enumerate(timestamps):
+                    output_path = os.path.join(folder, f"out{index}.jpg")
+                    ffmpeg.input(m.filename, ss=timestamp).filter('scale', 512, -1).output(output_path, vframes=1, loglevel="quiet").run()
+
+                    try:
+                        thumbnails_media.append(Media(
+                            filename=output_path)
+                            .set("id", f"thumbnail_{index}")
+                            .set("timestamp", "%.3fs" % timestamp)
+                        )
+                    except Exception as e:
+                        logger.error(f"error creating thumbnail {index} for media: {e}")
+
+                to_enrich.media[m_id].set("thumbnails", thumbnails_media)

src/auto_archiver/modules/timestamping_enricher/__init__.py

@@ -0,0 +1 @@
+from .timestamping_enricher import TimestampingEnricher

src/auto_archiver/modules/timestamping_enricher/__manifest__.py

@@ -0,0 +1,54 @@
+{
+    "name": "Timestamping Enricher",
+    "type": ["enricher"],
+    "requires_setup": True,
+    "external_dependencies": {
+        "python": [
+            "loguru",
+            "slugify",
+            "tsp_client",
+            "asn1crypto",
+            "certvalidator",
+            "certifi"
+        ],
+    },
+    "configs": {
+        "tsa_urls": {
+            "default": [
+                    # [Adobe Approved Trust List] and [Windows Cert Store]
+                    "http://timestamp.digicert.com",
+                    "http://timestamp.identrust.com",
+                    # "https://timestamp.entrust.net/TSS/RFC3161sha2TS", # not valid for timestamping
+                    # "https://timestamp.sectigo.com", # wait 15 seconds between each request.
+
+                    # [Adobe: European Union Trusted Lists].
+                    # "https://timestamp.sectigo.com/qualified", # wait 15 seconds between each request.
+
+                    # [Windows Cert Store]
+                    "http://timestamp.globalsign.com/tsa/r6advanced1",
+                    # [Adobe: European Union Trusted Lists] and [Windows Cert Store]
+                    # "http://ts.quovadisglobal.com/eu", # not valid for timestamping
+                    # "http://tsa.belgium.be/connect", # self-signed certificate in certificate chain
+                    # "https://timestamp.aped.gov.gr/qtss", # self-signed certificate in certificate chain
+                    # "http://tsa.sep.bg", # self-signed certificate in certificate chain
+                    # "http://tsa.izenpe.com", #unable to get local issuer certificate
+                    # "http://kstamp.keynectis.com/KSign", # unable to get local issuer certificate
+                    "http://tss.accv.es:8318/tsa",
+                ],
+            "help": "List of RFC3161 Time Stamp Authorities to use, separate with commas if passed via the command line.",
+        }
+    },
+    "description": """
+    Generates RFC3161-compliant timestamp tokens using Time Stamp Authorities (TSA) for archived files.
+
+    ### Features
+    - Creates timestamp tokens to prove the existence of files at a specific time, useful for legal and authenticity purposes.
+    - Aggregates file hashes into a text file and timestamps the concatenated data.
+    - Uses multiple Time Stamp Authorities (TSAs) to ensure reliability and redundancy.
+    - Validates timestamping certificates against trusted Certificate Authorities (CAs) using the `certifi` trust store.
+
+    ### Notes
+    - Should be run after the `hash_enricher` to ensure file hashes are available.
+    - Requires internet access to interact with the configured TSAs.
+    """
+}

src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py

@@ -0,0 +1,105 @@
+import os
+from loguru import logger
+from tsp_client import TSPSigner, SigningSettings, TSPVerifier
+from tsp_client.algorithms import DigestAlgorithm
+from importlib.metadata import version
+from asn1crypto.cms import ContentInfo
+from certvalidator import CertificateValidator, ValidationContext
+from asn1crypto import pem
+import certifi
+
+from auto_archiver.base_processors import Enricher
+from auto_archiver.core import Metadata, ArchivingContext, Media
+from auto_archiver.base_processors import Extractor
+
+
+class TimestampingEnricher(Enricher):
+    """
+    Uses several RFC3161 Time Stamp Authorities to generate a timestamp token that will be preserved. This can be used to prove that a certain file existed at a certain time, useful for legal purposes, for example, to prove that a certain file was not tampered with after a certain date.
+
+    The information that gets timestamped is concatenation (via paragraphs) of the file hashes existing in the current archive. It will depend on which archivers and enrichers ran before this one. Inner media files (like thumbnails) are not included in the .txt file. It should run AFTER the hash_enricher.
+
+    See https://gist.github.com/Manouchehri/fd754e402d98430243455713efada710 for list of timestamp authorities.
+    """
+    name = "timestamping_enricher"
+
+    def __init__(self, config: dict) -> None:
+        super().__init__(config)
+
+    def enrich(self, to_enrich: Metadata) -> None:
+        url = to_enrich.get_url()
+        logger.debug(f"RFC3161 timestamping existing files for {url=}")
+
+        # create a new text file with the existing media hashes
+        hashes = [m.get("hash").replace("SHA-256:", "").replace("SHA3-512:", "") for m in to_enrich.media if m.get("hash")]
+
+        if not len(hashes):
+            logger.warning(f"No hashes found in {url=}")
+            return
+        
+        tmp_dir = ArchivingContext.get_tmp_dir()
+        hashes_fn = os.path.join(tmp_dir, "hashes.txt")
+
+        data_to_sign = "\n".join(hashes)
+        with open(hashes_fn, "w") as f: 
+            f.write(data_to_sign)
+        hashes_media = Media(filename=hashes_fn)
+
+        timestamp_tokens = []
+        from slugify import slugify
+        for tsa_url in self.tsa_urls:
+            try:
+                signing_settings = SigningSettings(tsp_server=tsa_url, digest_algorithm=DigestAlgorithm.SHA256)
+                signer = TSPSigner()
+                message = bytes(data_to_sign, encoding='utf8')
+                # send TSQ and get TSR from the TSA server
+                signed = signer.sign(message=message, signing_settings=signing_settings)
+                # fail if there's any issue with the certificates, uses certifi list of trusted CAs
+                TSPVerifier(certifi.where()).verify(signed, message=message)
+                # download and verify timestamping certificate
+                cert_chain = self.download_and_verify_certificate(signed)
+                # continue with saving the timestamp token
+                tst_fn = os.path.join(tmp_dir, f"timestamp_token_{slugify(tsa_url)}")
+                with open(tst_fn, "wb") as f: f.write(signed)
+                timestamp_tokens.append(Media(filename=tst_fn).set("tsa", tsa_url).set("cert_chain", cert_chain))
+            except Exception as e:
+                logger.warning(f"Error while timestamping {url=} with {tsa_url=}: {e}")
+
+        if len(timestamp_tokens):
+            hashes_media.set("timestamp_authority_files", timestamp_tokens)
+            hashes_media.set("certifi v", version("certifi"))
+            hashes_media.set("tsp_client v", version("tsp_client"))
+            hashes_media.set("certvalidator v", version("certvalidator"))
+            to_enrich.add_media(hashes_media, id="timestamped_hashes")
+            to_enrich.set("timestamped", True)
+            logger.success(f"{len(timestamp_tokens)} timestamp tokens created for {url=}")
+        else:
+            logger.warning(f"No successful timestamps for {url=}")
+
+    def download_and_verify_certificate(self, signed: bytes) -> list[Media]:
+        # returns the leaf certificate URL, fails if not set
+        tst = ContentInfo.load(signed)
+
+        trust_roots = []
+        with open(certifi.where(), 'rb') as f:
+            for _, _, der_bytes in pem.unarmor(f.read(), multiple=True):
+                trust_roots.append(der_bytes)
+        context = ValidationContext(trust_roots=trust_roots)
+
+        certificates = tst["content"]["certificates"]
+        first_cert = certificates[0].dump()
+        intermediate_certs = []
+        for i in range(1, len(certificates)): # cannot use list comprehension [1:]
+            intermediate_certs.append(certificates[i].dump())
+
+        validator = CertificateValidator(first_cert, intermediate_certs=intermediate_certs, validation_context=context)
+        path = validator.validate_usage({'digital_signature'}, extended_key_usage={'time_stamping'})
+
+        cert_chain = []
+        for cert in path:
+            cert_fn = os.path.join(ArchivingContext.get_tmp_dir(), f"{str(cert.serial_number)[:20]}.crt")
+            with open(cert_fn, "wb") as f:
+                f.write(cert.dump())
+            cert_chain.append(Media(filename=cert_fn).set("subject", cert.subject.native["common_name"]))
+
+        return cert_chain

src/auto_archiver/modules/twitter_api_extractor/__init__.py

@@ -0,0 +1 @@
+from .twitter_api_extractor import TwitterApiExtractor