diff --git a/README.md b/README.md index 1bd6ddd..c52c464 100644 --- a/README.md +++ b/README.md @@ -218,7 +218,7 @@ configurations: ## Running on Google Sheets Feeder (gsheet_feeder) The `--gsheet_feeder.sheet` property is the name of the Google Sheet to check for URLs. This sheet must have been shared with the Google Service account used by `gspread`. -This sheet must also have specific columns (case-insensitive) in the `header` as specified in [Gsheet.configs](src/auto_archiver/utils/gsheet.py). The default names of these columns and their purpose is: +This sheet must also have specific columns (case-insensitive) in the `header` as specified in [gsheet_feeder.__manifest__.py](src/auto_archiver/modules/gsheet_feeder/__manifest__.py). The default names of these columns and their purpose is: Inputs: diff --git a/src/auto_archiver/archivers/__init__.py b/src/auto_archiver/archivers/__init__.py deleted file mode 100644 index 54515ec..0000000 --- a/src/auto_archiver/archivers/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -""" -Archivers are responsible for retrieving the content from various external platforms. -They act as specialized modules, each tailored to interact with a specific platform, -service, or data source. The archivers collectively enable the tool to comprehensively -collect and preserve a variety of content types, such as posts, images, videos and metadata. - -""" -from .archiver import Archiver diff --git a/src/auto_archiver/base_processors/__init__.py b/src/auto_archiver/base_processors/__init__.py new file mode 100644 index 0000000..4995457 --- /dev/null +++ b/src/auto_archiver/base_processors/__init__.py @@ -0,0 +1,6 @@ +from .database import Database +from .enricher import Enricher +from .feeder import Feeder +from .storage import Storage +from .extractor import Extractor +from .formatter import Formatter \ No newline at end of file diff --git a/src/auto_archiver/databases/database.py b/src/auto_archiver/base_processors/database.py similarity index 96% rename from src/auto_archiver/databases/database.py rename to src/auto_archiver/base_processors/database.py index 30cba7e..28f0061 100644 --- a/src/auto_archiver/databases/database.py +++ b/src/auto_archiver/base_processors/database.py @@ -3,13 +3,13 @@ from dataclasses import dataclass from abc import abstractmethod, ABC from typing import Union -from ..core import Metadata, Step +from auto_archiver.core import Metadata, Step @dataclass class Database(Step, ABC): - name = "database" + name = "database" def __init__(self, config: dict) -> None: # without this STEP.__init__ is not called super().__init__(config) diff --git a/src/auto_archiver/base_processors/enricher.py b/src/auto_archiver/base_processors/enricher.py new file mode 100644 index 0000000..d26eedf --- /dev/null +++ b/src/auto_archiver/base_processors/enricher.py @@ -0,0 +1,31 @@ +""" +Enrichers are modular components that enhance archived content by adding +context, metadata, or additional processing. + +These add additional information to the context, such as screenshots, hashes, and metadata. +They are designed to work within the archiving pipeline, operating on `Metadata` objects after +the archiving step and before storage or formatting. + +Enrichers are optional but highly useful for making the archived data more powerful. +""" +from __future__ import annotations +from dataclasses import dataclass +from abc import abstractmethod, ABC +from auto_archiver.core import Metadata, Step + +@dataclass +class Enricher(Step, ABC): + """Base classes and utilities for enrichers in the Auto-Archiver system.""" + name = "enricher" + + def __init__(self, config: dict) -> None: + # without this STEP.__init__ is not called + super().__init__(config) + + + # only for typing... + def init(name: str, config: dict) -> Enricher: + return Step.init(name, config, Enricher) + + @abstractmethod + def enrich(self, to_enrich: Metadata) -> None: pass diff --git a/src/auto_archiver/archivers/archiver.py b/src/auto_archiver/base_processors/extractor.py similarity index 81% rename from src/auto_archiver/archivers/archiver.py rename to src/auto_archiver/base_processors/extractor.py index b5f3f40..c772325 100644 --- a/src/auto_archiver/archivers/archiver.py +++ b/src/auto_archiver/base_processors/extractor.py @@ -1,7 +1,7 @@ -""" The `archiver` module defines the base functionality for implementing archivers in the media archiving framework. - This class provides common utility methods and a standard interface for archivers. +""" The `extractor` module defines the base functionality for implementing extractors in the media archiving framework. + This class provides common utility methods and a standard interface for extractors. - Factory method to initialize an archiver instance based on its name. + Factory method to initialize an extractor instance based on its name. """ @@ -15,32 +15,32 @@ import mimetypes, requests from loguru import logger from retrying import retry -from ..core import Metadata, Step, ArchivingContext +from ..core import Metadata, ArchivingContext @dataclass -class Archiver: +class Extractor: """ - Base class for implementing archivers in the media archiving framework. + Base class for implementing extractors in the media archiving framework. Subclasses must implement the `download` method to define platform-specific behavior. """ def setup(self) -> None: - # used when archivers need to login or do other one-time setup + # used when extractors need to login or do other one-time setup pass def cleanup(self) -> None: - # called when archivers are done, or upon errors, cleanup any resources + # called when extractors are done, or upon errors, cleanup any resources pass def sanitize_url(self, url: str) -> str: # used to clean unnecessary URL parameters OR unfurl redirect links return url - + def suitable(self, url: str) -> bool: """ - Returns True if this archiver can handle the given URL - + Returns True if this extractor can handle the given URL + Should be overridden by subclasses """ return True @@ -84,10 +84,10 @@ class Archiver: for chunk in d.iter_content(chunk_size=8192): f.write(chunk) return to_filename - + except requests.RequestException as e: logger.warning(f"Failed to fetch the Media URL: {e}") @abstractmethod def download(self, item: Metadata) -> Metadata: - pass + pass \ No newline at end of file diff --git a/src/auto_archiver/feeders/feeder.py b/src/auto_archiver/base_processors/feeder.py similarity index 86% rename from src/auto_archiver/feeders/feeder.py rename to src/auto_archiver/base_processors/feeder.py index 4aa263f..7fbd6b1 100644 --- a/src/auto_archiver/feeders/feeder.py +++ b/src/auto_archiver/base_processors/feeder.py @@ -1,8 +1,8 @@ from __future__ import annotations from dataclasses import dataclass from abc import abstractmethod -from ..core import Metadata -from ..core import Step +from auto_archiver.core import Metadata +from auto_archiver.core import Step @dataclass diff --git a/src/auto_archiver/formatters/formatter.py b/src/auto_archiver/base_processors/formatter.py similarity index 90% rename from src/auto_archiver/formatters/formatter.py rename to src/auto_archiver/base_processors/formatter.py index b10477e..4c59af8 100644 --- a/src/auto_archiver/formatters/formatter.py +++ b/src/auto_archiver/base_processors/formatter.py @@ -1,7 +1,7 @@ from __future__ import annotations from dataclasses import dataclass from abc import abstractmethod -from ..core import Metadata, Media, Step +from auto_archiver.core import Metadata, Media, Step @dataclass diff --git a/src/auto_archiver/storages/storage.py b/src/auto_archiver/base_processors/storage.py similarity index 50% rename from src/auto_archiver/storages/storage.py rename to src/auto_archiver/base_processors/storage.py index c9b55e0..da6b2ef 100644 --- a/src/auto_archiver/storages/storage.py +++ b/src/auto_archiver/base_processors/storage.py @@ -4,10 +4,10 @@ from dataclasses import dataclass from typing import IO, Optional import os -from ..utils.misc import random_str +from auto_archiver.utils.misc import random_str -from ..core import Media, Step, ArchivingContext, Metadata -from ..enrichers import HashEnricher +from auto_archiver.core import Media, Step, ArchivingContext, Metadata +from auto_archiver.modules.hash_enricher.hash_enricher import HashEnricher from loguru import logger from slugify import slugify @@ -15,29 +15,6 @@ from slugify import slugify @dataclass class Storage(Step): name = "storage" - PATH_GENERATOR_OPTIONS = ["flat", "url", "random"] - FILENAME_GENERATOR_CHOICES = ["random", "static"] - - def __init__(self, config: dict) -> None: - # without this STEP.__init__ is not called - super().__init__(config) - assert self.path_generator in Storage.PATH_GENERATOR_OPTIONS, f"path_generator must be one of {Storage.PATH_GENERATOR_OPTIONS}" - assert self.filename_generator in Storage.FILENAME_GENERATOR_CHOICES, f"filename_generator must be one of {Storage.FILENAME_GENERATOR_CHOICES}" - - @staticmethod - def configs() -> dict: - return { - "path_generator": { - "default": "url", - "help": "how to store the file in terms of directory structure: 'flat' sets to root; 'url' creates a directory based on the provided URL; 'random' creates a random directory.", - "choices": Storage.PATH_GENERATOR_OPTIONS - }, - "filename_generator": { - "default": "random", - "help": "how to name stored files: 'random' creates a random string; 'static' uses a replicable strategy such as a hash.", - "choices": Storage.FILENAME_GENERATOR_CHOICES - } - } def init(name: str, config: dict) -> Storage: # only for typing... @@ -68,19 +45,27 @@ class Storage(Step): folder = ArchivingContext.get("folder", "") filename, ext = os.path.splitext(media.filename) - # path_generator logic - if self.path_generator == "flat": + # Handle path_generator logic + path_generator = ArchivingContext.get("path_generator", "url") + if path_generator == "flat": path = "" - filename = slugify(filename) # in case it comes with os.sep - elif self.path_generator == "url": path = slugify(url) - elif self.path_generator == "random": + filename = slugify(filename) # Ensure filename is slugified + elif path_generator == "url": + path = slugify(url) + elif path_generator == "random": path = ArchivingContext.get("random_path", random_str(24), True) + else: + raise ValueError(f"Invalid path_generator: {path_generator}") - # filename_generator logic - if self.filename_generator == "random": filename = random_str(24) - elif self.filename_generator == "static": + # Handle filename_generator logic + filename_generator = ArchivingContext.get("filename_generator", "random") + if filename_generator == "random": + filename = random_str(24) + elif filename_generator == "static": he = HashEnricher({"hash_enricher": {"algorithm": ArchivingContext.get("hash_enricher.algorithm"), "chunksize": 1.6e7}}) hd = he.calculate_hash(media.filename) filename = hd[:24] + else: + raise ValueError(f"Invalid filename_generator: {filename_generator}") media.key = os.path.join(folder, path, f"{filename}{ext}") diff --git a/src/auto_archiver/core/config.py b/src/auto_archiver/core/config.py index 1c19ae2..24f6a61 100644 --- a/src/auto_archiver/core/config.py +++ b/src/auto_archiver/core/config.py @@ -20,7 +20,7 @@ from typing import Any, List # configurable_parents = [ # Feeder, # Enricher, -# Archiver, +# Extractor, # Database, # Storage, # Formatter @@ -28,7 +28,7 @@ from typing import Any, List # ] # feeder: Feeder # formatter: Formatter -# archivers: List[Archiver] = field(default_factory=[]) +# extractors: List[Extractor] = field(default_factory=[]) # enrichers: List[Enricher] = field(default_factory=[]) # storages: List[Storage] = field(default_factory=[]) # databases: List[Database] = field(default_factory=[]) diff --git a/src/auto_archiver/databases/__init__.py b/src/auto_archiver/databases/__init__.py deleted file mode 100644 index 5aaa679..0000000 --- a/src/auto_archiver/databases/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -""" Databases are used to store the outputs from running the Autp Archiver. - - -""" \ No newline at end of file diff --git a/src/auto_archiver/enrichers/__init__.py b/src/auto_archiver/enrichers/__init__.py deleted file mode 100644 index 67cb0e5..0000000 --- a/src/auto_archiver/enrichers/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -""" -Enrichers are modular components that enhance archived content by adding -context, metadata, or additional processing. - -These add additional information to the context, such as screenshots, hashes, and metadata. -They are designed to work within the archiving pipeline, operating on `Metadata` objects after -the archiving step and before storage or formatting. - -Enrichers are optional but highly useful for making the archived data more powerful. - - -""" diff --git a/src/auto_archiver/enrichers/enricher.py b/src/auto_archiver/enrichers/enricher.py deleted file mode 100644 index f195f23..0000000 --- a/src/auto_archiver/enrichers/enricher.py +++ /dev/null @@ -1,22 +0,0 @@ -""" Base classes and utilities for enrichers in the Auto-Archiver system. -""" -from __future__ import annotations -from dataclasses import dataclass -from abc import abstractmethod, ABC -from ..core import Metadata, Step - -@dataclass -class Enricher(Step, ABC): - name = "enricher" - - def __init__(self, config: dict) -> None: - # without this STEP.__init__ is not called - super().__init__(config) - - - # only for typing... - def init(name: str, config: dict) -> Enricher: - return Step.init(name, config, Enricher) - - @abstractmethod - def enrich(self, to_enrich: Metadata) -> None: pass diff --git a/src/auto_archiver/feeders/__init__.py b/src/auto_archiver/feeders/__init__.py deleted file mode 100644 index 3eb33d7..0000000 --- a/src/auto_archiver/feeders/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -""" Feeders handle the input of media into the Auto Archiver. - -""" diff --git a/src/auto_archiver/formatters/__init__.py b/src/auto_archiver/formatters/__init__.py deleted file mode 100644 index 1a9dcd0..0000000 --- a/src/auto_archiver/formatters/__init__.py +++ /dev/null @@ -1 +0,0 @@ -""" Formatters for the output of the content. """ diff --git a/src/auto_archiver/modules/api_db/__init__.py b/src/auto_archiver/modules/api_db/__init__.py new file mode 100644 index 0000000..2070b06 --- /dev/null +++ b/src/auto_archiver/modules/api_db/__init__.py @@ -0,0 +1 @@ +from api_db import AAApiDb \ No newline at end of file diff --git a/src/auto_archiver/modules/api_db/__manifest__.py b/src/auto_archiver/modules/api_db/__manifest__.py new file mode 100644 index 0000000..c89165f --- /dev/null +++ b/src/auto_archiver/modules/api_db/__manifest__.py @@ -0,0 +1,33 @@ +{ + "name": "Auto-Archiver API Database", + "type": ["database"], + "entry_point": "api_db:AAApiDb", + "requires_setup": True, + "external_dependencies": { + "python": ["requests", + "loguru"], + }, + "configs": { + "api_endpoint": {"default": None, "help": "API endpoint where calls are made to"}, + "api_token": {"default": None, "help": "API Bearer token."}, + "public": {"default": False, "help": "whether the URL should be publicly available via the API"}, + "author_id": {"default": None, "help": "which email to assign as author"}, + "group_id": {"default": None, "help": "which group of users have access to the archive in case public=false as author"}, + "allow_rearchive": {"default": True, "help": "if False then the API database will be queried prior to any archiving operations and stop if the link has already been archived", "type": "bool",}, + "store_results": {"default": True, "help": "when set, will send the results to the API database.", "type": "bool",}, + "tags": {"default": [], "help": "what tags to add to the archived URL",} + }, + "description": """ + Provides integration with the Auto-Archiver API for querying and storing archival data. + +### Features +- **API Integration**: Supports querying for existing archives and submitting results. +- **Duplicate Prevention**: Avoids redundant archiving when `allow_rearchive` is disabled. +- **Configurable**: Supports settings like API endpoint, authentication token, tags, and permissions. +- **Tagging and Metadata**: Adds tags and manages metadata for archives. +- **Optional Storage**: Archives results conditionally based on configuration. + +### Setup +Requires access to an Auto-Archiver API instance and a valid API token. + """, +} diff --git a/src/auto_archiver/databases/api_db.py b/src/auto_archiver/modules/api_db/api_db.py similarity index 69% rename from src/auto_archiver/databases/api_db.py rename to src/auto_archiver/modules/api_db/api_db.py index 4304855..d2b43b7 100644 --- a/src/auto_archiver/databases/api_db.py +++ b/src/auto_archiver/modules/api_db/api_db.py @@ -2,8 +2,8 @@ from typing import Union import requests, os from loguru import logger -from . import Database -from ..core import Metadata +from auto_archiver.base_processors import Database +from auto_archiver.core import Metadata class AAApiDb(Database): @@ -19,18 +19,7 @@ class AAApiDb(Database): self.store_results = bool(self.store_results) self.assert_valid_string("api_endpoint") - @staticmethod - def configs() -> dict: - return { - "api_endpoint": {"default": None, "help": "API endpoint where calls are made to"}, - "api_token": {"default": None, "help": "API Bearer token."}, - "public": {"default": False, "help": "whether the URL should be publicly available via the API"}, - "author_id": {"default": None, "help": "which email to assign as author"}, - "group_id": {"default": None, "help": "which group of users have access to the archive in case public=false as author"}, - "allow_rearchive": {"default": True, "help": "if False then the API database will be queried prior to any archiving operations and stop if the link has already been archived"}, - "store_results": {"default": True, "help": "when set, will send the results to the API database."}, - "tags": {"default": [], "help": "what tags to add to the archived URL", "cli_set": lambda cli_val, cur_val: set(cli_val.split(","))}, - } + def fetch(self, item: Metadata) -> Union[Metadata, bool]: """ query the database for the existence of this item. Helps avoid re-archiving the same URL multiple times. diff --git a/src/auto_archiver/modules/atlos/__init__.py b/src/auto_archiver/modules/atlos/__init__.py new file mode 100644 index 0000000..de7fead --- /dev/null +++ b/src/auto_archiver/modules/atlos/__init__.py @@ -0,0 +1 @@ +from .atlos import AtlosStorage \ No newline at end of file diff --git a/src/auto_archiver/modules/atlos/__manifest__.py b/src/auto_archiver/modules/atlos/__manifest__.py new file mode 100644 index 0000000..459fefe --- /dev/null +++ b/src/auto_archiver/modules/atlos/__manifest__.py @@ -0,0 +1,40 @@ +{ + "name": "atlos_storage", + "type": ["storage"], + "requires_setup": True, + "external_dependencies": {"python": ["loguru", "requests"], "bin": [""]}, + "configs": { + "path_generator": { + "default": "url", + "help": "how to store the file in terms of directory structure: 'flat' sets to root; 'url' creates a directory based on the provided URL; 'random' creates a random directory.", + }, + "filename_generator": { + "default": "random", + "help": "how to name stored files: 'random' creates a random string; 'static' uses a replicable strategy such as a hash.", + }, + "api_token": { + "default": None, + "help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/", + "type": "str", + }, + "atlos_url": { + "default": "https://platform.atlos.org", + "help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.", + "type": "str", + }, + }, + "description": """ + AtlosStorage: A storage module for saving media files to the Atlos platform. + + ### Features + - Uploads media files to Atlos using Atlos-specific APIs. + - Automatically calculates SHA-256 hashes of media files for integrity verification. + - Skips uploads for files that already exist on Atlos with the same hash. + - Supports attaching metadata, such as `atlos_id`, to the uploaded files. + - Provides CDN-like URLs for accessing uploaded media. + + ### Notes + - Requires Atlos API configuration, including `atlos_url` and `api_token`. + - Files are linked to an `atlos_id` in the metadata, ensuring proper association with Atlos source materials. + """, +} diff --git a/src/auto_archiver/storages/atlos.py b/src/auto_archiver/modules/atlos/atlos.py similarity index 91% rename from src/auto_archiver/storages/atlos.py rename to src/auto_archiver/modules/atlos/atlos.py index 3b13aa0..6a175d3 100644 --- a/src/auto_archiver/storages/atlos.py +++ b/src/auto_archiver/modules/atlos/atlos.py @@ -4,9 +4,9 @@ from loguru import logger import requests import hashlib -from ..core import Media, Metadata -from ..storages import Storage -from ..utils import get_atlos_config_options +from auto_archiver.core import Media, Metadata +from auto_archiver.base_processors import Storage +from auto_archiver.utils import get_atlos_config_options class AtlosStorage(Storage): @@ -15,10 +15,6 @@ class AtlosStorage(Storage): def __init__(self, config: dict) -> None: super().__init__(config) - @staticmethod - def configs() -> dict: - return dict(Storage.configs(), **get_atlos_config_options()) - def get_cdn_url(self, _media: Media) -> str: # It's not always possible to provide an exact URL, because it's # possible that the media once uploaded could have been copied to diff --git a/src/auto_archiver/modules/atlos_db/__init__.py b/src/auto_archiver/modules/atlos_db/__init__.py new file mode 100644 index 0000000..1552e39 --- /dev/null +++ b/src/auto_archiver/modules/atlos_db/__init__.py @@ -0,0 +1 @@ +from atlos_db import AtlosDb \ No newline at end of file diff --git a/src/auto_archiver/modules/atlos_db/__manifest__.py b/src/auto_archiver/modules/atlos_db/__manifest__.py new file mode 100644 index 0000000..42ce560 --- /dev/null +++ b/src/auto_archiver/modules/atlos_db/__manifest__.py @@ -0,0 +1,36 @@ +{ + "name": "Atlos Database", + "type": ["database"], + "entry_point": "atlos_db:AtlosDb", + "requires_setup": True, + "external_dependencies": + {"python": ["loguru", + ""], + "bin": [""]}, + "configs": { + "api_token": { + "default": None, + "help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/", + }, + "atlos_url": { + "default": "https://platform.atlos.org", + "help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.", + "type": "str" + }, + }, + "description": """ +Handles integration with the Atlos platform for managing archival results. + +### Features +- Outputs archival results to the Atlos API for storage and tracking. +- Updates failure status with error details when archiving fails. +- Processes and formats metadata, including ISO formatting for datetime fields. +- Skips processing for items without an Atlos ID. + +### Setup +Required configs: +- atlos_url: Base URL for the Atlos API. +- api_token: Authentication token for API access. +""" +, +} diff --git a/src/auto_archiver/databases/atlos_db.py b/src/auto_archiver/modules/atlos_db/atlos_db.py similarity index 93% rename from src/auto_archiver/databases/atlos_db.py rename to src/auto_archiver/modules/atlos_db/atlos_db.py index 16c4910..2e24491 100644 --- a/src/auto_archiver/databases/atlos_db.py +++ b/src/auto_archiver/modules/atlos_db/atlos_db.py @@ -1,13 +1,14 @@ import os + from typing import Union from loguru import logger from csv import DictWriter from dataclasses import asdict import requests -from . import Database -from ..core import Metadata -from ..utils import get_atlos_config_options +from auto_archiver.base_processors import Database +from auto_archiver.core import Metadata +from auto_archiver.utils import get_atlos_config_options class AtlosDb(Database): @@ -21,10 +22,6 @@ class AtlosDb(Database): # without this STEP.__init__ is not called super().__init__(config) - @staticmethod - def configs() -> dict: - return get_atlos_config_options() - def failed(self, item: Metadata, reason: str) -> None: """Update DB accordingly for failure""" # If the item has no Atlos ID, there's nothing for us to do diff --git a/src/auto_archiver/modules/atlos_db/base_configs.py b/src/auto_archiver/modules/atlos_db/base_configs.py new file mode 100644 index 0000000..f672f82 --- /dev/null +++ b/src/auto_archiver/modules/atlos_db/base_configs.py @@ -0,0 +1,13 @@ +def get_atlos_config_options(): + return { + "api_token": { + "default": None, + "help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/", + "type": str + }, + "atlos_url": { + "default": "https://platform.atlos.org", + "help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.", + "type": str + }, + } \ No newline at end of file diff --git a/src/auto_archiver/modules/atlos_feeder/__init__.py b/src/auto_archiver/modules/atlos_feeder/__init__.py new file mode 100644 index 0000000..67b243a --- /dev/null +++ b/src/auto_archiver/modules/atlos_feeder/__init__.py @@ -0,0 +1 @@ +from .atlos_feeder import AtlosFeeder \ No newline at end of file diff --git a/src/auto_archiver/modules/atlos_feeder/__manifest__.py b/src/auto_archiver/modules/atlos_feeder/__manifest__.py new file mode 100644 index 0000000..0d90c8b --- /dev/null +++ b/src/auto_archiver/modules/atlos_feeder/__manifest__.py @@ -0,0 +1,34 @@ +{ + "name": "Atlos Feeder", + "type": ["feeder"], + "requires_setup": True, + "external_dependencies": { + "python": ["loguru", "requests"], + }, + "configs": { + "api_token": { + "default": None, + "help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/", + "type": "str" + }, + "atlos_url": { + "default": "https://platform.atlos.org", + "help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.", + "type": "str" + }, + }, + "description": """ + AtlosFeeder: A feeder module that integrates with the Atlos API to fetch source material URLs for archival. + + ### Features + - Connects to the Atlos API to retrieve a list of source material URLs. + - Filters source materials based on visibility, processing status, and metadata. + - Converts filtered source materials into `Metadata` objects with the relevant `atlos_id` and URL. + - Iterates through paginated results using a cursor for efficient API interaction. + + ### Notes + - Requires an Atlos API endpoint and a valid API token for authentication. + - Ensures only unprocessed, visible, and ready-to-archive URLs are returned. + - Handles pagination transparently when retrieving data from the Atlos API. + """ +} diff --git a/src/auto_archiver/feeders/atlos_feeder.py b/src/auto_archiver/modules/atlos_feeder/atlos_feeder.py similarity index 88% rename from src/auto_archiver/feeders/atlos_feeder.py rename to src/auto_archiver/modules/atlos_feeder/atlos_feeder.py index d3acc00..262f21b 100644 --- a/src/auto_archiver/feeders/atlos_feeder.py +++ b/src/auto_archiver/modules/atlos_feeder/atlos_feeder.py @@ -1,9 +1,9 @@ from loguru import logger import requests -from . import Feeder -from ..core import Metadata, ArchivingContext -from ..utils import get_atlos_config_options +from auto_archiver.base_processors import Feeder +from auto_archiver.core import Metadata, ArchivingContext +from auto_archiver.utils import get_atlos_config_options class AtlosFeeder(Feeder): @@ -15,10 +15,6 @@ class AtlosFeeder(Feeder): if type(self.api_token) != str: raise Exception("Atlos Feeder did not receive an Atlos API token") - @staticmethod - def configs() -> dict: - return get_atlos_config_options() - def __iter__(self) -> Metadata: # Get all the urls from the Atlos API count = 0 diff --git a/src/auto_archiver/modules/cli_feeder/__init__.py b/src/auto_archiver/modules/cli_feeder/__init__.py new file mode 100644 index 0000000..9c85787 --- /dev/null +++ b/src/auto_archiver/modules/cli_feeder/__init__.py @@ -0,0 +1 @@ +from .cli_feeder import CLIFeeder \ No newline at end of file diff --git a/src/auto_archiver/modules/cli_feeder/__manifest__.py b/src/auto_archiver/modules/cli_feeder/__manifest__.py new file mode 100644 index 0000000..febebd0 --- /dev/null +++ b/src/auto_archiver/modules/cli_feeder/__manifest__.py @@ -0,0 +1,23 @@ +{ + "name": "CLI Feeder", + "type": ["feeder"], + "requires_setup": False, + "external_dependencies": { + "python": ["loguru"], + }, + "configs": { + "urls": { + "default": None, + "help": "URL(s) to archive, either a single URL or a list of urls, should not come from config.yaml", + }, + }, + "description": """ + Processes URLs to archive passed via the command line and feeds them into the archiving pipeline. + + ### Features + - Takes a single URL or a list of URLs provided via the command line. + - Converts each URL into a `Metadata` object and yields it for processing. + - Ensures URLs are processed only if they are explicitly provided. + + """ +} diff --git a/src/auto_archiver/feeders/cli_feeder.py b/src/auto_archiver/modules/cli_feeder/cli_feeder.py similarity index 59% rename from src/auto_archiver/feeders/cli_feeder.py rename to src/auto_archiver/modules/cli_feeder/cli_feeder.py index b2f0add..7d0d01f 100644 --- a/src/auto_archiver/feeders/cli_feeder.py +++ b/src/auto_archiver/modules/cli_feeder/cli_feeder.py @@ -1,7 +1,7 @@ from loguru import logger -from . import Feeder -from ..core import Metadata, ArchivingContext +from auto_archiver.base_processors import Feeder +from auto_archiver.core import Metadata, ArchivingContext class CLIFeeder(Feeder): @@ -13,16 +13,6 @@ class CLIFeeder(Feeder): if type(self.urls) != list or len(self.urls) == 0: raise Exception("CLI Feeder did not receive any URL to process") - @staticmethod - def configs() -> dict: - return { - "urls": { - "default": None, - "help": "URL(s) to archive, either a single URL or a list of urls, should not come from config.yaml", - "cli_set": lambda cli_val, cur_val: list(set(cli_val.split(","))) - }, - } - def __iter__(self) -> Metadata: for url in self.urls: logger.debug(f"Processing {url}") diff --git a/src/auto_archiver/modules/console_db/__init__.py b/src/auto_archiver/modules/console_db/__init__.py new file mode 100644 index 0000000..343f09c --- /dev/null +++ b/src/auto_archiver/modules/console_db/__init__.py @@ -0,0 +1 @@ +from .console_db import ConsoleDb \ No newline at end of file diff --git a/src/auto_archiver/modules/console_db/__manifest__.py b/src/auto_archiver/modules/console_db/__manifest__.py new file mode 100644 index 0000000..cd40496 --- /dev/null +++ b/src/auto_archiver/modules/console_db/__manifest__.py @@ -0,0 +1,22 @@ +{ + "name": "Console Database", + "type": ["database"], + "requires_setup": False, + "external_dependencies": { + "python": ["loguru"], + }, + "description": """ +Provides a simple database implementation that outputs archival results and status updates to the console. + +### Features +- Logs the status of archival tasks directly to the console, including: + - started + - failed (with error details) + - aborted + - done (with optional caching status) +- Useful for debugging or lightweight setups where no external database is required. + +### Setup +No additional configuration is required. +""", +} diff --git a/src/auto_archiver/databases/console_db.py b/src/auto_archiver/modules/console_db/console_db.py similarity index 86% rename from src/auto_archiver/databases/console_db.py rename to src/auto_archiver/modules/console_db/console_db.py index bd45f95..9dfeb2c 100644 --- a/src/auto_archiver/databases/console_db.py +++ b/src/auto_archiver/modules/console_db/console_db.py @@ -1,7 +1,7 @@ from loguru import logger -from . import Database -from ..core import Metadata +from auto_archiver.base_processors import Database +from auto_archiver.core import Metadata class ConsoleDb(Database): @@ -14,10 +14,6 @@ class ConsoleDb(Database): # without this STEP.__init__ is not called super().__init__(config) - @staticmethod - def configs() -> dict: - return {} - def started(self, item: Metadata) -> None: logger.warning(f"STARTED {item}") diff --git a/src/auto_archiver/modules/csv_db/__init__.py b/src/auto_archiver/modules/csv_db/__init__.py new file mode 100644 index 0000000..1092cb2 --- /dev/null +++ b/src/auto_archiver/modules/csv_db/__init__.py @@ -0,0 +1 @@ +from .csv_db import CSVDb \ No newline at end of file diff --git a/src/auto_archiver/modules/csv_db/__manifest__.py b/src/auto_archiver/modules/csv_db/__manifest__.py new file mode 100644 index 0000000..1fe2d7d --- /dev/null +++ b/src/auto_archiver/modules/csv_db/__manifest__.py @@ -0,0 +1,22 @@ +{ + "name": "csv_db", + "type": ["database"], + "requires_setup": False, + "external_dependencies": {"python": ["loguru"] + }, + "configs": { + "csv_file": {"default": "db.csv", "help": "CSV file name"} + }, + "description": """ +Handles exporting archival results to a CSV file. + +### Features +- Saves archival metadata as rows in a CSV file. +- Automatically creates the CSV file with a header if it does not exist. +- Appends new metadata entries to the existing file. + +### Setup +Required config: +- csv_file: Path to the CSV file where results will be stored (default: "db.csv"). +""", +} diff --git a/src/auto_archiver/databases/csv_db.py b/src/auto_archiver/modules/csv_db/csv_db.py similarity index 81% rename from src/auto_archiver/databases/csv_db.py rename to src/auto_archiver/modules/csv_db/csv_db.py index f0d7153..eec4ec6 100644 --- a/src/auto_archiver/databases/csv_db.py +++ b/src/auto_archiver/modules/csv_db/csv_db.py @@ -3,8 +3,8 @@ from loguru import logger from csv import DictWriter from dataclasses import asdict -from . import Database -from ..core import Metadata +from auto_archiver.base_processors import Database +from auto_archiver.core import Metadata class CSVDb(Database): @@ -18,11 +18,6 @@ class CSVDb(Database): super().__init__(config) self.assert_valid_string("csv_file") - @staticmethod - def configs() -> dict: - return { - "csv_file": {"default": "db.csv", "help": "CSV file name"} - } def done(self, item: Metadata, cached: bool=False) -> None: """archival result ready - should be saved to DB""" diff --git a/src/auto_archiver/modules/csv_feeder/__init__.py b/src/auto_archiver/modules/csv_feeder/__init__.py new file mode 100644 index 0000000..161b78d --- /dev/null +++ b/src/auto_archiver/modules/csv_feeder/__init__.py @@ -0,0 +1 @@ +from .csv_feeder import CSVFeeder \ No newline at end of file diff --git a/src/auto_archiver/modules/csv_feeder/__manifest__.py b/src/auto_archiver/modules/csv_feeder/__manifest__.py new file mode 100644 index 0000000..4d19b70 --- /dev/null +++ b/src/auto_archiver/modules/csv_feeder/__manifest__.py @@ -0,0 +1,32 @@ +{ + "name": "CSV Feeder", + "type": ["feeder"], + "requires_setup": False, + "external_dependencies": { + "python": ["loguru"], + "bin": [""] + }, + "configs": { + "files": { + "default": None, + "help": "Path to the input file(s) to read the URLs from, comma separated. \ + Input files should be formatted with one URL per line", + }, + "column": { + "default": None, + "help": "Column number or name to read the URLs from, 0-indexed", + } + }, + "description": """ + Reads URLs from CSV files and feeds them into the archiving process. + + ### Features + - Supports reading URLs from multiple input files, specified as a comma-separated list. + - Allows specifying the column number or name to extract URLs from. + - Skips header rows if the first value is not a valid URL. + - Integrates with the `ArchivingContext` to manage URL feeding. + + ### Setu N + - Input files should be formatted with one URL per line. + """ +} diff --git a/src/auto_archiver/feeders/csv_feeder.py b/src/auto_archiver/modules/csv_feeder/csv_feeder.py similarity index 53% rename from src/auto_archiver/feeders/csv_feeder.py rename to src/auto_archiver/modules/csv_feeder/csv_feeder.py index 00bf7d7..7bff16e 100644 --- a/src/auto_archiver/feeders/csv_feeder.py +++ b/src/auto_archiver/modules/csv_feeder/csv_feeder.py @@ -1,27 +1,13 @@ from loguru import logger import csv -from . import Feeder -from ..core import Metadata, ArchivingContext -from ..utils import url_or_none +from auto_archiver.base_processors import Feeder +from auto_archiver.core import Metadata, ArchivingContext +from auto_archiver.utils import url_or_none class CSVFeeder(Feeder): - @staticmethod - def configs() -> dict: - return { - "files": { - "default": None, - "help": "Path to the input file(s) to read the URLs from, comma separated. \ - Input files should be formatted with one URL per line", - "cli_set": lambda cli_val, cur_val: list(set(cli_val.split(","))) - }, - "column": { - "default": None, - "help": "Column number or name to read the URLs from, 0-indexed", - } - } - + name = "csv_feeder" def __iter__(self) -> Metadata: url_column = self.column or 0 diff --git a/src/auto_archiver/modules/gdrive_storage/__init__.py b/src/auto_archiver/modules/gdrive_storage/__init__.py new file mode 100644 index 0000000..2765e4b --- /dev/null +++ b/src/auto_archiver/modules/gdrive_storage/__init__.py @@ -0,0 +1 @@ +from .gdrive_storage import GDriveStorage \ No newline at end of file diff --git a/src/auto_archiver/modules/gdrive_storage/__manifest__.py b/src/auto_archiver/modules/gdrive_storage/__manifest__.py new file mode 100644 index 0000000..b81b717 --- /dev/null +++ b/src/auto_archiver/modules/gdrive_storage/__manifest__.py @@ -0,0 +1,43 @@ +{ + "name": "Google Drive Storage", + "type": ["storage"], + "requires_setup": True, + "external_dependencies": { + "python": [ + "loguru", + "google-api-python-client", + "google-auth", + "google-auth-oauthlib", + "google-auth-httplib2" + ], + }, + "configs": { + "path_generator": { + "default": "url", + "help": "how to store the file in terms of directory structure: 'flat' sets to root; 'url' creates a directory based on the provided URL; 'random' creates a random directory.", + "choices": ["flat", "url", "random"], + }, + "filename_generator": { + "default": "random", + "help": "how to name stored files: 'random' creates a random string; 'static' uses a replicable strategy such as a hash.", + "choices": ["random", "static"], + }, + "root_folder_id": {"default": None, "help": "root google drive folder ID to use as storage, found in URL: 'https://drive.google.com/drive/folders/FOLDER_ID'"}, + "oauth_token": {"default": None, "help": "JSON filename with Google Drive OAuth token: check auto-archiver repository scripts folder for create_update_gdrive_oauth_token.py. NOTE: storage used will count towards owner of GDrive folder, therefore it is best to use oauth_token_filename over service_account."}, + "service_account": {"default": "secrets/service_account.json", "help": "service account JSON file path, same as used for Google Sheets. NOTE: storage used will count towards the developer account."}, + }, + "description": """ + GDriveStorage: A storage module for saving archived content to Google Drive. + + ### Features + - Saves media files to Google Drive, organizing them into folders based on the provided path structure. + - Supports OAuth token-based authentication or service account credentials for API access. + - Automatically creates folders in Google Drive if they don't exist. + - Retrieves CDN URLs for stored files, enabling easy sharing and access. + + ### Notes + - Requires setup with either a Google OAuth token or a service account JSON file. + - Files are uploaded to the specified `root_folder_id` and organized by the `media.key` structure. + - Automatically handles Google Drive API token refreshes for long-running jobs. + """ +} diff --git a/src/auto_archiver/storages/gd.py b/src/auto_archiver/modules/gdrive_storage/gdrive_storage.py similarity index 89% rename from src/auto_archiver/storages/gd.py rename to src/auto_archiver/modules/gdrive_storage/gdrive_storage.py index 61c5b21..4bcdb90 100644 --- a/src/auto_archiver/storages/gd.py +++ b/src/auto_archiver/modules/gdrive_storage/gdrive_storage.py @@ -9,8 +9,8 @@ from google.oauth2 import service_account from google.oauth2.credentials import Credentials from google.auth.transport.requests import Request -from ..core import Media -from . import Storage +from auto_archiver.core import Media +from auto_archiver.base_processors import Storage class GDriveStorage(Storage): @@ -58,16 +58,6 @@ class GDriveStorage(Storage): self.service = build('drive', 'v3', credentials=creds) - @staticmethod - def configs() -> dict: - return dict( - Storage.configs(), - ** { - "root_folder_id": {"default": None, "help": "root google drive folder ID to use as storage, found in URL: 'https://drive.google.com/drive/folders/FOLDER_ID'"}, - "oauth_token": {"default": None, "help": "JSON filename with Google Drive OAuth token: check auto-archiver repository scripts folder for create_update_gdrive_oauth_token.py. NOTE: storage used will count towards owner of GDrive folder, therefore it is best to use oauth_token_filename over service_account."}, - "service_account": {"default": "secrets/service_account.json", "help": "service account JSON file path, same as used for Google Sheets. NOTE: storage used will count towards the developer account."}, - }) - def get_cdn_url(self, media: Media) -> str: """ only support files saved in a folder for GD diff --git a/src/auto_archiver/modules/generic_extractor/bluesky.py b/src/auto_archiver/modules/generic_extractor/bluesky.py index 7aa9c39..c75c373 100644 --- a/src/auto_archiver/modules/generic_extractor/bluesky.py +++ b/src/auto_archiver/modules/generic_extractor/bluesky.py @@ -1,17 +1,12 @@ -import os -import mimetypes - -import requests from loguru import logger -from auto_archiver.core.context import ArchivingContext -from auto_archiver.archivers.archiver import Archiver +from auto_archiver.base_processors.extractor import Extractor from auto_archiver.core.metadata import Metadata, Media from .dropin import GenericDropin, InfoExtractor class Bluesky(GenericDropin): - def create_metadata(self, post: dict, ie_instance: InfoExtractor, archiver: Archiver, url: str) -> Metadata: + def create_metadata(self, post: dict, ie_instance: InfoExtractor, archiver: Extractor, url: str) -> Metadata: result = Metadata() result.set_url(url) result.set_title(post["record"]["text"]) @@ -42,7 +37,7 @@ class Bluesky(GenericDropin): - def _download_bsky_embeds(self, post: dict, archiver: Archiver) -> list[Media]: + def _download_bsky_embeds(self, post: dict, archiver: Extractor) -> list[Media]: """ Iterates over image(s) or video in a Bluesky post and downloads them """ diff --git a/src/auto_archiver/modules/generic_extractor/dropin.py b/src/auto_archiver/modules/generic_extractor/dropin.py index 37f3faf..99cd71b 100644 --- a/src/auto_archiver/modules/generic_extractor/dropin.py +++ b/src/auto_archiver/modules/generic_extractor/dropin.py @@ -1,6 +1,6 @@ from yt_dlp.extractor.common import InfoExtractor from auto_archiver.core.metadata import Metadata -from auto_archiver.archivers.archiver import Archiver +from auto_archiver.base_processors.extractor import Extractor class GenericDropin: """Base class for dropins for the generic extractor. @@ -30,7 +30,7 @@ class GenericDropin: raise NotImplementedError("This method should be implemented in the subclass") - def create_metadata(self, post: dict, ie_instance: InfoExtractor, archiver: Archiver, url: str) -> Metadata: + def create_metadata(self, post: dict, ie_instance: InfoExtractor, archiver: Extractor, url: str) -> Metadata: """ This method should create a Metadata object from the post data. """ diff --git a/src/auto_archiver/modules/generic_extractor/generic_extractor.py b/src/auto_archiver/modules/generic_extractor/generic_extractor.py index 276475f..8ceaabc 100644 --- a/src/auto_archiver/modules/generic_extractor/generic_extractor.py +++ b/src/auto_archiver/modules/generic_extractor/generic_extractor.py @@ -5,10 +5,10 @@ from yt_dlp.extractor.common import InfoExtractor from loguru import logger -from auto_archiver.archivers.archiver import Archiver +from auto_archiver.base_processors.extractor import Extractor from ...core import Metadata, Media, ArchivingContext -class GenericExtractor(Archiver): +class GenericExtractor(Extractor): name = "youtubedl_archiver" #left as is for backwards compat _dropins = {} diff --git a/src/auto_archiver/modules/generic_extractor/truth.py b/src/auto_archiver/modules/generic_extractor/truth.py index bf19dce..f52a748 100644 --- a/src/auto_archiver/modules/generic_extractor/truth.py +++ b/src/auto_archiver/modules/generic_extractor/truth.py @@ -2,7 +2,7 @@ from typing import Type from auto_archiver.utils import traverse_obj from auto_archiver.core.metadata import Metadata, Media -from auto_archiver.archivers.archiver import Archiver +from auto_archiver.base_processors.extractor import Extractor from yt_dlp.extractor.common import InfoExtractor from dateutil.parser import parse as parse_dt @@ -19,7 +19,7 @@ class Truth(GenericDropin): def skip_ytdlp_download(self, url, ie_instance: Type[InfoExtractor]) -> bool: return True - def create_metadata(self, post: dict, ie_instance: InfoExtractor, archiver: Archiver, url: str) -> Metadata: + def create_metadata(self, post: dict, ie_instance: InfoExtractor, archiver: Extractor, url: str) -> Metadata: """ Creates metadata from a truth social post diff --git a/src/auto_archiver/modules/generic_extractor/twitter.py b/src/auto_archiver/modules/generic_extractor/twitter.py index ce6c28d..11399d4 100644 --- a/src/auto_archiver/modules/generic_extractor/twitter.py +++ b/src/auto_archiver/modules/generic_extractor/twitter.py @@ -6,7 +6,7 @@ from slugify import slugify from auto_archiver.core.metadata import Metadata, Media from auto_archiver.utils import UrlUtil -from auto_archiver.archivers.archiver import Archiver +from auto_archiver.base_processors.extractor import Extractor from .dropin import GenericDropin, InfoExtractor @@ -32,7 +32,7 @@ class Twitter(GenericDropin): twid = ie_instance._match_valid_url(url).group('id') return ie_instance._extract_status(twid=twid) - def create_metadata(self, tweet: dict, ie_instance: InfoExtractor, archiver: Archiver, url: str) -> Metadata: + def create_metadata(self, tweet: dict, ie_instance: InfoExtractor, archiver: Extractor, url: str) -> Metadata: result = Metadata() try: if not tweet.get("user") or not tweet.get("created_at"): diff --git a/src/auto_archiver/modules/gsheet_db/__init__.py b/src/auto_archiver/modules/gsheet_db/__init__.py new file mode 100644 index 0000000..01fdee6 --- /dev/null +++ b/src/auto_archiver/modules/gsheet_db/__init__.py @@ -0,0 +1 @@ +from .gsheet_db import GsheetsDb \ No newline at end of file diff --git a/src/auto_archiver/modules/gsheet_db/__manifest__.py b/src/auto_archiver/modules/gsheet_db/__manifest__.py new file mode 100644 index 0000000..8c54fe5 --- /dev/null +++ b/src/auto_archiver/modules/gsheet_db/__manifest__.py @@ -0,0 +1,36 @@ +{ + "name": "Google Sheets Database", + "type": ["database"], + "requires_setup": True, + "external_dependencies": { + "python": ["loguru", "gspread", "python-slugify"], + }, + "configs": { + "allow_worksheets": { + "default": set(), + "help": "(CSV) only worksheets whose name is included in allow are included (overrides worksheet_block), leave empty so all are allowed", + }, + "block_worksheets": { + "default": set(), + "help": "(CSV) explicitly block some worksheets from being processed", + }, + "use_sheet_names_in_stored_paths": { + "default": True, + "help": "if True the stored files path will include 'workbook_name/worksheet_name/...'", + } + }, + "description": """ + GsheetsDatabase: + Handles integration with Google Sheets for tracking archival tasks. + +### Features +- Updates a Google Sheet with the status of the archived URLs, including in progress, success or failure, and method used. +- Saves metadata such as title, text, timestamp, hashes, screenshots, and media URLs to designated columns. +- Formats media-specific metadata, such as thumbnails and PDQ hashes for the sheet. +- Skips redundant updates for empty or invalid data fields. + +### Notes +- Currently works only with metadata provided by GsheetFeeder. +- Requires configuration of a linked Google Sheet and appropriate API credentials. + """ +} diff --git a/src/auto_archiver/databases/gsheet_db.py b/src/auto_archiver/modules/gsheet_db/gsheet_db.py similarity index 96% rename from src/auto_archiver/databases/gsheet_db.py rename to src/auto_archiver/modules/gsheet_db/gsheet_db.py index 98e72dc..239bc06 100644 --- a/src/auto_archiver/databases/gsheet_db.py +++ b/src/auto_archiver/modules/gsheet_db/gsheet_db.py @@ -1,12 +1,13 @@ from typing import Union, Tuple + import datetime from urllib.parse import quote from loguru import logger -from . import Database -from ..core import Metadata, Media, ArchivingContext -from ..utils import GWorksheet +from auto_archiver.base_processors import Database +from auto_archiver.core import Metadata, Media, ArchivingContext +from auto_archiver.modules.gsheet_feeder import GWorksheet class GsheetsDb(Database): @@ -20,10 +21,6 @@ class GsheetsDb(Database): # without this STEP.__init__ is not called super().__init__(config) - @staticmethod - def configs() -> dict: - return {} - def started(self, item: Metadata) -> None: logger.warning(f"STARTED {item}") gw, row = self._retrieve_gsheet(item) @@ -108,5 +105,4 @@ class GsheetsDb(Database): elif self.sheet_id: print(self.sheet_id) - return gw, row diff --git a/src/auto_archiver/modules/gsheet_feeder/__init__.py b/src/auto_archiver/modules/gsheet_feeder/__init__.py new file mode 100644 index 0000000..bb4230a --- /dev/null +++ b/src/auto_archiver/modules/gsheet_feeder/__init__.py @@ -0,0 +1,2 @@ +from .gworksheet import GWorksheet +from .gsheet_feeder import GsheetsFeeder \ No newline at end of file diff --git a/src/auto_archiver/modules/gsheet_feeder/__manifest__.py b/src/auto_archiver/modules/gsheet_feeder/__manifest__.py new file mode 100644 index 0000000..685a8fd --- /dev/null +++ b/src/auto_archiver/modules/gsheet_feeder/__manifest__.py @@ -0,0 +1,65 @@ +{ + "name": "Google Sheets Feeder", + "type": ["feeder"], + "entry_point": "GsheetsFeeder", + "requires_setup": True, + "external_dependencies": { + "python": ["loguru", "gspread", "python-slugify"], + }, + "configs": { + "sheet": {"default": None, "help": "name of the sheet to archive"}, + "sheet_id": {"default": None, "help": "(alternative to sheet name) the id of the sheet to archive"}, + "header": {"default": 1, "help": "index of the header row (starts at 1)"}, + "service_account": {"default": "secrets/service_account.json", "help": "service account JSON file path"}, + "columns": { + "default": { + 'url': 'link', + 'status': 'archive status', + 'folder': 'destination folder', + 'archive': 'archive location', + 'date': 'archive date', + 'thumbnail': 'thumbnail', + 'timestamp': 'upload timestamp', + 'title': 'upload title', + 'text': 'text content', + 'screenshot': 'screenshot', + 'hash': 'hash', + 'pdq_hash': 'perceptual hashes', + 'wacz': 'wacz', + 'replaywebpage': 'replaywebpage', + }, + "help": "names of columns in the google sheet (stringified JSON object)", + "type": "auto_archiver.utils.json_loader", + }, + "allow_worksheets": { + "default": set(), + "help": "(CSV) only worksheets whose name is included in allow are included (overrides worksheet_block), leave empty so all are allowed", + }, + "block_worksheets": { + "default": set(), + "help": "(CSV) explicitly block some worksheets from being processed", + }, + "use_sheet_names_in_stored_paths": { + "default": True, + "help": "if True the stored files path will include 'workbook_name/worksheet_name/...'", + "type": "bool", + } + }, + "description": """ + GsheetsFeeder + A Google Sheets-based feeder for the Auto Archiver. + + This reads data from Google Sheets and filters rows based on user-defined rules. + The filtered rows are processed into `Metadata` objects. + + ### Features + - Validates the sheet structure and filters rows based on input configurations. + - Processes only worksheets allowed by the `allow_worksheets` and `block_worksheets` configurations. + - Ensures only rows with valid URLs and unprocessed statuses are included for archival. + - Supports organizing stored files into folder paths based on sheet and worksheet names. + + ### Notes + - Requires a Google Service Account JSON file for authentication. Suggested location is `secrets/gsheets_service_account.json`. + - Create the sheet using the template provided in the docs. + """ +} diff --git a/src/auto_archiver/feeders/gsheet_feeder.py b/src/auto_archiver/modules/gsheet_feeder/gsheet_feeder.py similarity index 61% rename from src/auto_archiver/feeders/gsheet_feeder.py rename to src/auto_archiver/modules/gsheet_feeder/gsheet_feeder.py index 1c4fc32..b57174f 100644 --- a/src/auto_archiver/feeders/gsheet_feeder.py +++ b/src/auto_archiver/modules/gsheet_feeder/gsheet_feeder.py @@ -8,45 +8,62 @@ The filtered rows are processed into `Metadata` objects. - validates the sheet's structure and filters rows based on input configurations. - Ensures only rows with valid URLs and unprocessed statuses are included. """ -import gspread, os +import os +import gspread from loguru import logger from slugify import slugify -# from . import Enricher -from . import Feeder -from ..core import Metadata, ArchivingContext -from ..utils import Gsheets, GWorksheet +from auto_archiver.base_processors import Feeder +from auto_archiver.core import Metadata, ArchivingContext +from . import GWorksheet -class GsheetsFeeder(Gsheets, Feeder): +class GsheetsFeeder(Feeder): name = "gsheet_feeder" - def __init__(self, config: dict) -> None: - # without this STEP.__init__ is not called - super().__init__(config) - self.gsheets_client = gspread.service_account(filename=self.service_account) + def __init__(self) -> None: + """ + Initializes the GsheetsFeeder with preloaded configurations. + """ + super().__init__() + # Initialize the gspread client with the provided service account file + # self.gsheets_client = gspread.service_account(filename=self.config["service_account"]) + # + # # Set up feeder-specific configurations from the config + # self.sheet_name = config.get("sheet") + # self.sheet_id = config.get("sheet_id") + # self.header = config.get("header", 1) + # self.columns = config.get("columns", {}) + # assert self.sheet_name or self.sheet_id, ( + # "You need to define either a 'sheet' name or a 'sheet_id' in your manifest." + # ) + + + # # Configuration attributes + # self.sheet = config.get("sheet") + # self.sheet_id = config.get("sheet_id") + # self.header = config.get("header", 1) + # self.columns = config.get("columns", {}) + # self.allow_worksheets = config.get("allow_worksheets", set()) + # self.block_worksheets = config.get("block_worksheets", set()) + # self.use_sheet_names_in_stored_paths = config.get("use_sheet_names_in_stored_paths", True) + + # Ensure the header is an integer + # try: + # self.header = int(self.header) + # except ValueError: + # pass + # assert isinstance(self.header, int), f"Header must be an integer, got {type(self.header)}" + # assert self.sheet or self.sheet_id, "Either 'sheet' or 'sheet_id' must be defined." + # + + def open_sheet(self): + if self.sheet: + return self.gsheets_client.open(self.sheet) + else: # self.sheet_id + return self.gsheets_client.open_by_key(self.sheet_id) - @staticmethod - def configs() -> dict: - return dict( - Gsheets.configs(), - ** { - "allow_worksheets": { - "default": set(), - "help": "(CSV) only worksheets whose name is included in allow are included (overrides worksheet_block), leave empty so all are allowed", - "cli_set": lambda cli_val, cur_val: set(cli_val.split(",")) - }, - "block_worksheets": { - "default": set(), - "help": "(CSV) explicitly block some worksheets from being processed", - "cli_set": lambda cli_val, cur_val: set(cli_val.split(",")) - }, - "use_sheet_names_in_stored_paths": { - "default": True, - "help": "if True the stored files path will include 'workbook_name/worksheet_name/...'", - } - }) def __iter__(self) -> Metadata: sh = self.open_sheet() diff --git a/src/auto_archiver/utils/gworksheet.py b/src/auto_archiver/modules/gsheet_feeder/gworksheet.py similarity index 100% rename from src/auto_archiver/utils/gworksheet.py rename to src/auto_archiver/modules/gsheet_feeder/gworksheet.py diff --git a/src/auto_archiver/modules/hash_enricher/__init__.py b/src/auto_archiver/modules/hash_enricher/__init__.py new file mode 100644 index 0000000..18ec885 --- /dev/null +++ b/src/auto_archiver/modules/hash_enricher/__init__.py @@ -0,0 +1 @@ +from .hash_enricher import HashEnricher \ No newline at end of file diff --git a/src/auto_archiver/modules/hash_enricher/__manifest__.py b/src/auto_archiver/modules/hash_enricher/__manifest__.py new file mode 100644 index 0000000..a7697b9 --- /dev/null +++ b/src/auto_archiver/modules/hash_enricher/__manifest__.py @@ -0,0 +1,28 @@ +{ + "name": "Hash Enricher", + "type": ["enricher"], + "requires_setup": False, + "external_dependencies": { + "python": ["loguru"], + }, + "configs": { + "algorithm": {"default": "SHA-256", "help": "hash algorithm to use", "choices": ["SHA-256", "SHA3-512"]}, + # TODO add non-negative requirement to match previous implementation? + "chunksize": {"default": 1.6e7, "help": "number of bytes to use when reading files in chunks (if this value is too large you will run out of RAM), default is 16MB"}, + }, + "description": """ +Generates cryptographic hashes for media files to ensure data integrity and authenticity. + +### Features +- Calculates cryptographic hashes (SHA-256 or SHA3-512) for media files stored in `Metadata` objects. +- Ensures content authenticity, integrity validation, and duplicate identification. +- Efficiently processes large files by reading file bytes in configurable chunk sizes. +- Supports dynamic configuration of hash algorithms and chunk sizes. +- Updates media metadata with the computed hash value in the format `:`. + +### Notes +- Default hash algorithm is SHA-256, but SHA3-512 is also supported. +- Chunk size defaults to 16 MB but can be adjusted based on memory requirements. +- Useful for workflows requiring hash-based content validation or deduplication. +""", +} diff --git a/src/auto_archiver/enrichers/hash_enricher.py b/src/auto_archiver/modules/hash_enricher/hash_enricher.py similarity index 83% rename from src/auto_archiver/enrichers/hash_enricher.py rename to src/auto_archiver/modules/hash_enricher/hash_enricher.py index 69973b7..8731b06 100644 --- a/src/auto_archiver/enrichers/hash_enricher.py +++ b/src/auto_archiver/modules/hash_enricher/hash_enricher.py @@ -10,8 +10,8 @@ making it suitable for handling large files efficiently. import hashlib from loguru import logger -from . import Enricher -from ..core import Metadata, ArchivingContext +from auto_archiver.base_processors import Enricher +from auto_archiver.core import Metadata, ArchivingContext class HashEnricher(Enricher): @@ -40,18 +40,15 @@ class HashEnricher(Enricher): else: self.chunksize = self.configs()["chunksize"]["default"] - self.chunksize = int(self.chunksize) + try: + self.chunksize = int(self.chunksize) + except ValueError: + raise ValueError(f"Invalid chunksize value: {self.chunksize}. Must be an integer.") + assert self.chunksize >= -1, "read length must be non-negative or -1" ArchivingContext.set("hash_enricher.algorithm", self.algorithm, keep_on_reset=True) - @staticmethod - def configs() -> dict: - return { - "algorithm": {"default": "SHA-256", "help": "hash algorithm to use", "choices": ["SHA-256", "SHA3-512"]}, - "chunksize": {"default": int(1.6e7), "help": "number of bytes to use when reading files in chunks (if this value is too large you will run out of RAM), default is 16MB"}, - } - def enrich(self, to_enrich: Metadata) -> None: url = to_enrich.get_url() logger.debug(f"calculating media hashes for {url=} (using {self.algorithm})") diff --git a/src/auto_archiver/modules/html_formatter/__init__.py b/src/auto_archiver/modules/html_formatter/__init__.py new file mode 100644 index 0000000..432ef33 --- /dev/null +++ b/src/auto_archiver/modules/html_formatter/__init__.py @@ -0,0 +1 @@ +from .html_formatter import HtmlFormatter \ No newline at end of file diff --git a/src/auto_archiver/modules/html_formatter/__manifest__.py b/src/auto_archiver/modules/html_formatter/__manifest__.py new file mode 100644 index 0000000..259a3d1 --- /dev/null +++ b/src/auto_archiver/modules/html_formatter/__manifest__.py @@ -0,0 +1,13 @@ +{ + "name": "HTML Formatter", + "type": ["formatter"], + "requires_setup": False, + "external_dependencies": { + "python": ["loguru", "jinja2"], + "bin": [""] + }, + "configs": { + "detect_thumbnails": {"default": True, "help": "if true will group by thumbnails generated by thumbnail enricher by id 'thumbnail_00'"} + }, + "description": """ """, +} diff --git a/src/auto_archiver/formatters/html_formatter.py b/src/auto_archiver/modules/html_formatter/html_formatter.py similarity index 87% rename from src/auto_archiver/formatters/html_formatter.py rename to src/auto_archiver/modules/html_formatter/html_formatter.py index 5d95474..15104b2 100644 --- a/src/auto_archiver/formatters/html_formatter.py +++ b/src/auto_archiver/modules/html_formatter/html_formatter.py @@ -7,11 +7,11 @@ from loguru import logger import json import base64 -from ..version import __version__ -from ..core import Metadata, Media, ArchivingContext -from . import Formatter -from ..enrichers import HashEnricher -from ..utils.misc import random_str +from auto_archiver.version import __version__ +from auto_archiver.core import Metadata, Media, ArchivingContext +from auto_archiver.base_processors import Formatter +from auto_archiver.modules.hash_enricher import HashEnricher +from auto_archiver.utils.misc import random_str @dataclass @@ -28,12 +28,6 @@ class HtmlFormatter(Formatter): }) self.template = self.environment.get_template("html_template.html") - @staticmethod - def configs() -> dict: - return { - "detect_thumbnails": {"default": True, "help": "if true will group by thumbnails generated by thumbnail enricher by id 'thumbnail_00'"} - } - def format(self, item: Metadata) -> Media: url = item.get_url() if item.is_empty(): diff --git a/src/auto_archiver/formatters/templates/__init__.py b/src/auto_archiver/modules/html_formatter/templates/__init__.py similarity index 100% rename from src/auto_archiver/formatters/templates/__init__.py rename to src/auto_archiver/modules/html_formatter/templates/__init__.py diff --git a/src/auto_archiver/formatters/templates/html_template.html b/src/auto_archiver/modules/html_formatter/templates/html_template.html similarity index 100% rename from src/auto_archiver/formatters/templates/html_template.html rename to src/auto_archiver/modules/html_formatter/templates/html_template.html diff --git a/src/auto_archiver/formatters/templates/macros.html b/src/auto_archiver/modules/html_formatter/templates/macros.html similarity index 100% rename from src/auto_archiver/formatters/templates/macros.html rename to src/auto_archiver/modules/html_formatter/templates/macros.html diff --git a/src/auto_archiver/modules/instagram_api_archiver/__init__.py b/src/auto_archiver/modules/instagram_api_archiver/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/auto_archiver/modules/instagram_api_extractor/__init__.py b/src/auto_archiver/modules/instagram_api_extractor/__init__.py new file mode 100644 index 0000000..8805c07 --- /dev/null +++ b/src/auto_archiver/modules/instagram_api_extractor/__init__.py @@ -0,0 +1 @@ +from .instagram_api_extractor import InstagramAPIExtractor diff --git a/src/auto_archiver/modules/instagram_api_archiver/__manifest__.py b/src/auto_archiver/modules/instagram_api_extractor/__manifest__.py similarity index 50% rename from src/auto_archiver/modules/instagram_api_archiver/__manifest__.py rename to src/auto_archiver/modules/instagram_api_extractor/__manifest__.py index 2bb3f67..cdaf635 100644 --- a/src/auto_archiver/modules/instagram_api_archiver/__manifest__.py +++ b/src/auto_archiver/modules/instagram_api_extractor/__manifest__.py @@ -1,15 +1,13 @@ { - "name": "Instagram API Archiver", + "name": "Instagram API Extractor", "type": ["extractor"], - "entry_point": "instagram_api_archiver:InstagramApiArchiver", - "depends": ["core"], "external_dependencies": {"python": ["requests", "loguru", "retrying", "tqdm",], }, - "no_setup_required": False, + "requires_setup": True, "configs": { "access_token": {"default": None, "help": "a valid instagrapi-api token"}, "api_endpoint": {"default": None, "help": "API endpoint to use"}, @@ -26,5 +24,22 @@ "help": "if true, will remove empty values from the json output", }, }, - "description": "", + "description": """ +Archives various types of Instagram content using the Instagrapi API. + +### Features +- Connects to an Instagrapi API deployment to fetch Instagram profiles, posts, stories, highlights, reels, and tagged content. +- Supports advanced configuration options, including: + - Full profile download (all posts, stories, highlights, and tagged content). + - Limiting the number of posts to fetch for large profiles. + - Minimising JSON output to remove empty fields and redundant data. +- Provides robust error handling and retries for API calls. +- Ensures efficient media scraping, including handling nested or carousel media items. +- Adds downloaded media and metadata to the result for further processing. + +### Notes +- Requires a valid Instagrapi API token (`access_token`) and API endpoint (`api_endpoint`). +- Full-profile downloads can be limited by setting `full_profile_max_posts`. +- Designed to fetch content in batches for large profiles, minimising API load. +""", } diff --git a/src/auto_archiver/modules/instagram_api_archiver/instagram_api_archiver.py b/src/auto_archiver/modules/instagram_api_extractor/instagram_api_extractor.py similarity index 93% rename from src/auto_archiver/modules/instagram_api_archiver/instagram_api_archiver.py rename to src/auto_archiver/modules/instagram_api_extractor/instagram_api_extractor.py index cc6e074..c1271fc 100644 --- a/src/auto_archiver/modules/instagram_api_archiver/instagram_api_archiver.py +++ b/src/auto_archiver/modules/instagram_api_extractor/instagram_api_extractor.py @@ -1,5 +1,5 @@ """ -The `instagram_api_archiver` module provides tools for archiving various types of Instagram content +The `instagram_api_extractor` module provides tools for archiving various types of Instagram content using the [Instagrapi API](https://github.com/subzeroid/instagrapi). Connects to an Instagrapi API deployment and allows for downloading Instagram user profiles, @@ -16,19 +16,19 @@ from loguru import logger from retrying import retry from tqdm import tqdm -from auto_archiver.archivers import Archiver +from auto_archiver.base_processors import Extractor from auto_archiver.core import Media from auto_archiver.core import Metadata -class InstagramAPIArchiver(Archiver): +class InstagramAPIExtractor(Extractor): """ Uses an https://github.com/subzeroid/instagrapi API deployment to fetch instagram posts data # TODO: improvement collect aggregates of locations[0].location and mentions for all posts """ - name = "instagram_api_archiver" + name = "instagram_api_extractor" global_pattern = re.compile( r"(?:(?:http|https):\/\/)?(?:www.)?(?:instagram.com)\/(stories(?:\/highlights)?|p|reel)?\/?([^\/\?]*)\/?(\d+)?" @@ -45,25 +45,6 @@ class InstagramAPIArchiver(Archiver): self.full_profile = bool(self.full_profile) self.minimize_json_output = bool(self.minimize_json_output) - @staticmethod - def configs() -> dict: - return { - "access_token": {"default": None, "help": "a valid instagrapi-api token"}, - "api_endpoint": {"default": None, "help": "API endpoint to use"}, - "full_profile": { - "default": False, - "help": "if true, will download all posts, tagged posts, stories, and highlights for a profile, if false, will only download the profile pic and information.", - }, - "full_profile_max_posts": { - "default": 0, - "help": "Use to limit the number of posts to download when full_profile is true. 0 means no limit. limit is applied softly since posts are fetched in batch, once to: posts, tagged posts, and highlights", - }, - "minimize_json_output": { - "default": True, - "help": "if true, will remove empty values from the json output", - }, - } - def download(self, item: Metadata) -> Metadata: url = item.get_url() diff --git a/src/auto_archiver/modules/instagram_archiver/__init__.py b/src/auto_archiver/modules/instagram_archiver/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/auto_archiver/modules/instagram_extractor/__init__.py b/src/auto_archiver/modules/instagram_extractor/__init__.py new file mode 100644 index 0000000..6f39171 --- /dev/null +++ b/src/auto_archiver/modules/instagram_extractor/__init__.py @@ -0,0 +1 @@ +from .instagram_extractor import InstagramExtractor \ No newline at end of file diff --git a/src/auto_archiver/modules/instagram_archiver/__manifest__.py b/src/auto_archiver/modules/instagram_extractor/__manifest__.py similarity index 84% rename from src/auto_archiver/modules/instagram_archiver/__manifest__.py rename to src/auto_archiver/modules/instagram_extractor/__manifest__.py index bd63ab4..f1857c2 100644 --- a/src/auto_archiver/modules/instagram_archiver/__manifest__.py +++ b/src/auto_archiver/modules/instagram_extractor/__manifest__.py @@ -1,13 +1,13 @@ { - "name": "Instagram Archiver", + "name": "Instagram Extractor", "type": ["extractor"], - "entry_point": "instagram_archiver:InstagramArchiver", - "depends": ["core"], "external_dependencies": { - "python": ["instaloader", - "loguru",], + "python": [ + "instaloader", + "loguru", + ], }, - "no_setup_required": False, + "requires_setup": True, "configs": { "username": {"default": None, "help": "a valid Instagram username"}, "password": { diff --git a/src/auto_archiver/modules/instagram_archiver/instagram_archiver.py b/src/auto_archiver/modules/instagram_extractor/instagram_extractor.py similarity index 87% rename from src/auto_archiver/modules/instagram_archiver/instagram_archiver.py rename to src/auto_archiver/modules/instagram_extractor/instagram_extractor.py index 4cf001d..2b9bece 100644 --- a/src/auto_archiver/modules/instagram_archiver/instagram_archiver.py +++ b/src/auto_archiver/modules/instagram_extractor/instagram_extractor.py @@ -7,15 +7,15 @@ import re, os, shutil, traceback import instaloader # https://instaloader.github.io/as-module.html from loguru import logger -from auto_archiver.archivers import Archiver +from auto_archiver.base_processors import Extractor from auto_archiver.core import Metadata from auto_archiver.core import Media -class InstagramArchiver(Archiver): +class InstagramExtractor(Extractor): """ Uses Instaloader to download either a post (inc images, videos, text) or as much as possible from a profile (posts, stories, highlights, ...) """ - name = "instagram_archiver" + name = "instagram_extractor" # NB: post regex should be tested before profile # https://regex101.com/r/MGPquX/1 @@ -45,16 +45,7 @@ class InstagramArchiver(Archiver): except Exception as e2: logger.error(f"Unable to finish login (retrying from file): {e2}\n{traceback.format_exc()}") - @staticmethod - def configs() -> dict: - return { - "username": {"default": None, "help": "a valid Instagram username"}, - "password": {"default": None, "help": "the corresponding Instagram account password"}, - "download_folder": {"default": "instaloader", "help": "name of a folder to temporarily download content to"}, - "session_file": {"default": "secrets/instaloader.session", "help": "path to the instagram session which saves session credentials"}, - #TODO: fine-grain - # "download_stories": {"default": True, "help": "if the link is to a user profile: whether to get stories information"}, - } + def download(self, item: Metadata) -> Metadata: url = item.get_url() @@ -76,7 +67,7 @@ class InstagramArchiver(Archiver): elif len(profile_matches): result = self.download_profile(url, profile_matches[0]) except Exception as e: - logger.error(f"Failed to download with instagram archiver due to: {e}, make sure your account credentials are valid.") + logger.error(f"Failed to download with instagram extractor due to: {e}, make sure your account credentials are valid.") finally: shutil.rmtree(self.download_folder, ignore_errors=True) return result diff --git a/src/auto_archiver/modules/instagram_tbot_archiver/__init__.py b/src/auto_archiver/modules/instagram_tbot_archiver/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/auto_archiver/modules/instagram_tbot_extractor/__init__.py b/src/auto_archiver/modules/instagram_tbot_extractor/__init__.py new file mode 100644 index 0000000..aa39e63 --- /dev/null +++ b/src/auto_archiver/modules/instagram_tbot_extractor/__init__.py @@ -0,0 +1 @@ +from .instagram_tbot_extractor import InstagramTbotExtractor diff --git a/src/auto_archiver/modules/instagram_tbot_archiver/__manifest__.py b/src/auto_archiver/modules/instagram_tbot_extractor/__manifest__.py similarity index 81% rename from src/auto_archiver/modules/instagram_tbot_archiver/__manifest__.py rename to src/auto_archiver/modules/instagram_tbot_extractor/__manifest__.py index cadb729..95d6808 100644 --- a/src/auto_archiver/modules/instagram_tbot_archiver/__manifest__.py +++ b/src/auto_archiver/modules/instagram_tbot_extractor/__manifest__.py @@ -1,8 +1,6 @@ { - "name": "Instagram Telegram Bot Archiver", + "name": "Instagram Telegram Bot Extractor", "type": ["extractor"], - "entry_point": "instagram_tbot_archiver:InstagramTbotArchiver", - "depends": ["core", "utils"], "external_dependencies": {"python": ["loguru", "telethon",], }, @@ -14,7 +12,7 @@ "timeout": {"default": 45, "help": "timeout to fetch the instagram content in seconds."}, }, "description": """ -The `InstagramTbotArchiver` module uses a Telegram bot (`instagram_load_bot`) to fetch and archive Instagram content, +The `InstagramTbotExtractor` module uses a Telegram bot (`instagram_load_bot`) to fetch and archive Instagram content, such as posts and stories. It leverages the Telethon library to interact with the Telegram API, sending Instagram URLs to the bot and downloading the resulting media and metadata. The downloaded content is stored as `Media` objects and returned as part of a `Metadata` object. @@ -27,7 +25,7 @@ returned as part of a `Metadata` object. ### Setup -To use the `InstagramTbotArchiver`, you need to provide the following configuration settings: +To use the `InstagramTbotExtractor`, you need to provide the following configuration settings: - **API ID and Hash**: Telegram API credentials obtained from [my.telegram.org/apps](https://my.telegram.org/apps). - **Session File**: Optional path to store the Telegram session file for future use. diff --git a/src/auto_archiver/modules/instagram_tbot_archiver/instagram_tbot_archiver.py b/src/auto_archiver/modules/instagram_tbot_extractor/instagram_tbot_extractor.py similarity index 81% rename from src/auto_archiver/modules/instagram_tbot_archiver/instagram_tbot_archiver.py rename to src/auto_archiver/modules/instagram_tbot_extractor/instagram_tbot_extractor.py index 9fdc208..36c8a06 100644 --- a/src/auto_archiver/modules/instagram_tbot_archiver/instagram_tbot_archiver.py +++ b/src/auto_archiver/modules/instagram_tbot_extractor/instagram_tbot_extractor.py @@ -1,5 +1,5 @@ """ -InstagramTbotArchiver Module +InstagramTbotExtractor Module This module provides functionality to archive Instagram content (posts, stories, etc.) using a Telegram bot (`instagram_load_bot`). It interacts with the Telegram API via the Telethon library to send Instagram URLs to the bot, which retrieves the @@ -15,18 +15,18 @@ from sqlite3 import OperationalError from loguru import logger from telethon.sync import TelegramClient -from auto_archiver.archivers import Archiver +from auto_archiver.base_processors import Extractor from auto_archiver.core import Metadata, Media, ArchivingContext from auto_archiver.utils import random_str -class InstagramTbotArchiver(Archiver): +class InstagramTbotExtractor(Extractor): """ calls a telegram bot to fetch instagram posts/stories... and gets available media from it https://github.com/adw0rd/instagrapi https://t.me/instagram_load_bot """ - name = "instagram_tbot_archiver" + name = "instagram_tbot_extractor" def __init__(self, config: dict) -> None: super().__init__(config) @@ -34,15 +34,6 @@ class InstagramTbotArchiver(Archiver): self.assert_valid_string("api_hash") self.timeout = int(self.timeout) - @staticmethod - def configs() -> dict: - return { - "api_id": {"default": None, "help": "telegram API_ID value, go to https://my.telegram.org/apps"}, - "api_hash": {"default": None, "help": "telegram API_HASH value, go to https://my.telegram.org/apps"}, - "session_file": {"default": "secrets/anon-insta", "help": "optional, records the telegram login session for future usage, '.session' will be appended to the provided value."}, - "timeout": {"default": 45, "help": "timeout to fetch the instagram content in seconds."}, - } - def setup(self) -> None: """ 1. makes a copy of session_file that is removed in cleanup @@ -58,7 +49,7 @@ class InstagramTbotArchiver(Archiver): try: self.client = TelegramClient(self.session_file, self.api_id, self.api_hash) except OperationalError as e: - logger.error(f"Unable to access the {self.session_file} session, please make sure you don't use the same session file here and in telethon_archiver. if you do then disable at least one of the archivers for the 1st time you setup telethon session: {e}") + logger.error(f"Unable to access the {self.session_file} session, please make sure you don't use the same session file here and in telethon_extractor. if you do then disable at least one of the archivers for the 1st time you setup telethon session: {e}") with self.client.start(): logger.success(f"SETUP {self.name} login works.") diff --git a/src/auto_archiver/modules/local_storage/__init__.py b/src/auto_archiver/modules/local_storage/__init__.py new file mode 100644 index 0000000..6746373 --- /dev/null +++ b/src/auto_archiver/modules/local_storage/__init__.py @@ -0,0 +1 @@ +from .local import LocalStorage \ No newline at end of file diff --git a/src/auto_archiver/modules/local_storage/__manifest__.py b/src/auto_archiver/modules/local_storage/__manifest__.py new file mode 100644 index 0000000..c012be0 --- /dev/null +++ b/src/auto_archiver/modules/local_storage/__manifest__.py @@ -0,0 +1,35 @@ +{ + "name": "Local Storage", + "type": ["storage"], + "requires_setup": False, + "external_dependencies": { + "python": ["loguru"], + }, + "configs": { + "path_generator": { + "default": "url", + "help": "how to store the file in terms of directory structure: 'flat' sets to root; 'url' creates a directory based on the provided URL; 'random' creates a random directory.", + "choices": ["flat", "url", "random"], + }, + "filename_generator": { + "default": "random", + "help": "how to name stored files: 'random' creates a random string; 'static' uses a replicable strategy such as a hash.", + "choices": ["random", "static"], + }, + "save_to": {"default": "./archived", "help": "folder where to save archived content"}, + "save_absolute": {"default": False, "help": "whether the path to the stored file is absolute or relative in the output result inc. formatters (WARN: leaks the file structure)"}, + }, + "description": """ + LocalStorage: A storage module for saving archived content locally on the filesystem. + + ### Features + - Saves archived media files to a specified folder on the local filesystem. + - Maintains file metadata during storage using `shutil.copy2`. + - Supports both absolute and relative paths for stored files, configurable via `save_absolute`. + - Automatically creates directories as needed for storing files. + + ### Notes + - Default storage folder is `./archived`, but this can be changed via the `save_to` configuration. + - The `save_absolute` option can reveal the file structure in output formats; use with caution. + """ +} diff --git a/src/auto_archiver/storages/local.py b/src/auto_archiver/modules/local_storage/local.py similarity index 69% rename from src/auto_archiver/storages/local.py rename to src/auto_archiver/modules/local_storage/local.py index aa08e49..530f111 100644 --- a/src/auto_archiver/storages/local.py +++ b/src/auto_archiver/modules/local_storage/local.py @@ -4,8 +4,8 @@ from typing import IO import os from loguru import logger -from ..core import Media -from ..storages import Storage +from auto_archiver.core import Media +from auto_archiver.base_processors import Storage class LocalStorage(Storage): @@ -15,15 +15,6 @@ class LocalStorage(Storage): super().__init__(config) os.makedirs(self.save_to, exist_ok=True) - @staticmethod - def configs() -> dict: - return dict( - Storage.configs(), - ** { - "save_to": {"default": "./archived", "help": "folder where to save archived content"}, - "save_absolute": {"default": False, "help": "whether the path to the stored file is absolute or relative in the output result inc. formatters (WARN: leaks the file structure)"}, - }) - def get_cdn_url(self, media: Media) -> str: # TODO: is this viable with Storage.configs on path/filename? dest = os.path.join(self.save_to, media.key) diff --git a/src/auto_archiver/modules/meta_enricher/__init__.py b/src/auto_archiver/modules/meta_enricher/__init__.py new file mode 100644 index 0000000..4e1d330 --- /dev/null +++ b/src/auto_archiver/modules/meta_enricher/__init__.py @@ -0,0 +1 @@ +from .meta_enricher import MetaEnricher diff --git a/src/auto_archiver/modules/meta_enricher/__manifest__.py b/src/auto_archiver/modules/meta_enricher/__manifest__.py new file mode 100644 index 0000000..10acf71 --- /dev/null +++ b/src/auto_archiver/modules/meta_enricher/__manifest__.py @@ -0,0 +1,22 @@ +{ + "name": "Archive Metadata Enricher", + "type": ["enricher"], + "requires_setup": False, + "external_dependencies": { + "python": ["loguru"], + }, + "description": """ + Adds metadata information about the archive operations, Adds metadata about archive operations, including file sizes and archive duration./ + To be included at the end of all enrichments. + + ### Features +- Calculates the total size of all archived media files, storing the result in human-readable and byte formats. +- Computes the duration of the archival process, storing the elapsed time in seconds. +- Ensures all enrichments are performed only if the `Metadata` object contains valid data. +- Adds detailed metadata to provide insights into file sizes and archival performance. + +### Notes +- Skips enrichment if no media or metadata is available in the `Metadata` object. +- File sizes are calculated using the `os.stat` module, ensuring accurate byte-level reporting. +""", +} diff --git a/src/auto_archiver/enrichers/meta_enricher.py b/src/auto_archiver/modules/meta_enricher/meta_enricher.py similarity index 93% rename from src/auto_archiver/enrichers/meta_enricher.py rename to src/auto_archiver/modules/meta_enricher/meta_enricher.py index b721bb5..f9b74f7 100644 --- a/src/auto_archiver/enrichers/meta_enricher.py +++ b/src/auto_archiver/modules/meta_enricher/meta_enricher.py @@ -2,8 +2,8 @@ import datetime import os from loguru import logger -from . import Enricher -from ..core import Metadata +from auto_archiver.base_processors import Enricher +from auto_archiver.core import Metadata class MetaEnricher(Enricher): @@ -17,10 +17,6 @@ class MetaEnricher(Enricher): # without this STEP.__init__ is not called super().__init__(config) - @staticmethod - def configs() -> dict: - return {} - def enrich(self, to_enrich: Metadata) -> None: url = to_enrich.get_url() if to_enrich.is_empty(): @@ -28,7 +24,7 @@ class MetaEnricher(Enricher): return logger.debug(f"calculating archive metadata information for {url=}") - + self.enrich_file_sizes(to_enrich) self.enrich_archive_duration(to_enrich) @@ -40,10 +36,10 @@ class MetaEnricher(Enricher): media.set("bytes", file_stats.st_size) media.set("size", self.human_readable_bytes(file_stats.st_size)) total_size += file_stats.st_size - + to_enrich.set("total_bytes", total_size) to_enrich.set("total_size", self.human_readable_bytes(total_size)) - + def human_readable_bytes(self, size: int) -> str: # receives number of bytes and returns human readble size diff --git a/src/auto_archiver/modules/metadata_enricher/__init__.py b/src/auto_archiver/modules/metadata_enricher/__init__.py new file mode 100644 index 0000000..020bd4a --- /dev/null +++ b/src/auto_archiver/modules/metadata_enricher/__init__.py @@ -0,0 +1 @@ +from .metadata_enricher import MetadataEnricher \ No newline at end of file diff --git a/src/auto_archiver/modules/metadata_enricher/__manifest__.py b/src/auto_archiver/modules/metadata_enricher/__manifest__.py new file mode 100644 index 0000000..bfc9b75 --- /dev/null +++ b/src/auto_archiver/modules/metadata_enricher/__manifest__.py @@ -0,0 +1,22 @@ +{ + "name": "Media Metadata Enricher", + "type": ["enricher"], + "requires_setup": False, + "external_dependencies": { + "python": ["loguru"], + "bin": ["exiftool"] + + }, + "description": """ + Extracts metadata information from files using ExifTool. + + ### Features + - Uses ExifTool to extract detailed metadata from media files. + - Processes file-specific data like camera settings, geolocation, timestamps, and other embedded metadata. + - Adds extracted metadata to the corresponding `Media` object within the `Metadata`. + + ### Notes + - Requires ExifTool to be installed and accessible via the system's PATH. + - Skips enrichment for files where metadata extraction fails. + """ +} diff --git a/src/auto_archiver/enrichers/metadata_enricher.py b/src/auto_archiver/modules/metadata_enricher/metadata_enricher.py similarity index 92% rename from src/auto_archiver/enrichers/metadata_enricher.py rename to src/auto_archiver/modules/metadata_enricher/metadata_enricher.py index 9fe257e..cb68b98 100644 --- a/src/auto_archiver/enrichers/metadata_enricher.py +++ b/src/auto_archiver/modules/metadata_enricher/metadata_enricher.py @@ -2,8 +2,8 @@ import subprocess import traceback from loguru import logger -from . import Enricher -from ..core import Metadata +from auto_archiver.base_processors import Enricher +from auto_archiver.core import Metadata class MetadataEnricher(Enricher): @@ -16,9 +16,6 @@ class MetadataEnricher(Enricher): # without this STEP.__init__ is not called super().__init__(config) - @staticmethod - def configs() -> dict: - return {} def enrich(self, to_enrich: Metadata) -> None: url = to_enrich.get_url() diff --git a/src/auto_archiver/modules/mute_formatter/__init__.py b/src/auto_archiver/modules/mute_formatter/__init__.py new file mode 100644 index 0000000..b92fce9 --- /dev/null +++ b/src/auto_archiver/modules/mute_formatter/__init__.py @@ -0,0 +1 @@ +from .mute_formatter import MuteFormatter diff --git a/src/auto_archiver/modules/mute_formatter/__manifest__.py b/src/auto_archiver/modules/mute_formatter/__manifest__.py new file mode 100644 index 0000000..af3f83a --- /dev/null +++ b/src/auto_archiver/modules/mute_formatter/__manifest__.py @@ -0,0 +1,9 @@ +m = { + "name": "Mute Formatter", + "type": ["formatter"], + "requires_setup": False, + "external_dependencies": { + }, + "description": """ Default formatter. + """, +} diff --git a/src/auto_archiver/formatters/mute_formatter.py b/src/auto_archiver/modules/mute_formatter/mute_formatter.py similarity index 100% rename from src/auto_archiver/formatters/mute_formatter.py rename to src/auto_archiver/modules/mute_formatter/mute_formatter.py diff --git a/src/auto_archiver/modules/pdq_hash_enricher/__init__.py b/src/auto_archiver/modules/pdq_hash_enricher/__init__.py new file mode 100644 index 0000000..b444197 --- /dev/null +++ b/src/auto_archiver/modules/pdq_hash_enricher/__init__.py @@ -0,0 +1 @@ +from .pdq_hash_enricher import PdqHashEnricher \ No newline at end of file diff --git a/src/auto_archiver/modules/pdq_hash_enricher/__manifest__.py b/src/auto_archiver/modules/pdq_hash_enricher/__manifest__.py new file mode 100644 index 0000000..7b418b1 --- /dev/null +++ b/src/auto_archiver/modules/pdq_hash_enricher/__manifest__.py @@ -0,0 +1,21 @@ +{ + "name": "PDQ Hash Enricher", + "type": ["enricher"], + "requires_setup": False, + "external_dependencies": { + "python": ["loguru", "pdqhash", "numpy", "Pillow"], + }, + "description": """ + PDQ Hash Enricher for generating perceptual hashes of media files. + + ### Features + - Calculates perceptual hashes for image files using the PDQ hashing algorithm. + - Enables detection of duplicate or near-duplicate visual content. + - Processes images stored in `Metadata` objects, adding computed hashes to the corresponding `Media` entries. + - Skips non-image media or files unsuitable for hashing (e.g., corrupted or unsupported formats). + + ### Notes + - Best used after enrichers like `thumbnail_enricher` or `screenshot_enricher` to ensure images are available. + - Uses the `pdqhash` library to compute 256-bit perceptual hashes, which are stored as hexadecimal strings. + """ +} diff --git a/src/auto_archiver/enrichers/pdq_hash_enricher.py b/src/auto_archiver/modules/pdq_hash_enricher/pdq_hash_enricher.py similarity index 95% rename from src/auto_archiver/enrichers/pdq_hash_enricher.py rename to src/auto_archiver/modules/pdq_hash_enricher/pdq_hash_enricher.py index 36f793d..7e3f467 100644 --- a/src/auto_archiver/enrichers/pdq_hash_enricher.py +++ b/src/auto_archiver/modules/pdq_hash_enricher/pdq_hash_enricher.py @@ -16,8 +16,8 @@ import numpy as np from PIL import Image, UnidentifiedImageError from loguru import logger -from . import Enricher -from ..core import Metadata +from auto_archiver.base_processors import Enricher +from auto_archiver.core import Metadata class PdqHashEnricher(Enricher): @@ -31,10 +31,6 @@ class PdqHashEnricher(Enricher): # Without this STEP.__init__ is not called super().__init__(config) - @staticmethod - def configs() -> dict: - return {} - def enrich(self, to_enrich: Metadata) -> None: url = to_enrich.get_url() logger.debug(f"calculating perceptual hashes for {url=}") diff --git a/src/auto_archiver/modules/s3_storage/__init__.py b/src/auto_archiver/modules/s3_storage/__init__.py new file mode 100644 index 0000000..1c826fd --- /dev/null +++ b/src/auto_archiver/modules/s3_storage/__init__.py @@ -0,0 +1 @@ +from .s3 import S3Storage \ No newline at end of file diff --git a/src/auto_archiver/modules/s3_storage/__manifest__.py b/src/auto_archiver/modules/s3_storage/__manifest__.py new file mode 100644 index 0000000..fc41eb3 --- /dev/null +++ b/src/auto_archiver/modules/s3_storage/__manifest__.py @@ -0,0 +1,49 @@ +{ + "name": "S3 Storage", + "type": ["storage"], + "requires_setup": True, + "external_dependencies": { + "python": ["boto3", "loguru"], + }, + "configs": { + "path_generator": { + "default": "url", + "help": "how to store the file in terms of directory structure: 'flat' sets to root; 'url' creates a directory based on the provided URL; 'random' creates a random directory.", + "choices": ["flat", "url", "random"], + }, + "filename_generator": { + "default": "random", + "help": "how to name stored files: 'random' creates a random string; 'static' uses a replicable strategy such as a hash.", + "choices": ["random", "static"], + }, + "bucket": {"default": None, "help": "S3 bucket name"}, + "region": {"default": None, "help": "S3 region name"}, + "key": {"default": None, "help": "S3 API key"}, + "secret": {"default": None, "help": "S3 API secret"}, + "random_no_duplicate": {"default": False, "help": f"if set, it will override `path_generator`, `filename_generator` and `folder`. It will check if the file already exists and if so it will not upload it again. Creates a new root folder path `{NO_DUPLICATES_FOLDER}`"}, + "endpoint_url": { + "default": 'https://{region}.digitaloceanspaces.com', + "help": "S3 bucket endpoint, {region} are inserted at runtime" + }, + "cdn_url": { + "default": 'https://{bucket}.{region}.cdn.digitaloceanspaces.com/{key}', + "help": "S3 CDN url, {bucket}, {region} and {key} are inserted at runtime" + }, + "private": {"default": False, "help": "if true S3 files will not be readable online"}, + }, + "description": """ + S3Storage: A storage module for saving media files to an S3-compatible object storage. + + ### Features + - Uploads media files to an S3 bucket with customizable configurations. + - Supports `random_no_duplicate` mode to avoid duplicate uploads by checking existing files based on SHA-256 hashes. + - Automatically generates unique paths for files when duplicates are found. + - Configurable endpoint and CDN URL for different S3-compatible providers. + - Supports both private and public file storage, with public files being readable online. + + ### Notes + - Requires S3 credentials (API key and secret) and a bucket name to function. + - The `random_no_duplicate` option ensures no duplicate uploads by leveraging hash-based folder structures. + - Uses `boto3` for interaction with the S3 API. + """ +} diff --git a/src/auto_archiver/storages/s3.py b/src/auto_archiver/modules/s3_storage/s3.py similarity index 66% rename from src/auto_archiver/storages/s3.py rename to src/auto_archiver/modules/s3_storage/s3.py index 5139068..a637259 100644 --- a/src/auto_archiver/storages/s3.py +++ b/src/auto_archiver/modules/s3_storage/s3.py @@ -2,10 +2,11 @@ from typing import IO import boto3, os -from ..utils.misc import random_str -from ..core import Media -from ..storages import Storage -from ..enrichers import HashEnricher +from auto_archiver.utils.misc import random_str +from auto_archiver.core import Media +from auto_archiver.base_processors import Storage +# TODO +from auto_archiver.modules.hash_enricher import HashEnricher from loguru import logger NO_DUPLICATES_FOLDER = "no-dups/" @@ -25,27 +26,6 @@ class S3Storage(Storage): if self.random_no_duplicate: logger.warning("random_no_duplicate is set to True, this will override `path_generator`, `filename_generator` and `folder`.") - @staticmethod - def configs() -> dict: - return dict( - Storage.configs(), - ** { - "bucket": {"default": None, "help": "S3 bucket name"}, - "region": {"default": None, "help": "S3 region name"}, - "key": {"default": None, "help": "S3 API key"}, - "secret": {"default": None, "help": "S3 API secret"}, - "random_no_duplicate": {"default": False, "help": f"if set, it will override `path_generator`, `filename_generator` and `folder`. It will check if the file already exists and if so it will not upload it again. Creates a new root folder path `{NO_DUPLICATES_FOLDER}`"}, - "endpoint_url": { - "default": 'https://{region}.digitaloceanspaces.com', - "help": "S3 bucket endpoint, {region} are inserted at runtime" - }, - "cdn_url": { - "default": 'https://{bucket}.{region}.cdn.digitaloceanspaces.com/{key}', - "help": "S3 CDN url, {bucket}, {region} and {key} are inserted at runtime" - }, - "private": {"default": False, "help": "if true S3 files will not be readable online"}, - }) - def get_cdn_url(self, media: Media) -> str: return self.cdn_url.format(bucket=self.bucket, region=self.region, key=media.key) diff --git a/src/auto_archiver/modules/screenshot_enricher/__init__.py b/src/auto_archiver/modules/screenshot_enricher/__init__.py new file mode 100644 index 0000000..393f726 --- /dev/null +++ b/src/auto_archiver/modules/screenshot_enricher/__init__.py @@ -0,0 +1 @@ +from .screenshot_enricher import ScreenshotEnricher diff --git a/src/auto_archiver/modules/screenshot_enricher/__manifest__.py b/src/auto_archiver/modules/screenshot_enricher/__manifest__.py new file mode 100644 index 0000000..c1a30e7 --- /dev/null +++ b/src/auto_archiver/modules/screenshot_enricher/__manifest__.py @@ -0,0 +1,30 @@ +{ + "name": "Screenshot Enricher", + "type": ["enricher"], + "requires_setup": True, + "external_dependencies": { + "python": ["loguru", "selenium"], + "bin": ["chromedriver"] + }, + "configs": { + "width": {"default": 1280, "help": "width of the screenshots"}, + "height": {"default": 720, "help": "height of the screenshots"}, + "timeout": {"default": 60, "help": "timeout for taking the screenshot"}, + "sleep_before_screenshot": {"default": 4, "help": "seconds to wait for the pages to load before taking screenshot"}, + "http_proxy": {"default": "", "help": "http proxy to use for the webdriver, eg http://proxy-user:password@proxy-ip:port"}, + "save_to_pdf": {"default": False, "help": "save the page as pdf along with the screenshot. PDF saving options can be adjusted with the 'print_options' parameter"}, + "print_options": {"default": {}, "help": "options to pass to the pdf printer"} + }, + "description": """ + Captures screenshots and optionally saves web pages as PDFs using a WebDriver. + + ### Features + - Takes screenshots of web pages, with configurable width, height, and timeout settings. + - Optionally saves pages as PDFs, with additional configuration for PDF printing options. + - Bypasses URLs detected as authentication walls. + - Integrates seamlessly with the metadata enrichment pipeline, adding screenshots and PDFs as media. + + ### Notes + - Requires a WebDriver (e.g., ChromeDriver) installed and accessible via the system's PATH. + """ +} diff --git a/src/auto_archiver/enrichers/screenshot_enricher.py b/src/auto_archiver/modules/screenshot_enricher/screenshot_enricher.py similarity index 62% rename from src/auto_archiver/enrichers/screenshot_enricher.py rename to src/auto_archiver/modules/screenshot_enricher/screenshot_enricher.py index b2ef096..0140875 100644 --- a/src/auto_archiver/enrichers/screenshot_enricher.py +++ b/src/auto_archiver/modules/screenshot_enricher/screenshot_enricher.py @@ -5,24 +5,15 @@ import base64 from selenium.common.exceptions import TimeoutException -from . import Enricher -from ..utils import Webdriver, UrlUtil, random_str -from ..core import Media, Metadata, ArchivingContext +from auto_archiver.base_processors import Enricher +from auto_archiver.utils import Webdriver, UrlUtil, random_str +from auto_archiver.core import Media, Metadata, ArchivingContext class ScreenshotEnricher(Enricher): name = "screenshot_enricher" - @staticmethod - def configs() -> dict: - return { - "width": {"default": 1280, "help": "width of the screenshots"}, - "height": {"default": 720, "help": "height of the screenshots"}, - "timeout": {"default": 60, "help": "timeout for taking the screenshot"}, - "sleep_before_screenshot": {"default": 4, "help": "seconds to wait for the pages to load before taking screenshot"}, - "http_proxy": {"default": "", "help": "http proxy to use for the webdriver, eg http://proxy-user:password@proxy-ip:port"}, - "save_to_pdf": {"default": False, "help": "save the page as pdf along with the screenshot. PDF saving options can be adjusted with the 'print_options' parameter"}, - "print_options": {"default": {}, "help": "options to pass to the pdf printer"} - } + def __init__(self, config: dict) -> None: + super().__init__(config) def enrich(self, to_enrich: Metadata) -> None: url = to_enrich.get_url() diff --git a/src/auto_archiver/modules/ssl_enricher/__init__.py b/src/auto_archiver/modules/ssl_enricher/__init__.py new file mode 100644 index 0000000..23d2bee --- /dev/null +++ b/src/auto_archiver/modules/ssl_enricher/__init__.py @@ -0,0 +1 @@ +from .ssl_enricher import SSLEnricher \ No newline at end of file diff --git a/src/auto_archiver/modules/ssl_enricher/__manifest__.py b/src/auto_archiver/modules/ssl_enricher/__manifest__.py new file mode 100644 index 0000000..f44fc94 --- /dev/null +++ b/src/auto_archiver/modules/ssl_enricher/__manifest__.py @@ -0,0 +1,22 @@ +{ + "name": "SSL Certificate Enricher", + "type": ["enricher"], + "requires_setup": False, + "external_dependencies": { + "python": ["loguru", "python-slugify"], + }, + "configs": { + "skip_when_nothing_archived": {"default": True, "help": "if true, will skip enriching when no media is archived"}, + }, + "description": """ + Retrieves SSL certificate information for a domain and stores it as a file. + + ### Features + - Fetches SSL certificates for domains using the HTTPS protocol. + - Stores certificates in PEM format and adds them as media to the metadata. + - Skips enrichment if no media has been archived, based on the `skip_when_nothing_archived` configuration. + + ### Notes + - Requires the target URL to use the HTTPS scheme; other schemes are not supported. + """ +} diff --git a/src/auto_archiver/enrichers/ssl_enricher.py b/src/auto_archiver/modules/ssl_enricher/ssl_enricher.py similarity index 73% rename from src/auto_archiver/enrichers/ssl_enricher.py rename to src/auto_archiver/modules/ssl_enricher/ssl_enricher.py index 396df2e..965f699 100644 --- a/src/auto_archiver/enrichers/ssl_enricher.py +++ b/src/auto_archiver/modules/ssl_enricher/ssl_enricher.py @@ -3,8 +3,8 @@ from slugify import slugify from urllib.parse import urlparse from loguru import logger -from . import Enricher -from ..core import Metadata, ArchivingContext, Media +from auto_archiver.base_processors import Enricher +from auto_archiver.core import Metadata, ArchivingContext, Media class SSLEnricher(Enricher): @@ -15,13 +15,7 @@ class SSLEnricher(Enricher): def __init__(self, config: dict) -> None: super().__init__(config) - self. skip_when_nothing_archived = bool(self.skip_when_nothing_archived) - - @staticmethod - def configs() -> dict: - return { - "skip_when_nothing_archived": {"default": True, "help": "if true, will skip enriching when no media is archived"}, - } + self.skip_when_nothing_archived = bool(self.skip_when_nothing_archived) def enrich(self, to_enrich: Metadata) -> None: if not to_enrich.media and self.skip_when_nothing_archived: return diff --git a/src/auto_archiver/modules/telegram_archiver/__init__.py b/src/auto_archiver/modules/telegram_archiver/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/auto_archiver/modules/telegram_extractor/__init__.py b/src/auto_archiver/modules/telegram_extractor/__init__.py new file mode 100644 index 0000000..1fd80c2 --- /dev/null +++ b/src/auto_archiver/modules/telegram_extractor/__init__.py @@ -0,0 +1 @@ +from .telegram_extractor import TelegramExtractor \ No newline at end of file diff --git a/src/auto_archiver/modules/telegram_archiver/__manifest__.py b/src/auto_archiver/modules/telegram_extractor/__manifest__.py similarity index 76% rename from src/auto_archiver/modules/telegram_archiver/__manifest__.py rename to src/auto_archiver/modules/telegram_extractor/__manifest__.py index b56477a..86b5e0f 100644 --- a/src/auto_archiver/modules/telegram_archiver/__manifest__.py +++ b/src/auto_archiver/modules/telegram_extractor/__manifest__.py @@ -1,9 +1,7 @@ { - "name": "Telegram Archiver", + "name": "Telegram Extractor", "type": ["extractor"], - "entry_point": "telegram_archiver:TelegramArchiver", "requires_setup": False, - "depends": ["core"], "external_dependencies": { "python": [ "requests", @@ -12,7 +10,7 @@ ], }, "description": """ - The `TelegramArchiver` retrieves publicly available media content from Telegram message links without requiring login credentials. + The `TelegramExtractor` retrieves publicly available media content from Telegram message links without requiring login credentials. It processes URLs to fetch images and videos embedded in Telegram messages, ensuring a structured output using `Metadata` and `Media` objects. Recommended for scenarios where login-based archiving is not viable, although `telethon_archiver` is advised for more comprehensive functionality. diff --git a/src/auto_archiver/modules/telegram_archiver/telegram_archiver.py b/src/auto_archiver/modules/telegram_extractor/telegram_extractor.py similarity index 89% rename from src/auto_archiver/modules/telegram_archiver/telegram_archiver.py rename to src/auto_archiver/modules/telegram_extractor/telegram_extractor.py index c793095..31bdaca 100644 --- a/src/auto_archiver/modules/telegram_archiver/telegram_archiver.py +++ b/src/auto_archiver/modules/telegram_extractor/telegram_extractor.py @@ -2,23 +2,20 @@ import requests, re, html from bs4 import BeautifulSoup from loguru import logger -from auto_archiver.archivers import Archiver +from auto_archiver.base_processors import Extractor from auto_archiver.core import Metadata, Media -class TelegramArchiver(Archiver): +class TelegramExtractor(Extractor): """ - Archiver for telegram that does not require login, but the telethon_archiver is much more advised, + Extractor for telegram that does not require login, but the telethon_extractor is much more advised, will only return if at least one image or one video is found """ - name = "telegram_archiver" + name = "telegram_extractor" def __init__(self, config: dict) -> None: super().__init__(config) - @staticmethod - def configs() -> dict: - return {} def download(self, item: Metadata) -> Metadata: url = item.get_url() diff --git a/src/auto_archiver/modules/telethon_archiver/__init__.py b/src/auto_archiver/modules/telethon_archiver/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/auto_archiver/modules/telethon_extractor/__init__.py b/src/auto_archiver/modules/telethon_extractor/__init__.py new file mode 100644 index 0000000..a837fdf --- /dev/null +++ b/src/auto_archiver/modules/telethon_extractor/__init__.py @@ -0,0 +1 @@ +from .telethon_extractor import TelethonArchiver \ No newline at end of file diff --git a/src/auto_archiver/modules/telethon_archiver/__manifest__.py b/src/auto_archiver/modules/telethon_extractor/__manifest__.py similarity index 84% rename from src/auto_archiver/modules/telethon_archiver/__manifest__.py rename to src/auto_archiver/modules/telethon_extractor/__manifest__.py index 82d56ba..5d71fdd 100644 --- a/src/auto_archiver/modules/telethon_archiver/__manifest__.py +++ b/src/auto_archiver/modules/telethon_extractor/__manifest__.py @@ -1,10 +1,8 @@ -# TODO rm dependency on json +import json { - "name": "telethon_archiver", + "name": "telethon_extractor", "type": ["extractor"], - "entry_point": "telethon_archiver:TelethonArchiver", "requires_setup": True, - "depends": [""], "external_dependencies": { "python": ["telethon", "loguru", @@ -21,12 +19,11 @@ "channel_invites": { "default": {}, "help": "(JSON string) private channel invite links (format: t.me/joinchat/HASH OR t.me/+HASH) and (optional but important to avoid hanging for minutes on startup) channel id (format: CHANNEL_ID taken from a post url like https://t.me/c/CHANNEL_ID/1), the telegram account will join any new channels on setup", - # TODO - #"cli_set": lambda cli_val, cur_val: dict(cur_val, **json.loads(cli_val)) + "type": "auto_archiver.utils.json_loader", } }, "description": """ -The `TelethonArchiver` uses the Telethon library to archive posts and media from Telegram channels and groups. +The `TelethonExtractor` uses the Telethon library to archive posts and media from Telegram channels and groups. It supports private and public channels, downloading grouped posts with media, and can join channels using invite links if provided in the configuration. @@ -38,7 +35,7 @@ if provided in the configuration. - Outputs structured metadata and media using `Metadata` and `Media` objects. ### Setup -To use the `TelethonArchiver`, you must configure the following: +To use the `TelethonExtractor`, you must configure the following: - **API ID and API Hash**: Obtain these from [my.telegram.org](https://my.telegram.org/apps). - **Session File**: Optional, but records login sessions for future use (default: `secrets/anon.session`). - **Bot Token**: Optional, allows access to additional content (e.g., large videos) but limits private channel archiving. diff --git a/src/auto_archiver/modules/telethon_archiver/telethon_archiver.py b/src/auto_archiver/modules/telethon_extractor/telethon_extractor.py similarity index 86% rename from src/auto_archiver/modules/telethon_archiver/telethon_archiver.py rename to src/auto_archiver/modules/telethon_extractor/telethon_extractor.py index 89668f3..8b49a10 100644 --- a/src/auto_archiver/modules/telethon_archiver/telethon_archiver.py +++ b/src/auto_archiver/modules/telethon_extractor/telethon_extractor.py @@ -8,13 +8,13 @@ from loguru import logger from tqdm import tqdm import re, time, json, os -from auto_archiver.archivers import Archiver +from auto_archiver.base_processors import Extractor from auto_archiver.core import Metadata, Media, ArchivingContext from auto_archiver.utils import random_str -class TelethonArchiver(Archiver): - name = "telethon_archiver" +class TelethonArchiver(Extractor): + name = "telethon_extractor" link_pattern = re.compile(r"https:\/\/t\.me(\/c){0,1}\/(.+)\/(\d+)") invite_pattern = re.compile(r"t.me(\/joinchat){0,1}\/\+?(.+)") @@ -23,20 +23,6 @@ class TelethonArchiver(Archiver): self.assert_valid_string("api_id") self.assert_valid_string("api_hash") - @staticmethod - def configs() -> dict: - return { - "api_id": {"default": None, "help": "telegram API_ID value, go to https://my.telegram.org/apps"}, - "api_hash": {"default": None, "help": "telegram API_HASH value, go to https://my.telegram.org/apps"}, - "bot_token": {"default": None, "help": "optional, but allows access to more content such as large videos, talk to @botfather"}, - "session_file": {"default": "secrets/anon", "help": "optional, records the telegram login session for future usage, '.session' will be appended to the provided value."}, - "join_channels": {"default": True, "help": "disables the initial setup with channel_invites config, useful if you have a lot and get stuck"}, - "channel_invites": { - "default": {}, - "help": "(JSON string) private channel invite links (format: t.me/joinchat/HASH OR t.me/+HASH) and (optional but important to avoid hanging for minutes on startup) channel id (format: CHANNEL_ID taken from a post url like https://t.me/c/CHANNEL_ID/1), the telegram account will join any new channels on setup", - "cli_set": lambda cli_val, cur_val: dict(cur_val, **json.loads(cli_val)) - } - } def setup(self) -> None: """ diff --git a/src/auto_archiver/modules/thumbnail_enricher/__init__.py b/src/auto_archiver/modules/thumbnail_enricher/__init__.py new file mode 100644 index 0000000..fe20719 --- /dev/null +++ b/src/auto_archiver/modules/thumbnail_enricher/__init__.py @@ -0,0 +1 @@ +from .thumbnail_enricher import ThumbnailEnricher diff --git a/src/auto_archiver/modules/thumbnail_enricher/__manifest__.py b/src/auto_archiver/modules/thumbnail_enricher/__manifest__.py new file mode 100644 index 0000000..2b0f167 --- /dev/null +++ b/src/auto_archiver/modules/thumbnail_enricher/__manifest__.py @@ -0,0 +1,27 @@ +{ + "name": "Thumbnail Enricher", + "type": ["enricher"], + "requires_setup": False, + "external_dependencies": { + "python": ["loguru", "ffmpeg-python"], + "bin": ["ffmpeg"] + }, + "configs": { + "thumbnails_per_minute": {"default": 60, "help": "how many thumbnails to generate per minute of video, can be limited by max_thumbnails"}, + "max_thumbnails": {"default": 16, "help": "limit the number of thumbnails to generate per video, 0 means no limit"}, + }, + "description": """ + Generates thumbnails for video files to provide visual previews. + + ### Features + - Processes video files and generates evenly distributed thumbnails. + - Calculates the number of thumbnails based on video duration, `thumbnails_per_minute`, and `max_thumbnails`. + - Distributes thumbnails equally across the video's duration and stores them as media objects. + - Adds metadata for each thumbnail, including timestamps and IDs. + + ### Notes + - Requires `ffmpeg` to be installed and accessible via the system's PATH. + - Handles videos without pre-existing duration metadata by probing with `ffmpeg`. + - Skips enrichment for non-video media files. + """ +} diff --git a/src/auto_archiver/enrichers/thumbnail_enricher.py b/src/auto_archiver/modules/thumbnail_enricher/thumbnail_enricher.py similarity index 86% rename from src/auto_archiver/enrichers/thumbnail_enricher.py rename to src/auto_archiver/modules/thumbnail_enricher/thumbnail_enricher.py index 5d8bee2..8c34502 100644 --- a/src/auto_archiver/enrichers/thumbnail_enricher.py +++ b/src/auto_archiver/modules/thumbnail_enricher/thumbnail_enricher.py @@ -9,9 +9,9 @@ and identify important moments without watching the entire video. import ffmpeg, os from loguru import logger -from . import Enricher -from ..core import Media, Metadata, ArchivingContext -from ..utils.misc import random_str +from auto_archiver.base_processors import Enricher +from auto_archiver.core import Media, Metadata, ArchivingContext +from auto_archiver.utils.misc import random_str class ThumbnailEnricher(Enricher): @@ -25,13 +25,6 @@ class ThumbnailEnricher(Enricher): super().__init__(config) self.thumbnails_per_second = int(self.thumbnails_per_minute) / 60 self.max_thumbnails = int(self.max_thumbnails) - - @staticmethod - def configs() -> dict: - return { - "thumbnails_per_minute": {"default": 60, "help": "how many thumbnails to generate per minute of video, can be limited by max_thumbnails"}, - "max_thumbnails": {"default": 16, "help": "limit the number of thumbnails to generate per video, 0 means no limit"}, - } def enrich(self, to_enrich: Metadata) -> None: """ diff --git a/src/auto_archiver/modules/timestamping_enricher/__init__.py b/src/auto_archiver/modules/timestamping_enricher/__init__.py new file mode 100644 index 0000000..62d358a --- /dev/null +++ b/src/auto_archiver/modules/timestamping_enricher/__init__.py @@ -0,0 +1 @@ +from .timestamping_enricher import TimestampingEnricher diff --git a/src/auto_archiver/modules/timestamping_enricher/__manifest__.py b/src/auto_archiver/modules/timestamping_enricher/__manifest__.py new file mode 100644 index 0000000..496d211 --- /dev/null +++ b/src/auto_archiver/modules/timestamping_enricher/__manifest__.py @@ -0,0 +1,54 @@ +{ + "name": "Timestamping Enricher", + "type": ["enricher"], + "requires_setup": True, + "external_dependencies": { + "python": [ + "loguru", + "slugify", + "tsp_client", + "asn1crypto", + "certvalidator", + "certifi" + ], + }, + "configs": { + "tsa_urls": { + "default": [ + # [Adobe Approved Trust List] and [Windows Cert Store] + "http://timestamp.digicert.com", + "http://timestamp.identrust.com", + # "https://timestamp.entrust.net/TSS/RFC3161sha2TS", # not valid for timestamping + # "https://timestamp.sectigo.com", # wait 15 seconds between each request. + + # [Adobe: European Union Trusted Lists]. + # "https://timestamp.sectigo.com/qualified", # wait 15 seconds between each request. + + # [Windows Cert Store] + "http://timestamp.globalsign.com/tsa/r6advanced1", + # [Adobe: European Union Trusted Lists] and [Windows Cert Store] + # "http://ts.quovadisglobal.com/eu", # not valid for timestamping + # "http://tsa.belgium.be/connect", # self-signed certificate in certificate chain + # "https://timestamp.aped.gov.gr/qtss", # self-signed certificate in certificate chain + # "http://tsa.sep.bg", # self-signed certificate in certificate chain + # "http://tsa.izenpe.com", #unable to get local issuer certificate + # "http://kstamp.keynectis.com/KSign", # unable to get local issuer certificate + "http://tss.accv.es:8318/tsa", + ], + "help": "List of RFC3161 Time Stamp Authorities to use, separate with commas if passed via the command line.", + } + }, + "description": """ + Generates RFC3161-compliant timestamp tokens using Time Stamp Authorities (TSA) for archived files. + + ### Features + - Creates timestamp tokens to prove the existence of files at a specific time, useful for legal and authenticity purposes. + - Aggregates file hashes into a text file and timestamps the concatenated data. + - Uses multiple Time Stamp Authorities (TSAs) to ensure reliability and redundancy. + - Validates timestamping certificates against trusted Certificate Authorities (CAs) using the `certifi` trust store. + + ### Notes + - Should be run after the `hash_enricher` to ensure file hashes are available. + - Requires internet access to interact with the configured TSAs. + """ +} diff --git a/src/auto_archiver/enrichers/timestamping_enricher.py b/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py similarity index 72% rename from src/auto_archiver/enrichers/timestamping_enricher.py rename to src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py index dffa1a3..c90d42c 100644 --- a/src/auto_archiver/enrichers/timestamping_enricher.py +++ b/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py @@ -8,9 +8,9 @@ from certvalidator import CertificateValidator, ValidationContext from asn1crypto import pem import certifi -from . import Enricher -from ..core import Metadata, ArchivingContext, Media -from ..archivers import Archiver +from auto_archiver.base_processors import Enricher +from auto_archiver.core import Metadata, ArchivingContext, Media +from auto_archiver.base_processors import Extractor class TimestampingEnricher(Enricher): @@ -26,37 +26,6 @@ class TimestampingEnricher(Enricher): def __init__(self, config: dict) -> None: super().__init__(config) - @staticmethod - def configs() -> dict: - return { - "tsa_urls": { - "default": [ - # [Adobe Approved Trust List] and [Windows Cert Store] - "http://timestamp.digicert.com", - "http://timestamp.identrust.com", - # "https://timestamp.entrust.net/TSS/RFC3161sha2TS", # not valid for timestamping - # "https://timestamp.sectigo.com", # wait 15 seconds between each request. - - # [Adobe: European Union Trusted Lists]. - # "https://timestamp.sectigo.com/qualified", # wait 15 seconds between each request. - - # [Windows Cert Store] - "http://timestamp.globalsign.com/tsa/r6advanced1", - - # [Adobe: European Union Trusted Lists] and [Windows Cert Store] - # "http://ts.quovadisglobal.com/eu", # not valid for timestamping - # "http://tsa.belgium.be/connect", # self-signed certificate in certificate chain - # "https://timestamp.aped.gov.gr/qtss", # self-signed certificate in certificate chain - # "http://tsa.sep.bg", # self-signed certificate in certificate chain - # "http://tsa.izenpe.com", #unable to get local issuer certificate - # "http://kstamp.keynectis.com/KSign", # unable to get local issuer certificate - "http://tss.accv.es:8318/tsa", - ], - "help": "List of RFC3161 Time Stamp Authorities to use, separate with commas if passed via the command line.", - "cli_set": lambda cli_val, cur_val: set(cli_val.split(",")) - } - } - def enrich(self, to_enrich: Metadata) -> None: url = to_enrich.get_url() logger.debug(f"RFC3161 timestamping existing files for {url=}") diff --git a/src/auto_archiver/modules/twitter_api_archiver/__init__.py b/src/auto_archiver/modules/twitter_api_archiver/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/auto_archiver/modules/twitter_api_extractor/__init__.py b/src/auto_archiver/modules/twitter_api_extractor/__init__.py new file mode 100644 index 0000000..7005965 --- /dev/null +++ b/src/auto_archiver/modules/twitter_api_extractor/__init__.py @@ -0,0 +1 @@ +from .twitter_api_extractor import TwitterApiExtractor \ No newline at end of file diff --git a/src/auto_archiver/modules/twitter_api_archiver/__manifest__.py b/src/auto_archiver/modules/twitter_api_extractor/__manifest__.py similarity index 88% rename from src/auto_archiver/modules/twitter_api_archiver/__manifest__.py rename to src/auto_archiver/modules/twitter_api_extractor/__manifest__.py index b415679..02d0d6c 100644 --- a/src/auto_archiver/modules/twitter_api_archiver/__manifest__.py +++ b/src/auto_archiver/modules/twitter_api_extractor/__manifest__.py @@ -1,9 +1,7 @@ { - "name": "Twitter API Archiver", + "name": "Twitter API Extractor", "type": ["extractor"], - "entry_point": "twitter_api_archiver:TwitterApiArchiver", "requires_setup": True, - "depends": ["core"], "external_dependencies": { "python": ["requests", "loguru", @@ -13,14 +11,15 @@ }, "configs": { "bearer_token": {"default": None, "help": "[deprecated: see bearer_tokens] twitter API bearer_token which is enough for archiving, if not provided you will need consumer_key, consumer_secret, access_token, access_secret"}, - "bearer_tokens": {"default": [], "help": " a list of twitter API bearer_token which is enough for archiving, if not provided you will need consumer_key, consumer_secret, access_token, access_secret, if provided you can still add those for better rate limits. CSV of bearer tokens if provided via the command line"}, + "bearer_tokens": {"default": [], "help": " a list of twitter API bearer_token which is enough for archiving, if not provided you will need consumer_key, consumer_secret, access_token, access_secret, if provided you can still add those for better rate limits. CSV of bearer tokens if provided via the command line", + }, "consumer_key": {"default": None, "help": "twitter API consumer_key"}, "consumer_secret": {"default": None, "help": "twitter API consumer_secret"}, "access_token": {"default": None, "help": "twitter API access_token"}, "access_secret": {"default": None, "help": "twitter API access_secret"}, }, "description": """ - The `TwitterApiArchiver` fetches tweets and associated media using the Twitter API. + The `TwitterApiExtractor` fetches tweets and associated media using the Twitter API. It supports multiple API configurations for extended rate limits and reliable access. Features include URL expansion, media downloads (e.g., images, videos), and structured output via `Metadata` and `Media` objects. Requires Twitter API credentials such as bearer tokens @@ -34,7 +33,7 @@ - Outputs structured metadata and media using `Metadata` and `Media` objects. ### Setup - To use the `TwitterApiArchiver`, you must provide valid Twitter API credentials via configuration: + To use the `TwitterApiExtractor`, you must provide valid Twitter API credentials via configuration: - **Bearer Token(s)**: A single token or a list for rate-limited API access. - **Consumer Key and Secret**: Required for user-authenticated API access. - **Access Token and Secret**: Complements the consumer key for enhanced API capabilities. diff --git a/src/auto_archiver/modules/twitter_api_archiver/twitter_api_archiver.py b/src/auto_archiver/modules/twitter_api_extractor/twitter_api_extractor.py similarity index 81% rename from src/auto_archiver/modules/twitter_api_archiver/twitter_api_archiver.py rename to src/auto_archiver/modules/twitter_api_extractor/twitter_api_extractor.py index eb607cc..ea669b4 100644 --- a/src/auto_archiver/modules/twitter_api_archiver/twitter_api_archiver.py +++ b/src/auto_archiver/modules/twitter_api_extractor/twitter_api_extractor.py @@ -8,11 +8,11 @@ from loguru import logger from pytwitter import Api from slugify import slugify -from auto_archiver.archivers import Archiver +from auto_archiver.base_processors import Extractor from auto_archiver.core import Metadata,Media -class TwitterApiArchiver(Archiver): - name = "twitter_api_archiver" +class TwitterApiExtractor(Extractor): + name = "twitter_api_extractor" link_pattern = re.compile(r"(?:twitter|x).com\/(?:\#!\/)?(\w+)\/status(?:es)?\/(\d+)") def __init__(self, config: dict) -> None: @@ -34,17 +34,6 @@ class TwitterApiArchiver(Archiver): access_token=self.access_token, access_secret=self.access_secret)) assert self.api_client is not None, "Missing Twitter API configurations, please provide either AND/OR (consumer_key, consumer_secret, access_token, access_secret) to use this archiver, you can provide both for better rate-limit results." - @staticmethod - def configs() -> dict: - return { - "bearer_token": {"default": None, "help": "[deprecated: see bearer_tokens] twitter API bearer_token which is enough for archiving, if not provided you will need consumer_key, consumer_secret, access_token, access_secret"}, - "bearer_tokens": {"default": [], "help": " a list of twitter API bearer_token which is enough for archiving, if not provided you will need consumer_key, consumer_secret, access_token, access_secret, if provided you can still add those for better rate limits. CSV of bearer tokens if provided via the command line", "cli_set": lambda cli_val, cur_val: list(set(cli_val.split(",")))}, - "consumer_key": {"default": None, "help": "twitter API consumer_key"}, - "consumer_secret": {"default": None, "help": "twitter API consumer_secret"}, - "access_token": {"default": None, "help": "twitter API access_token"}, - "access_secret": {"default": None, "help": "twitter API access_secret"}, - } - @property # getter .mimetype def api_client(self) -> str: return self.apis[self.api_index] diff --git a/src/auto_archiver/modules/vk_archiver/__init__.py b/src/auto_archiver/modules/vk_archiver/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/auto_archiver/modules/vk_extractor/__init__.py b/src/auto_archiver/modules/vk_extractor/__init__.py new file mode 100644 index 0000000..0f9bcad --- /dev/null +++ b/src/auto_archiver/modules/vk_extractor/__init__.py @@ -0,0 +1 @@ +from .vk_extractor import VkExtractor diff --git a/src/auto_archiver/modules/vk_archiver/__manifest__.py b/src/auto_archiver/modules/vk_extractor/__manifest__.py similarity index 90% rename from src/auto_archiver/modules/vk_archiver/__manifest__.py rename to src/auto_archiver/modules/vk_extractor/__manifest__.py index 69bf162..bdcaf99 100644 --- a/src/auto_archiver/modules/vk_archiver/__manifest__.py +++ b/src/auto_archiver/modules/vk_extractor/__manifest__.py @@ -1,7 +1,6 @@ { - "name": "VKontakte Archiver", + "name": "VKontakte Extractor", "type": ["extractor"], - "entry_point": "vk_archiver:VKArchiver", "requires_setup": True, "depends": ["core", "utils"], "external_dependencies": { @@ -14,7 +13,7 @@ "session_file": {"default": "secrets/vk_config.v2.json", "help": "valid VKontakte password"}, }, "description": """ -The `VkArchiver` fetches posts, text, and images from VK (VKontakte) social media pages. +The `VkExtractor` fetches posts, text, and images from VK (VKontakte) social media pages. This archiver is specialized for `/wall` posts and uses the `VkScraper` library to extract and download content. Note that VK videos are handled separately by the `YTDownloader`. diff --git a/src/auto_archiver/modules/vk_archiver/vk_archiver.py b/src/auto_archiver/modules/vk_extractor/vk_extractor.py similarity index 77% rename from src/auto_archiver/modules/vk_archiver/vk_archiver.py rename to src/auto_archiver/modules/vk_extractor/vk_extractor.py index 3cfb446..eb4c171 100644 --- a/src/auto_archiver/modules/vk_archiver/vk_archiver.py +++ b/src/auto_archiver/modules/vk_extractor/vk_extractor.py @@ -2,16 +2,16 @@ from loguru import logger from vk_url_scraper import VkScraper from auto_archiver.utils.misc import dump_payload -from auto_archiver.archivers import Archiver +from auto_archiver.base_processors import Extractor from auto_archiver.core import Metadata, Media, ArchivingContext -class VkArchiver(Archiver): +class VkExtractor(Extractor): """" VK videos are handled by YTDownloader, this archiver gets posts text and images. Currently only works for /wall posts """ - name = "vk_archiver" + name = "vk_extractor" def __init__(self, config: dict) -> None: super().__init__(config) @@ -19,14 +19,6 @@ class VkArchiver(Archiver): self.assert_valid_string("password") self.vks = VkScraper(self.username, self.password, session_file=self.session_file) - @staticmethod - def configs() -> dict: - return { - "username": {"default": None, "help": "valid VKontakte username"}, - "password": {"default": None, "help": "valid VKontakte password"}, - "session_file": {"default": "secrets/vk_config.v2.json", "help": "valid VKontakte password"}, - } - def download(self, item: Metadata) -> Metadata: url = item.get_url() diff --git a/src/auto_archiver/modules/wacz_enricher/__init__.py b/src/auto_archiver/modules/wacz_enricher/__init__.py new file mode 100644 index 0000000..686b8d8 --- /dev/null +++ b/src/auto_archiver/modules/wacz_enricher/__init__.py @@ -0,0 +1 @@ +from .wacz_enricher import WaczExtractorEnricher diff --git a/src/auto_archiver/modules/wacz_enricher/__manifest__.py b/src/auto_archiver/modules/wacz_enricher/__manifest__.py new file mode 100644 index 0000000..07983d9 --- /dev/null +++ b/src/auto_archiver/modules/wacz_enricher/__manifest__.py @@ -0,0 +1,39 @@ +{ + "name": "WACZ Enricher", + "type": ["enricher", "archiver"], + "requires_setup": True, + "external_dependencies": { + "python": [ + "loguru", + "jsonlines", + "warcio" + ], + # TODO? + "bin": [ + "docker" + ] + }, + "configs": { + "profile": {"default": None, "help": "browsertrix-profile (for profile generation see https://github.com/webrecorder/browsertrix-crawler#creating-and-using-browser-profiles)."}, + "docker_commands": {"default": None, "help":"if a custom docker invocation is needed"}, + "timeout": {"default": 120, "help": "timeout for WACZ generation in seconds"}, + "extract_media": {"default": False, "help": "If enabled all the images/videos/audio present in the WACZ archive will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched."}, + "extract_screenshot": {"default": True, "help": "If enabled the screenshot captured by browsertrix will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched."}, + "socks_proxy_host": {"default": None, "help": "SOCKS proxy host for browsertrix-crawler, use in combination with socks_proxy_port. eg: user:password@host"}, + "socks_proxy_port": {"default": None, "help": "SOCKS proxy port for browsertrix-crawler, use in combination with socks_proxy_host. eg 1234"}, + "proxy_server": {"default": None, "help": "SOCKS server proxy URL, in development"}, + }, + "description": """ + Creates .WACZ archives of web pages using the `browsertrix-crawler` tool, with options for media extraction and screenshot saving. + + ### Features + - Archives web pages into .WACZ format using Docker or direct invocation of `browsertrix-crawler`. + - Supports custom profiles for archiving private or dynamic content. + - Extracts media (images, videos, audio) and screenshots from the archive, optionally adding them to the enrichment pipeline. + - Generates metadata from the archived page's content and structure (e.g., titles, text). + + ### Notes + - Requires Docker for running `browsertrix-crawler` unless explicitly disabled. + - Configurable via parameters for timeout, media extraction, screenshots, and proxy settings. + """ +} diff --git a/src/auto_archiver/enrichers/wacz_enricher.py b/src/auto_archiver/modules/wacz_enricher/wacz_enricher.py similarity index 87% rename from src/auto_archiver/enrichers/wacz_enricher.py rename to src/auto_archiver/modules/wacz_enricher/wacz_enricher.py index 3c39056..9ba43ae 100644 --- a/src/auto_archiver/enrichers/wacz_enricher.py +++ b/src/auto_archiver/modules/wacz_enricher/wacz_enricher.py @@ -5,13 +5,12 @@ from zipfile import ZipFile from loguru import logger from warcio.archiveiterator import ArchiveIterator -from ..core import Media, Metadata, ArchivingContext -from . import Enricher -from ..archivers import Archiver -from ..utils import UrlUtil, random_str +from auto_archiver.core import Media, Metadata, ArchivingContext +from auto_archiver.base_processors import Extractor, Enricher +from auto_archiver.utils import UrlUtil, random_str -class WaczArchiverEnricher(Enricher, Archiver): +class WaczExtractorEnricher(Enricher, Extractor): """ Uses https://github.com/webrecorder/browsertrix-crawler to generate a .WACZ archive of the URL If used with [profiles](https://github.com/webrecorder/browsertrix-crawler#creating-and-using-browser-profiles) @@ -24,19 +23,6 @@ class WaczArchiverEnricher(Enricher, Archiver): # without this STEP.__init__ is not called super().__init__(config) - @staticmethod - def configs() -> dict: - return { - "profile": {"default": None, "help": "browsertrix-profile (for profile generation see https://github.com/webrecorder/browsertrix-crawler#creating-and-using-browser-profiles)."}, - "docker_commands": {"default": None, "help":"if a custom docker invocation is needed"}, - "timeout": {"default": 120, "help": "timeout for WACZ generation in seconds"}, - "extract_media": {"default": False, "help": "If enabled all the images/videos/audio present in the WACZ archive will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched."}, - "extract_screenshot": {"default": True, "help": "If enabled the screenshot captured by browsertrix will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched."}, - "socks_proxy_host": {"default": None, "help": "SOCKS proxy host for browsertrix-crawler, use in combination with socks_proxy_port. eg: user:password@host"}, - "socks_proxy_port": {"default": None, "help": "SOCKS proxy port for browsertrix-crawler, use in combination with socks_proxy_host. eg 1234"}, - "proxy_server": {"default": None, "help": "SOCKS server proxy URL, in development"}, - } - def setup(self) -> None: self.use_docker = os.environ.get('WACZ_ENABLE_DOCKER') or not os.environ.get('RUNNING_IN_DOCKER') self.docker_in_docker = os.environ.get('WACZ_ENABLE_DOCKER') and os.environ.get('RUNNING_IN_DOCKER') diff --git a/src/auto_archiver/modules/wayback_enricher/__init__.py b/src/auto_archiver/modules/wayback_enricher/__init__.py new file mode 100644 index 0000000..9782831 --- /dev/null +++ b/src/auto_archiver/modules/wayback_enricher/__init__.py @@ -0,0 +1 @@ +from .wayback_enricher import WaybackExtractorEnricher \ No newline at end of file diff --git a/src/auto_archiver/modules/wayback_enricher/__manifest__.py b/src/auto_archiver/modules/wayback_enricher/__manifest__.py new file mode 100644 index 0000000..b3af284 --- /dev/null +++ b/src/auto_archiver/modules/wayback_enricher/__manifest__.py @@ -0,0 +1,29 @@ +{ + "name": "Wayback Machine Enricher", + "type": ["enricher", "archiver"], + "requires_setup": True, + "external_dependencies": { + "python": ["loguru", "requests"], + }, + "configs": { + "timeout": {"default": 15, "help": "seconds to wait for successful archive confirmation from wayback, if more than this passes the result contains the job_id so the status can later be checked manually."}, + "if_not_archived_within": {"default": None, "help": "only tell wayback to archive if no archive is available before the number of seconds specified, use None to ignore this option. For more information: https://docs.google.com/document/d/1Nsv52MvSjbLb2PCpHlat0gkzw0EvtSgpKHu4mk0MnrA"}, + "key": {"default": None, "help": "wayback API key. to get credentials visit https://archive.org/account/s3.php"}, + "secret": {"default": None, "help": "wayback API secret. to get credentials visit https://archive.org/account/s3.php"}, + "proxy_http": {"default": None, "help": "http proxy to use for wayback requests, eg http://proxy-user:password@proxy-ip:port"}, + "proxy_https": {"default": None, "help": "https proxy to use for wayback requests, eg https://proxy-user:password@proxy-ip:port"}, + }, + "description": """ + Submits the current URL to the Wayback Machine for archiving and returns either a job ID or the completed archive URL. + + ### Features + - Archives URLs using the Internet Archive's Wayback Machine API. + - Supports conditional archiving based on the existence of prior archives within a specified time range. + - Provides proxies for HTTP and HTTPS requests. + - Fetches and confirms the archive URL or provides a job ID for later status checks. + + ### Notes + - Requires a valid Wayback Machine API key and secret. + - Handles rate-limiting by Wayback Machine and retries status checks with exponential backoff. + """ +} diff --git a/src/auto_archiver/enrichers/wayback_enricher.py b/src/auto_archiver/modules/wayback_enricher/wayback_enricher.py similarity index 77% rename from src/auto_archiver/enrichers/wayback_enricher.py rename to src/auto_archiver/modules/wayback_enricher/wayback_enricher.py index 305bfcf..6942727 100644 --- a/src/auto_archiver/enrichers/wayback_enricher.py +++ b/src/auto_archiver/modules/wayback_enricher/wayback_enricher.py @@ -2,12 +2,11 @@ import json from loguru import logger import time, requests -from . import Enricher -from ..archivers import Archiver -from ..utils import UrlUtil -from ..core import Metadata +from auto_archiver.base_processors import Extractor, Enricher +from auto_archiver.utils import UrlUtil +from auto_archiver.core import Metadata -class WaybackArchiverEnricher(Enricher, Archiver): +class WaybackExtractorEnricher(Enricher, Extractor): """ Submits the current URL to the webarchive and returns a job_id or completed archive. @@ -21,17 +20,6 @@ class WaybackArchiverEnricher(Enricher, Archiver): assert type(self.secret) == str and len(self.secret) > 0, "please provide a value for the wayback_enricher API key" assert type(self.secret) == str and len(self.secret) > 0, "please provide a value for the wayback_enricher API secret" - @staticmethod - def configs() -> dict: - return { - "timeout": {"default": 15, "help": "seconds to wait for successful archive confirmation from wayback, if more than this passes the result contains the job_id so the status can later be checked manually."}, - "if_not_archived_within": {"default": None, "help": "only tell wayback to archive if no archive is available before the number of seconds specified, use None to ignore this option. For more information: https://docs.google.com/document/d/1Nsv52MvSjbLb2PCpHlat0gkzw0EvtSgpKHu4mk0MnrA"}, - "key": {"default": None, "help": "wayback API key. to get credentials visit https://archive.org/account/s3.php"}, - "secret": {"default": None, "help": "wayback API secret. to get credentials visit https://archive.org/account/s3.php"}, - "proxy_http": {"default": None, "help": "http proxy to use for wayback requests, eg http://proxy-user:password@proxy-ip:port"}, - "proxy_https": {"default": None, "help": "https proxy to use for wayback requests, eg https://proxy-user:password@proxy-ip:port"}, - } - def download(self, item: Metadata) -> Metadata: # this new Metadata object is required to avoid duplication result = Metadata() diff --git a/src/auto_archiver/modules/whisper_enricher/__init__.py b/src/auto_archiver/modules/whisper_enricher/__init__.py new file mode 100644 index 0000000..d3d3526 --- /dev/null +++ b/src/auto_archiver/modules/whisper_enricher/__init__.py @@ -0,0 +1 @@ +from .whisper_enricher import WhisperEnricher \ No newline at end of file diff --git a/src/auto_archiver/modules/whisper_enricher/__manifest__.py b/src/auto_archiver/modules/whisper_enricher/__manifest__.py new file mode 100644 index 0000000..25eae25 --- /dev/null +++ b/src/auto_archiver/modules/whisper_enricher/__manifest__.py @@ -0,0 +1,30 @@ +{ + "name": "Whisper Enricher", + "type": ["enricher"], + "requires_setup": True, + "external_dependencies": { + "python": ["loguru", "requests"], + }, + "configs": { + "api_endpoint": {"default": None, "help": "WhisperApi api endpoint, eg: https://whisperbox-api.com/api/v1, a deployment of https://github.com/bellingcat/whisperbox-transcribe."}, + "api_key": {"default": None, "help": "WhisperApi api key for authentication"}, + "include_srt": {"default": False, "help": "Whether to include a subtitle SRT (SubRip Subtitle file) for the video (can be used in video players)."}, + "timeout": {"default": 90, "help": "How many seconds to wait at most for a successful job completion."}, + "action": {"default": "translate", "help": "which Whisper operation to execute", "choices": ["transcribe", "translate", "language_detection"]}, + }, + "description": """ + Integrates with a Whisper API service to transcribe, translate, or detect the language of audio and video files. + + ### Features + - Submits audio or video files to a Whisper API deployment for processing. + - Supports operations such as transcription, translation, and language detection. + - Optionally generates SRT subtitle files for video content. + - Integrates with S3-compatible storage systems to make files publicly accessible for processing. + - Handles job submission, status checking, artifact retrieval, and cleanup. + + ### Notes + - Requires a Whisper API endpoint and API key for authentication. + - Only compatible with S3-compatible storage systems for media file accessibility. + - Handles multiple jobs and retries for failed or incomplete processing. + """ +} diff --git a/src/auto_archiver/enrichers/whisper_enricher.py b/src/auto_archiver/modules/whisper_enricher/whisper_enricher.py similarity index 87% rename from src/auto_archiver/enrichers/whisper_enricher.py rename to src/auto_archiver/modules/whisper_enricher/whisper_enricher.py index c0089a4..d14c537 100644 --- a/src/auto_archiver/enrichers/whisper_enricher.py +++ b/src/auto_archiver/modules/whisper_enricher/whisper_enricher.py @@ -2,9 +2,9 @@ import traceback import requests, time from loguru import logger -from . import Enricher -from ..core import Metadata, Media, ArchivingContext -from ..storages import S3Storage +from auto_archiver.base_processors import Enricher +from auto_archiver.core import Metadata, Media, ArchivingContext +from auto_archiver.modules.s3_storage import S3Storage class WhisperEnricher(Enricher): @@ -22,17 +22,6 @@ class WhisperEnricher(Enricher): assert type(self.api_key) == str and len(self.api_key) > 0, "please provide a value for the whisper_enricher api_key" self.timeout = int(self.timeout) - @staticmethod - def configs() -> dict: - return { - "api_endpoint": {"default": None, "help": "WhisperApi api endpoint, eg: https://whisperbox-api.com/api/v1, a deployment of https://github.com/bellingcat/whisperbox-transcribe."}, - "api_key": {"default": None, "help": "WhisperApi api key for authentication"}, - "include_srt": {"default": False, "help": "Whether to include a subtitle SRT (SubRip Subtitle file) for the video (can be used in video players)."}, - "timeout": {"default": 90, "help": "How many seconds to wait at most for a successful job completion."}, - "action": {"default": "translate", "help": "which Whisper operation to execute", "choices": ["transcribe", "translate", "language_detection"]}, - - } - def enrich(self, to_enrich: Metadata) -> None: if not self._get_s3_storage(): logger.error("WhisperEnricher: To use the WhisperEnricher you need to use S3Storage so files are accessible publicly to the whisper service being called.") diff --git a/src/auto_archiver/storages/__init__.py b/src/auto_archiver/storages/__init__.py deleted file mode 100644 index 0765833..0000000 --- a/src/auto_archiver/storages/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -""" This module contains the storage classes for the auto-archiver. - -""" \ No newline at end of file diff --git a/src/auto_archiver/utils/__init__.py b/src/auto_archiver/utils/__init__.py index 788f159..d2063d0 100644 --- a/src/auto_archiver/utils/__init__.py +++ b/src/auto_archiver/utils/__init__.py @@ -1,9 +1,7 @@ """ Auto Archiver Utilities. """ # we need to explicitly expose the available imports here -from .gworksheet import GWorksheet from .misc import * from .webdriver import Webdriver -from .gsheet import Gsheets from .url import UrlUtil from .atlos import get_atlos_config_options diff --git a/src/auto_archiver/utils/gsheet.py b/src/auto_archiver/utils/gsheet.py index f84aab2..485344f 100644 --- a/src/auto_archiver/utils/gsheet.py +++ b/src/auto_archiver/utils/gsheet.py @@ -16,6 +16,7 @@ class Gsheets(Step): assert type(self.header) == int, f"header ({self.header}) value must be an integer not {type(self.header)}" assert self.sheet is not None or self.sheet_id is not None, "You need to define either a 'sheet' name or a 'sheet_id' in your orchestration file when using gsheets." + # TODO merge this into gsheets processors manifest @staticmethod def configs() -> dict: return { diff --git a/src/auto_archiver/utils/misc.py b/src/auto_archiver/utils/misc.py index e312fc6..e985e3e 100644 --- a/src/auto_archiver/utils/misc.py +++ b/src/auto_archiver/utils/misc.py @@ -53,4 +53,7 @@ def update_nested_dict(dictionary, update_dict): def random_str(length: int = 32) -> str: assert length <= 32, "length must be less than 32 as UUID4 is used" - return str(uuid.uuid4()).replace("-", "")[:length] \ No newline at end of file + return str(uuid.uuid4()).replace("-", "")[:length] + +def json_loader(cli_val): + return json.loads(cli_val) diff --git a/tests/archivers/test_archiver_base.py b/tests/archivers/test_archiver_base.py index d793706..6223879 100644 --- a/tests/archivers/test_archiver_base.py +++ b/tests/archivers/test_archiver_base.py @@ -1,9 +1,7 @@ import pytest -from auto_archiver.core import Metadata -from auto_archiver.core import Step from auto_archiver.core.metadata import Metadata -from auto_archiver.archivers.archiver import Archiver +from auto_archiver.base_processors.extractor import Extractor class TestArchiverBase(object): archiver_class: str = None @@ -13,7 +11,7 @@ class TestArchiverBase(object): def setup_archiver(self): assert self.archiver_class is not None, "self.archiver_class must be set on the subclass" assert self.config is not None, "self.config must be a dict set on the subclass" - self.archiver: Archiver = self.archiver_class({self.archiver_class.name: self.config}) + self.archiver: Extractor = self.archiver_class({self.archiver_class.name: self.config}) def assertValidResponseMetadata(self, test_response: Metadata, title: str, timestamp: str, status: str = ""): assert test_response is not False diff --git a/tests/databases/test_csv_db.py b/tests/databases/test_csv_db.py index 4395ef0..989f1e9 100644 --- a/tests/databases/test_csv_db.py +++ b/tests/databases/test_csv_db.py @@ -1,5 +1,5 @@ -from auto_archiver.databases.csv_db import CSVDb +from auto_archiver.modules.csv_db import CSVDb from auto_archiver.core import Metadata diff --git a/tests/enrichers/test_hash_enricher.py b/tests/enrichers/test_hash_enricher.py index 99f8117..1477cde 100644 --- a/tests/enrichers/test_hash_enricher.py +++ b/tests/enrichers/test_hash_enricher.py @@ -1,6 +1,6 @@ import pytest -from auto_archiver.enrichers.hash_enricher import HashEnricher +from auto_archiver.modules.hash_enricher import HashEnricher from auto_archiver.core import Metadata, Media @pytest.mark.parametrize("algorithm, filename, expected_hash", [ diff --git a/tests/formatters/test_html_formatter.py b/tests/formatters/test_html_formatter.py index 3540062..2719033 100644 --- a/tests/formatters/test_html_formatter.py +++ b/tests/formatters/test_html_formatter.py @@ -1,5 +1,4 @@ -from auto_archiver.core.context import ArchivingContext -from auto_archiver.formatters.html_formatter import HtmlFormatter +from auto_archiver.modules.html_formatter import HtmlFormatter from auto_archiver.core import Metadata, Media