Merge main into timestamping_enricher

src/auto_archiver/__main__.py

@@ -1,9 +1,13 @@
-""" Entry point for the auto_archiver package. """
+"""Entry point for the auto_archiver package."""
+
 from auto_archiver.core.orchestrator import ArchivingOrchestrator
 import sys
 
+
 def main():
-    for _ in ArchivingOrchestrator()._command_line_run(sys.argv[1:]): pass
+    for _ in ArchivingOrchestrator()._command_line_run(sys.argv[1:]):
+        pass
+
 
 if __name__ == "__main__":
     main()

src/auto_archiver/core/__init__.py

@@ -1,6 +1,5 @@
-""" Core modules to handle things such as orchestration, metadata and configs..
+"""Core modules to handle things such as orchestration, metadata and configs.."""
 
-"""
 from .metadata import Metadata
 from .media import Media
 from .base_module import BaseModule
@@ -14,4 +13,4 @@ from .enricher import Enricher
 from .feeder import Feeder
 from .storage import Storage
 from .extractor import Extractor
-from .formatter import Formatter
+from .formatter import Formatter

src/auto_archiver/core/base_module.py

@@ -1,9 +1,8 @@
-
 from __future__ import annotations
 
-from typing import  Mapping, Any, Type, TYPE_CHECKING
+from typing import Mapping, Any, TYPE_CHECKING
 from abc import ABC
-from copy import deepcopy, copy
+from copy import deepcopy
 from tempfile import TemporaryDirectory
 from auto_archiver.utils import url as UrlUtil
 from auto_archiver.core.consts import MODULE_TYPES as CONF_MODULE_TYPES
@@ -13,8 +12,8 @@ from loguru import logger
 if TYPE_CHECKING:
     from .module import ModuleFactory
 
-class BaseModule(ABC):
 
+class BaseModule(ABC):
     """
     Base module class. All modules should inherit from this class.
 
@@ -46,15 +45,13 @@ class BaseModule(ABC):
 
     @property
     def storages(self) -> list:
-        return self.config.get('storages', [])
+        return self.config.get("storages", [])
 
     def config_setup(self, config: dict):
-
-        authentication = config.get('authentication', {})
         # this is important. Each instance is given its own deepcopied config, so modules cannot
         # change values to affect other modules
         config = deepcopy(config)
-        authentication = deepcopy(config.pop('authentication', {}))
+        authentication = deepcopy(config.pop("authentication", {}))
 
         self.authentication = authentication
         self.config = config
@@ -62,18 +59,28 @@ class BaseModule(ABC):
             setattr(self, key, val)
 
     def setup(self):
-        # For any additional setup required by modules, e.g. autehntication
+        # For any additional setup required by modules outside of the configs in the manifesst,
+        # e.g. authentication
         pass
 
     def auth_for_site(self, site: str, extract_cookies=True) -> Mapping[str, Any]:
         """
         Returns the authentication information for a given site. This is used to authenticate
         with a site before extracting data. The site should be the domain of the site, e.g. 'twitter.com'
-        
+
         :param site: the domain of the site to get authentication information for
         :param extract_cookies: whether or not to extract cookies from the given browser/file and return the cookie jar (disabling can speed up processing if you don't actually need the cookies jar).
 
-        :returns: authdict dict of login information for the given site
+        :returns: authdict dict -> {
+            "username": str,
+            "password": str,
+            "api_key": str,
+            "api_secret": str,
+            "cookie": str,
+            "cookies_file": str,
+            "cookies_from_browser": str,
+            "cookies_jar": CookieJar
+        }
 
         **Global options:**\n
         * cookies_from_browser: str - the name of the browser to extract cookies from (e.g. 'chrome', 'firefox' - uses ytdlp under the hood to extract\n
@@ -87,15 +94,15 @@ class BaseModule(ABC):
         * cookie: str - a cookie string to use for login (specific to this site)\n
         * cookies_file: str - the path to a cookies file to use for login (specific to this site)\n
         * cookies_from_browser: str - the name of the browser to extract cookies from (specitic for this site)\n
+
         """
         # TODO: think about if/how we can deal with sites that have multiple domains (main one is x.com/twitter.com)
         # for now the user must enter them both, like "x.com,twitter.com" in their config. Maybe we just hard-code?
 
-        site = UrlUtil.domain_for_url(site).lstrip("www.")
+        site = UrlUtil.domain_for_url(site).removeprefix("www.")
         # add the 'www' version of the site to the list of sites to check
         authdict = {}
 
-
         for to_try in [site, f"www.{site}"]:
             if to_try in self.authentication:
                 authdict.update(self.authentication[to_try])
@@ -105,19 +112,22 @@ class BaseModule(ABC):
         if not authdict:
             for key in self.authentication.keys():
                 if key in site or site in key:
-                    logger.debug(f"Could not find exact authentication information for site '{site}'. \
-                                    did find information for '{key}' which is close, is this what you meant? \
-                                    If so, edit your authentication settings to make sure it exactly matches.")
+                    logger.debug(
+                        f"Could not find exact authentication information for site '{site}'. \
+did find information for '{key}' which is close, is this what you meant? \
+If so, edit your authentication settings to make sure it exactly matches."
+                    )
 
         def get_ytdlp_cookiejar(args):
             import yt_dlp
             from yt_dlp import parse_options
+
             logger.debug(f"Extracting cookies from settings: {args[1]}")
             # parse_options returns a named tuple as follows, we only need the ydl_options part
             # collections.namedtuple('ParsedOptions', ('parser', 'options', 'urls', 'ydl_opts'))
-            ytdlp_opts = getattr(parse_options(args), 'ydl_opts')
+            ytdlp_opts = getattr(parse_options(args), "ydl_opts")
             return yt_dlp.YoutubeDL(ytdlp_opts).cookiejar
-        
+
         get_cookiejar_options = None
 
         # order of priority:
@@ -126,22 +136,21 @@ class BaseModule(ABC):
         # 3. cookies_from_browser setting in global config
         # 4. cookies_file setting in global config
 
-        if 'cookies_from_browser' in authdict:
-            get_cookiejar_options = ['--cookies-from-browser', authdict['cookies_from_browser']]
-        elif 'cookies_file' in authdict:
-            get_cookiejar_options = ['--cookies', authdict['cookies_file']]
-        elif 'cookies_from_browser' in self.authentication:
-            authdict['cookies_from_browser'] = self.authentication['cookies_from_browser']
-            get_cookiejar_options = ['--cookies-from-browser', self.authentication['cookies_from_browser']]
-        elif 'cookies_file' in self.authentication:
-            authdict['cookies_file'] = self.authentication['cookies_file']
-            get_cookiejar_options = ['--cookies', self.authentication['cookies_file']]
+        if "cookies_from_browser" in authdict:
+            get_cookiejar_options = ["--cookies-from-browser", authdict["cookies_from_browser"]]
+        elif "cookies_file" in authdict:
+            get_cookiejar_options = ["--cookies", authdict["cookies_file"]]
+        elif "cookies_from_browser" in self.authentication:
+            authdict["cookies_from_browser"] = self.authentication["cookies_from_browser"]
+            get_cookiejar_options = ["--cookies-from-browser", self.authentication["cookies_from_browser"]]
+        elif "cookies_file" in self.authentication:
+            authdict["cookies_file"] = self.authentication["cookies_file"]
+            get_cookiejar_options = ["--cookies", self.authentication["cookies_file"]]
 
-        
         if get_cookiejar_options:
-            authdict['cookies_jar'] = get_ytdlp_cookiejar(get_cookiejar_options)
+            authdict["cookies_jar"] = get_ytdlp_cookiejar(get_cookiejar_options)
 
         return authdict
-    
+
     def repr(self):
-        return f"Module<'{self.display_name}' (config: {self.config[self.name]})>"
+        return f"Module<'{self.display_name}' (config: {self.config[self.name]})>"

src/auto_archiver/core/config.py

@@ -6,26 +6,28 @@ flexible setup in various environments.
 """
 
 import argparse
-from ruamel.yaml import YAML, CommentedMap, add_representer
+from ruamel.yaml import YAML, CommentedMap
 import json
+import os
 
 from loguru import logger
 
 from copy import deepcopy
 from auto_archiver.core.consts import MODULE_TYPES
 
-from typing import Any, List, Type, Tuple
 
 _yaml: YAML = YAML()
 
 DEFAULT_CONFIG_FILE = "secrets/orchestration.yaml"
 
-EMPTY_CONFIG = _yaml.load("""
+EMPTY_CONFIG = _yaml.load(
+    """
 # Auto Archiver Configuration
 
 # Steps are the modules that will be run in the order they are defined
-steps:""" + "".join([f"\n   {module}s: []" for module in MODULE_TYPES]) + \
-"""
+steps:"""
+    + "".join([f"\n   {module}s: []" for module in MODULE_TYPES])
+    + """
 
 # Global configuration
 
@@ -52,50 +54,54 @@ authentication: {}
 logging:
   level: INFO
 
-""")
+"""
+)
 # note: 'logging' is explicitly added above in order to better format the config file
 
 
 # Arg Parse Actions/Classes
 class AuthenticationJsonParseAction(argparse.Action):
     def __call__(self, parser, namespace, values, option_string=None):
-
         try:
             auth_dict = json.loads(values)
             setattr(namespace, self.dest, auth_dict)
         except json.JSONDecodeError as e:
-            raise argparse.ArgumentTypeError(f"Invalid JSON input for argument '{self.dest}': {e}")
+            raise argparse.ArgumentTypeError(f"Invalid JSON input for argument '{self.dest}': {e}") from e
 
         def load_from_file(path):
             try:
-                with open(path, 'r') as f:
+                with open(path, "r") as f:
                     try:
                         auth_dict = json.load(f)
                     except json.JSONDecodeError:
                         f.seek(0)
                         # maybe it's yaml, try that
                         auth_dict = _yaml.load(f)
-                    if auth_dict.get('authentication'):
-                        auth_dict = auth_dict['authentication']
-                    auth_dict['load_from_file']  = path
+                    if auth_dict.get("authentication"):
+                        auth_dict = auth_dict["authentication"]
+                    auth_dict["load_from_file"] = path
                     return auth_dict
-            except:
+            except Exception:
                 return None
 
-        if isinstance(auth_dict, dict) and auth_dict.get('from_file'):
-            auth_dict = load_from_file(auth_dict['from_file'])
+        if isinstance(auth_dict, dict) and auth_dict.get("from_file"):
+            auth_dict = load_from_file(auth_dict["from_file"])
         elif isinstance(auth_dict, str):
             # if it's a string
             auth_dict = load_from_file(auth_dict)
-        
+
         if not isinstance(auth_dict, dict):
-            raise argparse.ArgumentTypeError("Authentication must be a dictionary of site names and their authentication methods")
-        global_options = ['cookies_from_browser', 'cookies_file', 'load_from_file']
+            raise argparse.ArgumentTypeError(
+                "Authentication must be a dictionary of site names and their authentication methods"
+            )
+        global_options = ["cookies_from_browser", "cookies_file", "load_from_file"]
         for key, auth in auth_dict.items():
             if key in global_options:
                 continue
             if not isinstance(key, str) or not isinstance(auth, dict):
-                raise argparse.ArgumentTypeError(f"Authentication must be a dictionary of site names and their authentication methods. Valid global configs are {global_options}")
+                raise argparse.ArgumentTypeError(
+                    f"Authentication must be a dictionary of site names and their authentication methods. Valid global configs are {global_options}"
+                )
 
         setattr(namespace, self.dest, auth_dict)
 
@@ -106,8 +112,8 @@ class UniqueAppendAction(argparse.Action):
             if value not in getattr(namespace, self.dest):
                 getattr(namespace, self.dest).append(value)
 
-class DefaultValidatingParser(argparse.ArgumentParser):
 
+class DefaultValidatingParser(argparse.ArgumentParser):
     def error(self, message):
         """
         Override of error to format a nicer looking error message using logger
@@ -136,8 +142,10 @@ class DefaultValidatingParser(argparse.ArgumentParser):
 
         return super().parse_known_args(args, namespace)
 
+
 # Config Utils
 
+
 def to_dot_notation(yaml_conf: CommentedMap | dict) -> dict:
     dotdict = {}
 
@@ -151,6 +159,7 @@ def to_dot_notation(yaml_conf: CommentedMap | dict) -> dict:
     process_subdict(yaml_conf)
     return dotdict
 
+
 def from_dot_notation(dotdict: dict) -> dict:
     normal_dict = {}
 
@@ -171,9 +180,11 @@ def from_dot_notation(dotdict: dict) -> dict:
 def is_list_type(value):
     return isinstance(value, list) or isinstance(value, tuple) or isinstance(value, set)
 
+
 def is_dict_type(value):
     return isinstance(value, dict) or isinstance(value, CommentedMap)
 
+
 def merge_dicts(dotdict: dict, yaml_dict: CommentedMap) -> CommentedMap:
     yaml_dict: CommentedMap = deepcopy(yaml_dict)
 
@@ -184,7 +195,7 @@ def merge_dicts(dotdict: dict, yaml_dict: CommentedMap) -> CommentedMap:
                 yaml_subdict[key] = value
                 continue
 
-            if key == 'steps':
+            if key == "steps":
                 for module_type, modules in value.items():
                     # overwrite the 'steps' from the config file with the ones from the CLI
                     yaml_subdict[key][module_type] = modules
@@ -199,6 +210,7 @@ def merge_dicts(dotdict: dict, yaml_dict: CommentedMap) -> CommentedMap:
     update_dict(from_dot_notation(dotdict), yaml_dict)
     return yaml_dict
 
+
 def read_yaml(yaml_filename: str) -> CommentedMap:
     config = None
     try:
@@ -212,20 +224,26 @@ def read_yaml(yaml_filename: str) -> CommentedMap:
 
     return config
 
+
 # TODO: make this tidier/find a way to notify of which keys should not be stored
 
 
 def store_yaml(config: CommentedMap, yaml_filename: str) -> None:
     config_to_save = deepcopy(config)
 
+    ## if the save path is the default location (secrets) then create the 'secrets' folder
+    if os.path.dirname(yaml_filename) == "secrets":
+        os.makedirs("secrets", exist_ok=True)
+
     auth_dict = config_to_save.get("authentication", {})
-    if auth_dict and auth_dict.get('load_from_file'):
+    if auth_dict and auth_dict.get("load_from_file"):
         # remove all other values from the config, don't want to store it in the config file
         auth_dict = {"load_from_file": auth_dict["load_from_file"]}
 
-    config_to_save.pop('urls', None)
+    config_to_save.pop("urls", None)
     with open(yaml_filename, "w", encoding="utf-8") as outf:
         _yaml.dump(config_to_save, outf)
 
+
 def is_valid_config(config: CommentedMap) -> bool:
-    return config and config != EMPTY_CONFIG
+    return config and config != EMPTY_CONFIG

src/auto_archiver/core/consts.py

@@ -1,23 +1,19 @@
+class SetupError(ValueError):
+    pass
 
-MODULE_TYPES = [
-    'feeder',
-    'extractor',
-    'enricher',
-    'database',
-    'storage',
-    'formatter'
-]
+
+MODULE_TYPES = ["feeder", "extractor", "enricher", "database", "storage", "formatter"]
 
 MANIFEST_FILE = "__manifest__.py"
 
 DEFAULT_MANIFEST = {
-    'name': '', # the display name of the module
-    'author': 'Bellingcat', # creator of the module, leave this as Bellingcat or set your own name!
-    'type': [], # the type of the module, can be one or more of MODULE_TYPES
-    'requires_setup': True, # whether or not this module requires additional setup such as setting API Keys or installing additional softare
-    'description': '', # a description of the module
-    'dependencies': {}, # external dependencies, e.g. python packages or binaries, in dictionary format
-    'entry_point': '', # the entry point for the module, in the format 'module_name::ClassName'. This can be left blank to use the default entry point of module_name::ModuleName
-    'version': '1.0', # the version of the module
-    'configs': {} # any configuration options this module has, these will be exposed to the user in the config file or via the command line
-}
+    "name": "",  # the display name of the module
+    "author": "Bellingcat",  # creator of the module, leave this as Bellingcat or set your own name!
+    "type": [],  # the type of the module, can be one or more of MODULE_TYPES
+    "requires_setup": True,  # whether or not this module requires additional setup such as setting API Keys or installing additional software
+    "description": "",  # a description of the module
+    "dependencies": {},  # external dependencies, e.g. python packages or binaries, in dictionary format
+    "entry_point": "",  # the entry point for the module, in the format 'module_name::ClassName'. This can be left blank to use the default entry point of module_name::ModuleName
+    "version": "1.0",  # the version of the module
+    "configs": {},  # any configuration options this module has, these will be exposed to the user in the config file or via the command line
+}

src/auto_archiver/core/database.py

@@ -1,6 +1,6 @@
 """
 Database module for the auto-archiver that defines the interface for implementing database modules
-in the media archiving framework. 
+in the media archiving framework.
 """
 
 from __future__ import annotations
@@ -9,6 +9,7 @@ from typing import Union
 
 from auto_archiver.core import Metadata, BaseModule
 
+
 class Database(BaseModule):
     """
     Base class for implementing database modules in the media archiving framework.
@@ -20,7 +21,7 @@ class Database(BaseModule):
         """signals the DB that the given item archival has started"""
         pass
 
-    def failed(self, item: Metadata, reason:str) -> None:
+    def failed(self, item: Metadata, reason: str) -> None:
         """update DB accordingly for failure"""
         pass
 
@@ -34,6 +35,6 @@ class Database(BaseModule):
         return False
 
     @abstractmethod
-    def done(self, item: Metadata, cached: bool=False) -> None:
+    def done(self, item: Metadata, cached: bool = False) -> None:
         """archival result ready - should be saved to DB"""
         pass

src/auto_archiver/core/enricher.py

@@ -8,13 +8,15 @@ the archiving step and before storage or formatting.
 
 Enrichers are optional but highly useful for making the archived data more powerful.
 """
+
 from __future__ import annotations
 from abc import abstractmethod
 from auto_archiver.core import Metadata, BaseModule
 
+
 class Enricher(BaseModule):
     """Base classes and utilities for enrichers in the Auto Archiver system.
-    
+
     Enricher modules must implement the `enrich` method to define their behavior.
     """
 

src/auto_archiver/core/extractor.py

@@ -1,17 +1,15 @@
-""" The `extractor` module defines the base functionality for implementing extractors in the media archiving framework.
-    This class provides common utility methods and a standard interface for extractors.
+"""The `extractor` module defines the base functionality for implementing extractors in the media archiving framework.
+This class provides common utility methods and a standard interface for extractors.
 
-    Factory method to initialize an extractor instance based on its name.
+Factory method to initialize an extractor instance based on its name.
 
 
 """
+
 from __future__ import annotations
-from pathlib import Path
 from abc import abstractmethod
-from dataclasses import dataclass
 import mimetypes
 import os
-import mimetypes
 import requests
 from loguru import logger
 from retrying import retry
@@ -39,7 +37,7 @@ class Extractor(BaseModule):
         Used to clean unnecessary URL parameters OR unfurl redirect links
         """
         return url
-    
+
     def match_link(self, url: str) -> re.Match:
         """
         Returns a match object if the given URL matches the valid_url pattern or False/None if not.
@@ -58,7 +56,7 @@ class Extractor(BaseModule):
         """
         if self.valid_url:
             return self.match_link(url) is not None
-        
+
         return True
 
     def _guess_file_type(self, path: str) -> str:
@@ -74,16 +72,17 @@ class Extractor(BaseModule):
     @retry(wait_random_min=500, wait_random_max=3500, stop_max_attempt_number=5)
     def download_from_url(self, url: str, to_filename: str = None, verbose=True) -> str:
         """
-            downloads a URL to provided filename, or inferred from URL, returns local filename
+        downloads a URL to provided filename, or inferred from URL, returns local filename
         """
         if not to_filename:
-            to_filename = url.split('/')[-1].split('?')[0]
+            to_filename = url.split("/")[-1].split("?")[0]
             if len(to_filename) > 64:
                 to_filename = to_filename[-64:]
         to_filename = os.path.join(self.tmp_dir, to_filename)
-        if verbose: logger.debug(f"downloading {url[0:50]=} {to_filename=}")
+        if verbose:
+            logger.debug(f"downloading {url[0:50]=} {to_filename=}")
         headers = {
-            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36'
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36"
         }
         try:
             d = requests.get(url, stream=True, headers=headers, timeout=30)
@@ -91,12 +90,12 @@ class Extractor(BaseModule):
 
             # get mimetype from the response headers
             if not mimetypes.guess_type(to_filename)[0]:
-                content_type = d.headers.get('Content-Type') or self._guess_file_type(url)
+                content_type = d.headers.get("Content-Type") or self._guess_file_type(url)
                 extension = mimetypes.guess_extension(content_type)
                 if extension:
                     to_filename += extension
 
-            with open(to_filename, 'wb') as f:
+            with open(to_filename, "wb") as f:
                 for chunk in d.iter_content(chunk_size=8192):
                     f.write(chunk)
             return to_filename
@@ -108,8 +107,8 @@ class Extractor(BaseModule):
     def download(self, item: Metadata) -> Metadata | False:
         """
         Downloads the media from the given URL and returns a Metadata object with the downloaded media.
-        
+
         If the URL is not supported or the download fails, this method should return False.
 
         """
-        pass
+        pass

src/auto_archiver/core/feeder.py

@@ -1,5 +1,5 @@
 """
-The feeder base module defines the interface for implementing feeders in the media archiving framework. 
+The feeder base module defines the interface for implementing feeders in the media archiving framework.
 """
 
 from __future__ import annotations
@@ -7,8 +7,8 @@ from abc import abstractmethod
 from auto_archiver.core import Metadata
 from auto_archiver.core import BaseModule
 
-class Feeder(BaseModule):
 
+class Feeder(BaseModule):
     """
     Base class for implementing feeders in the media archiving framework.
 
@@ -19,7 +19,7 @@ class Feeder(BaseModule):
     def __iter__(self) -> Metadata:
         """
         Returns an iterator (use `yield`) over the items to be archived.
-        
+
         These should be instances of Metadata, typically created with Metadata().set_url(url).
         """
-        return None
+        return None

src/auto_archiver/core/formatter.py

@@ -12,7 +12,7 @@ from auto_archiver.core import Metadata, Media, BaseModule
 class Formatter(BaseModule):
     """
     Base class for implementing formatters in the media archiving framework.
-    
+
     Subclasses must implement the `format` method to define their behavior.
     """
 
@@ -21,4 +21,4 @@ class Formatter(BaseModule):
         """
         Formats a Metadata object into a user-viewable format (e.g. HTML) and stores it if needed.
         """
-        return None
+        return None

src/auto_archiver/core/media.py

@@ -6,7 +6,7 @@ nested media retrieval, and type validation.
 from __future__ import annotations
 import os
 import traceback
-from typing import Any, List
+from typing import Any, List, Iterator
 from dataclasses import dataclass, field
 from dataclasses_json import dataclass_json, config
 import mimetypes
@@ -21,14 +21,14 @@ class Media:
     Represents a media file with associated properties and storage details.
 
     Attributes:
-    - filename: The file path of the media.
-    - key: An optional identifier for the media.
+    - filename: The file path of the media as saved locally (temporarily, before uploading to the storage).
     - urls: A list of URLs where the media is stored or accessible.
     - properties: Additional metadata or transformations for the media.
     - _mimetype: The media's mimetype (e.g., image/jpeg, video/mp4).
     """
+
     filename: str
-    key: str = None
+    _key: str = None
     urls: List[str] = field(default_factory=list)
     properties: dict = field(default_factory=dict)
     _mimetype: str = None  # eg: image/jpeg
@@ -47,19 +47,20 @@ class Media:
             for any_media in self.all_inner_media(include_self=True):
                 s.store(any_media, url, metadata=metadata)
 
-    def all_inner_media(self, include_self=False):
+    def all_inner_media(self, include_self=False) -> Iterator[Media]:
         """Retrieves all media, including nested media within properties or transformations on original media.
         This function returns a generator for all the inner media.
 
         """
-        if include_self: yield self
+        if include_self:
+            yield self
         for prop in self.properties.values():
-            if isinstance(prop, Media): 
+            if isinstance(prop, Media):
                 for inner_media in prop.all_inner_media(include_self=True):
                     yield inner_media
             if isinstance(prop, list):
                 for prop_media in prop:
-                    if isinstance(prop_media, Media): 
+                    if isinstance(prop_media, Media):
                         for inner_media in prop_media.all_inner_media(include_self=True):
                             yield inner_media
 
@@ -67,6 +68,10 @@ class Media:
         # checks if the media is already stored in the given storage
         return len(self.urls) > 0 and len(self.urls) == len(in_storage.config["steps"]["storages"])
 
+    @property
+    def key(self) -> str:
+        return self._key
+
     def set(self, key: str, value: Any) -> Media:
         self.properties[key] = value
         return self
@@ -110,15 +115,17 @@ class Media:
         # checks for video streams with ffmpeg, or min file size for a video
         # self.is_video() should be used together with this method
         try:
-            streams = ffmpeg.probe(self.filename, select_streams='v')['streams']
+            streams = ffmpeg.probe(self.filename, select_streams="v")["streams"]
             logger.warning(f"STREAMS FOR {self.filename} {streams}")
             return any(s.get("duration_ts", 0) > 0 for s in streams)
-        except Error: return False # ffmpeg errors when reading bad files
+        except Error:
+            return False  # ffmpeg errors when reading bad files
         except Exception as e:
             logger.error(e)
             logger.error(traceback.format_exc())
             try:
                 fsize = os.path.getsize(self.filename)
                 return fsize > 20_000
-            except: pass
+            except Exception as e:
+                pass
         return True

src/auto_archiver/core/metadata.py

@@ -13,7 +13,7 @@ from __future__ import annotations
 import hashlib
 from typing import Any, List, Union, Dict
 from dataclasses import dataclass, field
-from dataclasses_json import dataclass_json, config
+from dataclasses_json import dataclass_json
 import datetime
 from urllib.parse import urlparse
 from dateutil.parser import parse as parse_dt
@@ -21,6 +21,7 @@ from loguru import logger
 
 from .media import Media
 
+
 @dataclass_json  # annotation order matters
 @dataclass
 class Metadata:
@@ -40,19 +41,23 @@ class Metadata:
         - If `True`, this instance's values are overwritten by `right`.
         - If `False`, the inverse applies.
         """
-        if not right: return self
+        if not right:
+            return self
         if overwrite_left:
             if right.status and len(right.status):
                 self.status = right.status
             self._context.update(right._context)
             for k, v in right.metadata.items():
-                assert k not in self.metadata or type(v) == type(self.get(k))
-                if type(v) not in [dict, list, set] or k not in self.metadata:
+                assert k not in self.metadata or type(v) is type(self.get(k))
+                if not isinstance(v, (dict, list, set)) or k not in self.metadata:
                     self.set(k, v)
                 else:  # key conflict
-                    if type(v) in [dict, set]: self.set(k, self.get(k) | v)
-                    elif type(v) == list: self.set(k, self.get(k) + v)
+                    if isinstance(v, (dict, set)):
+                        self.set(k, self.get(k) | v)
+                    elif type(v) is list:
+                        self.set(k, self.get(k) + v)
             self.media.extend(right.media)
+
         else:  # invert and do same logic
             return right.merge(self)
         return self
@@ -69,7 +74,7 @@ class Metadata:
 
     def append(self, key: str, val: Any) -> Metadata:
         if key not in self.metadata:
-            self.metadata[key] = []    
+            self.metadata[key] = []
         self.metadata[key] = val
         return self
 
@@ -80,24 +85,26 @@ class Metadata:
         return self.metadata.get(key, default)
 
     def success(self, context: str = None) -> Metadata:
-        if context: self.status = f"{context}: success"
-        else: self.status = "success"
+        if context:
+            self.status = f"{context}: success"
+        else:
+            self.status = "success"
         return self
 
     def is_success(self) -> bool:
         return "success" in self.status
 
     def is_empty(self) -> bool:
-        meaningfull_ids = set(self.metadata.keys()) - set(["_processed_at", "url", "total_bytes", "total_size", "archive_duration_seconds"])
+        meaningfull_ids = set(self.metadata.keys()) - set(
+            ["_processed_at", "url", "total_bytes", "total_size", "archive_duration_seconds"]
+        )
         return not self.is_success() and len(self.media) == 0 and len(meaningfull_ids) == 0
 
     @property  # getter .netloc
     def netloc(self) -> str:
         return urlparse(self.get_url()).netloc
 
-
-# custom getter/setters
-
+    # custom getter/setters
 
     def set_url(self, url: str) -> Metadata:
         assert type(url) is str and len(url) > 0, "invalid URL"
@@ -120,36 +127,43 @@ class Metadata:
         return self.get("title")
 
     def set_timestamp(self, timestamp: datetime.datetime) -> Metadata:
-        if type(timestamp) == str:
+        if isinstance(timestamp, str):
             timestamp = parse_dt(timestamp)
-        assert type(timestamp) == datetime.datetime, "set_timestamp expects a datetime instance"
+        assert isinstance(timestamp, datetime.datetime), "set_timestamp expects a datetime instance"
         return self.set("timestamp", timestamp)
 
-    def get_timestamp(self, utc=True, iso=True) -> datetime.datetime:
+    def get_timestamp(self, utc=True, iso=True) -> datetime.datetime | str | None:
         ts = self.get("timestamp")
-        if not ts: return 
+        if not ts:
+            return None
         try:
-            if type(ts) == str: ts = datetime.datetime.fromisoformat(ts)
-            if type(ts) == float: ts = datetime.datetime.fromtimestamp(ts)
-            if utc: ts = ts.replace(tzinfo=datetime.timezone.utc)
-            if iso: return ts.isoformat()
-            return ts
+            if isinstance(ts, str):
+                ts = datetime.datetime.fromisoformat(ts)
+            elif isinstance(ts, float):
+                ts = datetime.datetime.fromtimestamp(ts)
+            if utc:
+                ts = ts.replace(tzinfo=datetime.timezone.utc)
+            return ts.isoformat() if iso else ts
         except Exception as e:
             logger.error(f"Unable to parse timestamp {ts}: {e}")
-            return
+            return None
 
     def add_media(self, media: Media, id: str = None) -> Metadata:
         # adds a new media, optionally including an id
-        if media is None: return
+        if media is None:
+            return
         if id is not None:
-            assert not len([1 for m in self.media if m.get("id") == id]), f"cannot add 2 pieces of media with the same id {id}"
+            assert not len([1 for m in self.media if m.get("id") == id]), (
+                f"cannot add 2 pieces of media with the same id {id}"
+            )
             media.set("id", id)
         self.media.append(media)
         return media
 
     def get_media_by_id(self, id: str, default=None) -> Media:
         for m in self.media:
-            if m.get("id") == id: return m
+            if m.get("id") == id:
+                return m
         return default
 
     def remove_duplicate_media_by_hash(self) -> None:
@@ -159,7 +173,8 @@ class Metadata:
             with open(filename, "rb") as f:
                 while True:
                     buf = f.read(chunksize)
-                    if not buf: break
+                    if not buf:
+                        break
                     hash_algo.update(buf)
             return hash_algo.hexdigest()
 
@@ -167,15 +182,18 @@ class Metadata:
         new_media = []
         for m in self.media:
             h = m.get("hash")
-            if not h: h = calculate_hash_in_chunks(hashlib.sha256(), int(1.6e7), m.filename)
-            if len(h) and h in media_hashes: continue
+            if not h:
+                h = calculate_hash_in_chunks(hashlib.sha256(), int(1.6e7), m.filename)
+            if len(h) and h in media_hashes:
+                continue
             media_hashes.add(h)
             new_media.append(m)
         self.media = new_media
 
     def get_first_image(self, default=None) -> Media:
         for m in self.media:
-            if "image" in m.mimetype: return m
+            if "image" in m.mimetype:
+                return m
         return default
 
     def set_final_media(self, final: Media) -> Metadata:
@@ -193,22 +211,25 @@ class Metadata:
     def __str__(self) -> str:
         return self.__repr__()
 
-
     @staticmethod
     def choose_most_complete(results: List[Metadata]) -> Metadata:
         # returns the most complete result from a list of results
         # prioritizes results with more media, then more metadata
-        if len(results) == 0: return None
-        if len(results) == 1: return results[0]
+        if len(results) == 0:
+            return None
+        if len(results) == 1:
+            return results[0]
         most_complete = results[0]
         for r in results[1:]:
-            if len(r.media) > len(most_complete.media): most_complete = r
-            elif len(r.media) == len(most_complete.media) and len(r.metadata) > len(most_complete.metadata): most_complete = r
+            if len(r.media) > len(most_complete.media):
+                most_complete = r
+            elif len(r.media) == len(most_complete.media) and len(r.metadata) > len(most_complete.metadata):
+                most_complete = r
         return most_complete
 
     def set_context(self, key: str, val: Any) -> Metadata:
         self._context[key] = val
         return self
-    
+
     def get_context(self, key: str, default: Any = None) -> Any:
-        return self._context.get(key, default)
+        return self._context.get(key, default)

src/auto_archiver/core/module.py

@@ -3,10 +3,12 @@ Defines the Step abstract base class, which acts as a blueprint for steps in the
 by handling user configuration, validating the steps properties, and implementing dynamic instantiation.
 
 """
+
 from __future__ import annotations
+import subprocess
 
 from dataclasses import dataclass
-from typing import List, TYPE_CHECKING
+from typing import List, TYPE_CHECKING, Type
 import shutil
 import ast
 import copy
@@ -16,7 +18,7 @@ import os
 from os.path import join
 from loguru import logger
 import auto_archiver
-from auto_archiver.core.consts import DEFAULT_MANIFEST, MANIFEST_FILE
+from auto_archiver.core.consts import DEFAULT_MANIFEST, MANIFEST_FILE, SetupError
 
 if TYPE_CHECKING:
     from .base_module import BaseModule
@@ -24,17 +26,17 @@ if TYPE_CHECKING:
 
 HAS_SETUP_PATHS = False
 
-class ModuleFactory:
 
+class ModuleFactory:
     def __init__(self):
         self._lazy_modules = {}
 
     def setup_paths(self, paths: list[str]) -> None:
         """
         Sets up the paths for the modules to be loaded from
-        
+
         This is necessary for the modules to be imported correctly
-        
+
         """
         global HAS_SETUP_PATHS
 
@@ -46,45 +48,55 @@ class ModuleFactory:
 
             # see odoo/module/module.py -> initialize_sys_path
             if path not in auto_archiver.modules.__path__:
-                    if HAS_SETUP_PATHS == True:
-                        logger.warning(f"You are attempting to re-initialise the module paths with: '{path}' for a 2nd time. \
+                if HAS_SETUP_PATHS:
+                    logger.warning(
+                        f"You are attempting to re-initialise the module paths with: '{path}' for a 2nd time. \
                                        This could lead to unexpected behaviour. It is recommended to only use a single modules path. \
-                                       If you wish to load modules from different paths then load a 2nd python interpreter (e.g. using multiprocessing).")
-                    auto_archiver.modules.__path__.append(path)
+                                       If you wish to load modules from different paths then load a 2nd python interpreter (e.g. using multiprocessing)."
+                    )
+                auto_archiver.modules.__path__.append(path)
 
         # sort based on the length of the path, so that the longest path is last in the list
         auto_archiver.modules.__path__ = sorted(auto_archiver.modules.__path__, key=len, reverse=True)
 
         HAS_SETUP_PATHS = True
 
-    def get_module(self, module_name: str, config: dict) -> BaseModule:
+    def get_module(self, module_name: str, config: dict) -> Type[BaseModule]:
         """
         Gets and sets up a module using the provided config
-        
+
         This will actually load and instantiate the module, and load all its dependencies (i.e. not lazy)
-        
+
         """
         return self.get_module_lazy(module_name).load(config)
 
     def get_module_lazy(self, module_name: str, suppress_warnings: bool = False) -> LazyBaseModule:
         """
         Lazily loads a module, returning a LazyBaseModule
-        
+
         This has all the information about the module, but does not load the module itself or its dependencies
-        
+
         To load an actual module, call .setup() on a lazy module
-        
+
         """
         if module_name in self._lazy_modules:
             return self._lazy_modules[module_name]
 
         available = self.available_modules(limit_to_modules=[module_name], suppress_warnings=suppress_warnings)
         if not available:
-            raise IndexError(f"Module '{module_name}' not found. Are you sure it's installed/exists?")
+            message = f"Module '{module_name}' not found. Are you sure it's installed/exists?"
+            if "archiver" in module_name:
+                message += f" Did you mean '{module_name.replace('archiver', 'extractor')}'?"
+            elif "gsheet" in module_name:
+                message += " Did you mean 'gsheet_feeder_db'?"
+            elif "atlos" in module_name:
+                message += " Did you mean 'atlos_feeder_db_storage'?"
+            raise IndexError(message)
         return available[0]
 
-    def available_modules(self, limit_to_modules: List[str]= [], suppress_warnings: bool = False) -> List[LazyBaseModule]:
-        
+    def available_modules(
+        self, limit_to_modules: List[str] = [], suppress_warnings: bool = False
+    ) -> List[LazyBaseModule]:
         # search through all valid 'modules' paths. Default is 'modules' in the current directory
 
         # see odoo/modules/module.py -> get_modules
@@ -116,7 +128,7 @@ class ModuleFactory:
                 self._lazy_modules[possible_module] = lazy_module
 
                 all_modules.append(lazy_module)
-        
+
         if not suppress_warnings:
             for module in limit_to_modules:
                 if not any(module == m.name for m in all_modules):
@@ -124,15 +136,16 @@ class ModuleFactory:
 
         return all_modules
 
+
 @dataclass
 class LazyBaseModule:
-
     """
     A lazy module class, which only loads the manifest and does not load the module itself.
 
     This is useful for getting information about a module without actually loading it.
 
     """
+
     name: str
     description: str
     path: str
@@ -149,30 +162,30 @@ class LazyBaseModule:
 
     @property
     def type(self):
-        return self.manifest['type']
+        return self.manifest["type"]
 
     @property
     def entry_point(self):
-        if not self._entry_point and not self.manifest['entry_point']:
+        if not self._entry_point and not self.manifest["entry_point"]:
             # try to create the entry point from the module name
             self._entry_point = f"{self.name}::{self.name.replace('_', ' ').title().replace(' ', '')}"
         return self._entry_point
 
     @property
     def dependencies(self) -> dict:
-        return self.manifest['dependencies']
-    
+        return self.manifest["dependencies"]
+
     @property
     def configs(self) -> dict:
-        return self.manifest['configs']
-    
+        return self.manifest["configs"]
+
     @property
     def requires_setup(self) -> bool:
-        return self.manifest['requires_setup']
-    
+        return self.manifest["requires_setup"]
+
     @property
     def display_name(self) -> str:
-        return self.manifest['name']
+        return self.manifest["name"]
 
     @property
     def manifest(self) -> dict:
@@ -186,17 +199,16 @@ class LazyBaseModule:
             try:
                 manifest.update(ast.literal_eval(f.read()))
             except (ValueError, TypeError, SyntaxError, MemoryError, RecursionError) as e:
-                raise ValueError(f"Error loading manifest from file {self.path}/{MANIFEST_FILE}: {e}")
-            
+                raise ValueError(f"Error loading manifest from file {self.path}/{MANIFEST_FILE}: {e}") from e
+
         self._manifest = manifest
-        self._entry_point = manifest['entry_point']
-        self.description = manifest['description']
-        self.version = manifest['version']
+        self._entry_point = manifest["entry_point"]
+        self.description = manifest["description"]
+        self.version = manifest["version"]
 
         return manifest
 
     def load(self, config) -> BaseModule:
-
         if self._instance:
             return self._instance
 
@@ -207,19 +219,21 @@ class LazyBaseModule:
                     # clear out any empty strings that a user may have erroneously added
                     continue
                 if not check(dep):
-                    logger.error(f"Module '{self.name}' requires external dependency '{dep}' which is not available/setup. \
-                                 Have you installed the required dependencies for the '{self.name}' module? See the README for more information.")
-                    exit(1)
+                    logger.error(
+                        f"Module '{self.name}' requires external dependency '{dep}' which is not available/setup. \
+                                 Have you installed the required dependencies for the '{self.name}' module? See the documentation for more information."
+                    )
+                    raise SetupError()
 
         def check_python_dep(dep):
             # first check if it's a module:
             try:
                 m = self.module_factory.get_module_lazy(dep, suppress_warnings=True)
                 try:
-                # we must now load this module and set it up with the config
+                    # we must now load this module and set it up with the config
                     m.load(config)
                     return True
-                except:
+                except Exception:
                     logger.error(f"Unable to setup module '{dep}' for use in module '{self.name}'")
                     return False
             except IndexError:
@@ -228,13 +242,26 @@ class LazyBaseModule:
 
             return find_spec(dep)
 
-        check_deps(self.dependencies.get('python', []), check_python_dep)
-        check_deps(self.dependencies.get('bin', []), lambda dep: shutil.which(dep))
+        def check_bin_dep(dep):
+            dep_exists = shutil.which(dep)
 
+            if dep == "docker":
+                if os.environ.get("RUNNING_IN_DOCKER"):
+                    # this is only for the WACZ enricher, which requires docker
+                    # if we're already running in docker then we don't need docker
+                    return True
+
+                # check if docker daemon is running
+                return dep_exists and subprocess.run(["docker", "ps", "-q"]).returncode == 0
+
+            return dep_exists
+
+        check_deps(self.dependencies.get("python", []), check_python_dep)
+        check_deps(self.dependencies.get("bin", []), check_bin_dep)
 
         logger.debug(f"Loading module '{self.display_name}'...")
 
-        for qualname in [self.name, f'auto_archiver.modules.{self.name}']:
+        for qualname in [self.name, f"auto_archiver.modules.{self.name}"]:
             try:
                 # first import the whole module, to make sure it's working properly
                 __import__(qualname)
@@ -243,10 +270,10 @@ class LazyBaseModule:
                 pass
 
         # then import the file for the entry point
-        file_name, class_name = self.entry_point.split('::')
-        sub_qualname = f'{qualname}.{file_name}'
+        file_name, class_name = self.entry_point.split("::")
+        sub_qualname = f"{qualname}.{file_name}"
 
-        __import__(f'{qualname}.{file_name}', fromlist=[self.entry_point])
+        __import__(f"{qualname}.{file_name}", fromlist=[self.entry_point])
         # finally, get the class instance
         instance: BaseModule = getattr(sys.modules[sub_qualname], class_name)()
 
@@ -254,11 +281,11 @@ class LazyBaseModule:
         instance.name = self.name
         instance.display_name = self.display_name
         instance.module_factory = self.module_factory
-        
-        # merge the default config with the user config
-        default_config = dict((k, v['default']) for k, v in self.configs.items() if 'default' in v)
 
-        config[self.name] = default_config  | config.get(self.name, {})
+        # merge the default config with the user config
+        default_config = dict((k, v["default"]) for k, v in self.configs.items() if "default" in v)
+
+        config[self.name] = default_config | config.get(self.name, {})
         instance.config_setup(config)
         instance.setup()
 
@@ -267,4 +294,4 @@ class LazyBaseModule:
         return instance
 
     def __repr__(self):
-        return f"Module<'{self.display_name}' ({self.name})>"
+        return f"Module<'{self.display_name}' ({self.name})>"

src/auto_archiver/core/orchestrator.py

@@ -1,6 +1,6 @@
-""" Orchestrates all archiving steps, including feeding items,
-    archiving them with specific archivers, enrichment, storage,
-    formatting, database operations and clean up.
+"""Orchestrates all archiving steps, including feeding items,
+archiving them with specific archivers, enrichment, storage,
+formatting, database operations and clean up.
 
 """
 
@@ -15,24 +15,32 @@ from copy import copy
 
 from rich_argparse import RichHelpFormatter
 from loguru import logger
+import requests
 
 from .metadata import Metadata, Media
 from auto_archiver.version import __version__
-from .config import read_yaml, store_yaml, to_dot_notation, merge_dicts, is_valid_config, \
-    DefaultValidatingParser, UniqueAppendAction, AuthenticationJsonParseAction, DEFAULT_CONFIG_FILE
+from .config import (
+    read_yaml,
+    store_yaml,
+    to_dot_notation,
+    merge_dicts,
+    is_valid_config,
+    DefaultValidatingParser,
+    UniqueAppendAction,
+    AuthenticationJsonParseAction,
+    DEFAULT_CONFIG_FILE,
+)
 from .module import ModuleFactory, LazyBaseModule
 from . import validators, Feeder, Extractor, Database, Storage, Formatter, Enricher
-from .consts import MODULE_TYPES
+from .consts import MODULE_TYPES, SetupError
 from auto_archiver.utils.url import check_url_or_raise
 
 if TYPE_CHECKING:
     from .base_module import BaseModule
     from .module import LazyBaseModule
 
-class SetupError(ValueError):
-    pass
-class ArchivingOrchestrator:
 
+class ArchivingOrchestrator:
     # instance variables
     module_factory: ModuleFactory
     setup_finished: bool
@@ -62,20 +70,63 @@ class ArchivingOrchestrator:
             epilog="Check the code at https://github.com/bellingcat/auto-archiver",
             formatter_class=RichHelpFormatter,
         )
-        parser.add_argument('--help', '-h', action='store_true', dest='help', help='show a full help message and exit')
-        parser.add_argument('--version', action='version', version=__version__)
-        parser.add_argument('--config', action='store', dest="config_file", help='the filename of the YAML configuration file (defaults to \'config.yaml\')', default=DEFAULT_CONFIG_FILE)
-        parser.add_argument('--mode', action='store', dest='mode', type=str, choices=['simple', 'full'], help='the mode to run the archiver in', default='simple')
+        parser.add_argument("--help", "-h", action="store_true", dest="help", help="show a full help message and exit")
+        parser.add_argument("--version", action="version", version=__version__)
+        parser.add_argument(
+            "--config",
+            action="store",
+            dest="config_file",
+            help="the filename of the YAML configuration file (defaults to 'config.yaml')",
+            default=DEFAULT_CONFIG_FILE,
+        )
+        parser.add_argument(
+            "--mode",
+            action="store",
+            dest="mode",
+            type=str,
+            choices=["simple", "full"],
+            help="the mode to run the archiver in",
+            default="simple",
+        )
         # override the default 'help' so we can inject all the configs and show those
-        parser.add_argument('-s', '--store', dest='store', default=False, help='Store the created config in the config file', action=argparse.BooleanOptionalAction)
-        parser.add_argument('--module_paths', dest='module_paths', nargs='+', default=[], help='additional paths to search for modules', action=UniqueAppendAction)
+        parser.add_argument(
+            "-s",
+            "--store",
+            dest="store",
+            default=False,
+            help="Store the created config in the config file",
+            action=argparse.BooleanOptionalAction,
+        )
+        parser.add_argument(
+            "--module_paths",
+            dest="module_paths",
+            nargs="+",
+            default=[],
+            help="additional paths to search for modules",
+            action=UniqueAppendAction,
+        )
 
         self.basic_parser = parser
         return parser
 
+    def check_steps(self, config):
+        for module_type in MODULE_TYPES:
+            if not config["steps"].get(f"{module_type}s", []):
+                if (module_type == "feeder" or module_type == "formatter") and config["steps"].get(f"{module_type}"):
+                    raise SetupError(
+                        f"It appears you have '{module_type}' set under 'steps' in your configuration file, but as of version 0.13.0 of Auto Archiver, you must use '{module_type}s'. Change this in your configuration file and try again. \
+Here's how that would look: \n\nsteps:\n  {module_type}s:\n  - [your_{module_type}_name_here]\n  {'extractors:...' if module_type == 'feeder' else '...'}\n"
+                    )
+                if module_type == "extractor" and config["steps"].get("archivers"):
+                    raise SetupError(
+                        "As of version 0.13.0 of Auto Archiver, the 'archivers' step name has been changed to 'extractors'. Change this in your configuration file and try again. \
+Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_here]\n  enrichers:...\n"
+                    )
+                raise SetupError(
+                    f"No {module_type}s were configured. Make sure to set at least one {module_type} in your configuration file or on the command line (using --{module_type}s)"
+                )
+
     def setup_complete_parser(self, basic_config: dict, yaml_config: dict, unused_args: list[str]) -> None:
-
-
         # modules parser to get the overridden 'steps' values
         modules_parser = argparse.ArgumentParser(
             add_help=False,
@@ -83,7 +134,9 @@ class ArchivingOrchestrator:
         self.add_modules_args(modules_parser)
         cli_modules, unused_args = modules_parser.parse_known_args(unused_args)
         for module_type in MODULE_TYPES:
-            yaml_config['steps'][f"{module_type}s"] = getattr(cli_modules, f"{module_type}s", []) or yaml_config['steps'].get(f"{module_type}s", [])
+            yaml_config["steps"][f"{module_type}s"] = getattr(cli_modules, f"{module_type}s", []) or yaml_config[
+                "steps"
+            ].get(f"{module_type}s", [])
 
         parser = DefaultValidatingParser(
             add_help=False,
@@ -100,39 +153,38 @@ class ArchivingOrchestrator:
         # but should we add them? Or should we just add them to the 'complete' parser?
 
         if is_valid_config(yaml_config):
+            self.check_steps(yaml_config)
             # only load the modules enabled in config
             # TODO: if some steps are empty (e.g. 'feeders' is empty), should we default to the 'simple' ones? Or only if they are ALL empty?
             enabled_modules = []
             # first loads the modules from the config file, then from the command line
             for module_type in MODULE_TYPES:
-                enabled_modules.extend(yaml_config['steps'].get(f"{module_type}s", []))
+                enabled_modules.extend(yaml_config["steps"].get(f"{module_type}s", []))
 
             # clear out duplicates, but keep the order
             enabled_modules = list(dict.fromkeys(enabled_modules))
-            avail_modules = self.module_factory.available_modules(limit_to_modules=enabled_modules, suppress_warnings=True)
+            avail_modules = self.module_factory.available_modules(
+                limit_to_modules=enabled_modules, suppress_warnings=True
+            )
             self.add_individual_module_args(avail_modules, parser)
-        elif basic_config.mode == 'simple':
+        elif basic_config.mode == "simple":
             simple_modules = [module for module in self.module_factory.available_modules() if not module.requires_setup]
             self.add_individual_module_args(simple_modules, parser)
 
-            # for simple mode, we use the cli_feeder and any modules that don't require setup
-            if not yaml_config['steps']['feeders']:
-                yaml_config['steps']['feeders'] = ['cli_feeder']
-
             # add them to the config
             for module in simple_modules:
                 for module_type in module.type:
-                    yaml_config['steps'].setdefault(f"{module_type}s", []).append(module.name)
+                    yaml_config["steps"].setdefault(f"{module_type}s", []).append(module.name)
         else:
             # load all modules, they're not using the 'simple' mode
             all_modules = self.module_factory.available_modules()
             # add all the modules to the steps
             for module in all_modules:
                 for module_type in module.type:
-                    yaml_config['steps'].setdefault(f"{module_type}s", []).append(module.name)
+                    yaml_config["steps"].setdefault(f"{module_type}s", []).append(module.name)
 
             self.add_individual_module_args(all_modules, parser)
-        
+
         parser.set_defaults(**to_dot_notation(yaml_config))
 
         # reload the parser with the new arguments, now that we have them
@@ -158,41 +210,75 @@ class ArchivingOrchestrator:
             store_yaml(config, basic_config.config_file)
 
         return config
-    
+
     def add_modules_args(self, parser: argparse.ArgumentParser = None):
         if not parser:
             parser = self.parser
 
         # Module loading from the command line
         for module_type in MODULE_TYPES:
-            parser.add_argument(f'--{module_type}s', dest=f'{module_type}s', nargs='+', help=f'the {module_type}s to use', default=[], action=UniqueAppendAction)
+            parser.add_argument(
+                f"--{module_type}s",
+                dest=f"{module_type}s",
+                nargs="+",
+                help=f"the {module_type}s to use",
+                default=[],
+                action=UniqueAppendAction,
+            )
 
     def add_additional_args(self, parser: argparse.ArgumentParser = None):
         if not parser:
             parser = self.parser
 
-        # allow passing URLs directly on the command line
-        parser.add_argument('urls', nargs='*', default=[], help='URL(s) to archive, either a single URL or a list of urls, should not come from config.yaml')
-
-        parser.add_argument('--authentication', dest='authentication', help='A dictionary of sites and their authentication methods \
+        parser.add_argument(
+            "--authentication",
+            dest="authentication",
+            help="A dictionary of sites and their authentication methods \
                                                                             (token, username etc.) that extractors can use to log into \
                                                                             a website. If passing this on the command line, use a JSON string. \
-                                                                            You may also pass a path to a valid JSON/YAML file which will be parsed.',
-                                                                            default={},
-                                                                            nargs="?",
-                                                                            action=AuthenticationJsonParseAction)
+                                                                            You may also pass a path to a valid JSON/YAML file which will be parsed.",
+            default={},
+            nargs="?",
+            action=AuthenticationJsonParseAction,
+        )
 
         # logging arguments
-        parser.add_argument('--logging.level', action='store', dest='logging.level', choices=['INFO', 'DEBUG', 'ERROR', 'WARNING'], help='the logging level to use', default='INFO', type=str.upper)
-        parser.add_argument('--logging.file', action='store', dest='logging.file', help='the logging file to write to', default=None)
-        parser.add_argument('--logging.rotation', action='store', dest='logging.rotation', help='the logging rotation to use', default=None)
-
-    def add_individual_module_args(self, modules: list[LazyBaseModule] = None, parser: argparse.ArgumentParser = None) -> None:
+        parser.add_argument(
+            "--logging.level",
+            action="store",
+            dest="logging.level",
+            choices=["INFO", "DEBUG", "ERROR", "WARNING"],
+            help="the logging level to use",
+            default="INFO",
+            type=str.upper,
+        )
+        parser.add_argument(
+            "--logging.file", action="store", dest="logging.file", help="the logging file to write to", default=None
+        )
+        parser.add_argument(
+            "--logging.rotation",
+            action="store",
+            dest="logging.rotation",
+            help="the logging rotation to use",
+            default=None,
+        )
 
+    def add_individual_module_args(
+        self, modules: list[LazyBaseModule] = None, parser: argparse.ArgumentParser = None
+    ) -> None:
         if not modules:
             modules = self.module_factory.available_modules()
-        
+
         for module in modules:
+            if module.name == "cli_feeder":
+                # special case. For the CLI feeder, allow passing URLs directly on the command line without setting --cli_feeder.urls=
+                parser.add_argument(
+                    "urls",
+                    nargs="*",
+                    default=[],
+                    help="URL(s) to archive, either a single URL or a list of urls, should not come from config.yaml",
+                )
+                continue
 
             if not module.configs:
                 # this module has no configs, don't show anything in the help
@@ -202,21 +288,21 @@ class ArchivingOrchestrator:
             group = parser.add_argument_group(module.display_name or module.name, f"{module.description[:100]}...")
 
             for name, kwargs in module.configs.items():
-                if not kwargs.get('metavar', None):
+                if not kwargs.get("metavar", None):
                     # make a nicer metavar, metavar is what's used in the help, e.g. --cli_feeder.urls [METAVAR]
-                    kwargs['metavar'] = name.upper()
+                    kwargs["metavar"] = name.upper()
 
-                if kwargs.get('required', False):
+                if kwargs.get("required", False):
                     # required args shouldn't have a 'default' value, remove it
-                    kwargs.pop('default', None)
+                    kwargs.pop("default", None)
 
-                kwargs.pop('cli_set', None)
-                should_store = kwargs.pop('should_store', False)
-                kwargs['dest'] = f"{module.name}.{kwargs.pop('dest', name)}"
+                kwargs.pop("cli_set", None)
+                should_store = kwargs.pop("should_store", False)
+                kwargs["dest"] = f"{module.name}.{kwargs.pop('dest', name)}"
                 try:
-                    kwargs['type'] = getattr(validators, kwargs.get('type', '__invalid__'))
+                    kwargs["type"] = getattr(validators, kwargs.get("type", "__invalid__"))
                 except AttributeError:
-                    kwargs['type'] = __builtins__.get(kwargs.get('type'), str)
+                    kwargs["type"] = __builtins__.get(kwargs.get("type"), str)
                 arg = group.add_argument(f"--{module.name}.{name}", **kwargs)
                 arg.should_store = should_store
 
@@ -231,12 +317,11 @@ class ArchivingOrchestrator:
         self.basic_parser.exit()
 
     def setup_logging(self, config):
+        logging_config = config["logging"]
 
-        logging_config = config['logging']
-
-        if logging_config.get('enabled', True) is False:
+        if logging_config.get("enabled", True) is False:
             # disabled logging settings, they're set on a higher level
-            logger.disable('auto_archiver')
+            logger.disable("auto_archiver")
             return
 
         # setup loguru logging
@@ -246,67 +331,63 @@ class ArchivingOrchestrator:
             pass
 
         # add other logging info
-        if self.logger_id is None: # note - need direct comparison to None since need to consider falsy value 0
-            self.logger_id = logger.add(sys.stderr, level=logging_config['level'])
-            if log_file := logging_config['file']:
-                logger.add(log_file) if not logging_config['rotation'] else logger.add(log_file, rotation=logging_config['rotation'])
+        if self.logger_id is None:  # note - need direct comparison to None since need to consider falsy value 0
+            self.logger_id = logger.add(sys.stderr, level=logging_config["level"])
+            if log_file := logging_config["file"]:
+                logger.add(log_file) if not logging_config["rotation"] else logger.add(
+                    log_file, rotation=logging_config["rotation"]
+                )
 
     def install_modules(self, modules_by_type):
         """
-        Traverses all modules in 'steps' and loads them into the orchestrator, storing them in the 
+        Traverses all modules in 'steps' and loads them into the orchestrator, storing them in the
         orchestrator's attributes (self.feeders, self.extractors etc.). If no modules of a certain type
         are loaded, the program will exit with an error message.
         """
 
         invalid_modules = []
         for module_type in MODULE_TYPES:
-
             step_items = []
             modules_to_load = modules_by_type[f"{module_type}s"]
             if not modules_to_load:
-                raise SetupError(f"No {module_type}s were configured. Make sure to set at least one {module_type} in your configuration file or on the command line (using --{module_type}s)")
+                raise SetupError(
+                    f"No {module_type}s were configured. Make sure to set at least one {module_type} in your configuration file or on the command line (using --{module_type}s)"
+                )
 
             def check_steps_ok():
                 if not len(step_items):
                     if len(modules_to_load):
-                        logger.error(f"Unable to load any {module_type}s. Tried the following, but none were available: {modules_to_load}")
-                    raise SetupError(f"NO {module_type.upper()}S LOADED. Please check your configuration and try again.")
-                
+                        logger.error(
+                            f"Unable to load any {module_type}s. Tried the following, but none were available: {modules_to_load}"
+                        )
+                    raise SetupError(
+                        f"NO {module_type.upper()}S LOADED. Please check your configuration and try again."
+                    )
 
-                if (module_type == 'feeder' or module_type == 'formatter') and len(step_items) > 1:
-                    raise SetupError(f"Only one {module_type} is allowed, found {len(step_items)} {module_type}s. Please remove one of the following from your configuration file: {modules_to_load}")
+                if (module_type == "feeder" or module_type == "formatter") and len(step_items) > 1:
+                    raise SetupError(
+                        f"Only one {module_type} is allowed, found {len(step_items)} {module_type}s. Please remove one of the following from your configuration file: {modules_to_load}"
+                    )
 
             for module in modules_to_load:
-                if module == 'cli_feeder':
-                    # cli_feeder is a pseudo module, it just takes the command line args for [URLS]
-                    urls = self.config['urls']
-                    if not urls:
-                        raise SetupError("No URLs provided. Please provide at least one URL via the command line, or set up an alternative feeder. Use --help for more information.")
-
-                    def feed(self) -> Generator[Metadata]:
-                        for url in urls:
-                            logger.debug(f"Processing URL: '{url}'")
-                            yield Metadata().set_url(url)
-
-                    pseudo_module = type('CLIFeeder', (Feeder,), {
-                        'name': 'cli_feeder',
-                        'display_name': 'CLI Feeder',
-                        '__iter__': feed
-
-                    })()
-
-                    pseudo_module.__iter__ = feed
-                    step_items.append(pseudo_module)
-                    continue
-
                 if module in invalid_modules:
                     continue
 
+                # check to make sure that we're trying to load it as the correct type - i.e. make sure the user hasn't put it under the wrong 'step'
+                lazy_module: LazyBaseModule = self.module_factory.get_module_lazy(module)
+                if module_type not in lazy_module.type:
+                    types = ",".join(f"'{t}'" for t in lazy_module.type)
+                    raise SetupError(
+                        f"Configuration Error: Module '{module}' is not a {module_type}, but has the types: {types}. Please check you set this module up under the right step in your orchestration file."
+                    )
+
+                loaded_module = None
                 try:
-                    loaded_module: BaseModule = self.module_factory.get_module(module, self.config)
+                    loaded_module: BaseModule = lazy_module.load(self.config)
                 except (KeyboardInterrupt, Exception) as e:
-                    logger.error(f"Error during setup of modules: {e}\n{traceback.format_exc()}")
-                    if module_type == 'extractor' and loaded_module.name == module:
+                    if not isinstance(e, KeyboardInterrupt) and not isinstance(e, SetupError):
+                        logger.error(f"Error during setup of modules: {e}\n{traceback.format_exc()}")
+                    if loaded_module and module_type == "extractor":
                         loaded_module.cleanup()
                     raise e
 
@@ -321,11 +402,13 @@ class ArchivingOrchestrator:
 
     def load_config(self, config_file: str) -> dict:
         if not os.path.exists(config_file) and config_file != DEFAULT_CONFIG_FILE:
-            logger.error(f"The configuration file {config_file} was  not found. Make sure the file exists and try again, or run without the --config file to use the default settings.")
+            logger.error(
+                f"The configuration file {config_file} was  not found. Make sure the file exists and try again, or run without the --config file to use the default settings."
+            )
             raise FileNotFoundError(f"Configuration file {config_file} not found")
 
         return read_yaml(config_file)
-    
+
     def setup_config(self, args: list) -> dict:
         """
         Sets up the configuration file, merging the default config with the user's config
@@ -349,30 +432,51 @@ class ArchivingOrchestrator:
 
         return self.setup_complete_parser(basic_config, yaml_config, unused_args)
 
+    def check_for_updates(self):
+        response = requests.get("https://pypi.org/pypi/auto-archiver/json").json()
+        latest_version = response["info"]["version"]
+        # check version compared to current version
+        if latest_version != __version__:
+            if os.environ.get("RUNNING_IN_DOCKER"):
+                update_cmd = "`docker pull bellingcat/auto-archiver:latest`"
+            else:
+                update_cmd = "`pip install --upgrade auto-archiver`"
+            logger.warning("")
+            logger.warning("********* IMPORTANT: UPDATE AVAILABLE ********")
+            logger.warning(f"A new version of auto-archiver is available (v{latest_version}, you have {__version__})")
+            logger.warning(f"Make sure to update to the latest version using: {update_cmd}")
+            logger.warning("")
+
     def setup(self, args: list):
         """
         Function to configure all setup of the orchestrator: setup configs and load modules.
-        
+
         This method should only ever be called once
         """
 
+        self.check_for_updates()
+
         if self.setup_finished:
-            logger.warning("The `setup_config()` function should only ever be run once. \
+            logger.warning(
+                "The `setup_config()` function should only ever be run once. \
                            If you need to re-run the setup, please re-instantiate a new instance of the orchestrator. \
                            For code implementatations, you should call .setup_config() once then you may call .feed() \
-                           multiple times to archive multiple URLs.")
+                           multiple times to archive multiple URLs."
+            )
             return
 
         self.setup_basic_parser()
         self.config = self.setup_config(args)
 
         logger.info(f"======== Welcome to the AUTO ARCHIVER ({__version__}) ==========")
-        self.install_modules(self.config['steps'])
+        self.install_modules(self.config["steps"])
 
         # log out the modules that were loaded
         for module_type in MODULE_TYPES:
-            logger.info(f"{module_type.upper()}S: " + ", ".join(m.display_name for m in getattr(self, f"{module_type}s")))
-        
+            logger.info(
+                f"{module_type.upper()}S: " + ", ".join(m.display_name for m in getattr(self, f"{module_type}s"))
+            )
+
         self.setup_finished = True
 
     def _command_line_run(self, args: list) -> Generator[Metadata]:
@@ -380,9 +484,9 @@ class ArchivingOrchestrator:
         This is the main entry point for the orchestrator, when run from the command line.
 
         :param args: list of arguments to pass to the orchestrator - these are the command line args
-        
+
         You should not call this method from code implementations.
-          
+
         This method sets up the configuration, loads the modules, and runs the feed.
         If you wish to make code invocations yourself, you should use the 'setup' and 'feed' methods separately.
         To test configurations, without loading any modules you can also first call 'setup_configs'
@@ -400,7 +504,6 @@ class ArchivingOrchestrator:
             e.cleanup()
 
     def feed(self) -> Generator[Metadata]:
-        
         url_count = 0
         for feeder in self.feeders:
             for item in feeder:
@@ -431,9 +534,9 @@ class ArchivingOrchestrator:
             self.cleanup()
             exit()
         except Exception as e:
-            logger.error(f'Got unexpected error on item {item}: {e}\n{traceback.format_exc()}')
+            logger.error(f"Got unexpected error on item {item}: {e}\n{traceback.format_exc()}")
             for d in self.databases:
-                if type(e) == AssertionError:
+                if isinstance(e, AssertionError):
                     d.failed(item, str(e))
                 else:
                     d.failed(item, reason="unexpected error")
@@ -446,13 +549,13 @@ class ArchivingOrchestrator:
 
     def archive(self, result: Metadata) -> Union[Metadata, None]:
         """
-            Runs the archiving process for a single URL
-            1. Each archiver can sanitize its own URLs
-            2. Check for cached results in Databases, and signal start to the databases
-            3. Call Archivers until one succeeds
-            4. Call Enrichers
-            5. Store all downloaded/generated media
-            6. Call selected Formatter and store formatted if needed
+        Runs the archiving process for a single URL
+        1. Each archiver can sanitize its own URLs
+        2. Check for cached results in Databases, and signal start to the databases
+        3. Call Archivers until one succeeds
+        4. Call Enrichers
+        5. Store all downloaded/generated media
+        6. Call selected Formatter and store formatted if needed
         """
 
         original_url = result.get_url().strip()
@@ -468,7 +571,8 @@ class ArchivingOrchestrator:
             url = a.sanitize_url(url)
 
         result.set_url(url)
-        if original_url != url: result.set("original_url", original_url)
+        if original_url != url:
+            result.set("original_url", original_url)
 
         # 2 - notify start to DBs, propagate already archived if feature enabled in DBs
         cached_result = None
@@ -479,7 +583,8 @@ class ArchivingOrchestrator:
         if cached_result:
             logger.debug("Found previously archived entry")
             for d in self.databases:
-                try: d.done(cached_result, cached=True)
+                try:
+                    d.done(cached_result, cached=True)
                 except Exception as e:
                     logger.error(f"ERROR database {d.name}: {e}: {traceback.format_exc()}")
             return cached_result
@@ -489,13 +594,15 @@ class ArchivingOrchestrator:
             logger.info(f"Trying extractor {a.name} for {url}")
             try:
                 result.merge(a.download(result))
-                if result.is_success(): break
+                if result.is_success():
+                    break
             except Exception as e:
                 logger.error(f"ERROR archiver {a.name}: {e}: {traceback.format_exc()}")
 
         # 4 - call enrichers to work with archived content
         for e in self.enrichers:
-            try: e.enrich(result)
+            try:
+                e.enrich(result)
             except Exception as exc:
                 logger.error(f"ERROR enricher {e.name}: {exc}: {traceback.format_exc()}")
 
@@ -513,12 +620,12 @@ class ArchivingOrchestrator:
 
         # signal completion to databases and archivers
         for d in self.databases:
-            try: d.done(result)
+            try:
+                d.done(result)
             except Exception as e:
                 logger.error(f"ERROR database {d.name}: {e}: {traceback.format_exc()}")
 
         return result
-    
 
     def setup_authentication(self, config: dict) -> dict:
         """
@@ -527,7 +634,7 @@ class ArchivingOrchestrator:
         Split up strings into multiple sites if they are comma separated
         """
 
-        authentication = config.get('authentication', {})
+        authentication = config.get("authentication", {})
 
         # extract out concatenated sites
         for key, val in copy(authentication).items():
@@ -536,8 +643,8 @@ class ArchivingOrchestrator:
                     site = site.strip()
                     authentication[site] = val
                 del authentication[key]
-        
-        config['authentication'] = authentication
+
+        config["authentication"] = authentication
         return config
 
     # Helper Properties

src/auto_archiver/core/storage.py

@@ -1,5 +1,22 @@
 """
 Base module for Storage modules – modular components that store media objects in various locations.
+
+If you are looking to implement a new storage module, you should subclass the `Storage` class and
+implement the `get_cdn_url` and `uploadf` methods.
+
+Your module **must** also have two config variables 'path_generator' and 'filename_generator' which
+determine how the key is generated for the media object. The 'path_generator' and 'filename_generator'
+variables can be set to one of the following values:
+- 'flat': A flat structure with no subfolders
+- 'url': A structure based on the URL of the media object
+- 'random': A random structure
+
+The 'filename_generator' variable can be set to one of the following values:
+- 'random': A random string
+- 'static': A replicable strategy such as a hash
+
+If you don't want to use this naming convention, you can override the `set_key` method in your subclass.
+
 """
 
 from __future__ import annotations
@@ -15,18 +32,19 @@ from auto_archiver.utils.misc import random_str
 from auto_archiver.core import Media, BaseModule, Metadata
 from auto_archiver.modules.hash_enricher.hash_enricher import HashEnricher
 
+
 class Storage(BaseModule):
-    
     """
     Base class for implementing storage modules in the media archiving framework.
 
     Subclasses must implement the `get_cdn_url` and `uploadf` methods to define their behavior.
     """
 
-    def store(self, media: Media, url: str, metadata: Metadata=None) -> None:
-        if media.is_stored(in_storage=self): 
+    def store(self, media: Media, url: str, metadata: Metadata = None) -> None:
+        if media.is_stored(in_storage=self):
             logger.debug(f"{media.key} already stored, skipping")
             return
+
         self.set_key(media, url, metadata)
         self.upload(media, metadata=metadata)
         media.add_url(self.get_cdn_url(media))
@@ -42,42 +60,55 @@ class Storage(BaseModule):
     def uploadf(self, file: IO[bytes], key: str, **kwargs: dict) -> bool:
         """
         Uploads (or saves) a file to the storage service/location.
+
+        This method should not be called directly, but instead through the 'store' method,
+        which sets up the media for storage.
         """
         pass
 
     def upload(self, media: Media, **kwargs) -> bool:
-        logger.debug(f'[{self.__class__.__name__}] storing file {media.filename} with key {media.key}')
-        with open(media.filename, 'rb') as f:
+        """
+        Uploads a media object to the storage service.
+
+        This method should not be called directly, but instead be called through the 'store' method,
+        which sets up the media for storage.
+        """
+        logger.debug(f"[{self.__class__.__name__}] storing file {media.filename} with key {media.key}")
+        with open(media.filename, "rb") as f:
             return self.uploadf(f, media, **kwargs)
 
-    def set_key(self, media: Media, url, metadata: Metadata) -> None:
+    def set_key(self, media: Media, url: str, metadata: Metadata) -> None:
         """takes the media and optionally item info and generates a key"""
-        if media.key is not None and len(media.key) > 0: return
-        folder = metadata.get_context('folder', '')
+
+        if media.key is not None and len(media.key) > 0:
+            # media key is already set
+            return
+
+        folder = metadata.get_context("folder", "")
         filename, ext = os.path.splitext(media.filename)
 
         # Handle path_generator logic
-        path_generator = self.config.get("path_generator", "url")
+        path_generator = self.path_generator
         if path_generator == "flat":
             path = ""
-            filename = slugify(filename)  # Ensure filename is slugified
         elif path_generator == "url":
-            path = slugify(url)
+            path = slugify(url)[:70]
         elif path_generator == "random":
-            path = self.config.get("random_path", random_str(24), True)
+            path = random_str(24)
         else:
             raise ValueError(f"Invalid path_generator: {path_generator}")
 
         # Handle filename_generator logic
-        filename_generator = self.config.get("filename_generator", "random")
+        filename_generator = self.filename_generator
         if filename_generator == "random":
             filename = random_str(24)
         elif filename_generator == "static":
             # load the hash_enricher module
-            he = self.module_factory.get_module(HashEnricher, self.config)
+            he: HashEnricher = self.module_factory.get_module("hash_enricher", self.config)
             hd = he.calculate_hash(media.filename)
             filename = hd[:24]
         else:
             raise ValueError(f"Invalid filename_generator: {filename_generator}")
 
-        media.key = os.path.join(folder, path, f"{filename}{ext}")
+        key = os.path.join(folder, path, f"{filename}{ext}")
+        media._key = key

src/auto_archiver/core/validators.py

@@ -3,11 +3,13 @@ from pathlib import Path
 import argparse
 import json
 
+
 def example_validator(value):
     if "example" not in value:
         raise argparse.ArgumentTypeError(f"{value} is not a valid value for this argument")
     return value
 
+
 def positive_number(value):
     if value < 0:
         raise argparse.ArgumentTypeError(f"{value} is not a positive number")
@@ -19,5 +21,6 @@ def valid_file(value):
         raise argparse.ArgumentTypeError(f"File '{value}' does not exist.")
     return value
 
+
 def json_loader(cli_val):
-    return json.loads(cli_val)
+    return json.loads(cli_val)

src/auto_archiver/modules/api_db/__init__.py

@@ -1 +1 @@
-from .api_db import AAApiDb
+from .api_db import AAApiDb

src/auto_archiver/modules/api_db/__manifest__.py

@@ -11,8 +11,7 @@
             "required": True,
             "help": "API endpoint where calls are made to",
         },
-        "api_token": {"default": None,
-                      "help": "API Bearer token."},
+        "api_token": {"default": None, "help": "API Bearer token."},
         "public": {
             "default": False,
             "type": "bool",
@@ -24,9 +23,9 @@
             "help": "which group of users have access to the archive in case public=false as author",
         },
         "use_api_cache": {
-            "default": True,
+            "default": False,
             "type": "bool",
-            "help": "if False then the API database will be queried prior to any archiving operations and stop if the link has already been archived",
+            "help": "if True then the API database will be queried prior to any archiving operations and stop if the link has already been archived",
         },
         "store_results": {
             "default": True,

src/auto_archiver/modules/api_db/api_db.py

@@ -12,10 +12,11 @@ class AAApiDb(Database):
     """Connects to auto-archiver-api instance"""
 
     def fetch(self, item: Metadata) -> Union[Metadata, bool]:
-        """ query the database for the existence of this item.
-            Helps avoid re-archiving the same URL multiple times.
+        """query the database for the existence of this item.
+        Helps avoid re-archiving the same URL multiple times.
         """
-        if not self.use_api_cache: return
+        if not self.use_api_cache:
+            return
 
         params = {"url": item.get_url(), "limit": 15}
         headers = {"Authorization": f"Bearer {self.api_token}", "accept": "application/json"}
@@ -32,22 +33,25 @@ class AAApiDb(Database):
 
     def done(self, item: Metadata, cached: bool = False) -> None:
         """archival result ready - should be saved to DB"""
-        if not self.store_results: return
+        if not self.store_results:
+            return
         if cached:
             logger.debug(f"skipping saving archive of {item.get_url()} to the AA API because it was cached")
             return
         logger.debug(f"saving archive of {item.get_url()} to the AA API.")
 
         payload = {
-            'author_id': self.author_id,
-            'url': item.get_url(),
-            'public': self.public,
-            'group_id': self.group_id,
-            'tags': list(self.tags),
-            'result': item.to_json(),
+            "author_id": self.author_id,
+            "url": item.get_url(),
+            "public": self.public,
+            "group_id": self.group_id,
+            "tags": list(self.tags),
+            "result": item.to_json(),
         }
         headers = {"Authorization": f"Bearer {self.api_token}"}
-        response = requests.post(os.path.join(self.api_endpoint, "interop/submit-archive"), json=payload, headers=headers)
+        response = requests.post(
+            os.path.join(self.api_endpoint, "interop/submit-archive"), json=payload, headers=headers
+        )
 
         if response.status_code == 201:
             logger.success(f"AA API: {response.json()}")

src/auto_archiver/modules/atlos_db/__init__.py

@@ -1 +0,0 @@
-from .atlos_db import AtlosDb

src/auto_archiver/modules/atlos_db/__manifest__.py

@@ -1,38 +0,0 @@
-{
-    "name": "Atlos Database",
-    "type": ["database"],
-    "entry_point": "atlos_db::AtlosDb",
-    "requires_setup": True,
-    "dependencies":
-        {"python": ["loguru",
-                    ""],
-         "bin": [""]},
-    "configs": {
-        "api_token": {
-            "default": None,
-            "help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/",
-            "required": True,
-            "type": "str",
-        },
-        "atlos_url": {
-            "default": "https://platform.atlos.org",
-            "help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.",
-            "type": "str"
-        },
-    },
-    "description": """
-Handles integration with the Atlos platform for managing archival results.
-
-### Features
-- Outputs archival results to the Atlos API for storage and tracking.
-- Updates failure status with error details when archiving fails.
-- Processes and formats metadata, including ISO formatting for datetime fields.
-- Skips processing for items without an Atlos ID.
-
-### Setup
-Required configs:
-- atlos_url: Base URL for the Atlos API.
-- api_token: Authentication token for API access.
-"""
-,
-}

src/auto_archiver/modules/atlos_db/atlos_db.py

@@ -1,66 +0,0 @@
-from typing import Union
-
-import requests
-from loguru import logger
-
-from auto_archiver.core import Database
-from auto_archiver.core import Metadata
-
-
-class AtlosDb(Database):
-    """
-    Outputs results to Atlos
-    """
-
-    def failed(self, item: Metadata, reason: str) -> None:
-        """Update DB accordingly for failure"""
-        # If the item has no Atlos ID, there's nothing for us to do
-        if not item.metadata.get("atlos_id"):
-            logger.info(f"Item {item.get_url()} has no Atlos ID, skipping")
-            return
-
-        requests.post(
-            f"{self.atlos_url}/api/v2/source_material/metadata/{item.metadata['atlos_id']}/auto_archiver",
-            headers={"Authorization": f"Bearer {self.api_token}"},
-            json={"metadata": {"processed": True, "status": "error", "error": reason}},
-        ).raise_for_status()
-        logger.info(
-            f"Stored failure for {item.get_url()} (ID {item.metadata['atlos_id']}) on Atlos: {reason}"
-        )
-
-    def fetch(self, item: Metadata) -> Union[Metadata, bool]:
-        """check and fetch if the given item has been archived already, each
-        database should handle its own caching, and configuration mechanisms"""
-        return False
-
-    def _process_metadata(self, item: Metadata) -> dict:
-        """Process metadata for storage on Atlos. Will convert any datetime
-        objects to ISO format."""
-
-        return {
-            k: v.isoformat() if hasattr(v, "isoformat") else v
-            for k, v in item.metadata.items()
-        }
-
-    def done(self, item: Metadata, cached: bool = False) -> None:
-        """archival result ready - should be saved to DB"""
-
-        if not item.metadata.get("atlos_id"):
-            logger.info(f"Item {item.get_url()} has no Atlos ID, skipping")
-            return
-
-        requests.post(
-            f"{self.atlos_url}/api/v2/source_material/metadata/{item.metadata['atlos_id']}/auto_archiver",
-            headers={"Authorization": f"Bearer {self.api_token}"},
-            json={
-                "metadata": dict(
-                    processed=True,
-                    status="success",
-                    results=self._process_metadata(item),
-                )
-            },
-        ).raise_for_status()
-
-        logger.info(
-            f"Stored success for {item.get_url()} (ID {item.metadata['atlos_id']}) on Atlos"
-        )

src/auto_archiver/modules/atlos_feeder/__init__.py

@@ -1 +0,0 @@
-from .atlos_feeder import AtlosFeeder

src/auto_archiver/modules/atlos_feeder/__manifest__.py

@@ -1,34 +0,0 @@
-{
-    "name": "Atlos Feeder",
-    "type": ["feeder"],
-    "requires_setup": True,
-    "dependencies": {
-        "python": ["loguru", "requests"],
-    },
-    "configs": {
-        "api_token": {
-            "type": "str",
-            "required": True,
-            "help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/",
-        },
-        "atlos_url": {
-            "default": "https://platform.atlos.org",
-            "help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.",
-            "type": "str"
-        },
-    },
-    "description": """
-    AtlosFeeder: A feeder module that integrates with the Atlos API to fetch source material URLs for archival.
-
-    ### Features
-    - Connects to the Atlos API to retrieve a list of source material URLs.
-    - Filters source materials based on visibility, processing status, and metadata.
-    - Converts filtered source materials into `Metadata` objects with the relevant `atlos_id` and URL.
-    - Iterates through paginated results using a cursor for efficient API interaction.
-
-    ### Notes
-    - Requires an Atlos API endpoint and a valid API token for authentication.
-    - Ensures only unprocessed, visible, and ready-to-archive URLs are returned.
-    - Handles pagination transparently when retrieving data from the Atlos API.
-    """
-}

src/auto_archiver/modules/atlos_feeder/atlos_feeder.py

@@ -1,42 +0,0 @@
-import requests
-from loguru import logger
-
-from auto_archiver.core import Feeder
-from auto_archiver.core import Metadata
-
-
-class AtlosFeeder(Feeder):
-
-    def __iter__(self) -> Metadata:
-        # Get all the urls from the Atlos API
-        count = 0
-        cursor = None
-        while True:
-            response = requests.get(
-                f"{self.atlos_url}/api/v2/source_material",
-                headers={"Authorization": f"Bearer {self.api_token}"},
-                params={"cursor": cursor},
-            )
-            data = response.json()
-            response.raise_for_status()
-            cursor = data["next"]
-
-            for item in data["results"]:
-                if (
-                    item["source_url"] not in [None, ""]
-                    and (
-                        item["metadata"]
-                        .get("auto_archiver", {})
-                        .get("processed", False)
-                        != True
-                    )
-                    and item["visibility"] == "visible"
-                    and item["status"] not in ["processing", "pending"]
-                ):
-                    yield Metadata().set_url(item["source_url"]).set(
-                        "atlos_id", item["id"]
-                    )
-                    count += 1
-
-            if len(data["results"]) == 0 or cursor is None:
-                break

src/auto_archiver/modules/atlos_feeder_db_storage/__init__.py

@@ -0,0 +1 @@
+from .atlos_feeder_db_storage import AtlosFeederDbStorage

src/auto_archiver/modules/atlos_feeder_db_storage/__manifest__.py

@@ -0,0 +1,46 @@
+{
+    "name": "Atlos Feeder Database Storage",
+    "type": ["feeder", "database", "storage"],
+    "entry_point": "atlos_feeder_db_storage::AtlosFeederDbStorage",
+    "requires_setup": True,
+    "dependencies": {
+        "python": ["loguru", "requests"],
+    },
+    "configs": {
+        "api_token": {
+            "type": "str",
+            "required": True,
+            "help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/",
+        },
+        "atlos_url": {
+            "default": "https://platform.atlos.org",
+            "help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.",
+            "type": "str",
+        },
+    },
+    "description": """
+    A module that integrates with the Atlos API to fetch source material URLs for archival, uplaod extracted media,
+    
+    [Atlos](https://www.atlos.org/) is a visual investigation and archiving platform designed for investigative research, journalism, and open-source intelligence (OSINT). 
+    It helps users organize, analyze, and store media from various sources, making it easier to track and investigate digital evidence.
+    
+    To get started create a new project and obtain an API token from the settings page. You can group event's into Atlos's 'incidents'.
+    Here you can add 'source material' by URLn and the Atlos feeder will fetch these URLs for archival.
+    
+    You can use Atlos only as a 'feeder', however you can also implement the 'database' and 'storage' features to store the media files in Atlos which is recommended.
+    The Auto Archiver will retain the Atlos ID for each item, ensuring that the media and database outputs are uplaoded back into the relevant media item.
+    
+    
+    ### Features
+    - Connects to the Atlos API to retrieve a list of source material URLs.
+    - Iterates through the URLs from all source material items which are unprocessed, visible, and ready to archive.
+    - If the storage option is selected, it will store the media files alongside the original source material item in Atlos.
+    - Is the database option is selected it will output the results to the media item, as well as updating failure status with error details when archiving fails.
+    - Skips Storege/ database upload for items without an Atlos ID - restricting that you must use the Atlos feeder so that it has the Atlos ID to store the results with.
+
+    ### Notes
+    - Requires an Atlos account with a project and a valid API token for authentication.
+    - Ensures only unprocessed, visible, and ready-to-archive URLs are returned.
+    - Feches any media items within an Atlos project, regardless of separation into incidents.
+    """,
+}

src/auto_archiver/modules/atlos_feeder_db_storage/atlos_feeder_db_storage.py

@@ -0,0 +1,143 @@
+import hashlib
+import os
+from typing import IO, Iterator, Optional, Union
+
+import requests
+from loguru import logger
+
+from auto_archiver.core import Database, Feeder, Media, Metadata, Storage
+from auto_archiver.utils import calculate_file_hash
+
+
+class AtlosFeederDbStorage(Feeder, Database, Storage):
+    def setup(self) -> requests.Session:
+        """create and return a persistent session."""
+        self.session = requests.Session()
+
+    def _get(self, endpoint: str, params: Optional[dict] = None) -> dict:
+        """Wrapper for GET requests to the Atlos API."""
+        url = f"{self.atlos_url}{endpoint}"
+        response = self.session.get(url, headers={"Authorization": f"Bearer {self.api_token}"}, params=params)
+        response.raise_for_status()
+        return response.json()
+
+    def _post(
+        self,
+        endpoint: str,
+        json: Optional[dict] = None,
+        params: Optional[dict] = None,
+        files: Optional[dict] = None,
+    ) -> dict:
+        """Wrapper for POST requests to the Atlos API."""
+        url = f"{self.atlos_url}{endpoint}"
+        response = self.session.post(
+            url,
+            headers={"Authorization": f"Bearer {self.api_token}"},
+            json=json,
+            params=params,
+            files=files,
+        )
+        response.raise_for_status()
+        return response.json()
+
+    # ! Atlos Module - Feeder Methods
+
+    def __iter__(self) -> Iterator[Metadata]:
+        """Iterate over unprocessed, visible source materials from Atlos."""
+        cursor = None
+        while True:
+            data = self._get("/api/v2/source_material", params={"cursor": cursor})
+            cursor = data.get("next")
+            results = data.get("results", [])
+            for item in results:
+                if (
+                    item.get("source_url") not in [None, ""]
+                    and not item.get("metadata", {}).get("auto_archiver", {}).get("processed", False)
+                    and item.get("visibility") == "visible"
+                    and item.get("status") not in ["processing", "pending"]
+                ):
+                    yield Metadata().set_url(item["source_url"]).set("atlos_id", item["id"])
+            if not results or cursor is None:
+                break
+
+    # ! Atlos Module - Database Methods
+
+    def failed(self, item: Metadata, reason: str) -> None:
+        """Mark an item as failed in Atlos, if the ID exists."""
+        atlos_id = item.metadata.get("atlos_id")
+        if not atlos_id:
+            logger.info(f"Item {item.get_url()} has no Atlos ID, skipping")
+            return
+        self._post(
+            f"/api/v2/source_material/metadata/{atlos_id}/auto_archiver",
+            json={"metadata": {"processed": True, "status": "error", "error": reason}},
+        )
+        logger.info(f"Stored failure for {item.get_url()} (ID {atlos_id}) on Atlos: {reason}")
+
+    def fetch(self, item: Metadata) -> Union[Metadata, bool]:
+        """check and fetch if the given item has been archived already, each
+        database should handle its own caching, and configuration mechanisms"""
+        return False
+
+    def _process_metadata(self, item: Metadata) -> dict:
+        """Process metadata for storage on Atlos. Will convert any datetime
+        objects to ISO format."""
+        return {k: v.isoformat() if hasattr(v, "isoformat") else v for k, v in item.metadata.items()}
+
+    def done(self, item: Metadata, cached: bool = False) -> None:
+        """Mark an item as successfully archived in Atlos."""
+        atlos_id = item.metadata.get("atlos_id")
+        if not atlos_id:
+            logger.info(f"Item {item.get_url()} has no Atlos ID, skipping")
+            return
+        self._post(
+            f"/api/v2/source_material/metadata/{atlos_id}/auto_archiver",
+            json={
+                "metadata": {
+                    "processed": True,
+                    "status": "success",
+                    "results": self._process_metadata(item),
+                }
+            },
+        )
+        logger.info(f"Stored success for {item.get_url()} (ID {atlos_id}) on Atlos")
+
+    # ! Atlos Module - Storage Methods
+
+    def get_cdn_url(self, _media: Media) -> str:
+        """Return the base Atlos URL as the CDN URL."""
+        return self.atlos_url
+
+    def upload(self, media: Media, metadata: Optional[Metadata] = None, **_kwargs) -> bool:
+        """Upload a media file to Atlos if it has not been uploaded already."""
+        if metadata is None:
+            logger.error(f"No metadata provided for {media.filename}")
+            return False
+
+        atlos_id = metadata.get("atlos_id")
+        if not atlos_id:
+            logger.error(f"No Atlos ID found in metadata; can't store {media.filename} in Atlos.")
+            return False
+
+        media_hash = calculate_file_hash(media.filename, hash_algo=hashlib.sha256, chunksize=4096)
+
+        # Check whether the media has already been uploaded
+        source_material = self._get(f"/api/v2/source_material/{atlos_id}")["result"]
+        existing_media = [artifact.get("file_hash_sha256") for artifact in source_material.get("artifacts", [])]
+        if media_hash in existing_media:
+            logger.info(f"{media.filename} with SHA256 {media_hash} already uploaded to Atlos")
+            return True
+
+        # Upload the media to the Atlos API
+        with open(media.filename, "rb") as file_obj:
+            self._post(
+                f"/api/v2/source_material/upload/{atlos_id}",
+                params={"title": media.properties},
+                files={"file": (os.path.basename(media.filename), file_obj)},
+            )
+        logger.info(f"Uploaded {media.filename} to Atlos with ID {atlos_id} and title {media.key}")
+        return True
+
+    def uploadf(self, file: IO[bytes], key: str, **kwargs: dict) -> bool:
+        """Upload a file-like object; not implemented."""
+        pass

src/auto_archiver/modules/atlos_storage/__init__.py

@@ -1 +0,0 @@
-from .atlos_storage import AtlosStorage

src/auto_archiver/modules/atlos_storage/__manifest__.py

@@ -1,32 +0,0 @@
-{
-    "name": "Atlos Storage",
-    "type": ["storage"],
-    "requires_setup": True,
-    "dependencies": {
-        "python": ["loguru", "boto3"],
-        "bin": []
-    },
-    "description": """
-    Stores media files in a [Atlos](https://www.atlos.org/).
-
-    ### Features
-    - Saves media files to Atlos, organizing them into folders based on the provided path structure.
-
-    ### Notes
-    - Requires setup with Atlos credentials.
-    - Files are uploaded to the specified `root_folder_id` and organized by the `media.key` structure.
-    """,
-    "configs": {
-        "api_token": {
-            "default": None,
-            "help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/",
-            "required": True,
-            "type": "str"
-        },
-        "atlos_url": {
-            "default": "https://platform.atlos.org",
-            "help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.",
-            "type": "str"
-        },
-    }
-}

src/auto_archiver/modules/atlos_storage/atlos_storage.py

@@ -1,66 +0,0 @@
-import hashlib
-import os
-from typing import IO, Optional
-
-import requests
-from loguru import logger
-
-from auto_archiver.core import Media, Metadata
-from auto_archiver.core import Storage
-
-
-class AtlosStorage(Storage):
-
-    def get_cdn_url(self, _media: Media) -> str:
-        # It's not always possible to provide an exact URL, because it's
-        # possible that the media once uploaded could have been copied to
-        # another project.
-        return self.atlos_url
-    
-    def _hash(self, media: Media) -> str:
-        # Hash the media file using sha-256. We don't use the existing auto archiver
-        # hash because there's no guarantee that the configuerer is using sha-256, which
-        # is how Atlos hashes files.
-
-        sha256 = hashlib.sha256()
-        with open(media.filename, "rb") as f:
-            while True:
-                buf = f.read(4096)
-                if not buf: break
-                sha256.update(buf)
-        return sha256.hexdigest()
-
-    def upload(self, media: Media, metadata: Optional[Metadata]=None, **_kwargs) -> bool:
-        atlos_id = metadata.get("atlos_id")
-        if atlos_id is None:
-            logger.error(f"No Atlos ID found in metadata; can't store {media.filename} on Atlos")
-            return False
-        
-        media_hash = self._hash(media)
-        
-        # Check whether the media has already been uploaded
-        source_material = requests.get(
-            f"{self.atlos_url}/api/v2/source_material/{atlos_id}",
-            headers={"Authorization": f"Bearer {self.api_token}"},
-        ).json()["result"]
-        existing_media = [x["file_hash_sha256"] for x in source_material.get("artifacts", [])]
-        if media_hash in existing_media:
-            logger.info(f"{media.filename} with SHA256 {media_hash} already uploaded to Atlos")
-            return True
-        
-        # Upload the media to the Atlos API
-        requests.post(
-            f"{self.atlos_url}/api/v2/source_material/upload/{atlos_id}",
-            headers={"Authorization": f"Bearer {self.api_token}"},
-            params={
-                "title": media.properties
-            },
-            files={"file": (os.path.basename(media.filename), open(media.filename, "rb"))},
-        ).raise_for_status()
-
-        logger.info(f"Uploaded {media.filename} to Atlos with ID {atlos_id} and title {media.key}")
-        
-        return True
-
-    # must be implemented even if unused
-    def uploadf(self, file: IO[bytes], key: str, **kwargs: dict) -> bool: pass

src/auto_archiver/modules/cli_feeder/__manifest__.py

@@ -0,0 +1,22 @@
+{
+    "name": "Command Line Feeder",
+    "type": ["feeder"],
+    "entry_point": "cli_feeder::CLIFeeder",
+    "requires_setup": False,
+    "configs": {
+        "urls": {
+            "default": None,
+            "help": "URL(s) to archive, either a single URL or a list of urls, should not come from config.yaml",
+        },
+    },
+    "description": """
+The Command Line Feeder is the default enabled feeder for the Auto Archiver. It allows you to pass URLs directly to the orchestrator from the command line 
+without the need to specify any additional configuration or command line arguments:
+
+`auto-archiver --feeder cli_feeder -- "https://example.com/1/,https://example.com/2/"`
+
+You can pass multiple URLs by separating them with a space. The URLs will be processed in the order they are provided.
+
+`auto-archiver --feeder cli_feeder -- https://example.com/1/ https://example.com/2/`
+""",
+}

src/auto_archiver/modules/cli_feeder/cli_feeder.py

@@ -0,0 +1,23 @@
+from loguru import logger
+
+from auto_archiver.core.feeder import Feeder
+from auto_archiver.core.metadata import Metadata
+from auto_archiver.core.consts import SetupError
+
+
+class CLIFeeder(Feeder):
+    def setup(self) -> None:
+        self.urls = self.config["urls"]
+        if not self.urls:
+            raise SetupError(
+                "No URLs provided. Please provide at least one URL via the command line, or set up an alternative feeder. Use --help for more information."
+            )
+
+    def __iter__(self) -> Metadata:
+        urls = self.config["urls"]
+        for url in urls:
+            logger.debug(f"Processing {url}")
+            m = Metadata().set_url(url)
+            yield m
+
+        logger.success(f"Processed {len(urls)} URL(s)")

src/auto_archiver/modules/console_db/__init__.py

@@ -1 +1 @@
-from .console_db import ConsoleDb
+from .console_db import ConsoleDb

src/auto_archiver/modules/console_db/console_db.py

@@ -6,18 +6,18 @@ from auto_archiver.core import Metadata
 
 class ConsoleDb(Database):
     """
-        Outputs results to the console
+    Outputs results to the console
     """
 
     def started(self, item: Metadata) -> None:
-        logger.warning(f"STARTED {item}")
+        logger.info(f"STARTED {item}")
 
-    def failed(self, item: Metadata, reason:str) -> None:
+    def failed(self, item: Metadata, reason: str) -> None:
         logger.error(f"FAILED {item}: {reason}")
 
     def aborted(self, item: Metadata) -> None:
         logger.warning(f"ABORTED {item}")
 
-    def done(self, item: Metadata, cached: bool=False) -> None:
+    def done(self, item: Metadata, cached: bool = False) -> None:
         """archival result ready - should be saved to DB"""
-        logger.success(f"DONE {item}")
+        logger.success(f"DONE {item}")

src/auto_archiver/modules/csv_db/__init__.py

@@ -1 +1 @@
-from .csv_db import CSVDb
+from .csv_db import CSVDb

src/auto_archiver/modules/csv_db/__manifest__.py

@@ -2,12 +2,11 @@
     "name": "CSV Database",
     "type": ["database"],
     "requires_setup": False,
-    "dependencies": {"python": ["loguru"]
-                              },
-    'entry_point': 'csv_db::CSVDb',
+    "dependencies": {"python": ["loguru"]},
+    "entry_point": "csv_db::CSVDb",
     "configs": {
-            "csv_file": {"default": "db.csv", "help": "CSV file name"}
-        },
+        "csv_file": {"default": "db.csv", "help": "CSV file name to save metadata to"},
+    },
     "description": """
 Handles exporting archival results to a CSV file.
 

src/auto_archiver/modules/csv_db/csv_db.py

@@ -9,14 +9,15 @@ from auto_archiver.core import Metadata
 
 class CSVDb(Database):
     """
-        Outputs results to a CSV file
+    Outputs results to a CSV file
     """
 
-    def done(self, item: Metadata, cached: bool=False) -> None:
+    def done(self, item: Metadata, cached: bool = False) -> None:
         """archival result ready - should be saved to DB"""
         logger.success(f"DONE {item}")
         is_empty = not os.path.isfile(self.csv_file) or os.path.getsize(self.csv_file) == 0
         with open(self.csv_file, "a", encoding="utf-8") as outf:
             writer = DictWriter(outf, fieldnames=asdict(Metadata()))
-            if is_empty: writer.writeheader()
+            if is_empty:
+                writer.writeheader()
             writer.writerow(asdict(item))

src/auto_archiver/modules/csv_feeder/__init__.py

@@ -1 +1 @@
-from .csv_feeder import CSVFeeder
+from .csv_feeder import CSVFeeder

src/auto_archiver/modules/csv_feeder/__manifest__.py

@@ -1,27 +1,23 @@
 {
     "name": "CSV Feeder",
     "type": ["feeder"],
-    "requires_setup": False,
-    "dependencies": {
-        "python": ["loguru"],
-        "bin": [""]
-    },
-    'requires_setup': True,
-    'entry_point': "csv_feeder::CSVFeeder",
+    "dependencies": {"python": ["loguru"], "bin": [""]},
+    "requires_setup": True,
+    "entry_point": "csv_feeder::CSVFeeder",
     "configs": {
-            "files": {
-                "default": None,
-                "help": "Path to the input file(s) to read the URLs from, comma separated. \
+        "files": {
+            "default": None,
+            "help": "Path to the input file(s) to read the URLs from, comma separated. \
                         Input files should be formatted with one URL per line",
-                "required": True,
-                "type": "valid_file",
-                "nargs": "+",
-            },
-            "column": {
-                "default": None,
-                "help": "Column number or name to read the URLs from, 0-indexed",
-            }
+            "required": True,
+            "type": "valid_file",
+            "nargs": "+",
         },
+        "column": {
+            "default": None,
+            "help": "Column number or name to read the URLs from, 0-indexed",
+        },
+    },
     "description": """
     Reads URLs from CSV files and feeds them into the archiving process.
 
@@ -33,5 +29,5 @@
     ### Setup
     - Input files should be formatted with one URL per line, with or without a header row.
     - If you have a header row, you can specify the column number or name to read URLs from using the 'column' config option.
-    """
+    """,
 }

src/auto_archiver/modules/csv_feeder/csv_feeder.py

@@ -5,11 +5,10 @@ from auto_archiver.core import Feeder
 from auto_archiver.core import Metadata
 from auto_archiver.utils import url_or_none
 
+
 class CSVFeeder(Feeder):
-
     column = None
 
-
     def __iter__(self) -> Metadata:
         for file in self.files:
             with open(file, "r") as f:
@@ -20,9 +19,11 @@ class CSVFeeder(Feeder):
                     try:
                         url_column = first_row.index(url_column)
                     except ValueError:
-                        logger.error(f"Column {url_column} not found in header row: {first_row}. Did you set the 'column' config correctly?")
+                        logger.error(
+                            f"Column {url_column} not found in header row: {first_row}. Did you set the 'column' config correctly?"
+                        )
                         return
-                elif not(url_or_none(first_row[url_column])):
+                elif not (url_or_none(first_row[url_column])):
                     # it's a header row, but we've been given a column number already
                     logger.debug(f"Skipping header row: {first_row}")
                 else:
@@ -35,4 +36,4 @@ class CSVFeeder(Feeder):
                         continue
                     url = row[url_column]
                     logger.debug(f"Processing {url}")
-                    yield Metadata().set_url(url)
+                    yield Metadata().set_url(url)

src/auto_archiver/modules/gdrive_storage/__init__.py

@@ -1 +1 @@
-from .gdrive_storage import GDriveStorage
+from .gdrive_storage import GDriveStorage

src/auto_archiver/modules/gdrive_storage/__manifest__.py

@@ -19,14 +19,21 @@
         },
         "filename_generator": {
             "default": "static",
-            "help": "how to name stored files: 'random' creates a random string; 'static' uses a replicable strategy such as a hash.",
+            "help": "how to name stored files: 'random' creates a random string; 'static' uses a hash, with the settings of the 'hash_enricher' module (defaults to SHA256 if not enabled).",
             "choices": ["random", "static"],
         },
-        "root_folder_id": {"required": True,
-                           "help": "root google drive folder ID to use as storage, found in URL: 'https://drive.google.com/drive/folders/FOLDER_ID'"},
-        "oauth_token": {"default": None,
-                        "help": "JSON filename with Google Drive OAuth token: check auto-archiver repository scripts folder for create_update_gdrive_oauth_token.py. NOTE: storage used will count towards owner of GDrive folder, therefore it is best to use oauth_token_filename over service_account."},
-        "service_account": {"default": "secrets/service_account.json", "help": "service account JSON file path, same as used for Google Sheets. NOTE: storage used will count towards the developer account."},
+        "root_folder_id": {
+            "required": True,
+            "help": "root google drive folder ID to use as storage, found in URL: 'https://drive.google.com/drive/folders/FOLDER_ID'",
+        },
+        "oauth_token": {
+            "default": None,
+            "help": "JSON filename with Google Drive OAuth token: check auto-archiver repository scripts folder for create_update_gdrive_oauth_token.py. NOTE: storage used will count towards owner of GDrive folder, therefore it is best to use oauth_token_filename over service_account.",
+        },
+        "service_account": {
+            "default": "secrets/service_account.json",
+            "help": "service account JSON file path, same as used for Google Sheets. NOTE: storage used will count towards the developer account.",
+        },
     },
     "description": """
     
@@ -94,5 +101,5 @@ This module integrates Google Drive as a storage backend, enabling automatic fol
     https://davemateer.com/2022/04/28/google-drive-with-python#tokens
     
     
-"""
+""",
 }

src/auto_archiver/modules/gdrive_storage/gdrive_storage.py

@@ -1,4 +1,3 @@
-
 import json
 import os
 import time
@@ -15,12 +14,9 @@ from auto_archiver.core import Media
 from auto_archiver.core import Storage
 
 
-
-
 class GDriveStorage(Storage):
-
     def setup(self) -> None:
-        self.scopes = ['https://www.googleapis.com/auth/drive']
+        self.scopes = ["https://www.googleapis.com/auth/drive"]
         # Initialize Google Drive service
         self._setup_google_drive_service()
 
@@ -37,25 +33,25 @@ class GDriveStorage(Storage):
 
     def _initialize_with_oauth_token(self):
         """Initialize Google Drive service with OAuth token."""
-        with open(self.oauth_token, 'r') as stream:
+        with open(self.oauth_token, "r") as stream:
             creds_json = json.load(stream)
-            creds_json['refresh_token'] = creds_json.get("refresh_token", "")
+            creds_json["refresh_token"] = creds_json.get("refresh_token", "")
 
         creds = Credentials.from_authorized_user_info(creds_json, self.scopes)
         if not creds.valid and creds.expired and creds.refresh_token:
             creds.refresh(Request())
-            with open(self.oauth_token, 'w') as token_file:
+            with open(self.oauth_token, "w") as token_file:
                 logger.debug("Saving refreshed OAuth token.")
                 token_file.write(creds.to_json())
         elif not creds.valid:
             raise ValueError("Invalid OAuth token. Please regenerate the token.")
 
-        return build('drive', 'v3', credentials=creds)
+        return build("drive", "v3", credentials=creds)
 
     def _initialize_with_service_account(self):
         """Initialize Google Drive service with service account."""
         creds = service_account.Credentials.from_service_account_file(self.service_account, scopes=self.scopes)
-        return build('drive', 'v3', credentials=creds)
+        return build("drive", "v3", credentials=creds)
 
     def get_cdn_url(self, media: Media) -> str:
         """
@@ -79,7 +75,7 @@ class GDriveStorage(Storage):
         return f"https://drive.google.com/file/d/{file_id}/view?usp=sharing"
 
     def upload(self, media: Media, **kwargs) -> bool:
-        logger.debug(f'[{self.__class__.__name__}] storing file {media.filename} with key {media.key}')
+        logger.debug(f"[{self.__class__.__name__}] storing file {media.filename} with key {media.key}")
         """
         1. for each sub-folder in the path check if exists or create
         2. upload file to root_id/other_paths.../filename
@@ -95,25 +91,30 @@ class GDriveStorage(Storage):
             parent_id = upload_to
 
         # upload file to gd
-        logger.debug(f'uploading {filename=} to folder id {upload_to}')
-        file_metadata = {
-            'name': [filename],
-            'parents': [upload_to]
-        }
+        logger.debug(f"uploading {filename=} to folder id {upload_to}")
+        file_metadata = {"name": [filename], "parents": [upload_to]}
         media = MediaFileUpload(media.filename, resumable=True)
-        gd_file = self.service.files().create(supportsAllDrives=True, body=file_metadata, media_body=media, fields='id').execute()
-        logger.debug(f'uploadf: uploaded file {gd_file["id"]} successfully in folder={upload_to}')
+        gd_file = (
+            self.service.files()
+            .create(supportsAllDrives=True, body=file_metadata, media_body=media, fields="id")
+            .execute()
+        )
+        logger.debug(f"uploadf: uploaded file {gd_file['id']} successfully in folder={upload_to}")
 
     # must be implemented even if unused
-    def uploadf(self, file: IO[bytes], key: str, **kwargs: dict) -> bool: pass
+    def uploadf(self, file: IO[bytes], key: str, **kwargs: dict) -> bool:
+        pass
 
-    def _get_id_from_parent_and_name(self, parent_id: str,
-                                     name: str,
-                                     retries: int = 1,
-                                     sleep_seconds: int = 10,
-                                     use_mime_type: bool = False,
-                                     raise_on_missing: bool = True,
-                                     use_cache=False):
+    def _get_id_from_parent_and_name(
+        self,
+        parent_id: str,
+        name: str,
+        retries: int = 1,
+        sleep_seconds: int = 10,
+        use_mime_type: bool = False,
+        raise_on_missing: bool = True,
+        use_cache=False,
+    ):
         """
         Retrieves the id of a folder or file from its @name and the @parent_id folder
         Optionally does multiple @retries and sleeps @sleep_seconds between them
@@ -134,32 +135,39 @@ class GDriveStorage(Storage):
         debug_header: str = f"[searching {name=} in {parent_id=}]"
         query_string = f"'{parent_id}' in parents and name = '{name}' and trashed = false "
         if use_mime_type:
-            query_string += f" and mimeType='application/vnd.google-apps.folder' "
+            query_string += " and mimeType='application/vnd.google-apps.folder' "
 
         for attempt in range(retries):
-            results = self.service.files().list(
-                # both below for Google Shared Drives
-                supportsAllDrives=True,
-                includeItemsFromAllDrives=True,
-                q=query_string,
-                spaces='drive',  # ie not appDataFolder or photos
-                fields='files(id, name)'
-            ).execute()
-            items = results.get('files', [])
+            results = (
+                self.service.files()
+                .list(
+                    # both below for Google Shared Drives
+                    supportsAllDrives=True,
+                    includeItemsFromAllDrives=True,
+                    q=query_string,
+                    spaces="drive",  # ie not appDataFolder or photos
+                    fields="files(id, name)",
+                )
+                .execute()
+            )
+            items = results.get("files", [])
 
             if len(items) > 0:
-                logger.debug(f"{debug_header} found {len(items)} matches, returning last of {','.join([i['id'] for i in items])}")
-                _id = items[-1]['id']
-                if use_cache: self.api_cache[cache_key] = _id
+                logger.debug(
+                    f"{debug_header} found {len(items)} matches, returning last of {','.join([i['id'] for i in items])}"
+                )
+                _id = items[-1]["id"]
+                if use_cache:
+                    self.api_cache[cache_key] = _id
                 return _id
             else:
-                logger.debug(f'{debug_header} not found, attempt {attempt+1}/{retries}.')
+                logger.debug(f"{debug_header} not found, attempt {attempt + 1}/{retries}.")
                 if attempt < retries - 1:
-                    logger.debug(f'sleeping for {sleep_seconds} second(s)')
+                    logger.debug(f"sleeping for {sleep_seconds} second(s)")
                     time.sleep(sleep_seconds)
 
         if raise_on_missing:
-            raise ValueError(f'{debug_header} not found after {retries} attempt(s)')
+            raise ValueError(f"{debug_header} not found after {retries} attempt(s)")
         return None
 
     def _mkdir(self, name: str, parent_id: str):
@@ -167,12 +175,7 @@ class GDriveStorage(Storage):
         Creates a new GDrive folder @name inside folder @parent_id
         Returns id of the created folder
         """
-        logger.debug(f'Creating new folder with {name=} inside {parent_id=}')
-        file_metadata = {
-            'name': [name],
-            'mimeType': 'application/vnd.google-apps.folder',
-            'parents': [parent_id]
-        }
-        gd_folder = self.service.files().create(supportsAllDrives=True, body=file_metadata, fields='id').execute()
-        return gd_folder.get('id')
-
+        logger.debug(f"Creating new folder with {name=} inside {parent_id=}")
+        file_metadata = {"name": [name], "mimeType": "application/vnd.google-apps.folder", "parents": [parent_id]}
+        gd_folder = self.service.files().create(supportsAllDrives=True, body=file_metadata, fields="id").execute()
+        return gd_folder.get("id")

src/auto_archiver/modules/generic_extractor/__init__.py

@@ -1 +1 @@
-from .generic_extractor import GenericExtractor
+from .generic_extractor import GenericExtractor

src/auto_archiver/modules/generic_extractor/__manifest__.py

@@ -15,6 +15,9 @@ supported by `yt-dlp`, such as YouTube, Facebook, and others. It provides functi
 for retrieving videos, subtitles, comments, and other metadata, and it integrates with
 the broader archiving framework.
 
+For a full list of video platforms supported by `yt-dlp`, see the
+[official documentation](https://github.com/yt-dlp/yt-dlp/blob/master/supportedsites.md)
+
 ### Features
 - Supports downloading videos and playlists.
 - Retrieves metadata like titles, descriptions, upload dates, and durations.
@@ -28,6 +31,13 @@ the broader archiving framework.
  metadata objects. Some dropins are included in this generic_archiver by default, but
 custom dropins can be created to handle additional websites and passed to the archiver
 via the command line using the `--dropins` option (TODO!).
+
+### Auto-Updates
+
+The Generic Extractor will also automatically check for updates to `yt-dlp` (every 5 days by default).
+This can be configured using the `ytdlp_update_interval` setting (or disabled by setting it to -1).
+If you are having issues with the extractor, you can review the version of `yt-dlp` being used with `yt-dlp --version`.
+
 """,
     "configs": {
         "subtitles": {"default": True, "help": "download subtitles if available", "type": "bool"},
@@ -64,5 +74,22 @@ via the command line using the `--dropins` option (TODO!).
             "default": "inf",
             "help": "Use to limit the number of videos to download when a channel or long page is being extracted. 'inf' means no limit.",
         },
+        "extractor_args": {
+            "default": {},
+            "help": "Additional arguments to pass to the yt-dlp extractor. See https://github.com/yt-dlp/yt-dlp/blob/master/README.md#extractor-arguments.",
+            "type": "json_loader",
+        },
+        "ytdlp_update_interval": {
+            "default": 5,
+            "help": "How often to check for yt-dlp updates (days). If positive, will check and update yt-dlp every [num] days. Set it to -1 to disable, or 0 to always update on every run.",
+            "type": "int",
+        },
+        "ytdlp_args": {
+            "default": "",
+            "help": "Additional arguments to pass to yt-dlp, e.g. --no-check-certificate or --plugin-dirs.\
+See yt-dlp documentation here for more information: https://github.com/yt-dlp/yt-dlp?tab=readme-ov-file#general-options\
+Note: this is not to be confused with 'extractor_args' which are specific to the extractor itself.",
+            "type": "str",
+        },
     },
 }

src/auto_archiver/modules/generic_extractor/bluesky.py

@@ -4,15 +4,16 @@ from auto_archiver.core.extractor import Extractor
 from auto_archiver.core.metadata import Metadata, Media
 from .dropin import GenericDropin, InfoExtractor
 
-class Bluesky(GenericDropin):
 
+class Bluesky(GenericDropin):
     def create_metadata(self, post: dict, ie_instance: InfoExtractor, archiver: Extractor, url: str) -> Metadata:
         result = Metadata()
         result.set_url(url)
         result.set_title(post["record"]["text"])
         result.set_timestamp(post["record"]["createdAt"])
         for k, v in self._get_post_data(post).items():
-            if v: result.set(k, v)
+            if v:
+                result.set(k, v)
 
         # download if embeds present (1 video XOR >=1 images)
         for media in self._download_bsky_embeds(post, archiver):
@@ -23,12 +24,12 @@ class Bluesky(GenericDropin):
 
     def extract_post(self, url: str, ie_instance: InfoExtractor) -> dict:
         # TODO: If/when this PR (https://github.com/yt-dlp/yt-dlp/pull/12098) is merged on ytdlp, remove the comments and delete the code below
-        handle, video_id = ie_instance._match_valid_url(url).group('handle', 'id')
+        handle, video_id = ie_instance._match_valid_url(url).group("handle", "id")
         return ie_instance._extract_post(handle=handle, post_id=video_id)
 
     def _download_bsky_embeds(self, post: dict, archiver: Extractor) -> list[Media]:
         """
-        Iterates over image(s) or video in a Bluesky post and downloads them        
+        Iterates over image(s) or video in a Bluesky post and downloads them
         """
         media = []
         embed = post.get("record", {}).get("embed", {})
@@ -37,16 +38,15 @@ class Bluesky(GenericDropin):
 
         media_url = "https://bsky.social/xrpc/com.atproto.sync.getBlob?cid={}&did={}"
         for image_media in image_medias:
-            url = media_url.format(image_media['image']['ref']['$link'], post['author']['did'])
+            url = media_url.format(image_media["image"]["ref"]["$link"], post["author"]["did"])
             image_media = archiver.download_from_url(url)
             media.append(Media(image_media))
         for video_media in video_medias:
-            url = media_url.format(video_media['ref']['$link'], post['author']['did'])
+            url = media_url.format(video_media["ref"]["$link"], post["author"]["did"])
             video_media = archiver.download_from_url(url)
             media.append(Media(video_media))
         return media
 
-
     def _get_post_data(self, post: dict) -> dict:
         """
         Extracts relevant information returned by the .getPostThread api call (excluding text/created_at): author, mentions, tags, links.
@@ -74,4 +74,4 @@ class Bluesky(GenericDropin):
             res["tags"] = tags
         if links:
             res["links"] = links
-        return res
+        return res

src/auto_archiver/modules/generic_extractor/dropin.py

@@ -1,12 +1,14 @@
+from typing import Type
 from yt_dlp.extractor.common import InfoExtractor
 from auto_archiver.core.metadata import Metadata
 from auto_archiver.core.extractor import Extractor
 
+
 class GenericDropin:
     """Base class for dropins for the generic extractor.
-    
+
     In many instances, an extractor will exist in ytdlp, but it will only process videos.
-    Dropins can be created and used to make use of the already-written private code of a 
+    Dropins can be created and used to make use of the already-written private code of a
     specific extractor from ytdlp.
 
     The dropin should be able to handle the following methods:
@@ -23,26 +25,26 @@ class GenericDropin:
 
     """
 
+    extractor: Type[Extractor] = None
+
     def extract_post(self, url: str, ie_instance: InfoExtractor):
         """
         This method should return the post data from the url.
         """
         raise NotImplementedError("This method should be implemented in the subclass")
-    
 
     def create_metadata(self, post: dict, ie_instance: InfoExtractor, archiver: Extractor, url: str) -> Metadata:
         """
         This method should create a Metadata object from the post data.
         """
         raise NotImplementedError("This method should be implemented in the subclass")
-    
 
     def skip_ytdlp_download(self, url: str, ie_instance: InfoExtractor):
         """
         This method should return True if you want to skip the ytdlp download method.
         """
         return False
-    
+
     def keys_to_clean(self, video_data: dict, info_extractor: InfoExtractor):
         """
         This method should return a list of strings (keys) to clean from the video_data dict.
@@ -50,9 +52,25 @@ class GenericDropin:
         E.g. ["uploader", "uploader_id", "tiktok_specific_field"]
         """
         return []
-    
+
     def download_additional_media(self, video_data: dict, info_extractor: InfoExtractor, metadata: Metadata):
         """
         This method should download any additional media from the post.
         """
-        return metadata
+        return metadata
+
+    def suitable(self, url, info_extractor: InfoExtractor):
+        """
+        A method to allow dropins to override their InfoExtractor's 'suitable' method.
+        Dropins should override this method and return True if the url is suitable for the extractor
+        (based on being able to parse other URLs). See the `suitable_extractors` method in the
+        `GenericExtractor` class for how this is implemented.
+
+        The default behaviour of this method is to return the result of the InfoExtractor's 'suitable' method.
+
+        ### Example: An example of where this is useful is for the FacebookIE extractor in yt-dlp. By default,
+        it's 'suitable' method only returns True for video URLs. However, we can override this method in the
+        Facebook dropin to return True for all Facebook URLs (photo/post types). This way, the Facebook dropin
+        can be used for all Facebook URLs.
+        """
+        return info_extractor.suitable(url)

src/auto_archiver/modules/generic_extractor/facebook.py

@@ -1,18 +1,154 @@
+import re
 from .dropin import GenericDropin
+from auto_archiver.core.metadata import Metadata
+from yt_dlp.extractor.facebook import FacebookIE
+
+# TODO: Remove if / when  https://github.com/yt-dlp/yt-dlp/pull/12275 is merged
+from yt_dlp.utils import (
+    clean_html,
+    get_element_by_id,
+    traverse_obj,
+    get_first,
+    merge_dicts,
+    int_or_none,
+    parse_count,
+)
+
+
+def _extract_metadata(self, webpage, video_id):
+    post_data = [
+        self._parse_json(j, video_id, fatal=False)
+        for j in re.findall(r"data-sjs>({.*?ScheduledServerJS.*?})</script>", webpage)
+    ]
+    post = (
+        traverse_obj(
+            post_data,
+            (..., "require", ..., ..., ..., "__bbox", "require", ..., ..., ..., "__bbox", "result", "data"),
+            expected_type=dict,
+        )
+        or []
+    )
+    media = traverse_obj(
+        post,
+        (
+            ...,
+            "attachments",
+            ...,
+            lambda k, v: (k == "media" and str(v["id"]) == video_id and v["__typename"] == "Video"),
+        ),
+        expected_type=dict,
+    )
+    title = get_first(media, ("title", "text"))
+    description = get_first(media, ("creation_story", "comet_sections", "message", "story", "message", "text"))
+    page_title = title or self._html_search_regex(
+        (
+            r'<h2\s+[^>]*class="uiHeaderTitle"[^>]*>(?P<content>[^<]*)</h2>',
+            r'(?s)<span class="fbPhotosPhotoCaption".*?id="fbPhotoPageCaption"><span class="hasCaption">(?P<content>.*?)</span>',
+            self._meta_regex("og:title"),
+            self._meta_regex("twitter:title"),
+            r"<title>(?P<content>.+?)</title>",
+        ),
+        webpage,
+        "title",
+        default=None,
+        group="content",
+    )
+    description = description or self._html_search_meta(
+        ["description", "og:description", "twitter:description"], webpage, "description", default=None
+    )
+    uploader_data = (
+        get_first(media, ("owner", {dict}))
+        or get_first(
+            post, ("video", "creation_story", "attachments", ..., "media", lambda k, v: k == "owner" and v["name"])
+        )
+        or get_first(post, (..., "video", lambda k, v: k == "owner" and v["name"]))
+        or get_first(post, ("node", "actors", ..., {dict}))
+        or get_first(post, ("event", "event_creator", {dict}))
+        or get_first(post, ("video", "creation_story", "short_form_video_context", "video_owner", {dict}))
+        or {}
+    )
+    uploader = uploader_data.get("name") or (
+        clean_html(get_element_by_id("fbPhotoPageAuthorName", webpage))
+        or self._search_regex(
+            (r'ownerName\s*:\s*"([^"]+)"', *self._og_regexes("title")), webpage, "uploader", fatal=False
+        )
+    )
+    timestamp = int_or_none(self._search_regex(r'<abbr[^>]+data-utime=["\'](\d+)', webpage, "timestamp", default=None))
+    thumbnail = self._html_search_meta(["og:image", "twitter:image"], webpage, "thumbnail", default=None)
+    # some webpages contain unretrievable thumbnail urls
+    # like https://lookaside.fbsbx.com/lookaside/crawler/media/?media_id=10155168902769113&get_thumbnail=1
+    # in https://www.facebook.com/yaroslav.korpan/videos/1417995061575415/
+    if thumbnail and not re.search(r"\.(?:jpg|png)", thumbnail):
+        thumbnail = None
+    info_dict = {
+        "description": description,
+        "uploader": uploader,
+        "uploader_id": uploader_data.get("id"),
+        "timestamp": timestamp,
+        "thumbnail": thumbnail,
+        "view_count": parse_count(
+            self._search_regex(
+                (r'\bviewCount\s*:\s*["\']([\d,.]+)', r'video_view_count["\']\s*:\s*(\d+)'),
+                webpage,
+                "view count",
+                default=None,
+            )
+        ),
+        "concurrent_view_count": get_first(
+            post, (("video", (..., ..., "attachments", ..., "media")), "liveViewerCount", {int_or_none})
+        ),
+        **traverse_obj(
+            post,
+            (
+                lambda _, v: video_id in v["url"],
+                "feedback",
+                {
+                    "like_count": ("likers", "count", {int}),
+                    "comment_count": ("total_comment_count", {int}),
+                    "repost_count": ("share_count_reduced", {parse_count}),
+                },
+            ),
+            get_all=False,
+        ),
+    }
+
+    info_json_ld = self._search_json_ld(webpage, video_id, default={})
+    info_json_ld["title"] = (
+        re.sub(r"\s*\|\s*Facebook$", "", title or info_json_ld.get("title") or page_title or "")
+        or (description or "").replace("\n", " ")
+        or f"Facebook video #{video_id}"
+    )
+    return merge_dicts(info_json_ld, info_dict)
 
 
 class Facebook(GenericDropin):
-    def extract_post(self, url: str, ie_instance):
-        video_id = ie_instance._match_valid_url(url).group('id')
-        ie_instance._download_webpage(
-            url.replace('://m.facebook.com/', '://www.facebook.com/'), video_id)
-        webpage = ie_instance._download_webpage(url, ie_instance._match_valid_url(url).group('id'))
+    def extract_post(self, url: str, ie_instance: FacebookIE):
+        post_id_regex = r"(?P<id>pfbid[A-Za-z0-9]+|\d+|t\.(\d+\/\d+))"
+        post_id = re.search(post_id_regex, url).group("id")
+        webpage = ie_instance._download_webpage(url.replace("://m.facebook.com/", "://www.facebook.com/"), post_id)
 
-        # TODO: fix once https://github.com/yt-dlp/yt-dlp/pull/12275 is merged
-        post_data = ie_instance._extract_metadata(webpage)
+        # TODO: For long posts, this _extract_metadata only seems to return the first 100 or so characters, followed by ...
+
+        # TODO: If/when https://github.com/yt-dlp/yt-dlp/pull/12275 is merged, uncomment next line and delete the one after
+        # post_data = ie_instance._extract_metadata(webpage, post_id)
+        post_data = _extract_metadata(ie_instance, webpage, post_id)
         return post_data
-    
-    def create_metadata(self, post: dict, ie_instance, archiver, url):
-        metadata = archiver.create_metadata(url)
-        metadata.set_title(post.get('title')).set_content(post.get('description')).set_post_data(post)
-        return metadata
+
+    def create_metadata(self, post: dict, ie_instance: FacebookIE, archiver, url):
+        result = Metadata()
+        result.set_content(post.get("description", ""))
+        result.set_title(post.get("title", ""))
+        result.set("author", post.get("uploader", ""))
+        result.set_url(url)
+        return result
+
+    def suitable(self, url, info_extractor: FacebookIE):
+        regex = r"(?:https?://(?:[\w-]+\.)?(?:facebook\.com||facebookwkhpilnemxj7asaniu7vnjjbiltxjqhye3mhbshg7kx5tfyd\.onion)/)"
+        return re.match(regex, url)
+
+    def skip_ytdlp_download(self, url: str, is_instance: FacebookIE):
+        """
+        Skip using the ytdlp download method for Facebook *photo* posts, they have a URL with an id of t.XXXXX/XXXXX
+        """
+        if re.search(r"/t.\d+/\d+", url):
+            return True

src/auto_archiver/modules/generic_extractor/generic_extractor.py

@@ -1,31 +1,94 @@
-import datetime, os, yt_dlp, pysubs2
+import datetime
+import os
 import importlib
+import subprocess
+
 from typing import Generator, Type
+
+import yt_dlp
 from yt_dlp.extractor.common import InfoExtractor
+import pysubs2
 
 from loguru import logger
 
 from auto_archiver.core.extractor import Extractor
 from auto_archiver.core import Metadata, Media
+from auto_archiver.utils import get_datetime_from_str
+from .dropin import GenericDropin
+
+
+class SkipYtdlp(Exception):
+    pass
+
 
 class GenericExtractor(Extractor):
     _dropins = {}
 
+    def setup(self):
+        # check for file .ytdlp-update in the secrets folder
+        if self.ytdlp_update_interval < 0:
+            return
+
+        use_secrets = os.path.exists("secrets")
+        path = os.path.join("secrets" if use_secrets else "", ".ytdlp-update")
+        next_update_check = None
+        if os.path.exists(path):
+            with open(path, "r") as f:
+                next_update_check = datetime.datetime.fromisoformat(f.read())
+
+        if not next_update_check or next_update_check < datetime.datetime.now():
+            self.update_ytdlp()
+
+            next_update_check = datetime.datetime.now() + datetime.timedelta(days=self.ytdlp_update_interval)
+            with open(path, "w") as f:
+                f.write(next_update_check.isoformat())
+
+    def update_ytdlp(self):
+        logger.info("Checking and updating yt-dlp...")
+        logger.info(
+            f"Tip: change the 'ytdlp_update_interval' setting to control how often yt-dlp is updated. Set to -1 to disable or 0 to enable on every run. Current setting: {self.ytdlp_update_interval}"
+        )
+        from importlib.metadata import version as get_version
+
+        old_version = get_version("yt-dlp")
+        try:
+            # try and update with pip (this works inside poetry environment and in a normal virtualenv)
+            result = subprocess.run(["pip", "install", "--upgrade", "yt-dlp"], check=True, capture_output=True)
+
+            if "Successfully installed yt-dlp" in result.stdout.decode():
+                new_version = importlib.metadata.version("yt-dlp")
+                logger.info(f"yt-dlp successfully (from {old_version} to {new_version})")
+                importlib.reload(yt_dlp)
+            else:
+                logger.info("yt-dlp already up to date")
+
+        except Exception as e:
+            logger.error(f"Error updating yt-dlp: {e}")
+
     def suitable_extractors(self, url: str) -> Generator[str, None, None]:
         """
         Returns a list of valid extractors for the given URL"""
         for info_extractor in yt_dlp.YoutubeDL()._ies.values():
-            if info_extractor.suitable(url) and info_extractor.working():
+            if not info_extractor.working():
+                continue
+
+            # check if there's a dropin and see if that declares whether it's suitable
+            dropin: GenericDropin = self.dropin_for_name(info_extractor.ie_key())
+            if dropin and dropin.suitable(url, info_extractor):
                 yield info_extractor
-        
+            elif info_extractor.suitable(url):
+                yield info_extractor
+
     def suitable(self, url: str) -> bool:
         """
         Checks for valid URLs out of all ytdlp extractors.
         Returns False for the GenericIE, which as labelled by yt-dlp: 'Generic downloader that works on some sites'
         """
         return any(self.suitable_extractors(url))
-    
-    def download_additional_media(self, video_data: dict, info_extractor: InfoExtractor, metadata: Metadata) -> Metadata:
+
+    def download_additional_media(
+        self, video_data: dict, info_extractor: InfoExtractor, metadata: Metadata
+    ) -> Metadata:
         """
         Downloads additional media like images, comments, subtitles, etc.
 
@@ -34,7 +97,7 @@ class GenericExtractor(Extractor):
 
         # Just get the main thumbnail. More thumbnails are available in
         # video_data['thumbnails'] should they be required
-        thumbnail_url = video_data.get('thumbnail')
+        thumbnail_url = video_data.get("thumbnail")
         if thumbnail_url:
             try:
                 cover_image_path = self.download_from_url(thumbnail_url)
@@ -57,15 +120,65 @@ class GenericExtractor(Extractor):
         Clean up the ytdlp generic video data to make it more readable and remove unnecessary keys that ytdlp adds
         """
 
-        base_keys = ['formats', 'thumbnail', 'display_id', 'epoch', 'requested_downloads',
-                     'duration_string', 'thumbnails', 'http_headers', 'webpage_url_basename', 'webpage_url_domain',
-                     'extractor', 'extractor_key', 'playlist', 'playlist_index', 'duration_string', 'protocol', 'requested_subtitles',
-                     'format_id', 'acodec', 'vcodec', 'ext', 'epoch', '_has_drm', 'filesize', 'audio_ext', 'video_ext', 'vbr', 'abr',
-                     'resolution', 'dynamic_range', 'aspect_ratio', 'cookies', 'format', 'quality', 'preference', 'artists',
-                     'channel_id', 'subtitles', 'tbr', 'url', 'original_url', 'automatic_captions', 'playable_in_embed', 'live_status',
-                     '_format_sort_fields', 'chapters', 'requested_formats', 'format_note',
-                     'audio_channels', 'asr', 'fps', 'was_live', 'is_live', 'heatmap', 'age_limit', 'stretched_ratio']
-        
+        base_keys = [
+            "formats",
+            "thumbnail",
+            "display_id",
+            "epoch",
+            "requested_downloads",
+            "duration_string",
+            "thumbnails",
+            "http_headers",
+            "webpage_url_basename",
+            "webpage_url_domain",
+            "extractor",
+            "extractor_key",
+            "playlist",
+            "playlist_index",
+            "duration_string",
+            "protocol",
+            "requested_subtitles",
+            "format_id",
+            "acodec",
+            "vcodec",
+            "ext",
+            "epoch",
+            "_has_drm",
+            "filesize",
+            "audio_ext",
+            "video_ext",
+            "vbr",
+            "abr",
+            "resolution",
+            "dynamic_range",
+            "aspect_ratio",
+            "cookies",
+            "format",
+            "quality",
+            "preference",
+            "artists",
+            "channel_id",
+            "subtitles",
+            "tbr",
+            "url",
+            "original_url",
+            "automatic_captions",
+            "playable_in_embed",
+            "live_status",
+            "_format_sort_fields",
+            "chapters",
+            "requested_formats",
+            "format_note",
+            "audio_channels",
+            "asr",
+            "fps",
+            "was_live",
+            "is_live",
+            "heatmap",
+            "age_limit",
+            "stretched_ratio",
+        ]
+
         dropin = self.dropin_for_name(info_extractor.ie_key())
         if dropin:
             try:
@@ -74,8 +187,8 @@ class GenericExtractor(Extractor):
                 pass
 
         return base_keys
-    
-    def add_metadata(self, video_data: dict, info_extractor: InfoExtractor, url:str, result: Metadata) -> Metadata:
+
+    def add_metadata(self, video_data: dict, info_extractor: InfoExtractor, url: str, result: Metadata) -> Metadata:
         """
         Creates a Metadata object from the given video_data
         """
@@ -84,29 +197,43 @@ class GenericExtractor(Extractor):
         result = self.download_additional_media(video_data, info_extractor, result)
 
         # keep both 'title' and 'fulltitle', but prefer 'title', falling back to 'fulltitle' if it doesn't exist
-        result.set_title(video_data.pop('title', video_data.pop('fulltitle', "")))
-        result.set_url(url)
+        if not result.get_title():
+            result.set_title(video_data.pop("title", video_data.pop("fulltitle", "")))
 
+        if not result.get("url"):
+            result.set_url(url)
+
+        if "description" in video_data and not result.get("content"):
+            result.set_content(video_data["description"])
         # extract comments if enabled
         if self.comments:
-            result.set("comments", [{
-                "text": c["text"],
-                "author": c["author"], 
-                "timestamp": datetime.datetime.fromtimestamp(c.get("timestamp"), tz = datetime.timezone.utc)
-            } for c in video_data.get("comments", [])])
+            result.set(
+                "comments",
+                [
+                    {
+                        "text": c["text"],
+                        "author": c["author"],
+                        "timestamp": datetime.datetime.fromtimestamp(c.get("timestamp"), tz=datetime.timezone.utc),
+                    }
+                    for c in video_data.get("comments", [])
+                ],
+            )
 
         # then add the common metadata
-        if timestamp := video_data.pop("timestamp", None):
-            timestamp = datetime.datetime.fromtimestamp(timestamp, tz = datetime.timezone.utc).isoformat()
+        timestamp = video_data.pop("timestamp", None)
+        if timestamp and not result.get("timestamp"):
+            timestamp = datetime.datetime.fromtimestamp(timestamp, tz=datetime.timezone.utc).isoformat()
             result.set_timestamp(timestamp)
-        if upload_date := video_data.pop("upload_date", None):
-            upload_date = datetime.datetime.strptime(upload_date, '%Y%m%d').replace(tzinfo=datetime.timezone.utc)
+
+        upload_date = video_data.pop("upload_date", None)
+        if upload_date and not result.get("upload_date"):
+            upload_date = get_datetime_from_str(upload_date, "%Y%m%d").replace(tzinfo=datetime.timezone.utc)
             result.set("upload_date", upload_date)
-        
+
         # then clean away any keys we don't want
         for clean_key in self.keys_to_clean(info_extractor, video_data):
             video_data.pop(clean_key, None)
-        
+
         # then add the rest of the video data
         for k, v in video_data.items():
             if v:
@@ -124,25 +251,28 @@ class GenericExtractor(Extractor):
 
         if not dropin:
             # TODO: add a proper link to 'how to create your own dropin'
-            logger.debug(f"""Could not find valid dropin for {info_extractor.IE_NAME}.
+            logger.debug(f"""Could not find valid dropin for {info_extractor.ie_key()}.
                      Why not try creating your own, and make sure it has a valid function called 'create_metadata'. Learn more: https://auto-archiver.readthedocs.io/en/latest/user_guidelines.html#""")
             return False
-        
+
         post_data = dropin.extract_post(url, ie_instance)
-        return dropin.create_metadata(post_data, ie_instance, self, url)
-
-    def get_metadata_for_video(self, data: dict, info_extractor: Type[InfoExtractor], url: str, ydl: yt_dlp.YoutubeDL) -> Metadata:
+        result = dropin.create_metadata(post_data, ie_instance, self, url)
+        return self.add_metadata(post_data, info_extractor, url, result)
 
+    def get_metadata_for_video(
+        self, data: dict, info_extractor: Type[InfoExtractor], url: str, ydl: yt_dlp.YoutubeDL
+    ) -> Metadata:
         # this time download
-        ydl.params['getcomments'] = self.comments
-        #TODO: for playlist or long lists of videos, how to download one at a time so they can be stored before the next one is downloaded?
+        ydl.params["getcomments"] = self.comments
+        # TODO: for playlist or long lists of videos, how to download one at a time so they can be stored before the next one is downloaded?
         data = ydl.extract_info(url, ie_key=info_extractor.ie_key(), download=True)
         if "entries" in data:
             entries = data.get("entries", [])
             if not len(entries):
-                logger.warning('YoutubeDLArchiver could not find any video')
+                logger.warning("YoutubeDLArchiver could not find any video")
                 return False
-        else: entries = [data]
+        else:
+            entries = [data]
 
         result = Metadata()
 
@@ -150,17 +280,18 @@ class GenericExtractor(Extractor):
             try:
                 filename = ydl.prepare_filename(entry)
                 if not os.path.exists(filename):
-                    filename = filename.split('.')[0] + '.mkv'
+                    filename = filename.split(".")[0] + ".mkv"
 
                 new_media = Media(filename)
                 for x in ["duration", "original_url", "fulltitle", "description", "upload_date"]:
-                    if x in entry: new_media.set(x, entry[x])
+                    if x in entry:
+                        new_media.set(x, entry[x])
 
                 # read text from subtitles if enabled
                 if self.subtitles:
-                    for lang, val in (data.get('requested_subtitles') or {}).items():
-                        try:    
-                            subs = pysubs2.load(val.get('filepath'), encoding="utf-8")
+                    for lang, val in (data.get("requested_subtitles") or {}).items():
+                        try:
+                            subs = pysubs2.load(val.get("filepath"), encoding="utf-8")
                             text = " ".join([line.text for line in subs])
                             new_media.set(f"subtitles_{lang}", text)
                         except Exception as e:
@@ -170,8 +301,8 @@ class GenericExtractor(Extractor):
                 logger.error(f"Error processing entry {entry}: {e}")
 
         return self.add_metadata(data, info_extractor, url, result)
-    
-    def dropin_for_name(self, dropin_name: str, additional_paths = [], package=__package__) -> Type[InfoExtractor]:
+
+    def dropin_for_name(self, dropin_name: str, additional_paths=[], package=__package__) -> GenericDropin:
         dropin_name = dropin_name.lower()
 
         if dropin_name == "generic":
@@ -179,8 +310,10 @@ class GenericExtractor(Extractor):
             return None
 
         dropin_class_name = dropin_name.title()
+
         def _load_dropin(dropin):
             dropin_class = getattr(dropin, dropin_class_name)()
+            dropin.extractor = self
             return self._dropins.setdefault(dropin_name, dropin_class)
 
         try:
@@ -202,7 +335,7 @@ class GenericExtractor(Extractor):
                 return _load_dropin(dropin)
             except (FileNotFoundError, ModuleNotFoundError):
                 pass
-        
+
         # fallback to loading the dropins within auto-archiver
         try:
             return _load_dropin(importlib.import_module(f".{dropin_name}", package=package))
@@ -214,43 +347,53 @@ class GenericExtractor(Extractor):
     def download_for_extractor(self, info_extractor: InfoExtractor, url: str, ydl: yt_dlp.YoutubeDL) -> Metadata:
         """
         Tries to download the given url using the specified extractor
-        
+
         It first tries to use ytdlp directly to download the video. If the post is not a video, it will then try to
         use the extractor's _extract_post method to get the post metadata if possible.
         """
         # when getting info without download, we also don't need the comments
-        ydl.params['getcomments'] = False
+        ydl.params["getcomments"] = False
         result = False
 
         dropin_submodule = self.dropin_for_name(info_extractor.ie_key())
 
         try:
-            if dropin_submodule and dropin_submodule.skip_ytdlp_download(info_extractor, url):
-                raise Exception(f"Skipping using ytdlp to download files for {info_extractor.ie_key()}")
+            if dropin_submodule and dropin_submodule.skip_ytdlp_download(url, info_extractor):
+                logger.debug(f"Skipping using ytdlp to download files for {info_extractor.ie_key()}")
+                raise SkipYtdlp()
 
             # don't download since it can be a live stream
             data = ydl.extract_info(url, ie_key=info_extractor.ie_key(), download=False)
-            if data.get('is_live', False) and not self.livestreams:
+            if data.get("is_live", False) and not self.livestreams:
                 logger.warning("Livestream detected, skipping due to 'livestreams' configuration setting")
                 return False
             # it's a valid video, that the youtubdedl can download out of the box
             result = self.get_metadata_for_video(data, info_extractor, url, ydl)
 
         except Exception as e:
-            if info_extractor.ie_key() == "generic":
+            if info_extractor.IE_NAME == "generic":
                 # don't clutter the logs with issues about the 'generic' extractor not having a dropin
                 return False
 
-            logger.debug(f'Issue using "{info_extractor.IE_NAME}" extractor to download video (error: {repr(e)}), attempting to use extractor to get post data instead')
+            if not isinstance(e, SkipYtdlp):
+                logger.debug(
+                    f'Issue using "{info_extractor.IE_NAME}" extractor to download video (error: {repr(e)}), attempting to use dropin to get post data instead'
+                )
+
             try:
                 result = self.get_metadata_for_post(info_extractor, url, ydl)
             except (yt_dlp.utils.DownloadError, yt_dlp.utils.ExtractorError) as post_e:
-                logger.error(f'Error downloading metadata for post: {post_e}')
+                logger.error("Error downloading metadata for post: {error}", error=str(post_e))
                 return False
             except Exception as generic_e:
-                logger.debug(f'Attempt to extract using ytdlp extractor "{info_extractor.IE_NAME}" failed:  \n  {repr(generic_e)}', exc_info=True)
+                logger.debug(
+                    'Attempt to extract using ytdlp extractor "{name}" failed:  \n  {error}',
+                    name=info_extractor.IE_NAME,
+                    error=str(generic_e),
+                    exc_info=True,
+                )
                 return False
-        
+
         if result:
             extractor_name = "yt-dlp"
             if info_extractor:
@@ -266,43 +409,70 @@ class GenericExtractor(Extractor):
     def download(self, item: Metadata) -> Metadata:
         url = item.get_url()
 
-        #TODO: this is a temporary hack until this issue is closed: https://github.com/yt-dlp/yt-dlp/issues/11025
+        # TODO: this is a temporary hack until this issue is closed: https://github.com/yt-dlp/yt-dlp/issues/11025
         if url.startswith("https://ya.ru"):
             url = url.replace("https://ya.ru", "https://yandex.ru")
             item.set("replaced_url", url)
 
+        ydl_options = [
+            "-o",
+            os.path.join(self.tmp_dir, "%(id)s.%(ext)s"),
+            "--quiet",
+            "--no-playlist" if not self.allow_playlist else "--yes-playlist",
+            "--write-subs" if self.subtitles else "--no-write-subs",
+            "--write-auto-subs" if self.subtitles else "--no-write-auto-subs",
+            "--live-from-start" if self.live_from_start else "--no-live-from-start",
+        ]
+
+        # proxy handling
+        if self.proxy:
+            ydl_options.extend(["--proxy", self.proxy])
+
+        # max_downloads handling
+        if self.max_downloads != "inf":
+            ydl_options.extend(["--max-downloads", str(self.max_downloads)])
+            ydl_options.extend(["--playlist-end", str(self.max_downloads)])
 
-        ydl_options = {'outtmpl': os.path.join(self.tmp_dir, f'%(id)s.%(ext)s'), 
-                       'quiet': False, 'noplaylist': not self.allow_playlist ,
-                       'writesubtitles': self.subtitles,'writeautomaticsub': self.subtitles,
-                       "live_from_start": self.live_from_start, "proxy": self.proxy,
-                       "max_downloads": self.max_downloads, "playlistend": self.max_downloads}
-        
         # set up auth
         auth = self.auth_for_site(url, extract_cookies=False)
-
-        # order of importance: username/pasword -> api_key -> cookie -> cookies_from_browser -> cookies_file
+        # order of importance: username/password -> api_key -> cookie -> cookies_from_browser -> cookies_file
         if auth:
-            if 'username' in auth and 'password' in auth:
-                logger.debug(f'Using provided auth username and password for {url}')
-                ydl_options['username'] = auth['username']
-                ydl_options['password'] = auth['password']
-            elif 'cookie' in auth:
-                logger.debug(f'Using provided auth cookie for {url}')
-                yt_dlp.utils.std_headers['cookie'] = auth['cookie']
-            elif 'cookies_from_browser' in auth:
-                logger.debug(f'Using extracted cookies from browser {auth["cookies_from_browser"]} for {url}')
-                ydl_options['cookiesfrombrowser'] = auth['cookies_from_browser']
-            elif 'cookies_file' in auth:
-                logger.debug(f'Using cookies from file {auth["cookies_file"]} for {url}')
-                ydl_options['cookiefile'] = auth['cookies_file']
+            if "username" in auth and "password" in auth:
+                logger.debug(f"Using provided auth username and password for {url}")
+                ydl_options.extend(("--username", auth["username"]))
+                ydl_options.extend(("--password", auth["password"]))
+            elif "cookie" in auth:
+                logger.debug(f"Using provided auth cookie for {url}")
+                yt_dlp.utils.std_headers["cookie"] = auth["cookie"]
+            elif "cookies_from_browser" in auth:
+                logger.debug(f"Using extracted cookies from browser {auth['cookies_from_browser']} for {url}")
+                ydl_options.extend(("--cookies-from-browser", auth["cookies_from_browser"]))
+            elif "cookies_file" in auth:
+                logger.debug(f"Using cookies from file {auth['cookies_file']} for {url}")
+                ydl_options.extend(("--cookies", auth["cookies_file"]))
 
-        ydl = yt_dlp.YoutubeDL(ydl_options) # allsubtitles and subtitleslangs not working as expected, so default lang is always "en"
+        # Applying user-defined extractor_args
+        if self.extractor_args:
+            for key, args in self.extractor_args.items():
+                logger.debug(f"Setting extractor_args: {key}")
+                if isinstance(args, dict):
+                    arg_str = ";".join(f"{k}={v}" for k, v in args.items())
+                else:
+                    arg_str = str(args)
+                ydl_options.extend(["--extractor-args", f"{key}:{arg_str}"])
+
+        if self.ytdlp_args:
+            logger.debug("Adding additional ytdlp arguments: {self.ytdlp_args}")
+            ydl_options += self.ytdlp_args.split(" ")
+
+        *_, validated_options = yt_dlp.parse_options(ydl_options)
+        ydl = yt_dlp.YoutubeDL(
+            validated_options
+        )  # allsubtitles and subtitleslangs not working as expected, so default lang is always "en"
 
         for info_extractor in self.suitable_extractors(url):
             result = self.download_for_extractor(info_extractor, url, ydl)
             if result:
                 return result
 
-
         return False

src/auto_archiver/modules/generic_extractor/tiktok.py

@@ -0,0 +1,83 @@
+import requests
+from loguru import logger
+
+from yt_dlp.extractor.tiktok import TikTokIE, TikTokLiveIE, TikTokVMIE, TikTokUserIE
+
+from auto_archiver.core import Metadata, Media
+from datetime import datetime, timezone
+from .dropin import GenericDropin
+
+
+class Tiktok(GenericDropin):
+    """
+    TikTok droping for the Generic Extractor that uses an unofficial API if/when ytdlp fails.
+    It's useful for capturing content that requires a login, like sensitive content.
+    """
+
+    TIKWM_ENDPOINT = "https://www.tikwm.com/api/?url={url}"
+
+    def suitable(self, url, info_extractor) -> bool:
+        """This dropin (which uses Tikvm) is suitable for *all* Tiktok type URLs - videos, lives, VMs, and users.
+        Return the 'suitable' method from the TikTokIE class."""
+        return any(extractor().suitable(url) for extractor in (TikTokIE, TikTokLiveIE, TikTokVMIE, TikTokUserIE))
+
+    def extract_post(self, url: str, ie_instance):
+        logger.debug(f"Using Tikwm API to attempt to download tiktok video from {url=}")
+
+        endpoint = self.TIKWM_ENDPOINT.format(url=url)
+
+        r = requests.get(endpoint)
+        if r.status_code != 200:
+            raise ValueError(f"unexpected status code '{r.status_code}' from tikwm.com for {url=}:")
+
+        try:
+            json_response = r.json()
+        except ValueError:
+            raise ValueError(f"failed to parse JSON response from tikwm.com for {url=}")
+
+        if not json_response.get("msg") == "success" or not (api_data := json_response.get("data", {})):
+            raise ValueError(f"failed to get a valid response from tikwm.com for {url=}: {repr(json_response)}")
+
+        # tries to get the non-watermarked version first
+        video_url = api_data.pop("play", api_data.pop("wmplay", None))
+        if not video_url:
+            raise ValueError(f"no valid video URL found in response from tikwm.com for {url=}")
+
+        api_data["video_url"] = video_url
+        return api_data
+
+    def keys_to_clean(self, video_data: dict, info_extractor):
+        return ["video_url", "title", "create_time", "author", "cover", "origin_cover", "ai_dynamic_cover", "duration"]
+
+    def create_metadata(self, post: dict, ie_instance, archiver, url):
+        # prepare result, start by downloading video
+        result = Metadata()
+        video_url = post.pop("video_url")
+
+        # get the cover if possible
+        cover_url = post.pop("origin_cover", post.pop("cover", post.pop("ai_dynamic_cover", None)))
+        if cover_url and (cover_downloaded := archiver.download_from_url(cover_url)):
+            result.add_media(Media(cover_downloaded))
+
+        # get the video or fail
+        video_downloaded = archiver.download_from_url(video_url, f"vid_{post.get('id', '')}")
+        if not video_downloaded:
+            logger.error(f"failed to download video from {video_url}")
+            return False
+        video_media = Media(video_downloaded)
+        if duration := post.get("duration", None):
+            video_media.set("duration", duration)
+        result.add_media(video_media)
+
+        # add remaining metadata
+        result.set_title(post.get("title", ""))
+
+        if created_at := post.get("create_time", None):
+            result.set_timestamp(datetime.fromtimestamp(created_at, tz=timezone.utc))
+
+        if author := post.get("author", None):
+            result.set("author", author)
+
+        result.set("api_data", post)
+
+        return result

src/auto_archiver/modules/generic_extractor/truth.py

@@ -9,11 +9,11 @@ from dateutil.parser import parse as parse_dt
 
 from .dropin import GenericDropin
 
-class Truth(GenericDropin):
 
+class Truth(GenericDropin):
     def extract_post(self, url, ie_instance: InfoExtractor) -> dict:
         video_id = ie_instance._match_id(url)
-        truthsocial_url = f'https://truthsocial.com/api/v1/statuses/{video_id}'
+        truthsocial_url = f"https://truthsocial.com/api/v1/statuses/{video_id}"
         return ie_instance._download_json(truthsocial_url, video_id)
 
     def skip_ytdlp_download(self, url, ie_instance: Type[InfoExtractor]) -> bool:
@@ -22,31 +22,42 @@ class Truth(GenericDropin):
     def create_metadata(self, post: dict, ie_instance: InfoExtractor, archiver: Extractor, url: str) -> Metadata:
         """
         Creates metadata from a truth social post
-        
+
         Only used for posts that contain no media. ytdlp.TruthIE extractor can handle posts with media
-        
+
         Format is:
-        
+
         {'id': '109598702184774628', 'created_at': '2022-12-29T19:51:18.161Z', 'in_reply_to_id': None, 'quote_id': None, 'in_reply_to_account_id': None, 'sensitive': False, 'spoiler_text': '', 'visibility': 'public', 'language': 'en', 'uri': 'https://truthsocial.com/@bbcnewa/109598702184774628', 'url': 'https://truthsocial.com/@bbcnewa/109598702184774628', 'content': '<p>Pele, regarded by many as football\'s greatest ever player, has died in Brazil at the age of 82. <a href="https://www.bbc.com/sport/football/42751517" rel="nofollow noopener noreferrer" target="_blank"><span class="invisible">https://www.</span><span class="ellipsis">bbc.com/sport/football/4275151</span><span class="invisible">7</span></a></p>', 'account': {'id': '107905163010312793', 'username': 'bbcnewa', 'acct': 'bbcnewa', 'display_name': 'BBC News', 'locked': False, 'bot': False, 'discoverable': True, 'group': False, 'created_at': '2022-03-05T17:42:01.159Z', 'note': '<p>News, features and analysis by the BBC</p>', 'url': 'https://truthsocial.com/@bbcnewa', 'avatar': 'https://static-assets-1.truthsocial.com/tmtg:prime-ts-assets/accounts/avatars/107/905/163/010/312/793/original/e7c07550dc22c23a.jpeg', 'avatar_static': 'https://static-assets-1.truthsocial.com/tmtg:prime-ts-assets/accounts/avatars/107/905/163/010/312/793/original/e7c07550dc22c23a.jpeg', 'header': 'https://static-assets-1.truthsocial.com/tmtg:prime-ts-assets/accounts/headers/107/905/163/010/312/793/original/a00eeec2b57206c7.jpeg', 'header_static': 'https://static-assets-1.truthsocial.com/tmtg:prime-ts-assets/accounts/headers/107/905/163/010/312/793/original/a00eeec2b57206c7.jpeg', 'followers_count': 1131, 'following_count': 3, 'statuses_count': 9, 'last_status_at': '2024-11-12', 'verified': False, 'location': '', 'website': 'https://www.bbc.com/news', 'unauth_visibility': True, 'chats_onboarded': True, 'feeds_onboarded': True, 'accepting_messages': False, 'show_nonmember_group_statuses': None, 'emojis': [], 'fields': [], 'tv_onboarded': True, 'tv_account': False}, 'media_attachments': [], 'mentions': [], 'tags': [], 'card': None, 'group': None, 'quote': None, 'in_reply_to': None, 'reblog': None, 'sponsored': False, 'replies_count': 1, 'reblogs_count': 0, 'favourites_count': 2, 'favourited': False, 'reblogged': False, 'muted': False, 'pinned': False, 'bookmarked': False, 'poll': None, 'emojis': []}
         """
 
         result = Metadata()
         result.set_url(url)
-        timestamp = post['created_at'] # format is 2022-12-29T19:51:18.161Z
+        timestamp = post["created_at"]  # format is 2022-12-29T19:51:18.161Z
         result.set_timestamp(parse_dt(timestamp))
-        result.set('description', post['content'])
-        result.set('author', post['account']['username'])
+        result.set("description", post["content"])
+        result.set("author", post["account"]["username"])
 
-        for key in ['replies_count', 'reblogs_count', 'favourites_count', ('account', 'followers_count'), ('account', 'following_count'), ('account', 'statuses_count'), ('account', 'display_name'), 'language', 'in_reply_to_account', 'replies_count']:
+        for key in [
+            "replies_count",
+            "reblogs_count",
+            "favourites_count",
+            ("account", "followers_count"),
+            ("account", "following_count"),
+            ("account", "statuses_count"),
+            ("account", "display_name"),
+            "language",
+            "in_reply_to_account",
+            "replies_count",
+        ]:
             if isinstance(key, tuple):
                 store_key = " ".join(key)
             else:
                 store_key = key
             result.set(store_key, traverse_obj(post, key))
-        
-        # add the media
-        for media in post.get('media_attachments', []):
-            filename = archiver.download_from_url(media['url'])
-            result.add_media(Media(filename), id=media.get('id'))
 
-        return result
+        # add the media
+        for media in post.get("media_attachments", []):
+            filename = archiver.download_from_url(media["url"])
+            result.add_media(Media(filename), id=media.get("id"))
+
+        return result

src/auto_archiver/modules/generic_extractor/twitter.py

@@ -1,18 +1,17 @@
-import re, mimetypes, json
-from datetime import datetime
+import re
+import mimetypes
 
 from loguru import logger
 from slugify import slugify
 
 from auto_archiver.core.metadata import Metadata, Media
-from auto_archiver.utils import url as UrlUtil
+from auto_archiver.utils import url as UrlUtil, get_datetime_from_str
 from auto_archiver.core.extractor import Extractor
 
 from .dropin import GenericDropin, InfoExtractor
 
+
 class Twitter(GenericDropin):
-
-
     def choose_variant(self, variants):
         # choosing the highest quality possible
         variant, width, height = None, 0, 0
@@ -27,44 +26,48 @@ class Twitter(GenericDropin):
             else:
                 variant = var if not variant else variant
         return variant
-    
+
     def extract_post(self, url: str, ie_instance: InfoExtractor):
-        twid = ie_instance._match_valid_url(url).group('id')
+        twid = ie_instance._match_valid_url(url).group("id")
         return ie_instance._extract_status(twid=twid)
 
+    def keys_to_clean(self, video_data, info_extractor):
+        return ["user", "created_at", "entities", "favorited", "translator_type"]
+
     def create_metadata(self, tweet: dict, ie_instance: InfoExtractor, archiver: Extractor, url: str) -> Metadata:
         result = Metadata()
         try:
             if not tweet.get("user") or not tweet.get("created_at"):
-                raise ValueError(f"Error retreiving post. Are you sure it exists?")
-            timestamp = datetime.strptime(tweet["created_at"], "%a %b %d %H:%M:%S %z %Y")
+                raise ValueError("Error retreiving post. Are you sure it exists?")
+            timestamp = get_datetime_from_str(tweet["created_at"], "%a %b %d %H:%M:%S %z %Y")
         except (ValueError, KeyError) as ex:
             logger.warning(f"Unable to parse tweet: {str(ex)}\nRetreived tweet data: {tweet}")
             return False
-                
-        result\
-            .set_title(tweet.get('full_text', ''))\
-            .set_content(json.dumps(tweet, ensure_ascii=False))\
-            .set_timestamp(timestamp)
+
+        full_text = tweet.pop("full_text", "")
+        author = tweet["user"].get("name", "")
+        result.set("author", author).set_url(url)
+
+        result.set_title(f"{author} - {full_text}").set_content(full_text).set_timestamp(timestamp)
         if not tweet.get("entities", {}).get("media"):
-            logger.debug('No media found, archiving tweet text only')
+            logger.debug("No media found, archiving tweet text only")
             result.status = "twitter-ytdl"
             return result
         for i, tw_media in enumerate(tweet["entities"]["media"]):
             media = Media(filename="")
             mimetype = ""
             if tw_media["type"] == "photo":
-                media.set("src", UrlUtil.twitter_best_quality_url(tw_media['media_url_https']))
+                media.set("src", UrlUtil.twitter_best_quality_url(tw_media["media_url_https"]))
                 mimetype = "image/jpeg"
             elif tw_media["type"] == "video":
-                variant = self.choose_variant(tw_media['video_info']['variants'])
-                media.set("src", variant['url'])
-                mimetype = variant['content_type']
+                variant = self.choose_variant(tw_media["video_info"]["variants"])
+                media.set("src", variant["url"])
+                mimetype = variant["content_type"]
             elif tw_media["type"] == "animated_gif":
-                variant = tw_media['video_info']['variants'][0]
-                media.set("src", variant['url'])
-                mimetype = variant['content_type']
+                variant = tw_media["video_info"]["variants"][0]
+                media.set("src", variant["url"])
+                mimetype = variant["content_type"]
             ext = mimetypes.guess_extension(mimetype)
-            media.filename = archiver.download_from_url(media.get("src"), f'{slugify(url)}_{i}{ext}')
+            media.filename = archiver.download_from_url(media.get("src"), f"{slugify(url)}_{i}{ext}")
             result.add_media(media)
-        return result
+        return result

src/auto_archiver/modules/gsheet_db/__init__.py

@@ -1 +0,0 @@
-from .gsheet_db import GsheetsDb

src/auto_archiver/modules/gsheet_db/__manifest__.py

@@ -1,38 +0,0 @@
-{
-    "name": "Google Sheets Database",
-    "type": ["database"],
-    "entry_point": "gsheet_db::GsheetsDb",
-    "requires_setup": True,
-    "dependencies": {
-        "python": ["loguru", "gspread", "slugify"],
-    },
-    "configs": {
-        "allow_worksheets": {
-            "default": set(),
-            "help": "(CSV) only worksheets whose name is included in allow are included (overrides worksheet_block), leave empty so all are allowed",
-        },
-        "block_worksheets": {
-            "default": set(),
-            "help": "(CSV) explicitly block some worksheets from being processed",
-        },
-        "use_sheet_names_in_stored_paths": {
-            "default": True,
-            "type": "bool",
-            "help": "if True the stored files path will include 'workbook_name/worksheet_name/...'",
-        }
-    },
-    "description": """
-    GsheetsDatabase:
-    Handles integration with Google Sheets for tracking archival tasks.
-
-### Features
-- Updates a Google Sheet with the status of the archived URLs, including in progress, success or failure, and method used.
-- Saves metadata such as title, text, timestamp, hashes, screenshots, and media URLs to designated columns.
-- Formats media-specific metadata, such as thumbnails and PDQ hashes for the sheet.
-- Skips redundant updates for empty or invalid data fields.
-
-### Notes
-- Currently works only with metadata provided by GsheetFeeder. 
-- Requires configuration of a linked Google Sheet and appropriate API credentials.
-    """
-}

src/auto_archiver/modules/gsheet_db/gsheet_db.py

@@ -1,114 +0,0 @@
-from typing import Union, Tuple
-from urllib.parse import quote
-
-from loguru import logger
-
-from auto_archiver.core import Database
-from auto_archiver.core import Metadata, Media
-from auto_archiver.modules.gsheet_feeder import GWorksheet
-from auto_archiver.utils.misc import get_current_timestamp
-
-
-class GsheetsDb(Database):
-    """
-    NB: only works if GsheetFeeder is used.
-    could be updated in the future to support non-GsheetFeeder metadata
-    """
-
-    def started(self, item: Metadata) -> None:
-        logger.warning(f"STARTED {item}")
-        gw, row = self._retrieve_gsheet(item)
-        gw.set_cell(row, "status", "Archive in progress")
-
-    def failed(self, item: Metadata, reason: str) -> None:
-        logger.error(f"FAILED {item}")
-        self._safe_status_update(item, f"Archive failed {reason}")
-
-    def aborted(self, item: Metadata) -> None:
-        logger.warning(f"ABORTED {item}")
-        self._safe_status_update(item, "")
-
-    def fetch(self, item: Metadata) -> Union[Metadata, bool]:
-        """check if the given item has been archived already"""
-        return False
-
-    def done(self, item: Metadata, cached: bool = False) -> None:
-        """archival result ready - should be saved to DB"""
-        logger.success(f"DONE {item.get_url()}")
-        gw, row = self._retrieve_gsheet(item)
-        # self._safe_status_update(item, 'done')
-
-        cell_updates = []
-        row_values = gw.get_row(row)
-
-        def batch_if_valid(col, val, final_value=None):
-            final_value = final_value or val
-            try:
-                if val and gw.col_exists(col) and gw.get_cell(row_values, col) == "":
-                    cell_updates.append((row, col, final_value))
-            except Exception as e:
-                logger.error(f"Unable to batch {col}={final_value} due to {e}")
-
-        status_message = item.status
-        if cached:
-            status_message = f"[cached] {status_message}"
-        cell_updates.append((row, "status", status_message))
-
-        media: Media = item.get_final_media()
-        if hasattr(media, "urls"):
-            batch_if_valid("archive", "\n".join(media.urls))
-        batch_if_valid("date", True, get_current_timestamp())
-        batch_if_valid("title", item.get_title())
-        batch_if_valid("text", item.get("content", ""))
-        batch_if_valid("timestamp", item.get_timestamp())
-        if media:
-            batch_if_valid("hash", media.get("hash", "not-calculated"))
-
-        # merge all pdq hashes into a single string, if present
-        pdq_hashes = []
-        all_media = item.get_all_media()
-        for m in all_media:
-            if pdq := m.get("pdq_hash"):
-                pdq_hashes.append(pdq)
-        if len(pdq_hashes):
-            batch_if_valid("pdq_hash", ",".join(pdq_hashes))
-
-        if (screenshot := item.get_media_by_id("screenshot")) and hasattr(
-            screenshot, "urls"
-        ):
-            batch_if_valid("screenshot", "\n".join(screenshot.urls))
-
-        if thumbnail := item.get_first_image("thumbnail"):
-            if hasattr(thumbnail, "urls"):
-                batch_if_valid("thumbnail", f'=IMAGE("{thumbnail.urls[0]}")')
-
-        if browsertrix := item.get_media_by_id("browsertrix"):
-            batch_if_valid("wacz", "\n".join(browsertrix.urls))
-            batch_if_valid(
-                "replaywebpage",
-                "\n".join(
-                    [
-                        f"https://replayweb.page/?source={quote(wacz)}#view=pages&url={quote(item.get_url())}"
-                        for wacz in browsertrix.urls
-                    ]
-                ),
-            )
-
-        gw.batch_set_cell(cell_updates)
-
-    def _safe_status_update(self, item: Metadata, new_status: str) -> None:
-        try:
-            gw, row = self._retrieve_gsheet(item)
-            gw.set_cell(row, "status", new_status)
-        except Exception as e:
-            logger.debug(f"Unable to update sheet: {e}")
-
-    def _retrieve_gsheet(self, item: Metadata) -> Tuple[GWorksheet, int]:
-
-        if gsheet := item.get_context("gsheet"):
-            gw: GWorksheet = gsheet.get("worksheet")
-            row: int = gsheet.get("row")
-        elif self.sheet_id:
-            logger.error(f"Unable to retrieve Gsheet for {item.get_url()}, GsheetDB must be used alongside GsheetFeeder.")
-
-        return gw, row

src/auto_archiver/modules/gsheet_feeder/__init__.py

@@ -1,2 +0,0 @@
-from .gworksheet import GWorksheet
-from .gsheet_feeder import GsheetsFeeder

src/auto_archiver/modules/gsheet_feeder/gsheet_feeder.py

@@ -1,95 +0,0 @@
-"""
-GsheetsFeeder: A Google Sheets-based feeder for the Auto Archiver.
-
-This reads data from Google Sheets and filters rows based on user-defined rules.
-The filtered rows are processed into `Metadata` objects.
-
-### Key properties
-- validates the sheet's structure and filters rows based on input configurations.
-- Ensures only rows with valid URLs and unprocessed statuses are included.
-"""
-import os
-import gspread
-
-from loguru import logger
-from slugify import slugify
-
-from auto_archiver.core import Feeder
-from auto_archiver.core import Metadata
-from . import GWorksheet
-
-
-class GsheetsFeeder(Feeder):
-
-    def setup(self) -> None:
-        self.gsheets_client = gspread.service_account(filename=self.service_account)
-        # TODO mv to validators
-        if not self.sheet and not self.sheet_id:
-            raise ValueError("You need to define either a 'sheet' name or a 'sheet_id' in your manifest.")
-
-    def open_sheet(self):
-        if self.sheet:
-            return self.gsheets_client.open(self.sheet)
-        else:  # self.sheet_id
-            return self.gsheets_client.open_by_key(self.sheet_id)
-
-    def __iter__(self) -> Metadata:
-        sh = self.open_sheet()
-        for ii, worksheet in enumerate(sh.worksheets()):
-            if not self.should_process_sheet(worksheet.title):
-                logger.debug(f"SKIPPED worksheet '{worksheet.title}' due to allow/block rules")
-                continue
-            logger.info(f'Opening worksheet {ii=}: {worksheet.title=} header={self.header}')
-            gw = GWorksheet(worksheet, header_row=self.header, columns=self.columns)
-            if len(missing_cols := self.missing_required_columns(gw)):
-                logger.warning(f"SKIPPED worksheet '{worksheet.title}' due to missing required column(s) for {missing_cols}")
-                continue
-
-            # process and yield metadata here:
-            yield from self._process_rows(gw)
-            logger.success(f'Finished worksheet {worksheet.title}')
-
-    def _process_rows(self, gw: GWorksheet):
-        for row in range(1 + self.header, gw.count_rows() + 1):
-            url = gw.get_cell(row, 'url').strip()
-            if not len(url): continue
-            original_status = gw.get_cell(row, 'status')
-            status = gw.get_cell(row, 'status', fresh=original_status in ['', None])
-            # TODO: custom status parser(?) aka should_retry_from_status
-            if status not in ['', None]: continue
-
-            # All checks done - archival process starts here
-            m = Metadata().set_url(url)
-            self._set_context(m, gw, row)
-            yield m
-
-    def _set_context(self, m: Metadata, gw: GWorksheet, row: int) -> Metadata:
-        # TODO: Check folder value not being recognised
-        m.set_context("gsheet", {"row": row, "worksheet": gw})
-
-        if gw.get_cell_or_default(row, 'folder', "") is None:
-            folder = ''
-        else:
-            folder = slugify(gw.get_cell_or_default(row, 'folder', "").strip())
-        if len(folder):
-            if self.use_sheet_names_in_stored_paths:
-                m.set_context("folder", os.path.join(folder, slugify(self.sheet), slugify(gw.wks.title)))
-            else:
-                m.set_context("folder", folder)
-
-
-    def should_process_sheet(self, sheet_name: str) -> bool:
-        if len(self.allow_worksheets) and sheet_name not in self.allow_worksheets:
-            # ALLOW rules exist AND sheet name not explicitly allowed
-            return False
-        if len(self.block_worksheets) and sheet_name in self.block_worksheets:
-            # BLOCK rules exist AND sheet name is blocked
-            return False
-        return True
-
-    def missing_required_columns(self, gw: GWorksheet) -> list:
-        missing = []
-        for required_col in ['url', 'status']:
-            if not gw.col_exists(required_col):
-                missing.append(required_col)
-        return missing

src/auto_archiver/modules/gsheet_feeder_db/__init__.py

@@ -0,0 +1,2 @@
+from .gworksheet import GWorksheet
+from .gsheet_feeder_db import GsheetsFeederDB

src/auto_archiver/modules/gsheet_feeder_db/__manifest__.py

@@ -1,7 +1,7 @@
 {
-    "name": "Google Sheets Feeder",
-    "type": ["feeder"],
-    "entry_point": "gsheet_feeder::GsheetsFeeder",
+    "name": "Google Sheets Feeder Database",
+    "type": ["feeder", "database"],
+    "entry_point": "gsheet_feeder_db::GsheetsFeederDB",
     "requires_setup": True,
     "dependencies": {
         "python": ["loguru", "gspread", "slugify"],
@@ -53,8 +53,8 @@
         },
     },
     "description": """
-    GsheetsFeeder 
-    A Google Sheets-based feeder for the Auto Archiver.
+    GsheetsFeederDatabase
+    A Google Sheets-based feeder and optional database for the Auto Archiver.
 
     This reads data from Google Sheets and filters rows based on user-defined rules.
     The filtered rows are processed into `Metadata` objects.
@@ -64,11 +64,20 @@
     - Processes only worksheets allowed by the `allow_worksheets` and `block_worksheets` configurations.
     - Ensures only rows with valid URLs and unprocessed statuses are included for archival.
     - Supports organizing stored files into folder paths based on sheet and worksheet names.
+    - If the database is enabled, this updates the Google Sheet with the status of the archived URLs, including in progress, success or failure, and method used.
+    - Saves metadata such as title, text, timestamp, hashes, screenshots, and media URLs to designated columns.
+    - Formats media-specific metadata, such as thumbnails and PDQ hashes for the sheet.
+    - Skips redundant updates for empty or invalid data fields.
 
     ### Setup
-    - Requires a Google Service Account JSON file for authentication, which should be stored in `secrets/gsheets_service_account.json`.
-    To set up a service account, follow the instructions [here](https://gspread.readthedocs.io/en/latest/oauth2.html).
-    - Define the `sheet` or `sheet_id` configuration to specify the sheet to archive.
-    - Customize the column names in your Google sheet using the `columns` configuration.
+    1. Requires a Google Service Account JSON file for authentication.
+    To set up a service account, follow the instructions in the [how to](https://auto-archiver.readthedocs.io/en/latest/how_to/gsheets_setup.html),
+    or use the script:
+    ```
+    /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/bellingcat/auto-archiver/refs/heads/main/scripts/generate_google_services.sh)"
+    ```
+    2. Create a Google sheet with the required column(s) and then define the `sheet` or `sheet_id` configuration to specify this sheet.
+    3. Customize the column names in your Google sheet using the `columns` configuration.
+    4. The Google Sheet can be used solely as a feeder or as a feeder and database, but note you can't currently feed into the database from an alternate feeder.
     """,
 }

src/auto_archiver/modules/gsheet_feeder_db/gsheet_feeder_db.py

@@ -0,0 +1,198 @@
+"""
+GsheetsFeeder: A Google Sheets-based feeder for the Auto Archiver.
+
+This reads data from Google Sheets and filters rows based on user-defined rules.
+The filtered rows are processed into `Metadata` objects.
+
+### Key properties
+- validates the sheet's structure and filters rows based on input configurations.
+- Ensures only rows with valid URLs and unprocessed statuses are included.
+"""
+
+import os
+from typing import Tuple, Union
+from urllib.parse import quote
+
+import gspread
+from loguru import logger
+from slugify import slugify
+
+from auto_archiver.core import Feeder, Database, Media
+from auto_archiver.core import Metadata
+from auto_archiver.modules.gsheet_feeder_db import GWorksheet
+from auto_archiver.utils.misc import get_current_timestamp
+
+
+class GsheetsFeederDB(Feeder, Database):
+    def setup(self) -> None:
+        self.gsheets_client = gspread.service_account(filename=self.service_account)
+        # TODO mv to validators
+        if not self.sheet and not self.sheet_id:
+            raise ValueError("You need to define either a 'sheet' name or a 'sheet_id' in your manifest.")
+
+    def open_sheet(self):
+        if self.sheet:
+            return self.gsheets_client.open(self.sheet)
+        else:  # self.sheet_id
+            return self.gsheets_client.open_by_key(self.sheet_id)
+
+    def __iter__(self) -> Metadata:
+        sh = self.open_sheet()
+        for ii, worksheet in enumerate(sh.worksheets()):
+            if not self.should_process_sheet(worksheet.title):
+                logger.debug(f"SKIPPED worksheet '{worksheet.title}' due to allow/block rules")
+                continue
+            logger.info(f"Opening worksheet {ii=}: {worksheet.title=} header={self.header}")
+            gw = GWorksheet(worksheet, header_row=self.header, columns=self.columns)
+            if len(missing_cols := self.missing_required_columns(gw)):
+                logger.warning(
+                    f"SKIPPED worksheet '{worksheet.title}' due to missing required column(s) for {missing_cols}"
+                )
+                continue
+
+            # process and yield metadata here:
+            yield from self._process_rows(gw)
+            logger.success(f"Finished worksheet {worksheet.title}")
+
+    def _process_rows(self, gw: GWorksheet):
+        for row in range(1 + self.header, gw.count_rows() + 1):
+            url = gw.get_cell(row, "url").strip()
+            if not len(url):
+                continue
+            original_status = gw.get_cell(row, "status")
+            status = gw.get_cell(row, "status", fresh=original_status in ["", None])
+            # TODO: custom status parser(?) aka should_retry_from_status
+            if status not in ["", None]:
+                continue
+
+            # All checks done - archival process starts here
+            m = Metadata().set_url(url)
+            self._set_context(m, gw, row)
+            yield m
+
+    def _set_context(self, m: Metadata, gw: GWorksheet, row: int) -> Metadata:
+        # TODO: Check folder value not being recognised
+        m.set_context("gsheet", {"row": row, "worksheet": gw})
+
+        if gw.get_cell_or_default(row, "folder", "") is None:
+            folder = ""
+        else:
+            folder = slugify(gw.get_cell_or_default(row, "folder", "").strip())
+        if len(folder):
+            if self.use_sheet_names_in_stored_paths:
+                m.set_context("folder", os.path.join(folder, slugify(self.sheet), slugify(gw.wks.title)))
+            else:
+                m.set_context("folder", folder)
+
+    def should_process_sheet(self, sheet_name: str) -> bool:
+        if len(self.allow_worksheets) and sheet_name not in self.allow_worksheets:
+            # ALLOW rules exist AND sheet name not explicitly allowed
+            return False
+        if len(self.block_worksheets) and sheet_name in self.block_worksheets:
+            # BLOCK rules exist AND sheet name is blocked
+            return False
+        return True
+
+    def missing_required_columns(self, gw: GWorksheet) -> list:
+        missing = []
+        for required_col in ["url", "status"]:
+            if not gw.col_exists(required_col):
+                missing.append(required_col)
+        return missing
+
+    def started(self, item: Metadata) -> None:
+        logger.warning(f"STARTED {item}")
+        gw, row = self._retrieve_gsheet(item)
+        gw.set_cell(row, "status", "Archive in progress")
+
+    def failed(self, item: Metadata, reason: str) -> None:
+        logger.error(f"FAILED {item}")
+        self._safe_status_update(item, f"Archive failed {reason}")
+
+    def aborted(self, item: Metadata) -> None:
+        logger.warning(f"ABORTED {item}")
+        self._safe_status_update(item, "")
+
+    def fetch(self, item: Metadata) -> Union[Metadata, bool]:
+        """check if the given item has been archived already"""
+        return False
+
+    def done(self, item: Metadata, cached: bool = False) -> None:
+        """archival result ready - should be saved to DB"""
+        logger.success(f"DONE {item.get_url()}")
+        gw, row = self._retrieve_gsheet(item)
+        # self._safe_status_update(item, 'done')
+
+        cell_updates = []
+        row_values = gw.get_row(row)
+
+        def batch_if_valid(col, val, final_value=None):
+            final_value = final_value or val
+            try:
+                if val and gw.col_exists(col) and gw.get_cell(row_values, col) == "":
+                    cell_updates.append((row, col, final_value))
+            except Exception as e:
+                logger.error(f"Unable to batch {col}={final_value} due to {e}")
+
+        status_message = item.status
+        if cached:
+            status_message = f"[cached] {status_message}"
+        cell_updates.append((row, "status", status_message))
+
+        media: Media = item.get_final_media()
+        if hasattr(media, "urls"):
+            batch_if_valid("archive", "\n".join(media.urls))
+        batch_if_valid("date", True, get_current_timestamp())
+        batch_if_valid("title", item.get_title())
+        batch_if_valid("text", item.get("content", ""))
+        batch_if_valid("timestamp", item.get_timestamp())
+        if media:
+            batch_if_valid("hash", media.get("hash", "not-calculated"))
+
+        # merge all pdq hashes into a single string, if present
+        pdq_hashes = []
+        all_media = item.get_all_media()
+        for m in all_media:
+            if pdq := m.get("pdq_hash"):
+                pdq_hashes.append(pdq)
+        if len(pdq_hashes):
+            batch_if_valid("pdq_hash", ",".join(pdq_hashes))
+
+        if (screenshot := item.get_media_by_id("screenshot")) and hasattr(screenshot, "urls"):
+            batch_if_valid("screenshot", "\n".join(screenshot.urls))
+
+        if thumbnail := item.get_first_image("thumbnail"):
+            if hasattr(thumbnail, "urls"):
+                batch_if_valid("thumbnail", f'=IMAGE("{thumbnail.urls[0]}")')
+
+        if browsertrix := item.get_media_by_id("browsertrix"):
+            batch_if_valid("wacz", "\n".join(browsertrix.urls))
+            batch_if_valid(
+                "replaywebpage",
+                "\n".join(
+                    [
+                        f"https://replayweb.page/?source={quote(wacz)}#view=pages&url={quote(item.get_url())}"
+                        for wacz in browsertrix.urls
+                    ]
+                ),
+            )
+
+        gw.batch_set_cell(cell_updates)
+
+    def _safe_status_update(self, item: Metadata, new_status: str) -> None:
+        try:
+            gw, row = self._retrieve_gsheet(item)
+            gw.set_cell(row, "status", new_status)
+        except Exception as e:
+            logger.debug(f"Unable to update sheet: {e}")
+
+    def _retrieve_gsheet(self, item: Metadata) -> Tuple[GWorksheet, int]:
+        if gsheet := item.get_context("gsheet"):
+            gw: GWorksheet = gsheet.get("worksheet")
+            row: int = gsheet.get("row")
+        elif self.sheet_id:
+            logger.error(
+                f"Unable to retrieve Gsheet for {item.get_url()}, GsheetDB must be used alongside GsheetFeeder."
+            )
+
+        return gw, row

src/auto_archiver/modules/gsheet_feeder_db/gworksheet.py

@@ -5,23 +5,25 @@ class GWorksheet:
     """
     This class makes read/write operations to the a worksheet easier.
     It can read the headers from a custom row number, but the row references
-    should always include the offset of the header. 
-    eg: if header=4, row 5 will be the first with data. 
+    should always include the offset of the header.
+    eg: if header=4, row 5 will be the first with data.
     """
+
     COLUMN_NAMES = {
-        'url': 'link',
-        'status': 'archive status',
-        'folder': 'destination folder',
-        'archive': 'archive location',
-        'date': 'archive date',
-        'thumbnail': 'thumbnail',
-        'timestamp': 'upload timestamp',
-        'title': 'upload title',
-        'screenshot': 'screenshot',
-        'hash': 'hash',
-        'pdq_hash': 'perceptual hashes',
-        'wacz': 'wacz',
-        'replaywebpage': 'replaywebpage',
+        "url": "link",
+        "status": "archive status",
+        "folder": "destination folder",
+        "archive": "archive location",
+        "date": "archive date",
+        "thumbnail": "thumbnail",
+        "timestamp": "upload timestamp",
+        "title": "upload title",
+        "text": "text content",
+        "screenshot": "screenshot",
+        "hash": "hash",
+        "pdq_hash": "perceptual hashes",
+        "wacz": "wacz",
+        "replaywebpage": "replaywebpage",
     }
 
     def __init__(self, worksheet, columns=COLUMN_NAMES, header_row=1):
@@ -35,7 +37,7 @@ class GWorksheet:
 
     def _check_col_exists(self, col: str):
         if col not in self.columns:
-            raise Exception(f'Column {col} is not in the configured column names: {self.columns.keys()}')
+            raise Exception(f"Column {col} is not in the configured column names: {self.columns.keys()}")
 
     def _col_index(self, col: str):
         self._check_col_exists(col)
@@ -57,7 +59,7 @@ class GWorksheet:
 
     def get_cell(self, row, col: str, fresh=False):
         """
-        returns the cell value from (row, col), 
+        returns the cell value from (row, col),
         where row can be an index (1-based) OR list of values
         as received from self.get_row(row)
         if fresh=True, the sheet is queried again for this cell
@@ -66,11 +68,11 @@ class GWorksheet:
 
         if fresh:
             return self.wks.cell(row, col_index + 1).value
-        if type(row) == int:
+        if isinstance(row, int):
             row = self.get_row(row)
 
         if col_index >= len(row):
-            return ''
+            return ""
         return row[col_index]
 
     def get_cell_or_default(self, row, col: str, default: str = None, fresh=False, when_empty_use_default=True):
@@ -82,7 +84,7 @@ class GWorksheet:
             if when_empty_use_default and val.strip() == "":
                 return default
             return val
-        except:
+        except Exception:
             return default
 
     def set_cell(self, row: int, col: str, val):
@@ -95,13 +97,9 @@ class GWorksheet:
         receives a list of [(row:int, col:str, val)] and batch updates it, the parameters are the same as in the self.set_cell() method
         """
         cell_updates = [
-            {
-                'range': self.to_a1(row, col),
-                'values': [[str(val)[0:49999]]]
-            }
-            for row, col, val in cell_updates
+            {"range": self.to_a1(row, col), "values": [[str(val)[0:49999]]]} for row, col, val in cell_updates
         ]
-        self.wks.batch_update(cell_updates, value_input_option='USER_ENTERED')
+        self.wks.batch_update(cell_updates, value_input_option="USER_ENTERED")
 
     def to_a1(self, row: int, col: str):
         # row is 1-based

src/auto_archiver/modules/hash_enricher/__init__.py

@@ -1 +1 @@
-from .hash_enricher import HashEnricher
+from .hash_enricher import HashEnricher

src/auto_archiver/modules/hash_enricher/__manifest__.py

@@ -3,16 +3,17 @@
     "type": ["enricher"],
     "requires_setup": False,
     "dependencies": {
-                          "python": ["loguru"],
+        "python": ["loguru"],
     },
     "configs": {
-            "algorithm": {"default": "SHA-256", "help": "hash algorithm to use", "choices": ["SHA-256", "SHA3-512"]},
-            # TODO add non-negative requirement to match previous implementation?
-            "chunksize": {"default": 16000000,
-                          "help": "number of bytes to use when reading files in chunks (if this value is too large you will run out of RAM), default is 16MB",
-                          'type': 'int',
-                          },
+        "algorithm": {"default": "SHA-256", "help": "hash algorithm to use", "choices": ["SHA-256", "SHA3-512"]},
+        # TODO add non-negative requirement to match previous implementation?
+        "chunksize": {
+            "default": 16000000,
+            "help": "number of bytes to use when reading files in chunks (if this value is too large you will run out of RAM), default is 16MB",
+            "type": "int",
         },
+    },
     "description": """
 Generates cryptographic hashes for media files to ensure data integrity and authenticity.
 

src/auto_archiver/modules/hash_enricher/hash_enricher.py

@@ -1,4 +1,4 @@
-""" Hash Enricher for generating cryptographic hashes of media files.
+"""Hash Enricher for generating cryptographic hashes of media files.
 
 The `HashEnricher` calculates cryptographic hashes (e.g., SHA-256, SHA3-512)
 for media files stored in `Metadata` objects. These hashes are used for
@@ -7,6 +7,7 @@ exact duplicates. The hash is computed by reading the file's bytes in chunks,
 making it suitable for handling large files efficiently.
 
 """
+
 import hashlib
 from loguru import logger
 
@@ -20,7 +21,6 @@ class HashEnricher(Enricher):
     Calculates hashes for Media instances
     """
 
-
     def enrich(self, to_enrich: Metadata) -> None:
         url = to_enrich.get_url()
         logger.debug(f"calculating media hashes for {url=} (using {self.algorithm})")
@@ -35,5 +35,6 @@ class HashEnricher(Enricher):
             hash_algo = hashlib.sha256
         elif self.algorithm == "SHA3-512":
             hash_algo = hashlib.sha3_512
-        else: return ""
+        else:
+            return ""
         return calculate_file_hash(filename, hash_algo, self.chunksize)

src/auto_archiver/modules/html_formatter/__init__.py

@@ -1 +1 @@
-from .html_formatter import HtmlFormatter
+from .html_formatter import HtmlFormatter

src/auto_archiver/modules/html_formatter/__manifest__.py

@@ -2,12 +2,13 @@
     "name": "HTML Formatter",
     "type": ["formatter"],
     "requires_setup": False,
-    "dependencies": {
-                          "python": ["hash_enricher", "loguru", "jinja2"],
-                          "bin": [""]
-    },
+    "dependencies": {"python": ["hash_enricher", "loguru", "jinja2"], "bin": [""]},
     "configs": {
-            "detect_thumbnails": {"default": True, "help": "if true will group by thumbnails generated by thumbnail enricher by id 'thumbnail_00'"}
+        "detect_thumbnails": {
+            "default": True,
+            "help": "if true will group by thumbnails generated by thumbnail enricher by id 'thumbnail_00'",
+            "type": "bool",
         },
+    },
     "description": """ """,
 }

src/auto_archiver/modules/html_formatter/html_formatter.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
-import mimetypes, os, pathlib
+import mimetypes
+import os
+import pathlib
 from jinja2 import Environment, FileSystemLoader
 from urllib.parse import quote
 from loguru import logger
@@ -11,6 +13,7 @@ from auto_archiver.core import Metadata, Media
 from auto_archiver.core import Formatter
 from auto_archiver.utils.misc import random_str
 
+
 class HtmlFormatter(Formatter):
     environment: Environment = None
     template: any = None
@@ -21,9 +24,9 @@ class HtmlFormatter(Formatter):
         self.environment = Environment(loader=FileSystemLoader(template_dir), autoescape=True)
 
         # JinjaHelper class static methods are added as filters
-        self.environment.filters.update({
-            k: v.__func__ for k, v in JinjaHelpers.__dict__.items() if isinstance(v, staticmethod)
-        })
+        self.environment.filters.update(
+            {k: v.__func__ for k, v in JinjaHelpers.__dict__.items() if isinstance(v, staticmethod)}
+        )
 
         # Load a specific template or default to "html_template.html"
         template_name = self.config.get("template_name", "html_template.html")
@@ -36,11 +39,7 @@ class HtmlFormatter(Formatter):
             return
 
         content = self.template.render(
-            url=url,
-            title=item.get_title(),
-            media=item.media,
-            metadata=item.metadata,
-            version=__version__
+            url=url, title=item.get_title(), media=item.media, metadata=item.metadata, version=__version__
         )
 
         html_path = os.path.join(self.tmp_dir, f"formatted{random_str(24)}.html")
@@ -49,7 +48,7 @@ class HtmlFormatter(Formatter):
         final_media = Media(filename=html_path, _mimetype="text/html")
 
         # get the already instantiated hash_enricher module
-        he = self.module_factory.get_module('hash_enricher', self.config)
+        he = self.module_factory.get_module("hash_enricher", self.config)
         if len(hd := he.calculate_hash(final_media.filename)):
             final_media.set("hash", f"{he.algorithm}:{hd}")
 

src/auto_archiver/modules/instagram_api_extractor/__manifest__.py

@@ -2,18 +2,18 @@
     "name": "Instagram API Extractor",
     "type": ["extractor"],
     "entry_point": "instagram_api_extractor::InstagramAPIExtractor",
-    "dependencies":
-        {"python": ["requests",
-                    "loguru",
-                    "retrying",
-                    "tqdm",],
-         },
+    "dependencies": {
+        "python": [
+            "requests",
+            "loguru",
+            "retrying",
+            "tqdm",
+        ],
+    },
     "requires_setup": True,
     "configs": {
-        "access_token": {"default": None,
-                         "help": "a valid instagrapi-api token"},
-        "api_endpoint": {"required": True,
-                         "help": "API endpoint to use"},
+        "access_token": {"default": None, "help": "a valid instagrapi-api token"},
+        "api_endpoint": {"required": True, "help": "API endpoint to use"},
         "full_profile": {
             "default": False,
             "type": "bool",

src/auto_archiver/modules/instagram_api_extractor/instagram_api_extractor.py

@@ -36,21 +36,16 @@ class InstagramAPIExtractor(Extractor):
         if self.api_endpoint[-1] == "/":
             self.api_endpoint = self.api_endpoint[:-1]
 
-
     def download(self, item: Metadata) -> Metadata:
         url = item.get_url()
 
-        url.replace("instagr.com", "instagram.com").replace(
-            "instagr.am", "instagram.com"
-        )
+        url.replace("instagr.com", "instagram.com").replace("instagr.am", "instagram.com")
         insta_matches = self.valid_url.findall(url)
         logger.info(f"{insta_matches=}")
         if not len(insta_matches) or len(insta_matches[0]) != 3:
             return
         if len(insta_matches) > 1:
-            logger.warning(
-                f"Multiple instagram matches found in {url=}, using the first one"
-            )
+            logger.warning(f"Multiple instagram matches found in {url=}, using the first one")
             return
         g1, g2, g3 = insta_matches[0][0], insta_matches[0][1], insta_matches[0][2]
         if g1 == "":
@@ -73,23 +68,20 @@ class InstagramAPIExtractor(Extractor):
     def call_api(self, path: str, params: dict) -> dict:
         headers = {"accept": "application/json", "x-access-key": self.access_token}
         logger.debug(f"calling {self.api_endpoint}/{path} with {params=}")
-        return requests.get(
-            f"{self.api_endpoint}/{path}", headers=headers, params=params
-        ).json()
+        return requests.get(f"{self.api_endpoint}/{path}", headers=headers, params=params).json()
 
     def cleanup_dict(self, d: dict | list) -> dict:
         # repeats 3 times to remove nested empty values
         if not self.minimize_json_output:
             return d
-        if type(d) == list:
+        if isinstance(d, list):
             return [self.cleanup_dict(v) for v in d]
-        if type(d) != dict:
+        if not isinstance(d, dict):
             return d
         return {
             k: clean_v
             for k, v in d.items()
-            if (clean_v := self.cleanup_dict(v))
-            not in [0.0, 0, [], {}, "", None, "null"]
+            if (clean_v := self.cleanup_dict(v)) not in [0.0, 0, [], {}, "", None, "null"]
             and k not in ["x", "y", "width", "height"]
         }
 
@@ -103,7 +95,7 @@ class InstagramAPIExtractor(Extractor):
         result.set_title(user.get("full_name", username)).set("data", user)
         if pic_url := user.get("profile_pic_url_hd", user.get("profile_pic_url")):
             filename = self.download_from_url(pic_url)
-            result.add_media(Media(filename=filename), id=f"profile_picture")
+            result.add_media(Media(filename=filename), id="profile_picture")
 
         if self.full_profile:
             user_id = user.get("pk")
@@ -126,9 +118,7 @@ class InstagramAPIExtractor(Extractor):
             try:
                 self.download_all_tagged(result, user_id)
             except Exception as e:
-                result.append(
-                    "errors", f"Error downloading tagged posts for {username}"
-                )
+                result.append("errors", f"Error downloading tagged posts for {username}")
                 logger.error(f"Error downloading tagged posts for {username}: {e}")
 
             # download all highlights
@@ -143,7 +133,7 @@ class InstagramAPIExtractor(Extractor):
 
     def download_all_highlights(self, result, username, user_id):
         count_highlights = 0
-        highlights = self.call_api(f"v1/user/highlights", {"user_id": user_id})
+        highlights = self.call_api("v1/user/highlights", {"user_id": user_id})
         for h in highlights:
             try:
                 h_info = self._download_highlights_reusable(result, h.get("pk"))
@@ -153,26 +143,17 @@ class InstagramAPIExtractor(Extractor):
                     "errors",
                     f"Error downloading highlight id{h.get('pk')} for {username}",
                 )
-                logger.error(
-                    f"Error downloading highlight id{h.get('pk')} for {username}: {e}"
-                )
-            if (
-                self.full_profile_max_posts
-                and count_highlights >= self.full_profile_max_posts
-            ):
-                logger.info(
-                    f"HIGHLIGHTS reached full_profile_max_posts={self.full_profile_max_posts}"
-                )
+                logger.error(f"Error downloading highlight id{h.get('pk')} for {username}: {e}")
+            if self.full_profile_max_posts and count_highlights >= self.full_profile_max_posts:
+                logger.info(f"HIGHLIGHTS reached full_profile_max_posts={self.full_profile_max_posts}")
                 break
         result.set("#highlights", count_highlights)
 
-    def download_post(
-        self, result: Metadata, code: str = None, id: str = None, context: str = None
-    ) -> Metadata:
+    def download_post(self, result: Metadata, code: str = None, id: str = None, context: str = None) -> Metadata:
         if id:
-            post = self.call_api(f"v1/media/by/id", {"id": id})
+            post = self.call_api("v1/media/by/id", {"id": id})
         else:
-            post = self.call_api(f"v1/media/by/code", {"code": code})
+            post = self.call_api("v1/media/by/code", {"code": code})
         assert post, f"Post {id or code} not found"
 
         if caption_text := post.get("caption_text"):
@@ -192,15 +173,11 @@ class InstagramAPIExtractor(Extractor):
         return result.success("insta highlights")
 
     def _download_highlights_reusable(self, result: Metadata, id: str) -> dict:
-        full_h = self.call_api(f"v2/highlight/by/id", {"id": id})
+        full_h = self.call_api("v2/highlight/by/id", {"id": id})
         h_info = full_h.get("response", {}).get("reels", {}).get(f"highlight:{id}")
         assert h_info, f"Highlight {id} not found: {full_h=}"
 
-        if (
-            cover_media := h_info.get("cover_media", {})
-            .get("cropped_image_version", {})
-            .get("url")
-        ):
+        if cover_media := h_info.get("cover_media", {}).get("cropped_image_version", {}).get("url"):
             filename = self.download_from_url(cover_media)
             result.add_media(Media(filename=filename), id=f"cover_media highlight {id}")
 
@@ -210,9 +187,7 @@ class InstagramAPIExtractor(Extractor):
                 self.scrape_item(result, h, "highlight")
             except Exception as e:
                 result.append("errors", f"Error downloading highlight {h.get('id')}")
-                logger.error(
-                    f"Error downloading highlight, skipping {h.get('id')}: {e}"
-                )
+                logger.error(f"Error downloading highlight, skipping {h.get('id')}: {e}")
 
         return h_info
 
@@ -225,7 +200,7 @@ class InstagramAPIExtractor(Extractor):
         return result.success(f"insta stories {now}")
 
     def _download_stories_reusable(self, result: Metadata, username: str) -> list[dict]:
-        stories = self.call_api(f"v1/user/stories/by/username", {"username": username})
+        stories = self.call_api("v1/user/stories/by/username", {"username": username})
         if not stories or not len(stories):
             return []
         stories = stories[::-1]  # newest to oldest
@@ -244,10 +219,8 @@ class InstagramAPIExtractor(Extractor):
 
         post_count = 0
         while end_cursor != "":
-            posts = self.call_api(
-                f"v1/user/medias/chunk", {"user_id": user_id, "end_cursor": end_cursor}
-            )
-            if not len(posts) or not type(posts) == list or len(posts) != 2:
+            posts = self.call_api("v1/user/medias/chunk", {"user_id": user_id, "end_cursor": end_cursor})
+            if not posts or not isinstance(posts, list) or len(posts) != 2:
                 break
             posts, end_cursor = posts[0], posts[1]
             logger.info(f"parsing {len(posts)} posts, next {end_cursor=}")
@@ -260,13 +233,8 @@ class InstagramAPIExtractor(Extractor):
                     logger.error(f"Error downloading post, skipping {p.get('id')}: {e}")
                 pbar.update(1)
                 post_count += 1
-            if (
-                self.full_profile_max_posts
-                and post_count >= self.full_profile_max_posts
-            ):
-                logger.info(
-                    f"POSTS reached full_profile_max_posts={self.full_profile_max_posts}"
-                )
+            if self.full_profile_max_posts and post_count >= self.full_profile_max_posts:
+                logger.info(f"POSTS reached full_profile_max_posts={self.full_profile_max_posts}")
                 break
         result.set("#posts", post_count)
 
@@ -275,10 +243,8 @@ class InstagramAPIExtractor(Extractor):
         pbar = tqdm(desc="downloading tagged posts")
 
         tagged_count = 0
-        while next_page_id != None:
-            resp = self.call_api(
-                f"v2/user/tag/medias", {"user_id": user_id, "page_id": next_page_id}
-            )
+        while next_page_id is not None:
+            resp = self.call_api("v2/user/tag/medias", {"user_id": user_id, "page_id": next_page_id})
             posts = resp.get("response", {}).get("items", [])
             if not len(posts):
                 break
@@ -290,21 +256,12 @@ class InstagramAPIExtractor(Extractor):
                 try:
                     self.scrape_item(result, p, "tagged")
                 except Exception as e:
-                    result.append(
-                        "errors", f"Error downloading tagged post {p.get('id')}"
-                    )
-                    logger.error(
-                        f"Error downloading tagged post, skipping {p.get('id')}: {e}"
-                    )
+                    result.append("errors", f"Error downloading tagged post {p.get('id')}")
+                    logger.error(f"Error downloading tagged post, skipping {p.get('id')}: {e}")
                 pbar.update(1)
                 tagged_count += 1
-            if (
-                self.full_profile_max_posts
-                and tagged_count >= self.full_profile_max_posts
-            ):
-                logger.info(
-                    f"TAGS reached full_profile_max_posts={self.full_profile_max_posts}"
-                )
+            if self.full_profile_max_posts and tagged_count >= self.full_profile_max_posts:
+                logger.info(f"TAGS reached full_profile_max_posts={self.full_profile_max_posts}")
                 break
         result.set("#tagged", tagged_count)
 
@@ -318,9 +275,7 @@ class InstagramAPIExtractor(Extractor):
         context can be used to give specific id prefixes to media
         """
         if "clips_metadata" in item:
-            if reusable_text := item.get("clips_metadata", {}).get(
-                "reusable_text_attribute_string"
-            ):
+            if reusable_text := item.get("clips_metadata", {}).get("reusable_text_attribute_string"):
                 item["clips_metadata_text"] = reusable_text
             if self.minimize_json_output:
                 del item["clips_metadata"]

src/auto_archiver/modules/instagram_extractor/__init__.py

@@ -1 +1 @@
-from .instagram_extractor import InstagramExtractor
+from .instagram_extractor import InstagramExtractor

src/auto_archiver/modules/instagram_extractor/__manifest__.py

@@ -9,26 +9,30 @@
     },
     "requires_setup": True,
     "configs": {
-        "username": {"required": True,
-                     "help": "a valid Instagram username"},
+        "username": {"required": True, "help": "A valid Instagram username."},
         "password": {
             "required": True,
-            "help": "the corresponding Instagram account password",
+            "help": "The corresponding Instagram account password.",
         },
         "download_folder": {
             "default": "instaloader",
-            "help": "name of a folder to temporarily download content to",
+            "help": "Name of a folder to temporarily download content to.",
         },
         "session_file": {
             "default": "secrets/instaloader.session",
-            "help": "path to the instagram session which saves session credentials",
+            "help": "Path to the instagram session file which saves session credentials. If one doesn't exist this gives the path to store a new one.",
         },
         # TODO: fine-grain
         # "download_stories": {"default": True, "help": "if the link is to a user profile: whether to get stories information"},
     },
     "description": """
-    Uses the [Instaloader library](https://instaloader.github.io/as-module.html) to download content from Instagram. This class handles both individual posts
-    and user profiles, downloading as much information as possible, including images, videos, text, stories,
+    Uses the [Instaloader library](https://instaloader.github.io/as-module.html) to download content from Instagram. 
+    
+      > ⚠️ **Warning**  
+      > This module is not actively maintained due to known issues with blocking.  
+      > Prioritise usage of the [Instagram Tbot Extractor](./instagram_tbot_extractor.md) and [Instagram API Extractor](./instagram_api_extractor.md)
+  
+    This class handles both individual posts and user profiles, downloading as much information as possible, including images, videos, text, stories,
     highlights, and tagged posts. 
     Authentication is required via username/password or a session file.
                     

src/auto_archiver/modules/instagram_extractor/instagram_extractor.py

@@ -1,9 +1,12 @@
-""" Uses the Instaloader library to download content from Instagram. This class handles both individual posts
-    and user profiles, downloading as much information as possible, including images, videos, text, stories,
-    highlights, and tagged posts. Authentication is required via username/password or a session file.
+"""Uses the Instaloader library to download content from Instagram. This class handles both individual posts
+and user profiles, downloading as much information as possible, including images, videos, text, stories,
+highlights, and tagged posts. Authentication is required via username/password or a session file.
 
 """
-import re, os, shutil, traceback
+
+import re
+import os
+import shutil
 import instaloader
 from loguru import logger
 
@@ -11,14 +14,14 @@ from auto_archiver.core import Extractor
 from auto_archiver.core import Metadata
 from auto_archiver.core import Media
 
+
 class InstagramExtractor(Extractor):
     """
     Uses Instaloader to download either a post (inc images, videos, text) or as much as possible from a profile (posts, stories, highlights, ...)
     """
+
     # NB: post regex should be tested before profile
-
     valid_url = re.compile(r"(?:(?:http|https):\/\/)?(?:www.)?(?:instagram.com|instagr.am|instagr.com)\/")
-
     # https://regex101.com/r/MGPquX/1
     post_pattern = re.compile(r"{valid_url}(?:p|reel)\/(\w+)".format(valid_url=valid_url))
     # https://regex101.com/r/6Wbsxa/1
@@ -26,22 +29,26 @@ class InstagramExtractor(Extractor):
     # TODO: links to stories
 
     def setup(self) -> None:
+        logger.warning("Instagram Extractor is not actively maintained, and may not work as expected.")
+        logger.warning("Please consider using the Instagram Tbot Extractor or Instagram API Extractor instead.")
 
         self.insta = instaloader.Instaloader(
-            download_geotags=True, download_comments=True, compress_json=False, dirname_pattern=self.download_folder, filename_pattern="{date_utc}_UTC_{target}__{typename}"
+            download_geotags=True,
+            download_comments=True,
+            compress_json=False,
+            dirname_pattern=self.download_folder,
+            filename_pattern="{date_utc}_UTC_{target}__{typename}",
         )
         try:
             self.insta.load_session_from_file(self.username, self.session_file)
-        except Exception as e:
-            logger.error(f"Unable to login from session file: {e}\n{traceback.format_exc()}")
+        except Exception:
             try:
-                self.insta.login(self.username, config.instagram_self.password)
-                # TODO: wait for this issue to be fixed https://github.com/instaloader/instaloader/issues/1758
+                logger.debug("Session file failed", exc_info=True)
+                logger.info("No valid session file found - Attempting login with use and password.")
+                self.insta.login(self.username, self.password)
                 self.insta.save_session_to_file(self.session_file)
-            except Exception as e2:
-                logger.error(f"Unable to finish login (retrying from file): {e2}\n{traceback.format_exc()}")
-
-
+            except Exception as e:
+                logger.error(f"Failed to setup Instagram Extractor with Instagrapi. {e}")
 
     def download(self, item: Metadata) -> Metadata:
         url = item.get_url()
@@ -51,7 +58,8 @@ class InstagramExtractor(Extractor):
         profile_matches = self.profile_pattern.findall(url)
 
         # return if not a valid instagram link
-        if not len(post_matches) and not len(profile_matches): return
+        if not len(post_matches) and not len(profile_matches):
+            return
 
         result = None
         try:
@@ -63,7 +71,9 @@ class InstagramExtractor(Extractor):
             elif len(profile_matches):
                 result = self.download_profile(url, profile_matches[0])
         except Exception as e:
-            logger.error(f"Failed to download with instagram extractor due to: {e}, make sure your account credentials are valid.")
+            logger.error(
+                f"Failed to download with instagram extractor due to: {e}, make sure your account credentials are valid."
+            )
         finally:
             shutil.rmtree(self.download_folder, ignore_errors=True)
         return result
@@ -82,35 +92,50 @@ class InstagramExtractor(Extractor):
         profile = instaloader.Profile.from_username(self.insta.context, username)
         try:
             for post in profile.get_posts():
-                try: self.insta.download_post(post, target=f"profile_post_{post.owner_username}")
-                except Exception as e: logger.error(f"Failed to download post: {post.shortcode}: {e}")
-        except Exception as e: logger.error(f"Failed profile.get_posts: {e}")
+                try:
+                    self.insta.download_post(post, target=f"profile_post_{post.owner_username}")
+                except Exception as e:
+                    logger.error(f"Failed to download post: {post.shortcode}: {e}")
+        except Exception as e:
+            logger.error(f"Failed profile.get_posts: {e}")
 
         try:
             for post in profile.get_tagged_posts():
-                try: self.insta.download_post(post, target=f"tagged_post_{post.owner_username}")
-                except Exception as e: logger.error(f"Failed to download tagged post: {post.shortcode}: {e}")
-        except Exception as e: logger.error(f"Failed profile.get_tagged_posts: {e}")
+                try:
+                    self.insta.download_post(post, target=f"tagged_post_{post.owner_username}")
+                except Exception as e:
+                    logger.error(f"Failed to download tagged post: {post.shortcode}: {e}")
+        except Exception as e:
+            logger.error(f"Failed profile.get_tagged_posts: {e}")
 
         try:
             for post in profile.get_igtv_posts():
-                try: self.insta.download_post(post, target=f"igtv_post_{post.owner_username}")
-                except Exception as e: logger.error(f"Failed to download igtv post: {post.shortcode}: {e}")
-        except Exception as e: logger.error(f"Failed profile.get_igtv_posts: {e}")
+                try:
+                    self.insta.download_post(post, target=f"igtv_post_{post.owner_username}")
+                except Exception as e:
+                    logger.error(f"Failed to download igtv post: {post.shortcode}: {e}")
+        except Exception as e:
+            logger.error(f"Failed profile.get_igtv_posts: {e}")
 
         try:
             for story in self.insta.get_stories([profile.userid]):
                 for item in story.get_items():
-                    try: self.insta.download_storyitem(item, target=f"story_item_{story.owner_username}")
-                    except Exception as e: logger.error(f"Failed to download story item: {item}: {e}")
-        except Exception as e: logger.error(f"Failed get_stories: {e}")
+                    try:
+                        self.insta.download_storyitem(item, target=f"story_item_{story.owner_username}")
+                    except Exception as e:
+                        logger.error(f"Failed to download story item: {item}: {e}")
+        except Exception as e:
+            logger.error(f"Failed get_stories: {e}")
 
         try:
             for highlight in self.insta.get_highlights(profile.userid):
                 for item in highlight.get_items():
-                    try: self.insta.download_storyitem(item, target=f"highlight_item_{highlight.owner_username}")
-                    except Exception as e: logger.error(f"Failed to download highlight item: {item}: {e}")
-        except Exception as e: logger.error(f"Failed get_highlights: {e}")
+                    try:
+                        self.insta.download_storyitem(item, target=f"highlight_item_{highlight.owner_username}")
+                    except Exception as e:
+                        logger.error(f"Failed to download highlight item: {item}: {e}")
+        except Exception as e:
+            logger.error(f"Failed get_highlights: {e}")
 
         return self.process_downloads(url, f"@{username}", profile._asdict(), None)
 
@@ -122,7 +147,8 @@ class InstagramExtractor(Extractor):
             all_media = []
             for f in os.listdir(self.download_folder):
                 if os.path.isfile((filename := os.path.join(self.download_folder, f))):
-                    if filename[-4:] == ".txt": continue
+                    if filename[-4:] == ".txt":
+                        continue
                     all_media.append(Media(filename))
 
             assert len(all_media) > 1, "No uploaded media found"

src/auto_archiver/modules/instagram_tbot_extractor/__manifest__.py

@@ -1,16 +1,21 @@
 {
     "name": "Instagram Telegram Bot Extractor",
     "type": ["extractor"],
-    "dependencies": {"python": ["loguru", "telethon",],
-                              },
+    "dependencies": {
+        "python": [
+            "loguru",
+            "telethon",
+        ],
+    },
     "requires_setup": True,
     "configs": {
-            "api_id": {"default": None, "help": "telegram API_ID value, go to https://my.telegram.org/apps"},
-            "api_hash": {"default": None, "help": "telegram API_HASH value, go to https://my.telegram.org/apps"},
-            "session_file": {"default": "secrets/anon-insta", "help": "optional, records the telegram login session for future usage, '.session' will be appended to the provided value."},
-            "timeout": {"default": 45,
-                        "type": "int",
-                        "help": "timeout to fetch the instagram content in seconds."},
+        "api_id": {"default": None, "help": "telegram API_ID value, go to https://my.telegram.org/apps"},
+        "api_hash": {"default": None, "help": "telegram API_HASH value, go to https://my.telegram.org/apps"},
+        "session_file": {
+            "default": "secrets/anon-insta",
+            "help": "optional, records the telegram login session for future usage, '.session' will be appended to the provided value.",
+        },
+        "timeout": {"default": 45, "type": "int", "help": "timeout to fetch the instagram content in seconds."},
     },
     "description": """
 The `InstagramTbotExtractor` module uses a Telegram bot (`instagram_load_bot`) to fetch and archive Instagram content,

src/auto_archiver/modules/instagram_tbot_extractor/instagram_tbot_extractor.py

@@ -51,7 +51,7 @@ class InstagramTbotExtractor(Extractor):
         """Initializes the Telegram client."""
         try:
             self.client = TelegramClient(self.session_file, self.api_id, self.api_hash)
-        except OperationalError as e:
+        except OperationalError:
             logger.error(
                 f"Unable to access the {self.session_file} session. "
                 "Ensure that you don't use the same session file here and in telethon_extractor. "
@@ -65,15 +65,15 @@ class InstagramTbotExtractor(Extractor):
         session_file_name = self.session_file + ".session"
         if os.path.exists(session_file_name):
             os.remove(session_file_name)
-        
+
     def download(self, item: Metadata) -> Metadata:
         url = item.get_url()
-        if not "instagram.com" in url: return False
+        if "instagram.com" not in url:
+            return False
 
         result = Metadata()
         tmp_dir = self.tmp_dir
         with self.client.start():
-
             chat, since_id = self._send_url_to_bot(url)
             message = self._process_messages(chat, since_id, tmp_dir, result)
 
@@ -104,19 +104,20 @@ class InstagramTbotExtractor(Extractor):
         message = ""
         time.sleep(3)
         # media is added before text by the bot so it can be used as a stop-logic mechanism
-        while attempts < (self.timeout - 3) and (not message or not len(seen_media)):
+        while attempts < max(self.timeout - 3, 3) and (not message or not len(seen_media)):
             attempts += 1
             time.sleep(1)
             for post in self.client.iter_messages(chat, min_id=since_id):
                 since_id = max(since_id, post.id)
                 # Skip known filler message:
-                if post.message == 'The bot receives information through https://hikerapi.com/p/hJqpppqi':
+                if post.message == "The bot receives information through https://hikerapi.com/p/hJqpppqi":
                     continue
                 if post.media and post.id not in seen_media:
-                    filename_dest = os.path.join(tmp_dir, f'{chat.id}_{post.id}')
+                    filename_dest = os.path.join(tmp_dir, f"{chat.id}_{post.id}")
                     media = self.client.download_media(post.media, filename_dest)
                     if media:
                         result.add_media(Media(media))
                         seen_media.append(post.id)
-                if post.message: message += post.message
-        return message.strip()
+                if post.message:
+                    message += post.message
+        return message.strip()

src/auto_archiver/modules/local_storage/__init__.py

@@ -1 +1 @@
-from .local_storage import LocalStorage
+from .local_storage import LocalStorage

src/auto_archiver/modules/local_storage/__manifest__.py

@@ -13,11 +13,15 @@
         },
         "filename_generator": {
             "default": "static",
-            "help": "how to name stored files: 'random' creates a random string; 'static' uses a replicable strategy such as a hash.",
+            "help": "how to name stored files: 'random' creates a random string; 'static' uses a hash, with the settings of the 'hash_enricher' module (defaults to SHA256 if not enabled)",
             "choices": ["random", "static"],
         },
         "save_to": {"default": "./local_archive", "help": "folder where to save archived content"},
-        "save_absolute": {"default": False, "help": "whether the path to the stored file is absolute or relative in the output result inc. formatters (WARN: leaks the file structure)"},
+        "save_absolute": {
+            "default": False,
+            "type": "bool",
+            "help": "whether the path to the stored file is absolute or relative in the output result inc. formatters (Warning: saving an absolute path will show your computer's file structure)",
+        },
     },
     "description": """
     LocalStorage: A storage module for saving archived content locally on the filesystem.
@@ -31,5 +35,5 @@
     ### Notes
     - Default storage folder is `./archived`, but this can be changed via the `save_to` configuration.
     - The `save_absolute` option can reveal the file structure in output formats; use with caution.
-    """
+    """,
 }

src/auto_archiver/modules/local_storage/local_storage.py

@@ -1,4 +1,3 @@
-
 import shutil
 from typing import IO
 import os
@@ -6,25 +5,43 @@ from loguru import logger
 
 from auto_archiver.core import Media
 from auto_archiver.core import Storage
+from auto_archiver.core.consts import SetupError
 
 
 class LocalStorage(Storage):
+    def setup(self) -> None:
+        if len(self.save_to) > 200:
+            raise SetupError(
+                "Your save_to path is too long, this will cause issues saving files on your computer. Please use a shorter path."
+            )
 
     def get_cdn_url(self, media: Media) -> str:
-        # TODO: is this viable with Storage.configs on path/filename?
-        dest = os.path.join(self.save_to, media.key)
+        dest = media.key
+
         if self.save_absolute:
             dest = os.path.abspath(dest)
         return dest
 
+    def set_key(self, media, url, metadata):
+        # clarify we want to save the file to the save_to folder
+
+        old_folder = metadata.get("folder", "")
+        metadata.set_context("folder", os.path.join(self.save_to, metadata.get("folder", "")))
+        super().set_key(media, url, metadata)
+        # don't impact other storages that might want a different 'folder' set
+        metadata.set_context("folder", old_folder)
+
     def upload(self, media: Media, **kwargs) -> bool:
         # override parent so that we can use shutil.copy2 and keep metadata
-        dest = os.path.join(self.save_to, media.key)
+        dest = media.key
+
         os.makedirs(os.path.dirname(dest), exist_ok=True)
-        logger.debug(f'[{self.__class__.__name__}] storing file {media.filename} with key {media.key} to {dest}')
+        logger.debug(f"[{self.__class__.__name__}] storing file {media.filename} with key {media.key} to {dest}")
+
         res = shutil.copy2(media.filename, dest)
         logger.info(res)
         return True
 
     # must be implemented even if unused
-    def uploadf(self, file: IO[bytes], key: str, **kwargs: dict) -> bool: pass
+    def uploadf(self, file: IO[bytes], key: str, **kwargs: dict) -> bool:
+        pass

src/auto_archiver/modules/meta_enricher/__manifest__.py

@@ -3,7 +3,7 @@
     "type": ["enricher"],
     "requires_setup": False,
     "dependencies": {
-                          "python": ["loguru"],
+        "python": ["loguru"],
     },
     "description": """ 
     Adds metadata information about the archive operations, Adds metadata about archive operations, including file sizes and archive duration./

src/auto_archiver/modules/meta_enricher/meta_enricher.py

@@ -23,7 +23,9 @@ class MetaEnricher(Enricher):
         self.enrich_archive_duration(to_enrich)
 
     def enrich_file_sizes(self, to_enrich: Metadata):
-        logger.debug(f"calculating archive file sizes for url={to_enrich.get_url()} ({len(to_enrich.media)} media files)")
+        logger.debug(
+            f"calculating archive file sizes for url={to_enrich.get_url()} ({len(to_enrich.media)} media files)"
+        )
         total_size = 0
         for media in to_enrich.get_all_media():
             file_stats = os.stat(media.filename)
@@ -34,7 +36,6 @@ class MetaEnricher(Enricher):
         to_enrich.set("total_bytes", total_size)
         to_enrich.set("total_size", self.human_readable_bytes(total_size))
 
-
     def human_readable_bytes(self, size: int) -> str:
         # receives number of bytes and returns human readble size
         for unit in ["bytes", "KB", "MB", "GB", "TB"]:
@@ -46,4 +47,4 @@ class MetaEnricher(Enricher):
         logger.debug(f"calculating archive duration for url={to_enrich.get_url()} ")
 
         archive_duration = datetime.datetime.now(datetime.timezone.utc) - to_enrich.get("_processed_at")
-        to_enrich.set("archive_duration_seconds", archive_duration.seconds)
+        to_enrich.set("archive_duration_seconds", archive_duration.seconds)

src/auto_archiver/modules/metadata_enricher/__init__.py

@@ -1 +1 @@
-from .metadata_enricher import MetadataEnricher
+from .metadata_enricher import MetadataEnricher

src/auto_archiver/modules/metadata_enricher/__manifest__.py

@@ -2,10 +2,7 @@
     "name": "Media Metadata Enricher",
     "type": ["enricher"],
     "requires_setup": True,
-    "dependencies": {
-        "python": ["loguru"], 
-        "bin": ["exiftool"]
-    },
+    "dependencies": {"python": ["loguru"], "bin": ["exiftool"]},
     "description": """
     Extracts metadata information from files using ExifTool.
 
@@ -17,5 +14,5 @@
     ### Notes
     - Requires ExifTool to be installed and accessible via the system's PATH.
     - Skips enrichment for files where metadata extraction fails.
-    """
+    """,
 }

src/auto_archiver/modules/metadata_enricher/metadata_enricher.py

@@ -11,7 +11,6 @@ class MetadataEnricher(Enricher):
     Extracts metadata information from files using exiftool.
     """
 
-
     def enrich(self, to_enrich: Metadata) -> None:
         url = to_enrich.get_url()
         logger.debug(f"extracting EXIF metadata for {url=}")
@@ -23,13 +22,13 @@ class MetadataEnricher(Enricher):
     def get_metadata(self, filename: str) -> dict:
         try:
             # Run ExifTool command to extract metadata from the file
-            cmd = ['exiftool', filename]
+            cmd = ["exiftool", filename]
             result = subprocess.run(cmd, capture_output=True, text=True)
 
             # Process the output to extract individual metadata fields
             metadata = {}
             for line in result.stdout.splitlines():
-                field, value = line.strip().split(':', 1)
+                field, value = line.strip().split(":", 1)
                 metadata[field.strip()] = value.strip()
             return metadata
         except FileNotFoundError:

src/auto_archiver/modules/mute_formatter/__manifest__.py

@@ -2,8 +2,7 @@
     "name": "Mute Formatter",
     "type": ["formatter"],
     "requires_setup": True,
-    "dependencies": {
-    },
+    "dependencies": {},
     "description": """ Default formatter.
     """,
 }

src/auto_archiver/modules/mute_formatter/mute_formatter.py

@@ -5,5 +5,5 @@ from auto_archiver.core import Formatter
 
 
 class MuteFormatter(Formatter):
-
-    def format(self, item: Metadata) -> Media: return None
+    def format(self, item: Metadata) -> Media:
+        return None

src/auto_archiver/modules/opentimestamps_enricher/__manifest__.py

@@ -0,0 +1,100 @@
+{
+    "name": "OpenTimestamps Enricher",
+    "type": ["enricher"],
+    "requires_setup": True,
+    "dependencies": {
+        "python": [
+            "loguru",
+            "opentimestamps",
+        ],
+    },
+    "configs": {
+        "calendar_urls": {
+            "default": [
+                "https://alice.btc.calendar.opentimestamps.org",
+                "https://bob.btc.calendar.opentimestamps.org",
+                "https://finney.calendar.eternitywall.com",
+                # "https://ots.btc.catallaxy.com/", # ipv4 only
+            ],
+            "help": "List of OpenTimestamps calendar servers to use for timestamping. See here for a list of calendars maintained by opentimestamps:\
+https://opentimestamps.org/#calendars",
+            "type": "list",
+        },
+        "calendar_whitelist": {
+            "default": [],
+            "help": "Optional whitelist of calendar servers. Override this if you are using your own calendar servers. e.g. ['https://mycalendar.com']",
+            "type": "list",
+        },
+    },
+    "description": """
+    Creates OpenTimestamps proofs for archived files, providing blockchain-backed evidence of file existence at a specific time.
+
+    Uses OpenTimestamps – a service that timestamps data using the Bitcoin blockchain, providing a decentralized 
+    and secure way to prove that data existed at a certain point in time. A SHA256 hash of the file to be timestamped is used as the token
+    and sent to each of the 'timestamp calendars' for inclusion in the blockchain. The proof is then saved alongside the original file in a file with
+    the '.ots' extension.
+
+    ### Features
+    - Creates cryptographic timestamp proofs that link files to the Bitcoin
+    - Verifies timestamp proofs have been submitted to the blockchain (note: does not confirm they have been *added*)
+    - Can use multiple calendar servers to ensure reliability and redundancy
+    - Stores timestamp proofs alongside original files for future verification
+
+    ### Timestamp status
+    An opentimestamp, when submitted to a timestmap server will have a 'pending' status (Pending Attestation) as it waits to be added
+    to the blockchain. Once it has been added to the blockchain, it will have a 'confirmed' status (Bitcoin Block Timestamp).
+    This process typically takes several hours, depending on the calendar server and the current state of the Bitcoin network. As such,
+    the status of all timestamps added will be 'pending' until they are subsequently confirmed (see 'Upgrading Timestamps' below).
+
+    There are two possible statuses for a timestamp:
+    - `Pending`: The timestamp has been submitted to the calendar server but has not yet been confirmed in the Bitcoin blockchain.
+    - `Confirmed`: The timestamp has been confirmed in the Bitcoin blockchain.
+
+    ### Upgrading Timestamps
+    To upgrade a timestamp from 'pending' to 'confirmed', you can use the `ots upgrade` command from the opentimestamps-client package
+    (install it with `pip install opentimesptamps-client`).
+    Example: `ots upgrade my_file.ots`
+
+    Here is a useful script that could be used to upgrade all timestamps in a directory, which could be run on a cron job:
+```{code}  bash
+find . -name "*.ots" -type f | while read file; do
+    echo "Upgrading OTS $file"
+    ots upgrade $file
+done
+# The result might look like:
+# Upgrading OTS ./my_file.ots
+# Got 1 attestation(s) from https://alice.btc.calendar.opentimestamps.org
+# Success! Timestamp complete
+```
+
+```{note} Note: this will only upgrade the .ots files, and will not change the status text in any output .html files or any databases where the
+metadata is stored (e.g. Google Sheets, CSV database, API database etc.).
+```
+
+    ### Verifying Timestamps
+    The easiest way to verify a timestamp (ots) file is to install the opentimestamps-client command line tool and use the `ots verify` command.
+    Example: `ots verify my_file.ots`
+
+    ```{code}  bash
+$ ots verify my_file.ots
+Calendar https://bob.btc.calendar.opentimestamps.org: Pending confirmation in Bitcoin blockchain
+Calendar https://finney.calendar.eternitywall.com: Pending confirmation in Bitcoin blockchain
+Calendar https://alice.btc.calendar.opentimestamps.org: Timestamped by transaction 12345; waiting for 6 confirmations
+```
+
+    Note: if you're using a storage with `filename_generator` set to `static` or `random`, the files will be renamed when they are saved to the
+    final location meaning you will need to specify the original filename when verifying the timestamp with `ots verify -f original_filename my_file.ots`.
+
+    ### Choosing Calendar Servers
+
+    By default, the OpenTimestamps enricher uses a set of public calendar servers provided by the 'opentimestamps' project.
+    You can customize the list of calendar servers by providing URLs in the `calendar_urls` configuration option.
+
+    ### Calendar WhiteList
+
+    By default, the opentimestamps package only allows their own calendars to be used (see `DEFAULT_CALENDAR_WHITELIST` in `opentimestamps.calendar`),
+    if you want to use your own calendars, then you can override this setting in the `calendar_whitelist` configuration option.
+
+   
+    """,
+}

src/auto_archiver/modules/opentimestamps_enricher/opentimestamps_enricher.py

@@ -0,0 +1,172 @@
+import os
+
+from loguru import logger
+import opentimestamps
+from opentimestamps.calendar import RemoteCalendar, DEFAULT_CALENDAR_WHITELIST
+from opentimestamps.core.timestamp import Timestamp, DetachedTimestampFile
+from opentimestamps.core.notary import PendingAttestation, BitcoinBlockHeaderAttestation
+from opentimestamps.core.op import OpSHA256
+from opentimestamps.core import serialize
+from auto_archiver.core import Enricher
+from auto_archiver.core import Metadata, Media
+from auto_archiver.utils.misc import get_current_timestamp
+
+
+class OpentimestampsEnricher(Enricher):
+    def enrich(self, to_enrich: Metadata) -> None:
+        url = to_enrich.get_url()
+        logger.debug(f"OpenTimestamps timestamping files for {url=}")
+
+        # Get the media files to timestamp
+        media_files = [m for m in to_enrich.media if m.filename and not m.get("opentimestamps")]
+        if not media_files:
+            logger.warning(f"No files found to timestamp in {url=}")
+            return
+
+        timestamp_files = []
+        for media in media_files:
+            try:
+                # Get the file path from the media
+                file_path = media.filename
+                if not os.path.exists(file_path):
+                    logger.warning(f"File not found: {file_path}")
+                    continue
+
+                # Create timestamp for the file - hash is SHA256
+                # Note: hash is hard-coded to SHA256 and does not use hash_enricher to set it.
+                # SHA256 is the recommended hash, ref: https://github.com/bellingcat/auto-archiver/pull/247#discussion_r1992433181
+                logger.debug(f"Creating timestamp for {file_path}")
+                file_hash = None
+                with open(file_path, "rb") as f:
+                    file_hash = OpSHA256().hash_fd(f)
+
+                if not file_hash:
+                    logger.warning(f"Failed to hash file for timestamping, skipping: {file_path}")
+                    continue
+
+                # Create a timestamp with the file hash
+                timestamp = Timestamp(file_hash)
+
+                # Create a detached timestamp file with the hash operation and timestamp
+                detached_timestamp = DetachedTimestampFile(OpSHA256(), timestamp)
+
+                # Submit to calendar servers
+                submitted_to_calendar = False
+
+                logger.debug(f"Submitting timestamp to calendar servers for {file_path}")
+                calendars = []
+                whitelist = DEFAULT_CALENDAR_WHITELIST
+
+                if self.calendar_whitelist:
+                    whitelist = set(self.calendar_whitelist)
+
+                # Create calendar instances
+                calendar_urls = []
+                for url in self.calendar_urls:
+                    if url in whitelist:
+                        calendars.append(RemoteCalendar(url))
+                        calendar_urls.append(url)
+
+                # Submit the hash to each calendar
+                for calendar in calendars:
+                    try:
+                        calendar_timestamp = calendar.submit(file_hash)
+                        timestamp.merge(calendar_timestamp)
+                        logger.debug(f"Successfully submitted to calendar: {calendar.url}")
+                        submitted_to_calendar = True
+                    except Exception as e:
+                        logger.warning(f"Failed to submit to calendar {calendar.url}: {e}")
+
+                # If all calendar submissions failed, add pending attestations
+                if not submitted_to_calendar and not timestamp.attestations:
+                    logger.error(
+                        f"Failed to submit to any calendar for {file_path}. **This file will not be timestamped.**"
+                    )
+                    media.set("opentimestamps", False)
+                    continue
+
+                # Save the timestamp proof to a file
+                timestamp_path = os.path.join(self.tmp_dir, f"{os.path.basename(file_path)}.ots")
+                try:
+                    with open(timestamp_path, "wb") as f:
+                        # Create a serialization context and write to the file
+                        ctx = serialize.BytesSerializationContext()
+                        detached_timestamp.serialize(ctx)
+                        f.write(ctx.getbytes())
+                except Exception as e:
+                    logger.warning(f"Failed to serialize timestamp file: {e}")
+                    continue
+
+                # Create media for the timestamp file
+                timestamp_media = Media(filename=timestamp_path)
+                # explicitly set the mimetype, normally .ots files are 'application/vnd.oasis.opendocument.spreadsheet-template'
+                timestamp_media.mimetype = "application/vnd.opentimestamps"
+                timestamp_media.set("opentimestamps_version", opentimestamps.__version__)
+
+                verification_info = self.verify_timestamp(detached_timestamp)
+                for key, value in verification_info.items():
+                    timestamp_media.set(key, value)
+
+                media.set("opentimestamp_files", [timestamp_media])
+                timestamp_files.append(timestamp_media.filename)
+                # Update the original media to indicate it's been timestamped
+                media.set("opentimestamps", True)
+
+            except Exception as e:
+                logger.warning(f"Error while timestamping {media.filename}: {e}")
+
+        # Add timestamp files to the metadata
+        if timestamp_files:
+            to_enrich.set("opentimestamped", True)
+            to_enrich.set("opentimestamps_count", len(timestamp_files))
+            logger.success(f"{len(timestamp_files)} OpenTimestamps proofs created for {url=}")
+        else:
+            to_enrich.set("opentimestamped", False)
+            logger.warning(f"No successful timestamps created for {url=}")
+
+    def verify_timestamp(self, detached_timestamp):
+        """
+        Verify a timestamp and extract verification information.
+
+        Args:
+            detached_timestamp: The detached timestamp to verify.
+
+        Returns:
+            dict: Information about the verification result.
+        """
+        result = {}
+
+        # Check if we have attestations
+        attestations = list(detached_timestamp.timestamp.all_attestations())
+        result["attestation_count"] = len(attestations)
+
+        if attestations:
+            attestation_info = []
+            for msg, attestation in attestations:
+                info = {}
+
+                # Process different types of attestations
+                if isinstance(attestation, PendingAttestation):
+                    info["status"] = "pending"
+                    info["uri"] = attestation.uri
+
+                elif isinstance(attestation, BitcoinBlockHeaderAttestation):
+                    info["status"] = "confirmed"
+                    info["block_height"] = attestation.height
+
+                info["last_check"] = get_current_timestamp()
+
+                attestation_info.append(info)
+
+            result["attestations"] = attestation_info
+
+            # For at least one confirmed attestation
+            if any("confirmed" in a.get("status") for a in attestation_info):
+                result["verified"] = True
+            else:
+                result["verified"] = False
+        else:
+            result["verified"] = False
+        result["last_updated"] = get_current_timestamp()
+
+        return result

src/auto_archiver/modules/pdq_hash_enricher/__init__.py

@@ -1 +1 @@
-from .pdq_hash_enricher import PdqHashEnricher
+from .pdq_hash_enricher import PdqHashEnricher

src/auto_archiver/modules/pdq_hash_enricher/__manifest__.py

@@ -17,5 +17,5 @@
     ### Notes
     - Best used after enrichers like `thumbnail_enricher` or `screenshot_enricher` to ensure images are available.
     - Uses the `pdqhash` library to compute 256-bit perceptual hashes, which are stored as hexadecimal strings.
-    """
+    """,
 }

src/auto_archiver/modules/pdq_hash_enricher/pdq_hash_enricher.py

@@ -10,6 +10,7 @@ This enricher is typically used after thumbnail or screenshot enrichers
 to ensure images are available for hashing.
 
 """
+
 import traceback
 import pdqhash
 import numpy as np
@@ -34,7 +35,12 @@ class PdqHashEnricher(Enricher):
         for m in to_enrich.media:
             for media in m.all_inner_media(True):
                 media_id = media.get("id", "")
-                if media.is_image() and "screenshot" not in media_id and "warc-file-" not in media_id and len(hd := self.calculate_pdq_hash(media.filename)):
+                if (
+                    media.is_image()
+                    and "screenshot" not in media_id
+                    and "warc-file-" not in media_id
+                    and len(hd := self.calculate_pdq_hash(media.filename))
+                ):
                     media.set("pdq_hash", hd)
                     media_with_hashes.append(media.filename)
 
@@ -51,5 +57,7 @@ class PdqHashEnricher(Enricher):
                 hash = "".join(str(b) for b in hash_array)
                 return hex(int(hash, 2))[2:]
         except UnidentifiedImageError as e:
-            logger.error(f"Image {filename=} is likely corrupted or in unsupported format {e}: {traceback.format_exc()}")
+            logger.error(
+                f"Image {filename=} is likely corrupted or in unsupported format {e}: {traceback.format_exc()}"
+            )
         return ""

src/auto_archiver/modules/s3_storage/__init__.py

@@ -1 +1 @@
-from .s3_storage import S3Storage
+from .s3_storage import S3Storage

src/auto_archiver/modules/s3_storage/__manifest__.py

@@ -13,27 +13,27 @@
         },
         "filename_generator": {
             "default": "static",
-            "help": "how to name stored files: 'random' creates a random string; 'static' uses a replicable strategy such as a hash.",
+            "help": "how to name stored files: 'random' creates a random string; 'static' uses a hash, with the settings of the 'hash_enricher' module (defaults to SHA256 if not enabled).",
             "choices": ["random", "static"],
         },
         "bucket": {"default": None, "help": "S3 bucket name"},
         "region": {"default": None, "help": "S3 region name"},
         "key": {"default": None, "help": "S3 API key"},
         "secret": {"default": None, "help": "S3 API secret"},
-        "random_no_duplicate": {"default": False,
-                                "type": "bool",
-                                "help": "if set, it will override `path_generator`, `filename_generator` and `folder`. It will check if the file already exists and if so it will not upload it again. Creates a new root folder path `no-dups/`"},
+        "random_no_duplicate": {
+            "default": False,
+            "type": "bool",
+            "help": "if set, it will override `path_generator`, `filename_generator` and `folder`. It will check if the file already exists and if so it will not upload it again. Creates a new root folder path `no-dups/`",
+        },
         "endpoint_url": {
-            "default": 'https://{region}.digitaloceanspaces.com',
-            "help": "S3 bucket endpoint, {region} are inserted at runtime"
+            "default": "https://{region}.digitaloceanspaces.com",
+            "help": "S3 bucket endpoint, {region} are inserted at runtime",
         },
         "cdn_url": {
-            "default": 'https://{bucket}.{region}.cdn.digitaloceanspaces.com/{key}',
-            "help": "S3 CDN url, {bucket}, {region} and {key} are inserted at runtime"
+            "default": "https://{bucket}.{region}.cdn.digitaloceanspaces.com/{key}",
+            "help": "S3 CDN url, {bucket}, {region} and {key} are inserted at runtime",
         },
-        "private": {"default": False,
-                    "type": "bool",
-                    "help": "if true S3 files will not be readable online"},
+        "private": {"default": False, "type": "bool", "help": "if true S3 files will not be readable online"},
     },
     "description": """
     S3Storage: A storage module for saving media files to an S3-compatible object storage.
@@ -50,5 +50,5 @@
     - The `random_no_duplicate` option ensures no duplicate uploads by leveraging hash-based folder structures.
     - Uses `boto3` for interaction with the S3 API.
     - Depends on the `HashEnricher` module for hash calculation.
-    """
+    """,
 }

src/auto_archiver/modules/s3_storage/s3_storage.py

@@ -1,4 +1,3 @@
-
 from typing import IO
 
 import boto3
@@ -11,60 +10,62 @@ from auto_archiver.utils.misc import calculate_file_hash, random_str
 
 NO_DUPLICATES_FOLDER = "no-dups/"
 
-class S3Storage(Storage):
 
+class S3Storage(Storage):
     def setup(self) -> None:
         self.s3 = boto3.client(
-            's3',
+            "s3",
             region_name=self.region,
             endpoint_url=self.endpoint_url.format(region=self.region),
             aws_access_key_id=self.key,
-            aws_secret_access_key=self.secret
+            aws_secret_access_key=self.secret,
         )
         if self.random_no_duplicate:
-            logger.warning("random_no_duplicate is set to True, this will override `path_generator`, `filename_generator` and `folder`.")
+            logger.warning(
+                "random_no_duplicate is set to True, this will override `path_generator`, `filename_generator` and `folder`."
+            )
 
     def get_cdn_url(self, media: Media) -> str:
         return self.cdn_url.format(bucket=self.bucket, region=self.region, key=media.key)
 
     def uploadf(self, file: IO[bytes], media: Media, **kwargs: dict) -> None:
-        if not self.is_upload_needed(media): return True
+        if not self.is_upload_needed(media):
+            return True
 
         extra_args = kwargs.get("extra_args", {})
-        if not self.private and 'ACL' not in extra_args:
-            extra_args['ACL'] = 'public-read'
+        if not self.private and "ACL" not in extra_args:
+            extra_args["ACL"] = "public-read"
 
-        if 'ContentType' not in extra_args:
+        if "ContentType" not in extra_args:
             try:
                 if media.mimetype:
-                    extra_args['ContentType'] = media.mimetype
+                    extra_args["ContentType"] = media.mimetype
             except Exception as e:
                 logger.warning(f"Unable to get mimetype for {media.key=}, error: {e}")
         self.s3.upload_fileobj(file, Bucket=self.bucket, Key=media.key, ExtraArgs=extra_args)
         return True
-    
+
     def is_upload_needed(self, media: Media) -> bool:
         if self.random_no_duplicate:
             # checks if a folder with the hash already exists, if so it skips the upload
             hd = calculate_file_hash(media.filename)
             path = os.path.join(NO_DUPLICATES_FOLDER, hd[:24])
 
-            if existing_key:=self.file_in_folder(path):
-                media.key = existing_key
+            if existing_key := self.file_in_folder(path):
+                media._key = existing_key
                 media.set("previously archived", True)
                 logger.debug(f"skipping upload of {media.filename} because it already exists in {media.key}")
                 return False
-            
+
             _, ext = os.path.splitext(media.key)
-            media.key = os.path.join(path, f"{random_str(24)}{ext}")
+            media._key = os.path.join(path, f"{random_str(24)}{ext}")
         return True
 
-    def file_in_folder(self, path:str) -> str:
+    def file_in_folder(self, path: str) -> str:
         # checks if path exists and is not an empty folder
-        if not path.endswith('/'):
-            path = path + '/' 
-        resp = self.s3.list_objects(Bucket=self.bucket, Prefix=path, Delimiter='/', MaxKeys=1)
-        if 'Contents' in resp:
-            return resp['Contents'][0]['Key']
+        if not path.endswith("/"):
+            path = path + "/"
+        resp = self.s3.list_objects(Bucket=self.bucket, Prefix=path, Delimiter="/", MaxKeys=1)
+        if "Contents" in resp:
+            return resp["Contents"][0]["Key"]
         return False
-

src/auto_archiver/modules/screenshot_enricher/__manifest__.py

@@ -6,14 +6,29 @@
         "python": ["loguru", "selenium"],
     },
     "configs": {
-            "width": {"default": 1280, "help": "width of the screenshots"},
-            "height": {"default": 720, "help": "height of the screenshots"},
-            "timeout": {"default": 60, "help": "timeout for taking the screenshot"},
-            "sleep_before_screenshot": {"default": 4, "help": "seconds to wait for the pages to load before taking screenshot"},
-            "http_proxy": {"default": "", "help": "http proxy to use for the webdriver, eg http://proxy-user:password@proxy-ip:port"},
-            "save_to_pdf": {"default": False, "help": "save the page as pdf along with the screenshot. PDF saving options can be adjusted with the 'print_options' parameter"},
-            "print_options": {"default": {}, "help": "options to pass to the pdf printer"}
+        "width": {"default": 1280, "type": "int", "help": "width of the screenshots"},
+        "height": {"default": 1024, "type": "int", "help": "height of the screenshots"},
+        "timeout": {"default": 60, "type": "int", "help": "timeout for taking the screenshot"},
+        "sleep_before_screenshot": {
+            "default": 4,
+            "type": "int",
+            "help": "seconds to wait for the pages to load before taking screenshot",
         },
+        "http_proxy": {
+            "default": "",
+            "help": "http proxy to use for the webdriver, eg http://proxy-user:password@proxy-ip:port",
+        },
+        "save_to_pdf": {
+            "default": False,
+            "type": "bool",
+            "help": "save the page as pdf along with the screenshot. PDF saving options can be adjusted with the 'print_options' parameter",
+        },
+        "print_options": {
+            "default": {},
+            "help": "options to pass to the pdf printer, in JSON format. See https://www.selenium.dev/documentation/webdriver/interactions/print_page/ for more information",
+            "type": "json_loader",
+        },
+    },
     "description": """
     Captures screenshots and optionally saves web pages as PDFs using a WebDriver.
 
@@ -25,5 +40,5 @@
 
     ### Notes
     - Requires a WebDriver (e.g., ChromeDriver) installed and accessible via the system's PATH.
-    """
+    """,
 }

src/auto_archiver/modules/screenshot_enricher/screenshot_enricher.py

@@ -1,5 +1,6 @@
 from loguru import logger
-import time, os
+import time
+import os
 import base64
 
 from selenium.common.exceptions import TimeoutException
@@ -9,8 +10,8 @@ from auto_archiver.core import Enricher
 from auto_archiver.utils import Webdriver, url as UrlUtil, random_str
 from auto_archiver.core import Media, Metadata
 
-class ScreenshotEnricher(Enricher):
 
+class ScreenshotEnricher(Enricher):
     def __init__(self, webdriver_factory=None):
         super().__init__()
         self.webdriver_factory = webdriver_factory or Webdriver
@@ -18,15 +19,30 @@ class ScreenshotEnricher(Enricher):
     def enrich(self, to_enrich: Metadata) -> None:
         url = to_enrich.get_url()
 
-        if UrlUtil.is_auth_wall(url):
-            logger.debug(f"[SKIP] SCREENSHOT since url is behind AUTH WALL: {url=}")
-            return
-
         logger.debug(f"Enriching screenshot for {url=}")
         auth = self.auth_for_site(url)
+
+        # screenshot enricher only supports cookie-type auth (selenium)
+        has_valid_auth = auth and (auth.get("cookies") or auth.get("cookies_jar") or auth.get("cookie"))
+
+        if UrlUtil.is_auth_wall(url) and not has_valid_auth:
+            logger.warning(f"[SKIP] SCREENSHOT since url is behind AUTH WALL and no login details provided: {url=}")
+            if any(auth.get(key) for key in ["username", "password", "api_key", "api_secret"]):
+                logger.warning(
+                    f"Screenshot enricher only supports cookie-type authentication, you have provided {auth.keys()} which are not supported.\
+                               Consider adding 'cookie', 'cookies_file' or 'cookies_from_browser' to your auth for this site."
+                )
+            return
+
         with self.webdriver_factory(
-                self.width, self.height, self.timeout, facebook_accept_cookies='facebook.com' in url,
-                       http_proxy=self.http_proxy, print_options=self.print_options, auth=auth) as driver:
+            self.width,
+            self.height,
+            self.timeout,
+            facebook_accept_cookies="facebook.com" in url,
+            http_proxy=self.http_proxy,
+            print_options=self.print_options,
+            auth=auth,
+        ) as driver:
             try:
                 driver.get(url)
                 time.sleep(int(self.sleep_before_screenshot))
@@ -43,4 +59,3 @@ class ScreenshotEnricher(Enricher):
                 logger.info("TimeoutException loading page for screenshot")
             except Exception as e:
                 logger.error(f"Got error while loading webdriver for screenshot enricher: {e}")
-

src/auto_archiver/modules/ssl_enricher/__init__.py

@@ -1 +1 @@
-from .ssl_enricher import SSLEnricher
+from .ssl_enricher import SSLEnricher

src/auto_archiver/modules/ssl_enricher/__manifest__.py

@@ -5,9 +5,13 @@
     "dependencies": {
         "python": ["loguru", "slugify"],
     },
-    'entry_point': 'ssl_enricher::SSLEnricher',
+    "entry_point": "ssl_enricher::SSLEnricher",
     "configs": {
-        "skip_when_nothing_archived": {"default": True, "help": "if true, will skip enriching when no media is archived"},
+        "skip_when_nothing_archived": {
+            "default": True,
+            "type": "bool",
+            "help": "if true, will skip enriching when no media is archived",
+        },
     },
     "description": """
     Retrieves SSL certificate information for a domain and stores it as a file.
@@ -19,5 +23,5 @@
 
     ### Notes
     - Requires the target URL to use the HTTPS scheme; other schemes are not supported.
-    """
+    """,
 }

src/auto_archiver/modules/ssl_enricher/ssl_enricher.py

@@ -1,4 +1,5 @@
-import ssl, os
+import ssl
+import os
 from slugify import slugify
 from urllib.parse import urlparse
 from loguru import logger
@@ -13,16 +14,18 @@ class SSLEnricher(Enricher):
     """
 
     def enrich(self, to_enrich: Metadata) -> None:
-        if not to_enrich.media and self.skip_when_nothing_archived: return
-        
+        if not to_enrich.media and self.skip_when_nothing_archived:
+            return
+
         url = to_enrich.get_url()
         parsed = urlparse(url)
         assert parsed.scheme in ["https"], f"Invalid URL scheme {url=}"
-        
+
         domain = parsed.netloc
         logger.debug(f"fetching SSL certificate for {domain=} in {url=}")
 
         cert = ssl.get_server_certificate((domain, 443))
         cert_fn = os.path.join(self.tmp_dir, f"{slugify(domain)}.pem")
-        with open(cert_fn, "w") as f: f.write(cert)
+        with open(cert_fn, "w") as f:
+            f.write(cert)
         to_enrich.add_media(Media(filename=cert_fn), id="ssl_certificate")

src/auto_archiver/modules/telegram_extractor/__init__.py

@@ -1 +1 @@
-from .telegram_extractor import TelegramExtractor
+from .telegram_extractor import TelegramExtractor

src/auto_archiver/modules/telegram_extractor/telegram_extractor.py

@@ -1,4 +1,6 @@
-import requests, re, html
+import requests
+import re
+import html
 from bs4 import BeautifulSoup
 from loguru import logger
 
@@ -15,11 +17,11 @@ class TelegramExtractor(Extractor):
     def download(self, item: Metadata) -> Metadata:
         url = item.get_url()
         # detect URLs that we definitely cannot handle
-        if 't.me' != item.netloc:
+        if "t.me" != item.netloc:
             return False
 
         headers = {
-            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36'
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36"
         }
 
         # TODO: check if we can do this more resilient to variable URLs
@@ -27,11 +29,11 @@ class TelegramExtractor(Extractor):
             url += "?embed=1"
 
         t = requests.get(url, headers=headers)
-        s = BeautifulSoup(t.content, 'html.parser')
+        s = BeautifulSoup(t.content, "html.parser")
 
         result = Metadata()
         result.set_content(html.escape(str(t.content)))
-        if (timestamp := (s.find_all('time') or [{}])[0].get('datetime')):
+        if timestamp := (s.find_all("time") or [{}])[0].get("datetime"):
             result.set_timestamp(timestamp)
 
         video = s.find("video")
@@ -41,25 +43,26 @@ class TelegramExtractor(Extractor):
 
             image_urls = []
             for im in image_tags:
-                urls = [u.replace("'", "") for u in re.findall(r'url\((.*?)\)', im['style'])]
+                urls = [u.replace("'", "") for u in re.findall(r"url\((.*?)\)", im["style"])]
                 image_urls += urls
 
-            if not len(image_urls): return False
+            if not len(image_urls):
+                return False
             for img_url in image_urls:
                 result.add_media(Media(self.download_from_url(img_url)))
         else:
-            video_url = video.get('src')
+            video_url = video.get("src")
             m_video = Media(self.download_from_url(video_url))
             # extract duration from HTML
             try:
-                duration = s.find_all('time')[0].contents[0]
-                if ':' in duration:
-                    duration = float(duration.split(
-                        ':')[0]) * 60 + float(duration.split(':')[1])
+                duration = s.find_all("time")[0].contents[0]
+                if ":" in duration:
+                    duration = float(duration.split(":")[0]) * 60 + float(duration.split(":")[1])
                 else:
                     duration = float(duration)
                 m_video.set("duration", duration)
-            except: pass
+            except Exception:
+                pass
             result.add_media(m_video)
 
         return result.success("telegram")