Bump yaml in /scripts/settings

Bumps [yaml](https://github.com/eemeli/yaml) to 1.10.3 and updates ancestor dependency . These dependencies need to be updated together.


Updates `yaml` from 1.10.2 to 1.10.3
- [Release notes](https://github.com/eemeli/yaml/releases)
- [Commits](https://github.com/eemeli/yaml/compare/v1.10.2...v1.10.3)

Updates `yaml` from 2.8.2 to 2.9.0
- [Release notes](https://github.com/eemeli/yaml/releases)
- [Commits](https://github.com/eemeli/yaml/compare/v1.10.2...v1.10.3)

---
updated-dependencies:
- dependency-name: yaml
  dependency-version: 1.10.3
  dependency-type: indirect
- dependency-name: yaml
  dependency-version: 2.9.0
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>

.github/dependabot.yml

@@ -0,0 +1,40 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "/"
+    groups:
+      python:
+        patterns:
+          - "*"
+    schedule:
+      interval: "monthly"
+  
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    groups:
+      actions:
+        patterns:
+          - "*"
+    schedule:
+      interval: "monthly"
+
+  - package-ecosystem: "npm"
+    directory: "/scripts/settings/"
+    groups:
+      actions:
+        patterns:
+          - "*"
+    schedule:
+      interval: "monthly"
+
+  - package-ecosystem: "docker"
+    # Look for a `Dockerfile` in the `root` directory
+    directory: "/"
+    # Check for updates once a week
+    schedule:
+      interval: "monthly"

.github/workflows/docker-publish.yaml

@@ -11,7 +11,7 @@ on:
 
 env:
   # Use docker.io for Docker Hub if empty
-  REGISTRY: ghcr.io
+  REGISTRY: docker.io
   # github.repository as <account>/<repo>
   IMAGE_NAME: ${{ github.repository }}
 
@@ -22,7 +22,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v6
         
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
@@ -33,22 +33,24 @@ jobs:
         uses: docker/setup-buildx-action@v3
 
       - name: Log in to Docker Hub
-        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567
+        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96
+        uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051
         with:
           images: bellingcat/auto-archiver
       
       - name: Build and push Docker image
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v6
         with:
           context: .
           platforms: linux/amd64,linux/arm64
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:cache
+          cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:cache,mode=max

.github/workflows/python-publish.yaml

@@ -22,10 +22,10 @@ jobs:
 
     steps:
     - name: Checkout Repository
-      uses: actions/checkout@v4
+      uses: actions/checkout@v6
 
     - name: Set up Python
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v6
       with:
         python-version-file: pyproject.toml
 

.github/workflows/ruff.yaml

@@ -0,0 +1,34 @@
+name: Ruff Formatting & Linting
+
+on:
+  push:
+    branches: [ main ]
+    paths-ignore:
+      - "README.md"
+      - ".github"
+      - "poetry.lock"
+      - "scripts/settings"
+  pull_request:
+    branches: [ main ]
+    paths-ignore:
+      - "README.md"
+      - ".github"
+      - "poetry.lock"
+      - "scripts/settings"
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - name: Install Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install ruff
+
+      - name: Run Ruff
+        run: ruff check --output-format=github . && ruff format --check

.github/workflows/tests-core.yaml

@@ -5,9 +5,13 @@ on:
     branches: [ main ]
     paths:
       - src/**
+      - poetry.lock
+      - pyproject.toml
   pull_request:
     paths:
       - src/**
+      - poetry.lock
+      - pyproject.toml
 
 jobs:
   tests:
@@ -22,18 +26,28 @@ jobs:
         working-directory: ./
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
 
-      - name: Install Poetry
-        run: pipx install poetry
+      - name: Install ffmpeg
+        run: sudo apt-get update && sudo apt-get install -y ffmpeg
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: ${{ matrix.python-version }}
-          cache: 'poetry'
 
-      - name: Install dependencies
+      - name: Install latest Poetry
+        run: pipx install poetry
+      
+      - name: Cache Poetry and pip artifacts
+        uses: actions/cache@v5
+        with:
+          path: |
+            ~/.cache/pypoetry
+            ~/.cache/pip
+          key: poetry-${{ matrix.os }}-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }}
+
+      - name: Install dependencies from source only
         run: poetry install --no-interaction --with dev
 
       - name: Run Core Tests

.github/workflows/tests-download.yaml

@@ -20,21 +20,31 @@ jobs:
         working-directory: ./
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
 
-      - name: Install poetry
-        run: pipx install poetry
+      - name: Install ffmpeg
+        run: sudo apt-get update && sudo apt-get install -y ffmpeg
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: ${{ matrix.python-version }}
-          cache: 'poetry'
 
-      - name: Install dependencies
+      - name: Install latest Poetry
+        run: pipx install poetry
+
+      - name: Cache Poetry and pip artifacts
+        uses: actions/cache@v5
+        with:
+          path: |
+            ~/.cache/pypoetry
+            ~/.cache/pip
+          key: poetry-${{ matrix.os }}-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }}
+
+      - name: Install dependencies from source only
         run: poetry install --no-interaction --with dev
 
       - name: Run Download Tests
         run: poetry run pytest -ra -v -x -m "download"
         env:
-          TWITTER_BEARER_TOKEN: ${{ secrets.TWITTER_BEARER_TOKEN }}
+          TWITTER_BEARER_TOKEN: ${{ secrets.TWITTER_BEARER_TOKEN || '' }}

.gitignore

@@ -1,9 +1,11 @@
 tmp*/
 temp/
 .env*
+!.env*.example
 .DS_Store
 expmt/
 service_account.json
+service_account-*.json
 __pycache__/
 ._*
 anu.html
@@ -33,3 +35,10 @@ dist*
 docs/_build/
 docs/source/autoapi/
 docs/source/modules/autogen/
+scripts/settings_page.html
+scripts/settings/src/schema.json 
+.vite
+downloaded_files
+latest_logs
+# for launch.json
+.vscode

.pre-commit-config.yaml

@@ -0,0 +1,7 @@
+# Run Ruff formatter on commits.
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.9.10
+    hooks:
+      - id: ruff
+      - id: ruff-format

.readthedocs.yaml

@@ -7,8 +7,11 @@ version: 2
 
 build:
   os: ubuntu-22.04
+  apt_packages:
+    - ffmpeg
   tools:
     python: "3.10"
+    nodejs: "22"
   jobs:
     post_install:
       - pip install poetry
@@ -17,6 +20,11 @@ build:
       # See https://github.com/readthedocs/readthedocs.org/pull/11152/
       - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH poetry install --with docs
 
+      # generate the config editor page. Schema then HTML
+      - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH poetry run python scripts/generate_settings_schema.py
+      # install node dependencies and build the settings
+      - cd scripts/settings && npm install && npm run build && yes | cp -v dist/index.html ../../docs/source/installation/settings.html && cd ../..
+
 
 sphinx:
   configuration: docs/source/conf.py

Dockerfile

@@ -1,25 +1,17 @@
-FROM webrecorder/browsertrix-crawler:1.4.2 AS base
+FROM webrecorder/browsertrix-crawler:1.12.4 AS base
 
 ENV RUNNING_IN_DOCKER=1 \
     LANG=C.UTF-8 \
     LC_ALL=C.UTF-8 \
     PYTHONDONTWRITEBYTECODE=1 \
-    PYTHONFAULTHANDLER=1 \
-    PATH="/root/.local/bin:$PATH"
+    PYTHONFAULTHANDLER=1
+
+
+ARG TARGETARCH
 
 # Installing system dependencies
-RUN add-apt-repository ppa:mozillateam/ppa && \
-	apt-get update && \
-    apt-get install -y --no-install-recommends gcc ffmpeg fonts-noto exiftool && \
-	apt-get install -y --no-install-recommends firefox-esr && \
-    ln -s /usr/bin/firefox-esr /usr/bin/firefox && \
-    wget https://github.com/mozilla/geckodriver/releases/download/v0.35.0/geckodriver-v0.35.0-linux64.tar.gz && \
-    tar -xvzf geckodriver* -C /usr/local/bin && \
-    chmod +x /usr/local/bin/geckodriver && \
-    rm geckodriver-v* && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
-
+RUN	apt-get update && \
+    apt-get install -y --no-install-recommends gcc ffmpeg fonts-noto exiftool python3-tk
 
 # Poetry and runtime
 FROM base AS runtime
@@ -48,11 +40,21 @@ COPY ./src/ .
 RUN /poetry-venv/bin/poetry install --only main --no-cache
 
 
+# Run as non-root user to avoid permission issues with mounted volumes (see #342)
+# The base image already has an 'ubuntu' user at UID/GID 1000.
+# Ensure directories that need write access at runtime are writable.
+RUN chown 1000:1000 /app && \
+    chown -R 1000:1000 /app/.venv/lib/python3.12/site-packages/seleniumbase/drivers/ && \
+    mkdir -p /app/local_archive /app/secrets /tmp/archive && \
+    chown -R 1000:1000 /app/local_archive /app/secrets /tmp/archive
+
 # Update PATH to include virtual environment binaries
 # Allowing entry point to run the application directly with Python
 ENV VIRTUAL_ENV=/app/.venv \
     PATH="/app/.venv/bin:$PATH"
 
+USER 1000
+
 ENTRYPOINT ["python3", "-m", "auto_archiver"]
 
 # should be executed with 2 volumes (3 if local_storage is used)

Makefile

@@ -0,0 +1,79 @@
+# Variables
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = docs/source
+BUILDDIR      = docs/_build
+
+.PHONY: help
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+	@echo "Additional Commands:"
+	@echo "  make test         - Run all tests in 'tests/' with pytest"
+	@echo "  make ruff-check   - Run Ruff linting and formatting checks (safe)"
+	@echo "  make ruff-clean   - Auto-fix Ruff linting and formatting issues"
+	@echo "  make docs         - Generate documentation (same as 'make html')"
+	@echo "  make clean-docs   - Remove generated docs"
+	@echo "  make docker-build - Build the Auto Archiver Docker image"
+	@echo "  make docker-compose - Run Auto Archiver with Docker Compose"
+	@echo "  make docker-compose-rebuild - Rebuild and run Auto Archiver with Docker Compose"
+	@echo "  make show-docs    - Build and open the documentation in a browser"
+
+
+
+.PHONY: test
+test:
+	@echo "Running tests..."
+	@pytest tests --disable-warnings
+
+
+.PHONY: ruff-check
+ruff-check:
+	@echo "Checking code style with Ruff (safe)..."
+	@ruff check .
+
+
+.PHONY: ruff-clean
+ruff-clean:
+	@echo "Fixing lint and formatting issues with Ruff..."
+	@ruff check . --fix
+	@ruff format .
+
+
+.PHONY: docs
+docs:
+	@echo "Building documentation..."
+	@$(SPHINXBUILD) -M html "$(SOURCEDIR)" "$(BUILDDIR)"
+
+
+.PHONY: clean-docs
+clean-docs:
+	@echo "Cleaning up generated documentation files..."
+	@$(SPHINXBUILD) -M clean "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+	@rm -rf "$(SOURCEDIR)/autoapi/" "$(SOURCEDIR)/modules/autogen/"
+	@echo "Cleanup complete."
+
+
+.PHONY: show-docs
+show-docs:
+	@echo "Opening documentation in browser..."
+	@open "$(BUILDDIR)/html/index.html"
+
+.PHONY: docker-build
+docker-build:
+	@echo "Building local Auto Archiver Docker image..."
+	@docker compose build  # Uses the same build context as docker-compose.yml
+
+.PHONY: docker-compose
+docker-compose:
+	@echo "Running Auto Archiver with Docker Compose..."
+	@docker compose up
+
+.PHONY: docker-compose-rebuild
+docker-compose-rebuild:
+	@echo "Rebuilding and running Auto Archiver with Docker Compose..."
+	@docker compose up --build
+
+# Catch-all for Sphinx commands
+.PHONY: Makefile
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

README.md

@@ -1,16 +1,17 @@
 <h1 align="center">Auto Archiver</h1>
 
+[![Documentation Status](https://readthedocs.org/projects/auto-archiver/badge/?version=latest)](https://auto-archiver.readthedocs.io/en/latest/?badge=latest)
 [![PyPI version](https://badge.fury.io/py/auto-archiver.svg)](https://badge.fury.io/py/auto-archiver)
-[![Docker Image Version (latest by date)](https://img.shields.io/docker/v/bellingcat/auto-archiver?label=version&logo=docker)](https://hub.docker.com/r/bellingcat/auto-archiver)
+[![Docker Image Version (latest by date)](https://img.shields.io/docker/v/bellingcat/auto-archiver?sort=semver&logo=docker&color=#69F0AE)](https://hub.docker.com/r/bellingcat/auto-archiver)
 [![Core Test Status](https://github.com/bellingcat/auto-archiver/workflows/Core%20Tests/badge.svg)](https://github.com/bellingcat/auto-archiver/actions/workflows/tests-core.yaml)
-[![Download Test Status](https://github.com/bellingcat/auto-archiver/workflows/Download%20Tests/badge.svg)](https://github.com/bellingcat/auto-archiver/actions/workflows/tests-download.yaml)
+<!-- [![Download Test Status](https://github.com/bellingcat/auto-archiver/workflows/Download%20Tests/badge.svg)](https://github.com/bellingcat/auto-archiver/actions/workflows/tests-download.yaml) -->
+
 <!-- ![Docker Pulls](https://img.shields.io/docker/pulls/bellingcat/auto-archiver) -->
 <!-- [![PyPI download month](https://img.shields.io/pypi/dm/auto-archiver.svg)](https://pypi.python.org/pypi/auto-archiver/) -->
-<!-- [![Documentation Status](https://readthedocs.org/projects/vk-url-scraper/badge/?version=latest)](https://vk-url-scraper.readthedocs.io/en/latest/?badge=latest) -->
 
 
 
-Auto Archiver is a Python tool to automatically archive content on the web in a secure and verifiable way. It takes URLs from different sources (e.g. a CSV file, Google Sheets, command line etc.) and archives the content of each one. It can archive social media posts, videos, images and webpages. Content can enriched, then saved either locally or remotely (S3 bucket, Google Drive). The status of the archiving process can be appended to a CSV report, or if using Google Sheets – back to the original sheet.
+Auto Archiver is a Python tool to automatically archive content on the web in a secure and verifiable way. It takes URLs from different sources (e.g. a CSV file, Google Sheets, command line etc.) and archives the content of each one. It can archive social media posts, videos, images and webpages. Content can be enriched, then saved either locally or remotely (S3 bucket, Google Drive). The status of the archiving process can be appended to a CSV report, or if using Google Sheets – back to the original sheet.
 
 <div class="hidden_rtd">
   
@@ -23,11 +24,13 @@ Read the [article about Auto Archiver on bellingcat.com](https://www.bellingcat.
 
 ## Installation
 
-View the [Installation Guide](installation/installation.md) for full instructions
+View the [Installation Guide](https://auto-archiver.readthedocs.io/en/latest/installation/installation.html) for full instructions
+
+**Advanced:**
 
 To get started quickly using Docker:
 
-`docker pull bellingcat/auto-archiver && docker run`
+`docker pull bellingcat/auto-archiver && docker run -it --rm -v secrets:/app/secrets bellingcat/auto-archiver --config secrets/orchestration.yaml`
 
 Or pip:
 

docker-compose.yaml

@@ -1,4 +1,3 @@
-version: '3.8'
 
 services:
   auto-archiver:
@@ -7,10 +6,10 @@ services:
       context: .
       dockerfile: Dockerfile
     container_name: auto-archiver
+    # Override user to match host UID/GID and avoid permission issues on volumes.
+    # Set USER_ID and GROUP_ID env vars, or defaults to 1000:1000.
+    user: "${USER_ID:-1000}:${GROUP_ID:-1000}"
     volumes:
       - ./secrets:/app/secrets
       - ./local_archive:/app/local_archive
-    environment:
-      - WACZ_ENABLE_DOCKER=true
-      - RUNNING_IN_DOCKER=true
     command: --config secrets/orchestration.yaml

docs/Makefile

@@ -1,20 +0,0 @@
-# Minimal makefile for Sphinx documentation
-#
-
-# You can set these variables from the command line, and also
-# from the environment for the first two.
-SPHINXOPTS    ?=
-SPHINXBUILD   ?= sphinx-build
-SOURCEDIR     = source
-BUILDDIR      = _build
-
-# Put it first so that "make" without argument is like "make help".
-help:
-	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
-
-.PHONY: help Makefile
-
-# Catch-all target: route all unknown targets to Sphinx using the new
-# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
-%: Makefile
-	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/scripts/__init__.py

@@ -1 +1 @@
-from scripts import generate_module_docs
+from scripts import generate_module_docs

docs/scripts/scripts.py

@@ -1,82 +1,136 @@
 # iterate through all the modules in auto_archiver.modules and turn the __manifest__.py file into a markdown table
 from pathlib import Path
-from auto_archiver.core.module import available_modules
+from auto_archiver.core.module import ModuleFactory
 from auto_archiver.core.base_module import BaseModule
 from ruamel.yaml import YAML
+from ruamel.yaml.comments import CommentedMap
 import io
 
 MODULES_FOLDER = Path(__file__).parent.parent.parent.parent / "src" / "auto_archiver" / "modules"
 SAVE_FOLDER = Path(__file__).parent.parent / "source" / "modules" / "autogen"
 
 type_color = {
-    'feeder': "<span style='color: #FFA500'>[feeder](/core_modules.md#feeder-modules)</a></span>",
-    'extractor': "<span style='color: #00FF00'>[extractor](/core_modules.md#extractor-modules)</a></span>",
-    'enricher': "<span style='color: #0000FF'>[enricher](/core_modules.md#enricher-modules)</a></span>",
-    'database': "<span style='color: #FF00FF'>[database](/core_modules.md#database-modules)</a></span>",
-    'storage': "<span style='color: #FFFF00'>[storage](/core_modules.md#storage-modules)</a></span>",
-    'formatter': "<span style='color: #00FFFF'>[formatter](/core_modules.md#formatter-modules)</a></span>",
+    "feeder": "<span style='color: #FFA500'>[feeder](/core_modules.md#feeder-modules)</a></span>",
+    "extractor": "<span style='color: #00FF00'>[extractor](/core_modules.md#extractor-modules)</a></span>",
+    "enricher": "<span style='color: #0000FF'>[enricher](/core_modules.md#enricher-modules)</a></span>",
+    "database": "<span style='color: #FF00FF'>[database](/core_modules.md#database-modules)</a></span>",
+    "storage": "<span style='color: #FFFF00'>[storage](/core_modules.md#storage-modules)</a></span>",
+    "formatter": "<span style='color: #00FFFF'>[formatter](/core_modules.md#formatter-modules)</a></span>",
 }
 
 TABLE_HEADER = ("Option", "Description", "Default", "Type")
 
+EXAMPLE_YAML = """
+# steps configuration
+steps:
+...
+{steps_str}
+...
+
+# module configuration
+...
+
+{config_string}
+
+"""
+
+
 def generate_module_docs():
     yaml = YAML()
     SAVE_FOLDER.mkdir(exist_ok=True)
     modules_by_type = {}
 
     header_row = "| " + " | ".join(TABLE_HEADER) + "|\n" + "| --- " * len(TABLE_HEADER) + "|\n"
-    configs_cheatsheet = "\n## Configuration Options\n"
-    configs_cheatsheet += header_row
+    global_table = "\n## Configuration Options\n" + header_row
 
-    for module in sorted(available_modules(with_manifest=True), key=lambda x: (x.requires_setup, x.name)):
+    global_yaml = yaml.load("""\n# Module configuration\nplaceholder: {}""")
+
+    for module in sorted(ModuleFactory().available_modules(), key=lambda x: (x.requires_setup, x.name)):
         # generate the markdown file from the __manifest__.py file.
-
         manifest = module.manifest
-        for type in manifest['type']:
+        for type in manifest["type"]:
             modules_by_type.setdefault(type, []).append(module)
 
-        description = "\n".join(l.lstrip() for l in manifest['description'].split("\n"))
-        types = ", ".join(type_color[t] for t in manifest['type'])
+        description = "\n".join(line.lstrip() for line in manifest["description"].split("\n"))
+        types = ", ".join(type_color[t] for t in manifest["type"])
         readme_str = f"""
-# {manifest['name']}
+# {manifest["name"]}
 ```{{admonition}} Module type
 
 {types}
 ```
 {description}
-"""     
-        if not manifest['configs']:
-            readme_str += "\n*This module has no configuration options.*\n"
+"""
+        steps_str = "\n".join(f"  {t}s:\n  - {module.name}" for t in manifest["type"])
+
+        if manifest.get("autodoc_dropins"):
+            loaded_module = module.load({})
+            dropins = loaded_module.load_dropins()
+            dropin_str = "\n##### Available Dropins\n"
+            for dropin in dropins:
+                if not (ddoc := dropin.documentation()):
+                    continue
+                dropin_str += f"\n###### {ddoc.get('name', dropin.__name__)}\n\n"
+                dropin_str += f"{ddoc.get('description')}\n\n"
+                if ddoc.get("site"):
+                    dropin_str += f"**Site**: {ddoc['site']}\n\n"
+                if dauth := ddoc.get("authentication"):
+                    dropin_str += "**YAML configuration**:\n"
+                    dropin_auth_yaml = "authentication:\n...\n"
+                    for site, creds in dauth.items():
+                        dropin_auth_yaml += f"  {site}:\n"
+                        for k, v in creds.items():
+                            dropin_auth_yaml += f'    {k}: "{v}"\n'
+                    dropin_str += f"```{{code}} yaml\n{dropin_auth_yaml}...\n```\n"
+            readme_str += dropin_str
+
+        if not manifest["configs"]:
+            config_string = f"# No configuration options for {module.name}.*\n"
         else:
-            config_yaml = {}
             config_table = header_row
-            for key, value in manifest['configs'].items():
-                type = value.get('type', 'string')
-                if type == 'auto_archiver.utils.json_loader':
-                    value['type'] = 'json'
-                elif type == 'str':
+            config_yaml = {}
+
+            global_yaml[module.name] = CommentedMap()
+            global_yaml.yaml_set_comment_before_after_key(
+                module.name, f"\n\n{module.display_name} configuration options"
+            )
+
+            for key, value in manifest["configs"].items():
+                type = value.get("type", "string")
+                if type == "json_loader":
+                    value["type"] = "json"
+                elif type == "str":
                     type = "string"
-                
-                default = value.get('default', '')
+
+                default = value.get("default", "")
                 config_yaml[key] = default
-                help = "**Required**. " if value.get('required', False) else "Optional. "
-                help += value.get('help', '')
+
+                global_yaml[module.name][key] = default
+
+                if value.get("help", ""):
+                    global_yaml[module.name].yaml_add_eol_comment(value.get("help", ""), key)
+
+                help = "**Required**. " if value.get("required", False) else "Optional. "
+                help += value.get("help", "")
                 config_table += f"| `{module.name}.{key}` | {help} | {value.get('default', '')} | {type} |\n"
-                configs_cheatsheet += f"| `{module.name}.{key}` | {help} | {default} | {type} |\n"
+                global_table += f"| `{module.name}.{key}` | {help} | {default} | {type} |\n"
             readme_str += "\n## Configuration Options\n"
             readme_str += "\n### YAML\n"
-            yaml_string = io.BytesIO()
-            yaml.dump({module.name: config_yaml}, yaml_string)
-            
-            readme_str += f"```{{code}} yaml\n{yaml_string.getvalue().decode('utf-8')}\n```\n"
 
+            config_string = io.BytesIO()
+            yaml.dump({module.name: config_yaml}, config_string)
+            config_string = config_string.getvalue().decode("utf-8")
+        yaml_string = EXAMPLE_YAML.format(steps_str=steps_str, config_string=config_string)
+        readme_str += f"```{{code}} yaml\n{yaml_string}\n```\n"
+
+        if manifest["configs"]:
             readme_str += "\n### Command Line:\n"
             readme_str += config_table
 
         # add a link to the autodoc refs
         readme_str += f"\n[API Reference](../../../autoapi/{module.name}/index)\n"
         # create the module.type folder, use the first type just for where to store the file
-        for type in manifest['type']:
+        for type in manifest["type"]:
             type_folder = SAVE_FOLDER / type
             type_folder.mkdir(exist_ok=True)
             with open(type_folder / f"{module.name}.md", "w") as f:
@@ -84,8 +138,13 @@ def generate_module_docs():
                 f.write(readme_str)
         generate_index(modules_by_type)
 
+    del global_yaml["placeholder"]
+    global_string = io.BytesIO()
+    global_yaml = yaml.dump(global_yaml, global_string)
+    global_string = global_string.getvalue().decode("utf-8")
+    global_yaml = f"```yaml\n{global_string}\n```"
     with open(SAVE_FOLDER / "configs_cheatsheet.md", "w") as f:
-        f.write(configs_cheatsheet)
+        f.write("### Configuration File\n" + global_yaml + "\n### Command Line\n" + global_table)
 
 
 def generate_index(modules_by_type):
@@ -103,3 +162,7 @@ def generate_index(modules_by_type):
     with open(SAVE_FOLDER / "module_list.md", "w") as f:
         print("writing", SAVE_FOLDER / "module_list.md")
         f.write(readme_str)
+
+
+if __name__ == "__main__":
+    generate_module_docs()

docs/source/conf.py

@@ -3,9 +3,11 @@
 import sys
 import os
 from importlib.metadata import metadata
+from datetime import datetime
 
-sys.path.append(os.path.abspath('../scripts'))
+sys.path.append(os.path.abspath("../scripts"))
 from scripts import generate_module_docs
+from auto_archiver.version import __version__
 
 # -- Project Hooks -----------------------------------------------------------
 # convert the module __manifest__.py files into markdown files
@@ -15,35 +17,38 @@ generate_module_docs()
 # -- Project information -----------------------------------------------------
 package_metadata = metadata("auto-archiver")
 project = package_metadata["name"]
-authors = "Bellingcat"
+copyright = str(datetime.now().year)
+author = "Bellingcat"
 release = package_metadata["version"]
-language = 'en'
+language = "en"
 
 # -- General configuration ---------------------------------------------------
 extensions = [
-    "myst_parser",                  # Markdown support
-    "autoapi.extension",            # Generate API documentation from docstrings
-    "sphinxcontrib.mermaid",        # Mermaid diagrams
-    "sphinx.ext.viewcode",          # Source code links
+    "myst_parser",  # Markdown support
+    "autoapi.extension",  # Generate API documentation from docstrings
+    "sphinxcontrib.mermaid",  # Mermaid diagrams
+    "sphinx.ext.viewcode",  # Source code links
     "sphinx_copybutton",
-    "sphinx.ext.napoleon",          # Google-style and NumPy-style docstrings
+    "sphinx.ext.napoleon",  # Google-style and NumPy-style docstrings
     "sphinx.ext.autosectionlabel",
     # 'sphinx.ext.autosummary',       # Summarize module/class/function docs
 ]
 
-templates_path = ['_templates']
-exclude_patterns = []
+templates_path = ["_templates"]
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", ""]
 
 
 # -- AutoAPI Configuration ---------------------------------------------------
-autoapi_type = 'python'
+autoapi_type = "python"
 autoapi_dirs = ["../../src/auto_archiver/core/", "../../src/auto_archiver/utils/"]
 # get all the modules and add them to the autoapi_dirs
 autoapi_dirs.extend([f"../../src/auto_archiver/modules/{m}" for m in os.listdir("../../src/auto_archiver/modules")])
-autodoc_typehints = "signature"     # Include type hints in the signature
-autoapi_ignore = ["*/version.py", ]                 # Ignore specific modules
-autoapi_keep_files = True          # Option to retain intermediate JSON files for debugging
-autoapi_add_toctree_entry = True    # Include API docs in the TOC
+autodoc_typehints = "signature"  # Include type hints in the signature
+autoapi_ignore = [
+    "*/version.py",
+]  # Ignore specific modules
+autoapi_keep_files = True  # Option to retain intermediate JSON files for debugging
+autoapi_add_toctree_entry = True  # Include API docs in the TOC
 autoapi_python_use_implicit_namespaces = True
 autoapi_template_dir = "../_templates/autoapi"
 autoapi_options = [
@@ -56,13 +61,13 @@ autoapi_options = [
 
 # -- Markdown Support --------------------------------------------------------
 myst_enable_extensions = [
-    "deflist",              # Definition lists
-    "html_admonition",      # HTML-style admonitions
-    "html_image",           # Inline HTML images
-    "replacements",         # Substitutions like (C)
-    "smartquotes",          # Smart quotes
-    "linkify",              # Auto-detect links
-    "substitution",         # Text substitutions
+    "deflist",  # Definition lists
+    "html_admonition",  # HTML-style admonitions
+    "html_image",  # Inline HTML images
+    "replacements",  # Substitutions like (C)
+    "smartquotes",  # Smart quotes
+    "linkify",  # Auto-detect links
+    "substitution",  # Text substitutions
 ]
 myst_heading_anchors = 2
 myst_fence_as_directive = ["mermaid"]
@@ -73,7 +78,17 @@ source_suffix = {
 }
 
 # -- Options for HTML output -------------------------------------------------
-html_theme = 'sphinx_book_theme'
+html_theme = "sphinx_book_theme"
 html_static_path = ["../_static"]
 html_css_files = ["custom.css"]
+html_title = f"Auto Archiver v{__version__}"
+html_logo = "bc.png"
+html_theme_options = {
+    "repository_url": "https://github.com/bellingcat/auto-archiver",
+    "use_repository_button": True,
+}
 
+
+copybutton_prompt_text = r">>> |\.\.\."
+copybutton_prompt_is_regexp = True
+copybutton_only_copy_prompt_lines = False

docs/source/core_modules.md

@@ -1,8 +1,8 @@
 # Module Documentation
 
-These pages describe the core modules that come with `auto-archiver` and provide the main functionality for archiving websites on the internet. There are five core module types:
+These pages describe the core modules that come with Auto Archiver and provide the main functionality for archiving websites on the internet. There are five core module types:
 
-1. Feeders - these 'feed' information (the URLs) from various sources to the `auto-archiver` for processing
+1. Feeders - these 'feed' information (the URLs) from various sources to the Auto Archiver for processing
 2. Extractors - these 'extract' the page data for a given URL that is fed in by a feeder
 3. Enrichers - these 'enrich' the data extracted in the previous step with additional information
 4. Storage - these 'store' the data in a persistent location (on disk, Google Drive etc.)
@@ -24,4 +24,5 @@ modules/extractor
 modules/enricher
 modules/storage
 modules/database
+modules/formatter
 ```

docs/source/development/creating_modules.md

@@ -1,6 +1,6 @@
 # Creating Your Own Modules
 
-Modules are what's used to extend `auto-archiver` to process different websites or media, and/or transform the data in a way that suits your needs. In most cases, the [Core Modules](../core_modules.md) should be sufficient for every day use, but the most common use-cases for making your own Modules include:
+Modules are what's used to extend Auto Archiver to process different websites or media, and/or transform the data in a way that suits your needs. In most cases, the [Core Modules](../core_modules.md) should be sufficient for every day use, but the most common use-cases for making your own Modules include:
 
 1. Extracting data from a website which doesn't work with the current core extractors.
 2. Enriching or altering the data before saving with additional information that the core enrichers do not offer.
@@ -21,7 +21,7 @@ When done, you should have a module structure as follows:
 │   └── awesome_extractor.py
 ``` 
 
-Check out the [core modules](https://github.com/bellingcat/auto-archiver/tree/main/src/auto_archiver/modules) in the `auto-archiver` repository for examples of the folder structure for real-world modules.
+Check out the [core modules](https://github.com/bellingcat/auto-archiver/tree/main/src/auto_archiver/modules) in the Auto Archiver repository for examples of the folder structure for real-world modules.
 
 ## Populating the Manifest File
 

docs/source/development/developer_guidelines.md

@@ -21,7 +21,7 @@ This allows you to run the auto-archiver without the `poetry run` prefix.
 ### Optional Development Packages
 
 Install development packages (used for unit tests etc.) using:
-`poetry install -with dev`
+`poetry install --with dev`
 
 
 ```{toctree}
@@ -31,4 +31,6 @@ docker_development
 testing
 docs
 release
-```
+settings_page
+style_guide
+```

docs/source/development/docs.md

@@ -36,3 +36,12 @@ open docs/_build/html/index.html
 sphinx-autobuild docs/source docs/_build/html
 ```
 
+
+### Managing Readthedocs (RTD) Versions
+
+Version management is done at [https://app.readthedocs.org/projects/auto-archiver/](https://app.readthedocs.org/projects/auto-archiver/)
+(login required). Once logged in, you can create new versions, delete old versions or change visibility of versions. More info on
+[RTD](https://docs.readthedocs.com/platform/stable/versions.html).
+
+Currently, the Auto Archiver project is set up to automatically create a new docs version for each `vX.Y.Z` release. For more on this,
+see the RTD [instructions on automation](https://docs.readthedocs.com/platform/stable/guides/automation-rules.html) or edit the existing automation rule in the project settings.

docs/source/development/release.md

@@ -2,14 +2,32 @@
 
 ```{note} This is a work in progress.
 ```
+### Update the project version
 
-1. Update the version number in [version.py](src/auto_archiver/version.py)
-2. Go to github releases > new release > use `vx.y.z` for matching version notation
-   1. package is automatically updated in pypi
-   2. docker image is automatically pushed to dockerhup
+Update the version number in the project file: [pyproject.toml](../../pyproject.toml) following SemVer:
+```toml
+[project]
+name = "auto-archiver"
+version = "0.1.1"
+```
+Then commit and push the changes.
 
+* The package version is automatically updated in PyPi using the workflow [python-publish.yml](../../.github/workflows/python-publish.yml)
+* A Docker image is automatically pushed with the git tag to dockerhub using the workflow [docker-publish.yml](../../.github/workflows/docker-publish.yml)
+
+### Create the release on Git
+
+The release needs a git tag which should match the project version number, prefixed with a 'v'. For example, if the project version is `0.1.1`, the git tag should be `v0.1.1`.
+This can be done the usual way, or created within the Github UI when you create the release.
+
+Go to GitHub releases > new release > create the release with the new tag and the release notes.
 
 
 manual release to docker hub
   * `docker image tag auto-archiver bellingcat/auto-archiver:latest`
   * `docker push bellingcat/auto-archiver`
+
+
+### Building the Settings Page
+
+The Settings page is built as part of the python-publish workflow and packaged within the app.

docs/source/development/settings_page.md

@@ -0,0 +1,31 @@
+# Configuration Editor
+
+The [configuration editor](../installation/config_editor.md), is an easy-to-use UI for users to edit their auto-archiver settings.
+
+The single-file app is built using React and vite. To get started developing the package, follow these steps:
+
+1. Make sure you have Node v22 installed.
+
+```{note} Tip: if you don't have node installed:
+
+Use `nvm` to manage your node installations. Use: 
+`curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash` to install `nvm` and then `nvm i 22` to install Node v22
+```
+
+2. Generate the `schema.json` file for the currently installed modules using `python scripts/generate_settings_schema.py`
+3. Go to the settings folder `cd scripts/settings/` and build your environment with `npm i`
+4. Run a development version of the page with `npm run dev` and then open localhost:5173.
+5. Build a release version of the page with `npm run build`
+
+A release version creates a single-file app called `dist/index.html`. This file should be copied to `docs/source/installation/settings_base.html` so that it can be integrated into the sphinx docs.
+
+```{note}
+
+The single-file app dist/index.html does not include any `<html>` or `<head>` tags as it is designed to be built into a RTD docs page. Edit `index.html` in the settings folder if you wish to modify the built page.
+```
+
+## Readthedocs Integration
+
+The configuration editor is built as part of the RTD deployment (see `.readthedocs.yaml` file). This command is run every time RTD is built:
+
+`cd scripts/settings && npm install && npm run build && yes | cp dist/index.html ../../docs/source/installation/settings_base.html && cd ../..`

docs/source/development/style_guide.md

@@ -0,0 +1,70 @@
+# Style Guide
+
+
+The project uses [Ruff](https://docs.astral.sh/ruff/) for linting and  formatting.
+Our style configurations are set in the `pyproject.toml` file. If needed, you can modify them there.
+
+
+### **Formatting (Auto-Run Before Commit) 🛠️**  
+
+We have a pre-commit hook to run the formatter before you commit.
+This requires you to set it up once locally, then it will run automatically when you commit changes.
+
+```shell
+poetry run pre-commit install
+```
+
+Ruff can also be to run automatically.
+Alternative: Ruff can also be [integrated with most editors](https://docs.astral.sh/ruff/editors/setup/)  for real-time formatting.
+
+If you wish to disable the pre-commit hook (for example, if you want to commit some WIP code) you can use the `--no-verify` flag when you commit.
+For example: `git commit -m "WIP Code" --no-verify`
+
+### **Linting (Check Before Pushing) 🔍**
+
+We recommend you also run the linter before pushing code. 
+
+We have [Makefile](../../../Makefile) commands to run common tasks. 
+
+Tip: if you're on Windows you might need to install `make` first, or alternatively you can use ruff commands directly.
+
+
+**Lint Check:** This outputs a report of any issues found, without attempting to fix them: 
+```shell
+make ruff-check
+```
+
+Tip: To see a more detailed linting report, you can remove the following line from the `pyproject.toml` file:
+```toml
+[tool.ruff]
+
+# Remove this for a more detailed lint report
+output-format = "concise"
+```
+
+**Lint Fix:** This command will attempt to fix some of the issues it picked up with the lint check.
+
+Note not all warnings can be fixed automatically.
+
+⚠️ Warning: This can cause breaking changes. ⚠️
+
+Most fixes are safe, but some non-standard practices such as dynamic loading are not picked up by linters. Ensure you check any modifications by this before committing them.
+```shell
+make ruff-clean
+```
+
+**Changing Configurations ⚙️**
+
+
+Our rules are quite lenient for general usage, but if you want to run more rigorous checks you can then run checks with additional rules to see more nuanced errors which you can review manually.
+Check out the [ruff documentation](https://docs.astral.sh/ruff/configuration/) for the full list of rules.
+One example is to extend the selected rules for linting the `pyproject.toml` file:
+
+```toml 
+[tool.ruff.lint]
+# Extend the rules to check for by adding them to this option:
+# See documentation for more details: https://docs.astral.sh/ruff/rules/
+extend-select = ["B"]
+```
+
+Then re-run the `make ruff-check` command to see the new rules in action.

docs/source/development/testing.md

@@ -3,14 +3,14 @@
 `pytest` is used for testing. There are two main types of tests:
 
 1. 'core' tests which should be run on every change
-2. 'download' tests which hit the network. These tests will do things like make API calls (e.g. Twitter, Bluesky etc.) and should be run regularly to make sure that APIs have not changed.
+2. 'download' tests which hit the network. These tests will do things like make API calls (e.g. Twitter, Bluesky etc.) and should be run regularly to make sure that APIs have not changed, they take longer.
 
 
 ## Running Tests 
 
-1. Make sure you've installed the dev dependencies with `pytest install --with dev`
+1. Make sure you've installed the dev dependencies with `poetry install --with dev`
 2. Tests can be run as follows:
-```
+```{code} bash
 #### Command prefix of 'poetry run' removed here for simplicity
 # run core tests
 pytest -ra -v -m "not download"
@@ -18,4 +18,15 @@ pytest -ra -v -m "not download"
 pytest -ra -v -m "download"
 # run all tests
 pytest -ra -v
-```
+
+
+# run a specific test file
+pytest -ra -v tests/test_file.py
+# run a specific test function
+pytest -ra -v tests/test_file.py::test_function_name
+```
+
+3. Some tests require environment variables to be set. You can use the example `tests/.env.test.example` file as a template. Copy it to `tests/.env.test` and fill in the required values. This file will be loaded automatically by `pytest`.
+```{code} bash
+cp tests/.env.test.example tests/.env.test
+```

docs/source/flow_overview.md

@@ -8,7 +8,7 @@ The archiver archives web pages using the following workflow
 4. **Formatter** creates a report from all the archived content (HTML, PDF, ...)
 5. **Database** knows what's been archived and also stores the archive result (spreadsheet, CSV, or just the console)
 
-Each step in the workflow is handled by 'modules' that interact with the data in different ways. For example, the Twitter Extractor Module would extract information from the Twitter website. The Screenshot Enricher Module will take screenshots of the given page. See the [core modules page](core_modules.md) for an overview of all the modules that are available.
+Each step in the workflow is handled by 'modules' that interact with the data in different ways. For example, the Twitter Extractor Module would extract information from the Twitter website. The AntiBot Module will download HTML and take screenshots of the given page. See the [core modules page](core_modules.md) for an overview of all the modules that are available.
 
 Auto-archiver must have at least one module defined for each step of the workflow. This is done by setting the [configuration](installation/configurations.md) for your auto-archiver instance.
 

docs/source/how_to.md

@@ -1,47 +1,12 @@
 # How-To Guides
 
-## How to use Google Sheets to load and store archive information
-The `--gsheet_feeder.sheet` property is the name of the Google Sheet to check for URLs. 
-This sheet must have been shared with the Google Service account used by `gspread`. 
-This sheet must also have specific columns (case-insensitive) in the `header` - see the [Gsheet Feeder Docs](modules/autogen/feeder/gsheet_feeder.md) for more info. The default names of these columns and their purpose is:
-
-Inputs:
-
-* **Link** *(required)*: the URL of the post to archive
-* **Destination folder**: custom folder for archived file (regardless of storage)
-
-Outputs:
-* **Archive status** *(required)*: Status of archive operation
-* **Archive location**: URL of archived post
-* **Archive date**: Date archived
-* **Thumbnail**: Embeds a thumbnail for the post in the spreadsheet
-* **Timestamp**: Timestamp of original post
-* **Title**: Post title
-* **Text**: Post text
-* **Screenshot**: Link to screenshot of post
-* **Hash**: Hash of archived HTML file (which contains hashes of post media) - for checksums/verification
-* **Perceptual Hash**: Perceptual hashes of found images - these can be used for de-duplication of content
-* **WACZ**: Link to a WACZ web archive of post
-* **ReplayWebpage**: Link to a ReplayWebpage viewer of the WACZ archive
-
-For example, this is a spreadsheet configured with all of the columns for the auto archiver and a few URLs to archive. (Note that the column names are not case sensitive.)
-
-![A screenshot of a Google Spreadsheet with column headers defined as above, and several Youtube and Twitter URLs in the "Link" column](../demo-before.png)
-
-Now the auto archiver can be invoked, with this command in this example: `docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver:dockerize --config secrets/orchestration-global.yaml --gsheet_feeder.sheet "Auto archive test 2023-2"`. Note that the sheet name has been overridden/specified in the command line invocation.
-
-When the auto archiver starts running, it updates the "Archive status" column.
-
-![A screenshot of a Google Spreadsheet with column headers defined as above, and several Youtube and Twitter URLs in the "Link" column. The auto archiver has added "archive in progress" to one of the status columns.](../demo-progress.png)
-
-The links are downloaded and archived, and the spreadsheet is updated to the following:
-
-![A screenshot of a Google Spreadsheet with videos archived and metadata added per the description of the columns above.](../demo-after.png)
-
-Note that the first row is skipped, as it is assumed to be a header row (`--gsheet_feeder.header=1` and you can change it if you use more rows above). Rows with an empty URL column, or a non-empty archive column are also skipped. All sheets in the document will be checked.
-
-The "archive location" link contains the path of the archived file, in local storage, S3, or in Google Drive.
-
-![The archive result for a link in the demo sheet.](../demo-archive.png)
-
+The follow pages contain helpful how-to guides for common use cases of the Auto Archiver.
 ---
+
+```{toctree}
+:maxdepth: 1
+:glob:
+
+how_to/*
+
+```

docs/source/how_to/01_authentication_how_to.md

@@ -0,0 +1,222 @@
+# Logging in to sites
+
+This how-to guide shows you how you can use various authentication methods to allow you to login to a site you are trying to archive. This is useful for websites that require a user to be logged in to browse them, or for sites that restrict bots.
+
+In this How-To, we will authenticate on use Twitter/X.com using cookies, and on XXXX using username/password.
+
+
+
+## Using cookies to authenticate on Twitter/X
+
+It can be useful to archive tweets after logging in, since some tweets are only visible to authenticated users. One case is Tweets marked as 'Sensitive'.
+
+Take this tweet as an example: [https://x.com/SozinhoRamalho/status/1876710769913450647](https://x.com/SozinhoRamalho/status/1876710769913450647)
+
+This tweet has been marked as sensitive, so a normal run of Auto Archiver without a logged in session will fail to extract the tweet:
+
+```{code-block} console
+:emphasize-lines: 3,4,5,6
+
+>>> auto-archiver https://x.com/SozinhoRamalho/status/1876710769913450647                                                                                     ✭ ✱
+ ...
+ERROR: [twitter] 1876710769913450647: NSFW tweet requires authentication. Use --cookies, 
+--cookies-from-browser, --username and --password, --netrc-cmd, or --netrc (twitter) to
+ provide account credentials. See https://github.com/yt-dlp/yt-dlp/wiki/FAQ#how-do-i-pass-cookies-to-yt-dlp 
+ for how to manually pass cookies
+[twitter] 1876710769913450647: Downloading guest token
+[twitter] 1876710769913450647: Downloading GraphQL JSON
+2025-02-20 15:06:13.362 | ERROR    | auto_archiver.modules.generic_extractor.generic_extractor:download_for_extractor:248 - Error downloading metadata for post: NSFW tweet requires authentication. Use --cookies, --cookies-from-browser, --username and --password, --netrc-cmd, or --netrc (twitter) to provide account credentials. See  https://github.com/yt-dlp/yt-dlp/wiki/FAQ#how-do-i-pass-cookies-to-yt-dlp  for how to manually pass cookies
+[generic] Extracting URL: https://x.com/SozinhoRamalho/status/1876710769913450647
+[generic] 1876710769913450647: Downloading webpage
+WARNING: [generic] Falling back on generic information extractor
+[generic] 1876710769913450647: Extracting information
+ERROR: Unsupported URL: https://x.com/SozinhoRamalho/status/1876710769913450647
+2025-02-20 15:06:13.744 | INFO     | auto_archiver.core.orchestrator:archive:483 - Trying extractor telegram_extractor for https://x.com/SozinhoRamalho/status/1876710769913450647
+2025-02-20 15:06:13.744 | SUCCESS  | auto_archiver.modules.console_db.console_db:done:23 - DONE Metadata(status='nothing archived', metadata={'_processed_at': datetime.datetime(2025, 2, 20, 15, 6, 12, 473979, tzinfo=datetime.timezone.utc), 'url': 'https://x.com/SozinhoRamalho/status/1876710769913450647'}, media=[])
+...
+```
+
+To get round this limitation, we can use **cookies** (information about a logged in user) to mimic being logged in to Twitter. There are two ways to pass cookies to Auto Archiver. One is from a file, and the other is from a browser profile on your computer.
+
+In this tutorial, we will export the Twitter cookies from our browser and add them to Auto Archiver
+
+**1. Installing a cookie exporter extension**
+
+First, we need to install an extension in our browser to export the cookies for a certain site. The [FAQ on yt-dlp](https://github.com/yt-dlp/yt-dlp/wiki/FAQ#how-do-i-pass-cookies-to-yt-dlp) provides some suggestions: Get [cookies.txt LOCALLY](https://chromewebstore.google.com/detail/get-cookiestxt-locally/cclelndahbckbenkjhflpdbgdldlbecc) for Chrome or [cookies.txt](https://addons.mozilla.org/en-US/firefox/addon/cookies-txt/) for Firefox.
+
+**2. Export the cookies**
+
+```{note} See the note [here](../installation/authentication.md#recommendations-for-authentication) on why you shouldn't use your own personal account for archiving.
+```
+
+Once the extension is installed in your preferred browser, login to Twitter in this browser, and then activate the extension and export the cookies. You can choose to export all your cookies for your browser, or just cookies for this specific site. In the image below, we're only exporting cookies for Twitter/x.com:
+
+![extract cookies](extract_cookies.png)
+
+
+**3. Adding the cookies file to Auto Archiver**
+
+You now will have a file called `cookies.txt` (tip: name it `twitter_cookies.txt` if you only exported cookies for Twitter), which needs to be added to Auto Archiver.
+
+Do this by going into your Auto Archiver configuration file, and editing the `authentication` section. We will add the `cookies_file` option for the site `x.com,twitter.com`.
+
+```{note} For websites that have multiple URLs (like x.com and twitter.com) you can 'reuse' the same login information without duplicating it using a comma separated list of domain names.
+```
+
+I've saved my `twitter_cookies.txt` file in a `secrets` folder, so here's how my authentication section looks now:
+
+```{code} yaml
+:caption: orchestration.yaml
+
+...
+
+authentication:
+   x.com,twitter.com:
+      cookies_file: secrets/twitter_cookies.txt
+...
+```
+
+**4. Re-run your archiving with the cookies enabled**
+
+Now, the next time we re-run Auto Archiver, the cookies from our logged-in session will be used by Auto Archiver, and restricted/sensitive tweets can be downloaded!
+
+```{code} console
+>>> auto-archiver https://x.com/SozinhoRamalho/status/1876710769913450647                                                                                   ✭ ✱ ◼
+...
+2025-02-20 15:27:46.785 | WARNING  | auto_archiver.modules.console_db.console_db:started:13 - STARTED Metadata(status='no archiver', metadata={'_processed_at': datetime.datetime(2025, 2, 20, 15, 27, 46, 785304, tzinfo=datetime.timezone.utc), 'url': 'https://x.com/SozinhoRamalho/status/1876710769913450647'}, media=[])
+2025-02-20 15:27:46.785 | INFO     | auto_archiver.core.orchestrator:archive:483 - Trying extractor generic_extractor for https://x.com/SozinhoRamalho/status/1876710769913450647
+[twitter] Extracting URL: https://x.com/SozinhoRamalho/status/1876710769913450647
+...
+2025-02-20 15:27:53.134 | INFO     | auto_archiver.modules.local_storage.local_storage:upload:26 - ./local_archive/https-x-com-sozinhoramalho-status-1876710769913450647/06e8bacf27ac4bb983bf6280.html
+2025-02-20 15:27:53.135 | SUCCESS  | auto_archiver.modules.console_db.console_db:done:23 - DONE Metadata(status='yt-dlp_Twitter: success', 
+metadata={'_processed_at': datetime.datetime(2025, 2, 20, 15, 27, 48, 564738, tzinfo=datetime.timezone.utc), 'url': 
+'https://x.com/SozinhoRamalho/status/1876710769913450647', 'title': 'ignore tweet, testing sensitivity warning nudity https://t.co/t3u0hQsSB1', 
+...
+```
+
+
+### Finishing Touches
+
+You've now successfully exported your cookies from a logged-in session in your browser, and used them to authenticate with Twitter and download a sensitive tweet. Congratulations!
+
+Finally,Some important things to remember:
+
+1. It's best not to use your own personal account for archiving. [Here's why](../installation/authentication.md#recommendations-for-authentication).
+2. Cookies can be short-lived, so may need updating. Sometimes, a website session may 'expire' or a website may force you to login again. In these instances, you'll need to repeat the export step (step 2) after logging in again to update your cookies.
+
+## Authenticating on XXXX site with username/password
+
+```{note} 
+This section is still under construction 🚧
+```
+
+
+# Proof of Origin Tokens
+
+YouTube uses **Proof of Origin Tokens (POT)** as part of its bot detection system to verify that requests originate from valid clients. If a token is missing or invalid, some videos may return errors like "Sign in to confirm you're not a bot."
+
+yt-dlp provides [a detailed guide to POTs](https://github.com/yt-dlp/yt-dlp/wiki/PO-Token-Guide).
+
+### How Auto Archiver Uses POT
+This feature is enabled for the Generic Archiver via two yt-dlp plugins:
+
+- **Client-side plugin**: [yt-dlp-get-pot](https://github.com/coletdjnz/yt-dlp-get-pot)  
+  Detects when a token is required and requests one from a provider.
+
+- **Provider plugin**: [bgutil-ytdlp-pot-provider](https://github.com/Brainicism/bgutil-ytdlp-pot-provider)  
+  Includes both a Python plugin and a **Node.js server or script** to generate the token.
+
+These are installed in our Poetry environment.
+
+### Integration Methods
+
+**Docker (Recommended)**:
+
+When running the Auto Archiver using the Docker image, we use the [Node.js token generation script](https://github.com/Brainicism/bgutil-ytdlp-pot-provider/tree/master/server).
+This is to avoid managing a separate server process, and is handled automatically inside the Docker container when needed.
+
+This is already included in the Docker image, however if you need to disable this you can set the config option `bguils_po_token_method` under the `generic_extractor` section of your `orchestration.yaml` config file to "disabled".
+```yaml
+generic_extractor:
+  bguils_po_token_method: "disabled"
+```
+
+**PyPi/ Local**:
+
+When using the Auto Archiver PyPI package, or running locally, you will need additional system requirements to run the token generation script, namely either Docker, or Node.js and Yarn.
+
+See the [bgutil-ytdlp-pot-provider](https://github.com/Brainicism/bgutil-ytdlp-pot-provider?tab=readme-ov-file#a-http-server-option) documentation for more details.
+
+⚠️WARNING⚠️: This will add the server scripts to the home directory of wherever this is running.
+
+- You can set the config option `bguils_po_token_method` under the `generic_extractor` section of your `orchestration.yaml` config file to "script" to enable the token generation script process locally.
+- Alternatively you can run the bgutil-ytdlp-pot-provider server separately using their Docker image or Node.js server.
+
+### Notes
+
+- The token generation script is only triggered when needed by yt-dlp, so it should have no effect unless YouTube requests a POT.
+- If you're running the Auto Archiver in Docker, this is set up automatically.
+- If you're running locally, you'll need to run the setup script manually or enable the feature in your config.
+- You can set up both the server and the script, and the plugin will fallback on each other if needed. This is recommended for robustness!
+
+### Configurations: 
+
+## Configurations Summary
+
+| Option     | Behavior                                                                                                                                   | Docker Default? |
+|------------| ------------------------------------------------------------------------------------------------------------------------------------------ | --------------- |
+| `auto`     | Docker: Automatically downloads and uses the token generation script. Local: Does nothing; assumes a separate server is running externally. | ✅ Yes           |
+| `script`   | Explicitly downloads and uses the token generation script, even locally.                                                                   | ❌ No            |
+| `disabled` | Disables token generation completely.                                                                                                      | ❌ No            |
+
+Example configuration:
+
+ 
+```yaml
+generic_extractor:
+  # ...  
+  bguils_po_token_method: "script"
+  # For debugging add the verbose flag here:
+  ytdlp_args: "--no-abort-on-error --abort-on-error --verbose"
+
+```
+
+**Advanced Configuration:**
+
+If you change the default port of the bgutil-ytdlp-pot-provider server, you can pass the updated values using our `extractor_args` option for the gereric extractor.
+
+```yaml
+generic_extractor:
+  ytdlp_args: "--no-abort-on-error --abort-on-error --verbose"
+  ytdlp_update_interval: 5
+  bguils_po_token_method: "script"
+  extractor_args:
+    youtube:
+      getpot_bgutil_baseurl: "http://127.0.0.1:8080"
+      player_client: web,tv
+```
+For more details on this for bgutils see [here](https://github.com/Brainicism/bgutil-ytdlp-pot-provider?tab=readme-ov-file#usage)
+
+### Checking the logs
+
+To verify that the POT process working, look for the following lines in your log after adding the config option:
+
+```shell
+[GetPOT] BgUtilScript: Generating POT via script: /Users/you/bgutil-ytdlp-pot-provider/server/build/generate_once.js
+[debug] [GetPOT] BgUtilScript: Executing command to get POT via script: /Users/you/.nvm/versions/node/v20.18.0/bin/node /Users/you/bgutil-ytdlp-pot-provider/server/build/generate_once.js -v ymCMy8OflKM
+[debug] [GetPOT] BgUtilScript: stdout:
+{"poToken":"MlMxojNFhEJvUzGeHEkVRSK_luXtwcDnwSNIOgaUutqB7t99nmlNvtWgYayboopG6ZopZgmQ-6PJCWEMHv89MIiFGGlJRY25Fkwzxmia_8uYgf5AWf==","generatedAt":"2025-03-26T10:45:26.156Z","visitIdentifier":"ymCMy8OflKM"}
+[debug] [GetPOT] Fetching gvs PO Token for tv client
+```
+
+If it can't find the script or something, you'll see something like this:
+```shell
+[debug] [GetPOT] Fetching player PO Token for tv client
+WARNING: [GetPOT] BgUtilScript: Script path doesn't exist: /Users/you/bgutil-ytdlp-pot-provider/server/build/generate_once.js. Please make sure the script has been transpiled correctly.
+WARNING: [GetPOT] BgUtilHTTP: Error reaching GET http://127.0.0.1:4416/ping (caused by TransportError). Please make sure that the server is reachable at http://127.0.0.1:4416.
+[debug] [GetPOT] No player PO Token provider available for tv client
+```
+
+In this case check that the script has been transpiled correctly and is available at the path specified in the log, 
+or that the server is running and reachable.
+

docs/source/how_to/02_gsheets_setup.md

@@ -0,0 +1,190 @@
+# Using Google Sheets
+
+This guide explains how to set up Google Sheets to process URLs automatically and then store the archiving status back into the Google sheet. It is broadly split into 3 steps:
+
+1. Setting up your Google Sheet
+2. Setting up a service account so Auto Archiver can access the sheet
+3. Setting the Auto Archiver settings
+
+
+## 1. Setting up a Google Service Account
+
+Once your Google Sheet is set up, you need to create what's called a 'service account' that will allow the Auto Archiver to access it.
+
+To do this, you can either:
+* a) follow the steps in [this guide](https://gspread.readthedocs.io/en/latest/oauth2.html) all the way up until step 8. You should have downloaded a file called `service_account.json` and should save it in the `secrets/` folder
+* b) run the following script to automatically generate the file:
+```{code} bash
+https://raw.githubusercontent.com/bellingcat/auto-archiver/refs/heads/main/scripts/generate_google_services.sh | bash -s --
+```
+This uses gcloud to create a new project, a new user and downloads the service account automatically for you. The service account file will have the name `service_account-XXXXXXX.json` where XXXXXXX is a random 16 letter/digit string for the project created.
+
+```{note}
+To save the generated file to a different folder, pass an argument as follows:
+```{code} bash
+https://raw.githubusercontent.com/bellingcat/auto-archiver/refs/heads/main/scripts/generate_google_services.sh | bash -s -- /path/to/secrets
+```
+
+----------
+
+Once you've downloaded the file, you can save it to `secrets/service_account.json` (the default name), or to another file and then change the location in the settings (see step 4).
+
+Also make sure to **note down** the email address for this service account. You'll need that for step 3.
+
+```{note}
+The email address created in this step can be found either by opening the `service_account.json` file, or if you used b) the `generate_google_services.sh` script, then the script will have printed it out for you.
+
+The email address will look something like `user@project-name.iam.gserviceaccount.com`
+```
+
+
+## 2. Setting up your Google Sheet
+
+We recommend copying [this template Google Sheet](https://docs.google.com/spreadsheets/d/1NJZo_XZUBKTI1Ghlgi4nTPVvCfb0HXAs6j5tNGas72k/edit?usp=sharing) as a starting point for your project, as this matches all the columns required.
+
+But if you like, you can also create your own custom sheet. The only columns required are 'link', 'archive status', and 'archive location'. 'link' is the column with the URLs that you want the Auto Archiver to archive, the other two record the archival status and result. 
+
+Here's an overview of all the columns, and what a complete sheet would look like.
+
+**Inputs:**
+
+These are processed by the Gsheet Feeder and passed to the Auto Archiver.
+
+* **Link** *(required)*: the URL of the post that is to be archived
+* **Destination folder**: custom folder for archived file (regardless of storage)
+
+**Outputs:**
+
+These are updated by the Gsheet DB module during the archiving process.
+Note the required columns are only required if you are using the Gsheet DB module as well as the feeder.
+
+* **Archive status** *(required)*: Status of archive operation
+* **Archive location**: URL of archived post
+* **Archive date**: Date archived
+* **Thumbnail**: Embeds a thumbnail for the post in the spreadsheet
+* **Timestamp**: Timestamp of original post
+* **Title**: Post title
+* **Text**: Post text
+* **Screenshot**: Link to screenshot of post
+* **Hash**: Hash of archived HTML file (which contains hashes of post media) - for checksums/verification
+* **Perceptual Hash**: Perceptual hashes of found images - these can be used for de-duplication of content
+* **WACZ**: Link to a WACZ web archive of post
+* **ReplayWebpage**: Link to a ReplayWebpage viewer of the WACZ archive
+
+For example, this is a spreadsheet configured with all of the columns for the auto archiver and a few URLs to archive. 
+In this example the Ghseet Feeder and Gsheet DB are being used, and the archive is in progress.
+(Note that the column names are not case sensitive.)
+
+![A screenshot of a Google Spreadsheet with column headers defined as above, and several Youtube and Twitter URLs in the "Link" column](../../demo-before.png)
+
+We'll change the name of the 'Destination Folder' column in the Step 4a.
+
+## 3. Share your Google Sheet with your Service Account email address
+
+Remember that email address you copied in Step 1? Now that you've set up your Google sheet, click 'Share' in the top
+right hand corner and enter the email address. Make sure to give the account **Editor** access. Here's how that looks:
+
+![Share sheet](share_sheet.png)
+
+## 4. Setting up the configuration file
+
+The final step is to set your configuration. First, make sure you have `gsheet_feeder_db` set in the `steps.feeders` section of your config. If you wish to store the results of the archiving process back in your Google sheet, make sure to also put `gsheet_feeder_db` setting in the `steps.databases` section. Here's how this might look:
+
+```{code} yaml
+steps:
+    feeders:
+    - gsheet_feeder_db
+    ...
+    databases:
+    - gsheet_feeder_db # optional, if you also want to store the results in the Google sheet and tract the status of active archivals.
+    ...
+```
+
+Next, set up the `gsheet_feeder_db` configuration settings in the 'Configurations' part of the config `orchestration.yaml` file. Open up the file, and set the `gsheet_feeder_db.sheet` setting or the `gsheet_feeder_db.sheet_id` setting. The `sheet` should be the name of your sheet, as it shows in the top left of the sheet. 
+For example, the sheet [here](https://docs.google.com/spreadsheets/d/1NJZo_XZUBKTI1Ghlgi4nTPVvCfb0HXAs6j5tNGas72k/edit?gid=0#gid=0) is called 'Public Auto Archiver template'.
+
+If you saved your `service_account.json` file to anywhere other than the default location (`secrets/service_account.json`), then also make sure to change that now:
+
+Here's how this might look:
+
+```{code} yaml
+...
+gsheet_feeder_db:
+    sheet: 'My Awesome Sheet'
+    service_account: secrets/service_account-XXXXX.json # or leave as secrets/service_account.json
+    ...
+```
+
+You can also pass these settings directly on the command line without having to edit the file, here'a an example of how to do that (using docker):
+
+`docker run -it --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver:dockerize --gsheet_feeder_db.sheet "My Awesome Sheet 2"`. 
+
+Here, the sheet name has been overridden/specified in the command line invocation.
+
+### 4a. (Optional) Changing the column names
+
+In step 1, we said we would change the name of the 'Destination Folder'. Perhaps you don't like this name, or already have a sheet with a different name. In our example here, we want to name this column 'Save Folder'. To do this, we need to edit the `ghseet_feeder_db.column` setting in the configuration file. 
+For more information on this setting, see the [Gsheet Feeder Database docs](../modules/autogen/feeder/gsheet_feeder_db.md#configuration-options). We will first copy the default settings from the Gsheet Feeder docs for the 'column' settings, and then edit the 'Destination Folder' section to rename it 'Save Folder'. Our final configuration section looks like:
+
+```{code} yaml
+...
+gsheet_feeder_db:
+    sheet: 'My Awesome Sheet'
+    header: 1
+    service_account: secrets/service_account.json
+    columns:
+      url: link
+      status: archive status
+      folder: save folder # <-- note how this value has been changed
+      archive: archive location
+      date: archive date
+      thumbnail: thumbnail
+      timestamp: upload timestamp
+      title: upload title
+      text: text content
+      screenshot: screenshot
+      hash: hash
+      pdq_hash: perceptual hashes
+      wacz: wacz
+      replaywebpage: replaywebpage
+    
+```
+## 4. Running the Auto Archiver
+### Feeding the URLs to the Auto Archiver
+
+The URLs to be archived should be added to the Google Sheet, and optionally a folder value. Leave all the other configured columns empty (but you may add additional columns for your own use, as long as they don't conflict with the column names mapped in the configuration file).
+The Auto Archiver will archive  any URLs which have an empty 'status' column
+
+### Viewing the Results after archiving
+
+With the `ghseet_feeder_db` installed, once you start running the Auto Archiver, it will update the "Archive status" column.
+The status will be set to "Archive in progress" once the archival starts. If the archival is stopped during a run, either manually or because an error is raised the status value should be cleared.
+
+![A screenshot of a Google Spreadsheet with column headers defined as above, and several Youtube and Twitter URLs in the "Link" column. The auto archiver has added "archive in progress" to one of the status columns.](../../demo-progress.png)
+
+The links are downloaded and archived, and the spreadsheet is updated to the following:
+
+![A screenshot of a Google Spreadsheet with videos archived and metadata added per the description of the columns above.](../../demo-after.png)
+
+Note that the first row is skipped, as it is assumed to be a header row (`--gsheet_feeder_db.header=1` and you can change it if you use more rows above). Rows with an empty URL column, or a non-empty archive column are also skipped. All sheets in the document will be checked.
+
+The "archive location" link contains the path of the archived file, in local storage, S3, or in Google Drive.
+
+![The archive result for a link in the demo sheet.](../../demo-archive.png)
+
+### Troubleshooting
+
+**Hanging Archival in progress status**
+
+Occasionally system crashes or other unexpected events can cause the Auto Archiver to exit without cleaning up the status value.
+If you are sure that all archival processes have stopped but you still see "Archive in progress" in the status column, you can manually clear the status column to allow the Auto Archiver to retry that archival on the next run.
+
+**Nothing archived status**
+
+Sometimes this means the tool is genuinely unable to extract the content at this point in time, but sometimes it can be resolved with different configurations. 
+Try:
+  - Turning on additional 'extractor' types in the configuration file (this can appear as 'no archiver' in the status column). 
+  - Changing credentials or refreshing session files for extractors which require them
+  - Check if the extractors can accept any additional configurations such as adding a cookie file.
+
+

docs/source/how_to/03_logging.md

@@ -0,0 +1,104 @@
+# Keeping Logs
+
+Auto Archiver's logs can be helpful for debugging problematic archiving processes. This guide shows you how to use the logs configuration.
+
+## Setting up logging
+
+Logging settings can be set on the command line or using the orchestration config file ([learn more](../installation/configuration)). A special `logging` section defines the logging options.
+
+#### Enabling or Disabling Logging
+
+Logging to the console is enabled by default. If you want to globally disable Auto Archiver's logging, then you can set `enabled: false` in your `logging` config file:
+
+```{code} yaml
+:caption: orchestration.yaml
+...
+logging:
+   enabled: false
+...
+```
+
+```{note}
+This will disable all logs from Auto Archiver, but it does not disable logs for other tools that the Auto Archiver uses (for example: yt-dlp, firefox or ffmpeg). These logs will still appear in your console.
+```
+
+#### Logging Level
+
+There are 7 logging levels in total, with 5 of them used in this tool. They are: `DEBUG`, `INFO`, `SUCCESS`, `WARNING` and `ERROR`. If you select a level, only that and higher (more serious) levels will be included. `DEBUG` is the most verbose, while `ERROR` is the least verbose. 
+
+Change the warning level by setting the value in your orchestration config file:
+
+```{code} yaml
+:caption: orchestration.yaml
+
+...
+logging:
+    level: DEBUG # or INFO / WARNING / ERROR
+...
+```
+
+For normal usage, it is recommended to use the `INFO` level, or if you prefer quieter logs with less information, you can use the `WARNING` level. If you encounter issues with the archiving, then it's recommended to enable the `DEBUG` level.
+
+```{note} To learn about all logging levels, see the [loguru documentation](https://loguru.readthedocs.io/en/stable/api/logger.html)
+```
+
+### Logging Format
+By default, the console logs are formatted in a human-readable way and the file logs are formatted in JSON. This is new from version 1.1.1. If you want to change the format of the console logs to JSON too you can set the `format:` option in your logging settings. 
+
+```{code} yaml
+:caption: orchestration.yaml
+
+logging:
+    format: json
+```
+
+When the Auto Archiver is writing logs it will include context about specific tasks, so if you are archiving a URL from a Google Sheet, both the URL (and a unique `trace_id` for that URL's archiving attempt) and the Spreadsheet name and row will be included in the logs. This is useful for debugging and understanding what the Auto Archiver is doing.
+
+Using JSON allows you to easily parse the logs and extract specific information, tools like [`jq`](https://jqlang.org/) can be used to filter and search through the logs.
+
+### Logging to a file
+
+As default, auto-archiver will log to the console. But if you wish to store your logs for future reference, or you are running the auto-archiver from within code a implementation, then you may wish to enable file logging. This can be done by setting the `file:` config value in the logging settings.
+
+**Rotation:** For file logging, you can choose to 'rotate' your log files (creating new log files) so they do not get too large. Change this by setting the 'rotation' option in your logging settings. For a full list of rotation options, see the [loguru docs](https://loguru.readthedocs.io/en/stable/overview.html#easier-file-logging-with-rotation-retention-compression).
+
+```{code} yaml
+:caption: orchestration.yaml
+
+logging:
+    ...
+    file: /my/log/file.log
+    rotation: 1 day
+```
+
+### Logging each level to a different file
+If you want to log each level to a different file, you can do this by setting the `each_level_in_separate_file:` option to `true` and also setting your `file:` name, a new file will be created for each of the 5 levels used, by appending the `0_level` name to the file like so `your_file.log.1_error`. In this case the `level:` option is ignored, and all levels will be logged. 
+
+
+```{code} yaml
+:caption: orchestration.yaml
+
+logging:
+    each_level_in_separate_file: true
+    file: /my/logs/file.log 
+```
+This will create the following files:
+- `/my/logs/file.log.1_debug`
+- `/my/logs/file.log.2_info`
+- `/my/logs/file.log.3_success`
+- `/my/logs/file.log.4_warning`
+- `/my/logs/file.log.5_error`
+
+### Full logging example
+
+The below example logs only `DEBUG` logs to the console and to the file `/my/file.log`, rotating that file once per week:
+
+```{code} yaml
+:caption: orchestration.yaml
+
+logging:
+    level: DEBUG
+    format: json
+    file: /my/file.log
+    rotation: 1 week
+```

docs/source/how_to/04_run_instagrapi_server.md

@@ -0,0 +1,169 @@
+# InstagrAPI Server
+
+The instagram API Extractor requires access to a running instance of the InstagrAPI server. 
+We have a lightweight script with the endpoints required for our Instagram API Extractor module which you can run locally, or via Docker.
+
+
+
+⚠️ Warning: Remember that it's best not to use your own personal account for archiving. [Here's why](../installation/authentication.md#recommendations-for-authentication).
+## Quick Start: Using Docker
+
+We've provided a convenient shell script (`run_instagrapi_server.sh`) that simplifies the process of setting up and running the Instagrapi server in Docker. This script handles building the Docker image, setting up credentials, and starting the container.
+
+### 🔧 Running the script:
+
+Run this script either from the repository root or from within the `scripts/instagrapi_server` directory:
+
+```bash
+./scripts/instagrapi_server/run_instagrapi_server.sh
+```
+
+This script will:
+- Prompt for your Instagram username and password.
+- Create the necessary `.env` file.
+- Build the Docker image.
+- Start the Docker container and authenticate with Instagram, creating a session automatically.
+
+### ⏱ To run the server again later:
+```bash
+docker start ig-instasrv
+```
+
+### 🐛 Debugging:
+View logs:
+```bash
+docker logs ig-instasrv
+```
+
+
+### Overview: How the Setup Works
+
+1. You enter your Instagram credentials in a local `.env` file
+2. You run the server **once locally** to generate a session file
+3. After that, you can choose to run the server again locally or inside Docker without needing to log in again
+
+---
+
+## Optional: Manual / Local Setup
+
+If you'd prefer to run the server manually (without Docker), you can follow these steps:
+
+
+1. **Navigate to the server folder (and stay there for the rest of this guide)**:
+   ```bash
+   cd scripts/instagrapi_server
+   ```
+
+2. **Create a `secrets/` folder** (if it doesn't already exist in `scripts/instagrapi_server`):
+   ```bash
+   mkdir -p secrets
+   ```
+
+3. **Create a `.env` file** inside `secrets/` with your Instagram credentials:
+   ```dotenv
+   INSTAGRAM_USERNAME="your_username"
+   INSTAGRAM_PASSWORD="your_password"
+   ```
+
+4. **Install dependencies** using the pyproject.toml file:
+  
+   ```bash
+   poetry install --no-root
+   ```
+
+5. **Run the server locally**:
+   ```bash
+   poetry run uvicorn src.instaserver:app --port 8000
+   ```
+
+6. **Watch for the message**:
+   ```
+   Login successful, session saved.
+   ```
+
+✅ Your session is now saved to `secrets/instagrapi_session.json`.
+
+### To run it again locally:
+```bash
+poetry run uvicorn src.instaserver:app --port 8000
+```
+
+---
+
+## Adding the API Endpoint to Auto Archiver
+
+The server should now be running within that session, and accessible at  http://127.0.0.1:8000 
+
+You can set this in the Auto Archiver orchestration.yaml file like this:
+```yaml
+instagram_api_extractor:
+  api_endpoint: http://127.0.0.1:8000
+```
+
+
+---
+
+## 2. Running the Server Again
+
+Once the session file is created, you should be able to run the server without logging in again.
+
+### To run it locally (from scripts/instagrapi_server):
+```bash
+poetry run uvicorn src.instgrapinstance.instaserver:app --port 8000
+```
+
+---
+
+## 3. Running via Docker (After Setup is Complete, either locally or via the script)
+
+Once the `instagrapi_session.json` and `.env` files are set up, you can pass them Docker and it should authenticate successfully.
+
+### 🔨 Build the Docker image manually:
+```bash
+docker build -t instagrapi-server .
+```
+
+### ▶️ Run the container:
+```bash
+docker run -d \
+  --env-file secrets/.env \
+  -v "$(pwd)/secrets:/app/secrets" \
+  -p 8000:8000 \
+  --name ig-instasrv \
+  instagrapi-server
+```
+
+This passes the /secrets/ directory to docker as well as the environment variables from the `.env` file.
+
+
+
+---
+
+## 4. Optional Cleanup
+
+- **Stop the Docker container**:
+  ```bash
+  docker stop ig-instasrv
+  ```
+
+- **Remove the container**:
+  ```bash
+  docker rm ig-instasrv
+  ```
+
+- **Remove the Docker image**:
+  ```bash
+  docker rmi instagrapi-server
+  ```
+
+### ⏱ To run again later:
+```bash
+docker start ig-instasrv
+```
+
+---
+
+##  Notes
+
+- Never share your `.env` or `instagrapi_session.json` — these contain sensitive login data. 
+- If you want to reset your session, simply delete the `secrets/instagrapi_session.json` file and re-run the local server.

docs/source/how_to/05_upgrading_to_1_1_0.md

@@ -0,0 +1,54 @@
+# Upgrading from v1.0.1
+
+```{note} This how-to is only relevant for people who used Auto Archiver before June 2025 (versions prior to 1.1.0).
+
+If you are new to Auto Archiver, then you are already using the latest configuration format and this how-to is not relevant for you.
+```
+
+Versions 1.1.0+ of Auto Archiver has breaking changes in the configuration format, which means earlier configuration formats will not work without slight modifications.
+
+
+## Dropping `vk_extractor` module
+We have dropped the `vk_extractor` because of problems in a project we relied on. You will need to remove it from your configuration file, otherwise you will see an error like:
+
+```{code} console
+Module 'vk_extractor' not found. Are you sure it's installed/exists?
+```
+
+## Dropping `screenshot_enricher` module
+We have dropped the `screenshot_enricher` module because a new `antibot_extractor_enricher` (see below) module replaces its functionality more robustly and with less dependency hassle on geckodriver/firefox. You will need to remove it from your configuration file, otherwise you will see an error like:
+
+```{code} console
+Module 'screenshot_enricher' not found. Are you sure it's installed/exists?
+```
+
+
+## New `antibot_extractor_enricher` module and VkDropin
+We have added a new [`antibot_extractor_enricher`](../modules/autogen/extractor/antibot_extractor_enricher.md) module that uses a computer-controlled browser to extract content from websites that use anti-bot measures. You can add it to your configuration file like this:
+
+```{code} yaml
+steps:
+  extractors:
+    - antibot_extractor_enricher
+
+  # or alternatively, if you want to use it as an enricher:
+  enrichers:
+    - antibot_extractor_enricher
+```
+
+It will take a full page screenshot, a PDF capture, extract HTML source code, and any other relevant media. 
+
+It comes with Dropins that we will be adding and maintaining. 
+
+> Dropin: A module with site-specific behaviours that is loaded automatically. You don't need to add them to your configuration steps for them to run. Sometimes they need `authentication` configurations though.
+
+One such Dropin is the VkDropin which uses this automated browser to access VKontakte (VK) pages. You should add a username/password to the configuration file if you get authentication blocks from VK, to do so use the [authentication settings](authentication_how_to.md):
+
+```{code} yaml
+authentication:
+  vk.com:
+    username: your_username
+    password: your_password
+```
+
+See all available Dropins in [the source code](https://github.com/bellingcat/auto-archiver/tree/main/src/auto_archiver/modules/antibot_extractor_enricher/dropins). Usually each Dropin needs its own authentication settings, similarly to the VkDropin.

docs/source/how_to/06_new_config_format.md

@@ -0,0 +1,145 @@
+# Upgrading from v0.12
+
+```{note} This how-to is only relevant for people who used Auto Archiver before February 2025 (versions prior to 0.13).
+
+If you are new to Auto Archiver, then you are already using the latest configuration format and this how-to is not relevant for you.
+```
+
+Versions 0.13+ of Auto Archiver has breaking changes in the configuration format, which means earlier configuration formats will not work without slight modifications.
+
+## How do I know if I need to update my configuration format?
+
+There are two simple ways to check if you need to update your format:
+
+1. When you try and run auto-archiver using your existing configuration file, you get an error about no feeders or formatters being configured, like:
+
+```{code} console
+AssertionError: No feeders were configured. Make sure to set at least one feeder in
+your configuration file or on the command line (using --feeders)
+```
+
+2. Within your configuration file, you have a `feeder:` option. This is the old format. An example old format:
+```{code} yaml
+
+steps:
+  feeder: cli_feeder
+...
+```
+
+The next two sections outline the two methods you have for updating your file.
+
+## 1. Manually edit the configuration file and change the values.
+
+This is recommended if you want to keep all your old settings. Follow the steps below to change the relevant settings:
+
+#### a) Feeder & Formatter Steps Settings
+
+The feeder and formatter settings have been changed from a single string to a list.
+
+- `steps.feeder (string)` → `steps.feeders (list)`
+- `steps.formatter (string)` → `steps.formatters (list)`
+
+Example:
+
+```{code} yaml
+
+steps:
+   feeder: cli_feeder
+   ...
+   formatter: html_formatter
+
+# the above should be changed to:
+steps:
+   feeders:
+   - cli_feeder
+   ...
+   formatters:
+   - html_formatter
+```
+
+```{note} Auto Archiver still only supports one feeder and formatter, but from v0.13 onwards they must be added to the configuration file as a list.
+```
+
+#### b) Extractor (formerly Archiver) Steps Settings
+
+With v0.13 of Auto Archiver, `archivers` have been renamed to `extractors` to better reflect what they actually do - extract information from a URL. Change the configuration by renaming:
+
+- `steps.archivers` → `steps.extractors`
+
+The names of the actual modules have also changed, so for any extractor modules you have enabled, you will need to rename the `archiver` part to `extractor`. Some examples:
+
+- `telethon_archiver` → `telethon_extractor`
+- `wacz_archiver_enricher` → `wacz_extractor_enricher`
+- `wayback_archiver_enricher` → `wayback_extractor_enricher`
+
+
+#### c) Module Renaming
+
+
+The `youtube_archiver` has been renamed to `generic_extractor` as it is considered the default/fallback extractor. Read more about the [generic extractor](../modules/autogen/extractor/generic_extractor.md).
+
+The `atlos` modules have been merged into one, as have the `gsheets` feeder and database.
+
+- `atlos_feeder` → `atlos_feeder_db_storage`
+- `atlos_storage` → `atlos_feeder_db_storage`
+- `atlos_db` → `atlos_feeder_db_storage`
+- `gsheet_feeder` → `gsheet_feeder_db`
+- `gsheet_db` → `gsheet_feeder_db`
+
+
+Example:
+```{code} yaml
+steps:
+   feeders:
+   - gsheet_feeder_db # formerly gsheet_feeder
+   ...
+   extractors: # formerly 'archivers'
+   - telethon_extractor # formerly telethon_archiver
+   - generic_extractor # formerly youtube_archiver
+   - vk_extractor # formerly vk_archiver
+   databases:
+   - gsheet_feeder_db # formerly gsheet_db
+   ...
+
+```
+
+```{note}
+
+Don't forget to also rename the configuration settings. For example:
+
+```{code} yaml
+gsheet_feeder_db: # formerly gsheet_feeder
+  service_account: secrets/service_account.json
+  sheet: My Google Sheet
+...
+```
+
+#### d) Redundant / Obsolete Modules
+
+With v0.13 of Auto Archiver, the following modules have been removed and their features have been built in to the generic_extractor. You should remove them from the 'steps' section of your configuration file:
+
+* `twitter_archiver` - use the `generic_extractor` for general extraction, or the `twitter_api_extractor` for API access.
+* `tiktok_archiver` - use the `generic_extractor` to extract TikTok videos.
+
+
+## 2. Auto-generate a new config, then copy over your settings.
+
+Using this method, you can have Auto Archiver auto-generate a configuration file for you, then you can copy over the desired settings from your old config file. This is probably the easiest method and quickest to setup, but it may require some trial and error as you copy over your settings.
+
+First, move your existing `orchestration.yaml` file to a different folder or rename it.
+
+Then, you can generate a `simple` or `full` config using:
+
+```{code} console
+>>> # generate a simple config
+>>> auto-archiver 
+>>> # config will be written to orchestration.yaml
+>>> 
+>>> # generate a full config
+>>> auto-archiver --mode=full
+>>> 
+```
+
+After this, copy over any settings from your old config to the new config.
+
+

docs/source/index.md

@@ -8,10 +8,10 @@
 :caption: Contents:
 
 Overview <self>
-contributing
-installation/installation.rst
+installation/setup
 core_modules.md
 how_to
+contributing
 development/developer_guidelines
 autoapi/index.rst
 ```

docs/source/installation/authentication.md

@@ -0,0 +1,82 @@
+# Authentication
+
+The Authentication framework for auto-archiver allows you to add login details for various websites in a flexible way, directly from the configuration file.
+
+There are two main use cases for authentication:
+* Some websites require some kind of authentication in order to view the content. Examples include Facebook, Telegram etc.
+* Some websites use anti-bot systems to block bot-like tools from accessing the website. Adding real login information to auto-archiver can sometimes bypass this.
+
+```{note}
+
+The Authentication framework currently only works with the following modules:
+* [Generic Extractor](../modules/autogen/extractor/generic_extractor.md) - the main module for extracting content from websites
+* [Antibot Extractor/Enricher](../modules/autogen/extractor/antibot_extractor_enricher.md)
+
+To authenticate for WACZ archiving, see the instructions on the [](../modules/autogen/enricher/wacz_extractor_enricher.md) page.
+```
+
+## The Authentication Config
+
+You can save your authentication information directly inside your orchestration config file, or as a separate file (for security/multi-deploy purposes). Whether storing your settings inside the orchestration file, or as a separate file, the configuration format is the same. Currently, auto-archiver supports the following authentication types:
+
+**Username & Password:**
+- `username`: str - the username to use for login
+- `password`: str - the password to use for login
+
+**API**
+- `api_key`: str - the API key to use for login
+- `api_secret`: str - the API secret to use for login
+  
+**Cookies**
+- `cookie`: str - a cookie string to use for login (specific to this site)
+- `cookies_from_browser`: str - load cookies from this browser, for this site only.
+- `cookies_file`: str - load cookies from this file, for this site only.
+
+```{note} 
+
+Currently, the Username & Password, and API settings only work with the Generic and Antibot Extractors. Furthermore, many sites can still detect bots and block username/password logins. Twitter/X and YouTube are two prominent ones that block username/password logins.
+
+
+One of the 'Cookies' options is recommended for the most robust archiving, but it still isn't guaranteed to work.
+```
+
+```{code} yaml
+authentication:
+   # optional file to load authentication information from, for security or multi-system deploy purposes
+   load_from_file: path/to/authentication/file.txt
+   # optional setting to load cookies from the named browser on the system, for **ALL** websites
+   cookies_from_browser: firefox
+   # optional setting to load cookies from a cookies.txt/cookies.jar file, for **ALL** websites. See note below on extracting these
+   cookies_file: path/to/cookies.jar
+
+   mysite.com:
+      username: myusername
+      password: 123
+    
+   facebook.com:
+      cookie: single_cookie
+
+   othersite.com:
+      api_key: 123
+      api_secret: 1234
+  
+```
+
+
+### Recommendations for authentication
+
+1. **Store authentication information separately:**
+The authentication part of your configuration contains sensitive information. You should make efforts not to share this with others. For extra security, use the `load_from_file` option to keep your authentication settings out of your configuration file, ideally in a different folder.
+
+2. **Don't use your own personal credentials**
+Depending on the website you are extracting information from, there may be rules (Terms of Service) that prohibit you from scraping or extracting information using a bot. If you use your own personal account, there's a possibility it might get blocked/disabled. It's recommended to set up a separate, 'throwaway' account. In that way, if it gets blocked you can easily create another one to continue your archiving.
+
+
+### How to create a cookies.jar or pass cookies directly to auto-archiver
+
+auto-archiver uses yt-dlp's powerful cookies features under the hood. For instructions on how to extract a cookies.jar (or cookies.txt) file directly from your browser, see the FAQ in the [yt-dlp documentation](https://github.com/yt-dlp/yt-dlp/wiki/FAQ#how-do-i-pass-cookies-to-yt-dlp)
+
+```{note} For developers:
+
+For information on how to access and use authentication settings from within your module, see the `{generic_extractor}` for an example, or view the [`auth_for_site()` function in BaseModule](../autoapi/core/base_module/index.rst)
+```

docs/source/installation/config_editor.md

@@ -0,0 +1,5 @@
+# Configuration Editor
+
+```{raw} html
+:file: settings.html
+```

docs/source/installation/configurations.md

@@ -1,13 +1,18 @@
 
 # Configuration
 
-This section of the documentation provides guidelines for configuring the tool.
+The recommended way to configure auto-archiver for first-time users is to [run the Auto Archiver](setup.md#running) and have it auto-generate a default configuration for you. Then, if needed, you can edit the configuration file using one of the following methods.
 
-## Configuring using a file
 
-The recommended way to configure auto-archiver for long-term and deployed projects is a configuration file, typically called `orchestration.yaml`. This is a YAML file containing all the settings for your entire workflow.
+## 1. Configuration file
 
-The structure of orchestration file is split into 2 parts: `steps` (what [steps](../flow_overview.md) to use) and `configurations` (settings for different modules), here's a simplification:
+The configuration file is typically called `orchestration.yaml` and stored in the `secrets` folder on your desktop. The configuration file contains all the settings for your entire Auto Archiver workflow in one easy-to-find place.
+
+If you want to have Auto Archiver run with the recommended 'basic' setup, 
+
+### Advanced Configuration
+
+The structure of orchestration file is split into 2 parts: `steps` (what [steps](../flow_overview.md) to use) and `configurations` (settings for individual modules).
 
 A default `orchestration.yaml` will be created for you the first time you run auto-archiver (without any arguments). Here's what it looks like:
 
@@ -21,9 +26,9 @@ A default `orchestration.yaml` will be created for you the first time you run au
 
 </details>
 
-## Configuring from the Command Line
+## 2. Command Line configuration
 
-You can run auto-archiver directy from the command line, without the need for a configuration file, command line arguments are parsed using the format `module_name.config_value`. For example, a config value of `api_key` in the `instagram_extractor` module would be passed on the command line with the flag `--instagram_extractor.api_key=API_KEY`.
+You can run auto-archiver directly from the command line, without the need for a configuration file, command line arguments are parsed using the format `module_name.config_value`. For example, a config value of `api_key` in the `instagram_extractor` module would be passed on the command line with the flag `--instagram_extractor.api_key=API_KEY`.
 
 The command line arguments are useful for testing or editing config values and enabling/disabling modules on the fly. When you are happy with your settings, you can store them back in your configuration file by passing the `-s/--store` flag on the command line.
 

docs/source/installation/example_scripts/ubuntu_24_server_install.md

@@ -0,0 +1,160 @@
+
+### Bash script for Ubuntu 24 Server install
+
+> NOTE: this script has not been tested by the maintainers and results from the personal experience of a user. It is meant as a guide and not an out of the box script, as you will see it's aimed at a custom branches, users, and features like the Geckodriver which are removed as of version 1.0.2. 
+
+This acts as a handy guide on all requirements. This is built and tested on the 29th of May 2025 on Ubuntu Server 24.04.2 LTS (which is the current latest LTS)
+
+```bash
+#!/bin/sh
+
+# I usually run steps manually as logged in with the user: dave
+# which the application runs under which makes debugging easier
+
+cd ~
+sudo apt update -y
+sudo apt upgrade -y
+
+# Clone only my latest branch
+git clone -b v1-test --single-branch https://github.com/djhmateer/auto-archiver
+
+mkdir ~/auto-archiver/secrets
+sudo chown -R dave ~/auto-archiver
+
+sudo apt update -y
+sudo apt upgrade -y
+
+## Python 3.12.3 comes with Ubuntu 24.04.2
+
+# Poetry install 2.1.3 on 2nd June 25
+curl -sSL https://install.python-poetry.org | python3 -
+
+# had to restart here.. 
+sudo reboot
+
+# C++ compiler so pdqhash will install next
+sudo apt install build-essential python3-dev -y
+
+cd auto-archiver
+
+poetry install
+
+# FFMpeg
+# 6.1.1-3ubuntu5 on 2nd June 25
+sudo apt install ffmpeg -y
+
+## Firefox
+# 139.0+build2-0ubuntu0.24.04.1~mt1 on 2nd Jun 25
+# 16th Jun - don't need anymore as using Chrome in antibot
+# cd ~
+# sudo add-apt-repository ppa:mozillateam/ppa -y
+
+# echo '
+# Package: *
+# Pin: release o=LP-PPA-mozillateam
+# Pin-Priority: 1001
+# ' | sudo tee /etc/apt/preferences.d/mozilla-firefox
+
+# echo 'Unattended-Upgrade::Allowed-Origins:: "LP-PPA-mozillateam:${distro_codename}";' | sudo tee /etc/apt/apt.conf.d/51unattended-upgrades-firefox
+
+# sudo apt install firefox -y
+
+wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
+
+# Chrome
+cd ~
+# got problems here - fixed below
+# 137.0.7151.103 on 16th Jun 2025
+sudo dpkg -i google-chrome-stable_current_amd64.deb
+
+# fix dependencies on install above
+sudo apt-get install -f
+
+# had to click a lot on UI to get going.
+# to test
+# google-chrome
+
+## Gecko driver
+# check version numbers for new ones
+# https://github.com/mozilla/geckodriver/releases/
+wget https://github.com/mozilla/geckodriver/releases/download/v0.36.0/geckodriver-v0.36.0-linux64.tar.gz
+tar -xvzf geckodriver*
+chmod +x geckodriver
+sudo mv geckodriver /usr/local/bin/
+rm geckodriver*
+
+# Fonts so selenium via firefox can render other languages eg Burmese
+sudo apt install fonts-noto -y
+
+# Docker
+# Add Docker's official GPG key:
+sudo apt-get update -y
+sudo apt-get install ca-certificates curl -y
+sudo install -m 0755 -d /etc/apt/keyrings
+sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
+sudo chmod a+r /etc/apt/keyrings/docker.asc
+
+# Add the repository to Apt sources:
+echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
+  $(. /etc/os-release && echo "${UBUNTU_CODENAME:-$VERSION_CODENAME}") stable" | \
+  sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+
+sudo apt-get update -y
+
+sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -y
+
+# add dave user to docker group 
+sudo usermod -aG docker $USER
+
+# reboot otherwise can't pull images
+
+# https://github.com/webrecorder/browsertrix-crawler
+# https://hub.docker.com/r/webrecorder/browsertrix-crawler/tags
+# 1.6.2 on 4th Jun 2025
+docker pull webrecorder/browsertrix-crawler:latest
+
+# exif
+sudo apt install libimage-exiftool-perl -y
+
+
+## CRON run every minute
+# the cron job running as user dave will execute the shell script
+# I have many scripts running from cron_11 upwards.
+# patch in the correct number
+sudo chmod +x ~/auto-archiver/scripts/cron_15.sh
+
+# don't want service to run until a reboot otherwise problems with Gecko driver
+sudo service cron stop
+
+# runs the script every minute
+# notice put in a # to disable so will have to manually start it.
+cat <<EOT >> run-auto-archive
+#*/1 * * * * dave /home/dave/auto-archiver/scripts/cron_15.sh
+EOT
+
+sudo mv run-auto-archive /etc/cron.d
+sudo chown root /etc/cron.d/run-auto-archive
+sudo chmod 600 /etc/cron.d/run-auto-archive
+
+# Helper alias 'c' to open the above file
+echo "alias c='sudo vim /etc/cron.d/run-auto-archive'" >> ~/.bashrc
+
+# secrets folder copy
+# I run dev from:
+# \\wsl.localhost\Ubuntu-24.04\home\dave\code\auto-archiver\secrets\
+
+# orchestration.yaml - for aa config
+# service_account - for google spreadsheet
+# anon.session - for telethon so don't have to type in phone number
+# profile.tar.gz - for wacz to have a logged in profile for facebook, x.com and instagram to get data
+
+# Youtube - POT Tokens
+# https://github.com/Brainicism/bgutil-ytdlp-pot-provider
+docker run --name bgutil-provider --restart unless-stopped -d -p 4416:4416 brainicism/bgutil-ytdlp-pot-provider
+
+
+# test run
+cd ~/auto-archiver
+
+poetry run python src/auto_archiver --config secrets/orchestration-aa-demo-main.yaml
+```

docs/source/installation/faq.md

@@ -0,0 +1,59 @@
+# Frequently Asked Questions
+
+
+### Q: What websites does the Auto Archiver support?
+**A:** The Auto Archiver works for a large variety of sites. Firstly, the Auto Archiver can download
+and archive any video website supported by YT-DLP, a powerful video-downloading tool ([full list of of
+sites here](https://github.com/yt-dlp/yt-dlp/blob/master/supportedsites.md)). Aside from these sites,
+there are various different 'Extractors' for specific websites. See the full list of extractors that 
+are available on the [extractors](../modules/extractor.md) page. Some sites supported include:
+
+* Twitter
+* Instagram
+* Telegram
+* Tiktok
+* Bluesky
+
+```{note} What websites the Auto Archiver can archie depends on what extractors you have enabled in
+your configuration. See [configuration](./configurations.md) for more info.
+```
+
+### Q: Does the Auto Archiver only work for social media posts ?
+**A:** No, the Auto Archiver can archive any web page on the internet, not just social media posts.
+However, for social media posts Auto Archiver can extract more relevant/useful information (such as 
+post comments, likes, author etc.) which may not be available for a generic website. If you are looking
+to more generally archive webpages, then you should make sure to enable the [](../modules/autogen/extractor/wacz_extractor_enricher.md)
+and the [](../modules/autogen/extractor/wayback_extractor_enricher.md).
+
+### Q: What kind of data is stored for each webpage that's archived?
+**A:** This depends on the website archived, but more generally, for social media posts any videos and photos in
+the post will be archived. For video sites, the video will be downloaded separately. For most of these sites, additional
+metadata such as published date, uploader/author and ratings/comments will also be saved. Additionally, further data can be
+saved depending on the enrichers that you have enabled. Some other types of data saved are timestamps if you have the 
+[](../modules/autogen/enricher/timestamping_enricher.md) or [](../modules/autogen/enricher/opentimestamps_enricher.md) enabled,
+screenshots of the web page with the [](../modules/autogen/enricher/screenshot_enricher.md), and for videos, thumbnails of the
+video with the [](../modules/autogen/enricher/thumbnail_enricher.md). You can also store things like hashes (SHA256, or pdq hashes)
+with the various hash enrichers.
+
+### Q: Where is my data stored?
+**A:** With the default configuration, data is stored on your local computer in the `local_storage` folder. You can adjust these settings by
+changing the [storage modules](../modules/storage.md) you have enabled. For example, you could choose to store your data in an S3 bucket or 
+on Google Drive. 
+
+```{note}
+You can choose to store your data in multiple places, for example your local drive **and** an S3 bucket for redundancy.
+```
+
+### Q: What should I do is something doesn't work?
+**A:** First, read through the log files to see if you can find a specific reason why something isn't working. Learn more about logging
+and how to enable debug logging in the [Logging Howto](../how_to/logging.md).
+
+If you cannot find an answer in the logs, then try searching this documentation or existing / closed issues on the [Github Issue Tracker](https://github.com/bellingcat/auto-archiver/issues?q=is%3Aissue%20). If you still cannot find an answer, then consider opening an issue on the Github Issue Tracker or asking in the Bellingcat Discord
+'Auto Archiver' group.
+
+#### Common reasons why an archiving might not work:
+
+* The website may have temporarily adjusted its settings - sometimes sites like Telegram or Twitter adjust their scraping protection settings. Often,
+waiting a day or two and then trying again can work.
+* The site requires you to be logged in - you could try using cookies or authentication to bypass any blocks. See [](../installation/authentication.md) for more information.
+* The website you're trying to archive has changed its settings/structure. Make sure you're using the latest version of Auto Archiver and try again.

docs/source/installation/installation.md

@@ -1,91 +1,64 @@
-# Installing Auto Archiver
+# Installation
 
 ```{toctree}
-:depth: 1
-:hidden:
+:maxdepth: 1
 
-configurations.md
-config_cheatsheet.md
+upgrading.md
 ```
 
-There are 3  main ways to use the auto-archiver:
-1. Easiest: [via docker](#installing-with-docker)
+There are 3 main ways to use the auto-archiver. We recommend the 'docker' method for most uses. This installs all the requirements in one command.
+
+1. Easiest (recommended): [via docker](#installing-with-docker)
 2. Local Install: [using pip](#installing-locally-with-pip)
 3. Developer Install: [see the developer guidelines](../development/developer_guidelines)
 
-
-But **you always need a configuration/orchestration file**, which is where you'll configure where/what/how to archive. Make sure you read [orchestration](#orchestration).
-
-
-## Installing with Docker
+## 1. Installing with Docker
 
 [![dockeri.co](https://dockerico.blankenship.io/image/bellingcat/auto-archiver)](https://hub.docker.com/r/bellingcat/auto-archiver)
 
-Docker works like a virtual machine running inside your computer, it isolates everything and makes installation simple. Since it is an isolated environment when you need to pass it your orchestration file or get downloaded media out of docker you will need to connect folders on your machine with folders inside docker with the `-v` volume flag.
+Docker works like a virtual machine running inside your computer, making installation simple. You'll need to first set up Docker, and then download the Auto Archiver 'image':
 
 
-1. Install [docker](https://docs.docker.com/get-docker/)
-2. Pull the auto-archiver docker [image](https://hub.docker.com/r/bellingcat/auto-archiver) with `docker pull bellingcat/auto-archiver`
-3. Run the docker image locally in a container: `docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver --config secrets/orchestration.yaml` breaking this command down:
-   1. `docker run` tells docker to start a new container (an instance of the image)
-   2. `--rm` makes sure this container is removed after execution (less garbage locally)
-   3. `-v $PWD/secrets:/app/secrets` - your secrets folder
-      1. `-v` is a volume flag which means a folder that you have on your computer will be connected to a folder inside the docker container
-      2. `$PWD/secrets` points to a `secrets/` folder in your current working directory (where your console points to), we use this folder as a best practice to hold all the secrets/tokens/passwords/... you use
-      3. `/app/secrets` points to the path the docker container where this image can be found
-   4.  `-v $PWD/local_archive:/app/local_archive` - (optional) if you use local_storage
-       1.  `-v` same as above, this is a volume instruction
-       2.  `$PWD/local_archive` is a folder `local_archive/` in case you want to archive locally and have the files accessible outside docker
-       3.  `/app/local_archive` is a folder inside docker that you can reference in your orchestration.yml file 
+**a) Download and install docker**
 
-### Example invocations
+Go to the [Docker website](https://docs.docker.com/get-docker/) and download right version for your operating system. 
 
-The invocations below will run the auto-archiver Docker image using a configuration file that you have specified
+**b) Pull the Auto Archiver docker image**
+
+Open your command line terminal, and copy-paste / type:
 
 ```bash
-# all the configurations come from ./secrets/orchestration.yaml
-docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver --config secrets/orchestration.yaml
-# uses the same configurations but for another google docs sheet 
-# with a header on row 2 and with some different column names
-# notice that columns is a dictionary so you need to pass it as JSON and it will override only the values provided
-docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver --config secrets/orchestration.yaml --gsheet_feeder.sheet="use it on another sheets doc" --gsheet_feeder.header=2 --gsheet_feeder.columns='{"url": "link"}'
-# all the configurations come from orchestration.yaml and specifies that s3 files should be private
-docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver --config secrets/orchestration.yaml --s3_storage.private=1
+docker pull bellingcat/auto-archiver
 ```
 
-## Installing Locally with Pip
+This will download the docker image, which may take a while.
+
+That's it, all done! You're now ready to set up [your configuration file](configurations.md). Or, if you want to use the recommended defaults, then you can [run Auto Archiver immediately](setup.md#running-a-docker-install).
+
+------------
+
+## 2. Installing Locally with Pip
 
 1. Make sure you have python 3.10 or higher installed
 2. Install the package with your preferred package manager: `pip/pipenv/conda install auto-archiver` or `poetry add auto-archiver`
 3. Test it's installed with `auto-archiver --help`
-4. Install other local dependency requirements (for )
-5. Run it with your orchestration file and pass any flags you want in the command line `auto-archiver --config secrets/orchestration.yaml` if your orchestration file is inside a `secrets/`, which we advise
+4. Install other local dependency requirements (for example `ffmpeg`, `firefox`)
 
-### Example invocations
-
-Once all your [local requirements](#installing-local-requirements) are correctly installed, the
-
-```bash
-# all the configurations come from ./secrets/orchestration.yaml
-auto-archiver --config secrets/orchestration.yaml
-# uses the same configurations but for another google docs sheet 
-# with a header on row 2 and with some different column names
-# notice that columns is a dictionary so you need to pass it as JSON and it will override only the values provided
-auto-archiver --config secrets/orchestration.yaml --gsheet_feeder.sheet="use it on another sheets doc" --gsheet_feeder.header=2 --gsheet_feeder.columns='{"url": "link"}'
-# all the configurations come from orchestration.yaml and specifies that s3 files should be private
-auto-archiver --config secrets/orchestration.yaml --s3_storage.private=1
-```
+After this, you're ready to set up your [your configuration file](configurations.md), or if you want to use the recommended defaults, then you can [run Auto Archiver immediately](setup.md#running-a-local-install).
 
 ### Installing Local Requirements
 
 If using the local installation method, you will also need to install the following dependencies locally:
 
 1.[ffmpeg](https://www.ffmpeg.org/) - for handling of downloaded videos
-2. [firefox](https://www.mozilla.org/en-US/firefox/new/) and [geckodriver](https://github.com/mozilla/geckodriver/releases) on a path folder like `/usr/local/bin` - for taking webpage screenshots with the screenshot enricher
-3. (optional) [fonts-noto](https://fonts.google.com/noto) to deal with multiple unicode characters during selenium/geckodriver's screenshots: `sudo apt install fonts-noto -y`.
+<!-- 2. [firefox](https://www.mozilla.org/en-US/firefox/new/) and [geckodriver](https://github.com/mozilla/geckodriver/releases) on a path folder like `/usr/local/bin` - for taking webpage screenshots with the screenshot enricher -->
+3. (optional) [fonts-noto](https://fonts.google.com/noto) to deal with multiple unicode characters during selenium screenshots: `sudo apt install fonts-noto -y`.
 4. [Browsertrix Crawler docker image](https://hub.docker.com/r/webrecorder/browsertrix-crawler) for the WACZ enricher/archiver
 
 
+### Custom installation scripts
+- [Ubuntu 24 Server Install by @djhmateer](example_scripts/ubuntu_24_server_install.md) - a WYSIWYG example script from a user who set up the Auto Archiver on a fresh Ubuntu 24 server.
+
 
 ## Developer Install
 

docs/source/installation/requirements.md

@@ -0,0 +1,14 @@
+# Requirements
+
+Using the Auto Archiver is very simple, but ideally you have some familiarity with using the command line to run programs. ([Command line crash course](https://developer.mozilla.org/en-US/docs/Learn_web_development/Getting_started/Environment_setup/Command_line)).
+
+### System Requirements
+
+* Auto Archiver works on any Windows, macOS and Linux computer
+* If you're using the **local install** method, then you should make sure to have python3.10+ installed
+
+### Storage Requirements
+
+By default, Auto Archiver uses your local computer storage for any downloaded media (videos, images etc.). If you're downloading large files, this may take up a lot of your local computer's space (more than 5GB of space).
+
+If your storage space is limited, then you may want to set up an [alternative storage method](../modules/storage.md) for your media.

docs/source/installation/setup.md

@@ -0,0 +1,80 @@
+# Getting Started
+
+```{toctree}
+:hidden:
+
+installation.md
+configurations.md
+config_editor.md
+authentication.md
+requirements.md
+faq.md
+config_cheatsheet.md
+```
+
+## Getting Started
+
+To get started with Auto Archiver, there are 3 main steps you need to complete.
+
+1. [Install Auto Archiver](installation.md)
+2. [Setup up your configuration](configurations.md) (if you are ok with the default settings, you can skip this step)
+3. Run the archiving process<a id="running"></a>
+
+The way you run the Auto Archiver depends on how you installed it (docker install or local install)
+
+### Running a Docker Install
+
+If you installed Auto Archiver using docker, open up your terminal, and copy-paste / type the following command:
+
+```bash
+docker run -it --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver -- "https://example.com/1/"
+```
+
+breaking this command down:
+   1. `docker run` tells docker to start a new container (an instance of the image)
+   2. `-it` tells docker to run in 'interactive mode' so that we get nice colour logs
+   3. `--rm` makes sure this container is removed after execution (less garbage locally)
+   4. `-v $PWD/secrets:/app/secrets` - your secrets folder with settings
+      1. `-v` is a volume flag which means a folder that you have on your computer will be connected to a folder inside the docker container
+      2. `$PWD/secrets` points to a `secrets/` folder in your current working directory (where your console points to), we use this folder as a best practice to hold all the secrets/tokens/passwords/... you use
+      3. `/app/secrets` points to the path the docker container where this image can be found
+   5.  `-v $PWD/local_archive:/app/local_archive` - (optional) if you use local_storage
+       1.  `-v` same as above, this is a volume instruction
+       2.  `$PWD/local_archive` is a folder `local_archive/` in case you want to archive locally and have the files accessible outside docker
+       3.  `/app/local_archive` is a folder inside docker that you can reference in your orchestration.yml file 
+   6. ` -- "https://example.com/1/"` this will pass the URL to archive to the default [command line feeder](../modules/autogen/feeder/cli_feeder.md)
+
+### Example invocations
+
+The invocations below will run the auto-archiver Docker image using a configuration file that you have specified
+
+```bash
+# Have auto-archiver run with the default settings, generating a settings file in ./secrets/orchestration.yaml
+docker run -it --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver
+
+# uses the same configuration, but with the `gsheet_feeder`, a header on row 2 and with some different column names
+# Note this expects you to have followed the [Google Sheets setup](how_to/google_sheets.md) and added your service_account.json to the `secrets/` folder
+# notice that columns is a dictionary so you need to pass it as JSON and it will override only the values provided
+docker run -it --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver --feeders=gsheet_feeder --gsheet_feeder.sheet="use it on another sheets doc" --gsheet_feeder.header=2 --gsheet_feeder.columns='{"url": "link"}'
+# Runs auto-archiver for the first time, but in 'full' mode, enabling all modules to get a full settings file
+docker run -it --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver --mode full
+```
+
+------------
+
+### Running a Local Install
+
+### Example invocations
+
+Once all your [local requirements](#installing-local-requirements) are correctly installed, the
+
+```bash
+# all the configurations come from ./secrets/orchestration.yaml
+auto-archiver --config secrets/orchestration.yaml
+# uses the same configurations but for another google docs sheet 
+# with a header on row 2 and with some different column names
+# notice that columns is a dictionary so you need to pass it as JSON and it will override only the values provided
+auto-archiver --config secrets/orchestration.yaml --gsheet_feeder.sheet="use it on another sheets doc" --gsheet_feeder.header=2 --gsheet_feeder.columns='{"url": "link"}'
+# all the configurations come from orchestration.yaml and specifies that s3 files should be private
+auto-archiver --config secrets/orchestration.yaml --s3_storage.private=1
+```

docs/source/installation/upgrading.md

@@ -0,0 +1,30 @@
+
+# Upgrading
+
+If an update is available, then you will see a message in the logs when you
+run Auto Archiver. Here's what those logs look like:
+
+```{code} bash
+********* IMPORTANT: UPDATE AVAILABLE ********
+A new version of auto-archiver is available (v0.13.6, you have 0.13.4)
+Make sure to update to the latest version using: `pip install --upgrade auto-archiver`
+```
+
+Upgrading Auto Archiver depends on the way you installed it.
+
+## Docker
+
+To upgrade using docker, update the docker image with:
+
+```
+docker pull bellingcat/auto-archiver:latest 
+```
+
+## Pip
+
+To upgrade the pip package, use:
+
+```
+pip install --upgrade auto-archiver
+```
+

docs/source/modules/database.md

@@ -8,7 +8,7 @@ The default (enabled) databases are the CSV Database and the Console Database.
 ```
 
 ```{toctree}
-:depth: 1
+:maxdepth: 1
 :hidden:
 :glob:
 autogen/database/*

docs/source/modules/enricher.md

@@ -7,7 +7,7 @@ Enricher modules are used to add additional information to the items  that have
 ```
 
 ```{toctree}
-:depth: 1
+:maxdepth: 1
 :hidden:
 :glob:
 autogen/enricher/*

docs/source/modules/extractor.md

@@ -4,14 +4,15 @@ Extractor modules are used to extract the content of a given URL. Typically, one
 
 Extractors that are able to extract content from a wide range of websites include:
 1. Generic Extractor: parses videos and images on sites using the powerful yt-dlp library.
-2. Wayback Machine Extractor: sends pages to the Waygback machine for archiving, and stores the link.
-3. WACZ Extractor: runs a web browser to 'browse' the URL and save a copy of the page in WACZ format. 
+2. Antibot Extractor: uses a headless browser to bypass bot detection and extract content.
+3. WACZ Extractor: runs a web browser to 'browse' the URL and save a copy of the page in WACZ format.
+4. Wayback Machine Extractor: sends pages to the Wayback machine for archiving, and stores the archived link.
 
 ```{include} autogen/extractor.md
 ```
 
 ```{toctree}
-:depth: 1
+:maxdepth: 1
 :hidden:
 :glob:
 autogen/extractor/*

docs/source/modules/feeder.md

@@ -1,8 +1,8 @@
 # Feeder Modules
 
-Feeder modules are used to feed URLs into the `auto-archiver` for processing. Feeders can take these URLs from a variety of sources, such as a file, a database, or the command line.
+Feeder modules are used to feed URLs into the Auto Archiver for processing. Feeders can take these URLs from a variety of sources, such as a file, a database, or the command line.
 
-The default feeder is the command line feeder (`cli_feeder`), which allows you to input URLs directly into the `auto-archiver` from the command line.
+The default feeder is the command line feeder (`cli_feeder`), which allows you to input URLs directly into `auto-archiver` from the command line.
 
 Command line feeder usage:
 ```{code} bash
@@ -13,7 +13,7 @@ auto-archiver [options] -- URL1 URL2 ...
 ```
 
 ```{toctree}
-:depth: 1
+:maxdepth: 1
 :glob:
 :hidden:
 autogen/feeder/*

docs/source/modules/formatter.md

@@ -6,7 +6,7 @@ Formatter modules are used to format the data extracted from a URL into a specif
 ```
 
 ```{toctree}
-:depth: 1
+:maxdepth: 1
 :hidden:
 :glob:
 autogen/formatter/*

docs/source/modules/storage.md

@@ -8,7 +8,7 @@ The default is to store the files downloaded (e.g. images, videos) in a local di
 ```
 
 ```{toctree}
-:depth: 1
+:maxdepth: 1
 :hidden:
 :glob:
 autogen/storage/*

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [project]
 name = "auto-archiver"
-version = "0.13.0"
+version = "1.2.7"
 description = "Automatically archive links to videos, images, and social media content from Google Sheets (and more)."
 
 requires-python = ">=3.10,<3.13"
@@ -22,13 +22,11 @@ classifiers = [
 ]
 
 dependencies = [
-    "oscrypto @ git+https://github.com/wbond/oscrypto.git@d5f3437ed24257895ae1edd9e503cfb352e635a8",
     "gspread (>=0.0.0)",
     "beautifulsoup4 (>=0.0.0)",
     "bs4 (>=0.0.0)",
     "loguru (>=0.0.0)",
     "ffmpeg-python (>=0.0.0)",
-    "selenium (>=0.0.0)",
     "telethon (>=0.0.0)",
     "google-api-python-client (>=0.0.0)",
     "google-auth-httplib2 (>=0.0.0)",
@@ -42,28 +40,34 @@ dependencies = [
     "instaloader (>=0.0.0)",
     "tqdm (>=0.0.0)",
     "jinja2 (>=0.0.0)",
-    "pyOpenSSL (==24.2.1)",
-    "cryptography (>=41.0.0,<42.0.0)",
     "boto3 (>=1.28.0,<2.0.0)",
     "dataclasses-json (>=0.0.0)",
-    "yt-dlp (>=2025.1.26,<2026.0.0)",
     "numpy (==2.1.3)",
-    "vk-url-scraper (>=0.0.0)",
     "requests[socks] (>=0.0.0)",
     "warcio (>=0.0.0)",
     "jsonlines (>=0.0.0)",
     "pysubs2 (>=0.0.0)",
     "retrying (>=0.0.0)",
-    "tsp-client (>=0.0.0)",
-    "certvalidator (>=0.0.0)",
     "rich-argparse (>=1.6.0,<2.0.0)",
     "ruamel-yaml (>=0.18.10,<0.19.0)",
+    "rfc3161-client (>=1.0.5)",
+    "cryptography (>=46.0.3)",
+    "opentimestamps (>=0.4.5,<0.5.0)",
+    "bgutil-ytdlp-pot-provider (>=1.0.0)",
+    "yt-dlp[curl-cffi,default] (>=2025.5.22)",
+    "secretstorage (>=3.3.3,<4.0.0)",
+    "seleniumbase (>=4.36.4,<5.0.0)",
+    "pyautogui (>=0.9.54,<0.10.0)",
+    "pyperclip (>=1.9.0)", 
 ]
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^8.3.4"
 autopep8 = "^2.3.1"
 pytest-loguru = "^0.4.0"
+pytest-mock = "^3.14.0"
+ruff = "^0.15.2"
+pre-commit = "^4.1.0"
 
 [tool.poetry.group.docs.dependencies]
 sphinx = "^8.1.3"
@@ -89,4 +93,29 @@ documentation = "https://github.com/bellingcat/auto-archiver"
 markers = [
     "download: marks tests that download content from the network",
     "incremental: marks a class to run tests incrementally. If a test fails in the class, the remaining tests will be skipped",
-]
+]
+
+[tool.ruff]
+#exclude = ["docs"]
+line-length = 120
+# Remove this for a more detailed lint report
+output-format = "concise"
+# TODO: temp ignore rule for timestamping_enricher to allow for open PR
+exclude = ["src/auto_archiver/modules/timestamping_enricher/*"]
+
+
+[tool.ruff.lint]
+# Extend the rules to check for by adding them to this option:
+# See documentation for more details: https://docs.astral.sh/ruff/rules/
+#extend-select = ["B"]
+
+[tool.ruff.lint.per-file-ignores]
+# Ignore import violations in __init__.py files
+"__init__.py" = ["F401", "F403"]
+# Ignore 'useless expression' in manifest files.
+"__manifest__.py" = ["B018"]
+
+
+[tool.ruff.format]
+docstring-code-format = false
+

scripts/create_update_gdrive_oauth_token.py

@@ -1,5 +1,6 @@
 import os.path
-import click, json
+import click
+import json
 
 from google.auth.transport.requests import Request
 from google.oauth2.credentials import Credentials
@@ -70,11 +71,7 @@ def main(credentials, token):
         print(emailAddress)
 
         # Call the Drive v3 API and return some files
-        results = (
-            service.files()
-            .list(pageSize=10, fields="nextPageToken, files(id, name)")
-            .execute()
-        )
+        results = service.files().list(pageSize=10, fields="nextPageToken, files(id, name)").execute()
         items = results.get("files", [])
 
         if not items:

scripts/generate_google_services.sh

@@ -0,0 +1,135 @@
+#!/usr/bin/env bash
+
+set -e  # Exit on error
+
+
+UUID=$(LC_ALL=C tr -dc a-z0-9 </dev/urandom | head -c 16) 
+PROJECT_NAME="auto-archiver-$UUID"
+ACCOUNT_NAME="autoarchiver"
+KEY_FILE="service_account-$UUID.json"
+DEST_DIR="$1"
+
+echo "====================================================="
+echo "🔧 Auto-Archiver Google Services Setup Script"
+echo "====================================================="
+echo "This script will:"
+echo "  1. Install Google Cloud SDK if needed"
+echo "  2. Create a Google Cloud project named $PROJECT_NAME"
+echo "  3. Create a service account for Auto-Archiver"
+echo "  4. Generate a key file for API access"
+echo ""
+echo "  Tip: Pass a directory path as an argument to this script to move the key file there"
+echo "  e.g. ./generate_google_services.sh /path/to/secrets"
+echo "====================================================="
+
+# Check and install Google Cloud SDK based on platform
+install_gcloud_sdk() {
+    if command -v gcloud &> /dev/null; then
+        echo "✅ Google Cloud SDK is already installed"
+        return 0
+    fi
+
+    echo "📦 Installing Google Cloud SDK..."
+    
+    # Detect OS
+    case "$(uname -s)" in
+        Darwin*)
+            if command -v brew &> /dev/null; then
+                echo "🍺 Installing via Homebrew..."
+                brew install google-cloud-sdk --cask
+            else
+                echo "📥 Downloading Google Cloud SDK for macOS..."
+                curl -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-latest-darwin-x86_64.tar.gz
+                tar -xf google-cloud-cli-latest-darwin-x86_64.tar.gz
+                ./google-cloud-sdk/install.sh --quiet
+                rm google-cloud-cli-latest-darwin-x86_64.tar.gz
+                echo "🔄 Please restart your terminal and run this script again"
+                exit 0
+            fi
+            ;;
+        Linux*)
+            echo "📥 Downloading Google Cloud SDK for Linux..."
+            curl -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-latest-linux-x86_64.tar.gz
+            tar -xf google-cloud-cli-latest-linux-x86_64.tar.gz
+            ./google-cloud-sdk/install.sh --quiet
+            rm google-cloud-cli-latest-linux-x86_64.tar.gz
+            echo "🔄 Please restart your terminal and run this script again"
+            exit 0
+            ;;
+        CYGWIN*|MINGW*|MSYS*)
+            echo "⚠️ Windows detected. Please follow manual installation instructions at:"
+            echo "https://cloud.google.com/sdk/docs/install-sdk"
+            exit 1
+            ;;
+        *)
+            echo "⚠️ Unknown operating system. Please follow manual installation instructions at:"
+            echo "https://cloud.google.com/sdk/docs/install-sdk"
+            exit 1
+            ;;
+    esac
+    
+    echo "✅ Google Cloud SDK installed"
+}
+
+# Install Google Cloud SDK if needed
+install_gcloud_sdk
+
+# Login to Google Cloud
+if gcloud auth list --filter=status:ACTIVE --format="value(account)" | grep -q "@"; then
+    echo "✅ Already authenticated with Google Cloud"
+else
+    echo "🔑 Authenticating with Google Cloud..."
+    gcloud auth login
+fi
+
+# Create project
+echo "🌟 Creating Google Cloud project: $PROJECT_NAME"
+gcloud projects create $PROJECT_NAME
+
+# Create service account
+echo "👤 Creating service account: $ACCOUNT_NAME"
+gcloud iam service-accounts create $ACCOUNT_NAME --project $PROJECT_NAME
+
+# Enable required APIs (uncomment and add APIs as needed)
+echo "⬆️ Enabling required Google APIs..."
+gcloud services enable sheets.googleapis.com --project $PROJECT_NAME
+gcloud services enable drive.googleapis.com --project $PROJECT_NAME
+
+# Get the service account email
+echo "📧 Retrieving service account email..."
+ACCOUNT_EMAIL=$(gcloud iam service-accounts list --project $PROJECT_NAME --format="value(email)")
+
+# Create and download key
+echo "🔑 Generating service account key file: $KEY_FILE"
+gcloud iam service-accounts keys create $KEY_FILE --iam-account=$ACCOUNT_EMAIL
+
+# move the file to TARGET_DIR if provided
+if [[ -n "$DEST_DIR" ]]; then
+    # Expand `~` if used
+    DEST_DIR=$(eval echo "$DEST_DIR")
+
+    # Ensure the directory exists
+    if [[ ! -d "$DEST_DIR" ]]; then
+        mkdir -p "$DEST_DIR"
+    fi
+
+    DEST_PATH="$DEST_DIR/$KEY_FILE"
+    echo "🚚 Moving key file to: $DEST_PATH"
+    mv "$KEY_FILE" "$DEST_PATH"
+    KEY_FILE="$DEST_PATH"
+fi
+
+echo "====================================================="
+echo "✅ SETUP COMPLETE!"
+echo "====================================================="
+echo "📝 Important Information:"
+echo "  • Project Name: $PROJECT_NAME"
+echo "  • Service Account: $ACCOUNT_EMAIL"
+echo "  • Key File: $KEY_FILE"
+echo ""
+echo "📋 Next Steps:"
+echo "  1. Share any Google Sheets with this email address:"
+echo "     $ACCOUNT_EMAIL"
+echo "  2. Move $KEY_FILE to your auto-archiver secrets directory"
+echo "  3. Update your auto-archiver config to use this key file (if needed)"
+echo "====================================================="

scripts/generate_settings_schema.py

@@ -0,0 +1,63 @@
+import json
+import os
+import io
+
+from ruamel.yaml import YAML
+
+from auto_archiver.core.module import ModuleFactory
+from auto_archiver.core.consts import MODULE_TYPES
+from auto_archiver.core.config import EMPTY_CONFIG
+
+
+class SchemaEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, set):
+            return list(obj)
+        return json.JSONEncoder.default(self, obj)
+
+
+# Get available modules
+module_factory = ModuleFactory()
+available_modules = module_factory.available_modules()
+
+modules_by_type = {}
+# Categorize modules by type
+for module in available_modules:
+    for type in module.manifest.get("type", []):
+        modules_by_type.setdefault(type, []).append(module)
+
+all_modules_ordered_by_type = sorted(
+    available_modules, key=lambda x: (MODULE_TYPES.index(x.type[0]), not x.requires_setup)
+)
+
+yaml: YAML = YAML()
+
+config_string = io.BytesIO()
+yaml.dump(EMPTY_CONFIG, config_string)
+config_string = config_string.getvalue().decode("utf-8")
+output_schema = {
+    "modules": dict(
+        (
+            module.name,
+            {
+                "name": module.name,
+                "display_name": module.display_name,
+                "manifest": module.manifest,
+                "configs": module.configs or None,
+            },
+        )
+        for module in all_modules_ordered_by_type
+    ),
+    "steps": dict(
+        (f"{module_type}s", [module.name for module in modules_by_type[module_type]]) for module_type in MODULE_TYPES
+    ),
+    "configs": [m.name for m in all_modules_ordered_by_type if m.configs],
+    "module_types": MODULE_TYPES,
+    "empty_config": config_string,
+}
+
+current_file_dir = os.path.dirname(os.path.abspath(__file__))
+output_file = os.path.join(current_file_dir, "settings/src/schema.json")
+with open(output_file, "w") as file:
+    print(f"Writing schema to {output_file}")
+    json.dump(output_schema, file, indent=4, cls=SchemaEncoder)

scripts/instagrapi_server/.gitignore

@@ -0,0 +1,2 @@
+secrets*
+*instagrapi_session.json

scripts/instagrapi_server/Dockerfile

@@ -0,0 +1,19 @@
+FROM python:3.12-slim
+WORKDIR /app
+
+# Install Poetry
+RUN pip install --upgrade pip
+RUN pip install poetry
+
+# Copy all source code
+COPY . .
+
+# Prevent Poetry from creating a virtual environment
+RUN poetry config virtualenvs.create false
+
+# Install dependencies
+RUN poetry install --no-root
+
+
+# Use uvicorn to run the FastAPI app
+CMD ["poetry", "run", "uvicorn", "src.instaserver:app", "--host", "0.0.0.0", "--port", "8000"]

scripts/instagrapi_server/pyproject.toml

@@ -0,0 +1,18 @@
+[project]
+name = "instaserver"
+version = "0.1.0"
+description = "A FastAPI InstagrAPI server"
+package-mode = false
+requires-python = ">=3.10"
+dependencies = [
+    "fastapi (>=0.115.12,<0.116.0)",
+    "instagrapi (>=2.1.3,<3.0.0)",
+    "uvicorn (>=0.34.0,<0.35.0)",
+    "pillow (>=11.1.0,<12.0.0)",
+    "python-dotenv (>=1.1.0,<2.0.0)"
+]
+
+
+[build-system]
+requires = ["poetry-core>=2.0.0,<3.0.0"]
+build-backend = "poetry.core.masonry.api"

scripts/instagrapi_server/run_instagrapi_server.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+#
+# run_instagrapi_server.sh
+# Usage:
+#   From repo root:   ./scripts/instagrapi_server/run_instagrapi_server.sh
+#   Or from script dir: ./run_instagrapi_server.sh
+#
+
+set -e
+
+# Step 1: cd to the script's directory (contains Dockerfile and secrets/)
+cd "$(dirname "$0")" || exit 1
+
+# Create secrets/ if it doesn't exist
+if [[ ! -d "secrets" ]]; then
+  echo "Creating secrets/ directory..."
+  mkdir secrets
+fi
+
+echo "Enter your Instagram credentials to store in secrets/.env"
+read -rp "Instagram Username: " IGUSER
+read -rsp "Instagram Password: " IGPASS
+echo ""
+
+cat <<EOF > secrets/.env
+INSTAGRAM_USERNAME=$IGUSER
+INSTAGRAM_PASSWORD=$IGPASS
+EOF
+echo "Created secrets/.env with your credentials."
+
+# Build Docker image
+IMAGE_NAME="instagrapi-server"
+echo "Building Docker image '$IMAGE_NAME'..."
+docker build -t "$IMAGE_NAME" .
+
+# Run container
+CONTAINER_NAME="ig-instasrv"
+echo "Running container '$CONTAINER_NAME'..."
+docker run -d \
+  --env-file secrets/.env \
+  -v "$(pwd)/secrets:/app/secrets" \
+  -p 8000:8000 \
+  --name "$CONTAINER_NAME" \
+  "$IMAGE_NAME"
+
+echo "Done! Instagrapi server is running on port 8000."
+echo "Use 'docker logs $CONTAINER_NAME' to view logs."
+echo "Use 'docker stop $CONTAINER_NAME' and 'docker rm $CONTAINER_NAME' to stop/remove the container."

scripts/instagrapi_server/src/instaserver.py

@@ -0,0 +1,157 @@
+"""https://subzeroid.github.io/instagrapi/
+
+Run using the following command:
+ uvicorn src.instgrapinstance.instaserver:app --host 0.0.0.0 --port 8000 --reload
+"""
+
+import logging
+import os
+import sys
+from dotenv import load_dotenv
+
+from fastapi import FastAPI, HTTPException
+from instagrapi import Client
+from instagrapi.exceptions import LoginRequired, BadCredentials
+
+load_dotenv(dotenv_path="secrets/.env")
+logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
+
+INSTAGRAM_USERNAME = os.getenv("INSTAGRAM_USERNAME")
+INSTAGRAM_PASSWORD = os.getenv("INSTAGRAM_PASSWORD")
+SESSION_FILE = "secrets/instagrapi_session.json"
+
+app = FastAPI()
+cl = Client()
+
+
+@app.on_event("startup")
+def startup_event():
+    """Login automatically when server starts"""
+    try:
+        login_instagram()
+    except RuntimeError as e:
+        logging.error(f"API failed to start: {e}")
+        sys.exit(1)
+
+
+def login_instagram():
+    """Ensures Instagrapi is logged in and session is persistent"""
+    if not INSTAGRAM_USERNAME or not INSTAGRAM_PASSWORD:
+        raise RuntimeError("Instagram credentials are missing.")
+
+    if os.path.exists(SESSION_FILE):
+        try:
+            cl.load_settings(SESSION_FILE)
+            cl.get_timeline_feed()
+            logging.info("Using saved session.")
+            return
+        except LoginRequired:
+            logging.info("Session expired. Logging in again...")
+
+    try:
+        cl.login(INSTAGRAM_USERNAME, INSTAGRAM_PASSWORD)
+        cl.dump_settings(SESSION_FILE)
+        logging.info("Login successful, session saved.")
+    except BadCredentials as bc:
+        raise RuntimeError("Incorrect Instagram username or password.") from bc
+    except Exception as e:
+        raise RuntimeError(f"Login failed: {e}") from e
+
+
+@app.get("/v1/media/by/id")
+def get_media_by_id(id: str):
+    """Fetch post details by media ID"""
+    logging.info(f"Fetching media by ID: {id}")
+    try:
+        media = cl.media_info(id)
+        return media.model_dump()
+    except Exception as e:
+        logging.warning(f"Media not found for ID {id}: {e}")
+        raise HTTPException(status_code=404, detail="Post not found") from e
+
+
+@app.get("/v1/media/by/code")
+def get_media_by_code(code: str):
+    """Fetch post details by shortcode"""
+    logging.info(f"Fetching media by shortcode: {code}")
+    try:
+        media_id = cl.media_pk_from_code(code)
+        media = cl.media_info(media_id)
+        return media.model_dump()
+    except Exception as e:
+        logging.warning(f"Media not found for code {code}: {e}")
+        raise HTTPException(status_code=404, detail="Post not found") from e
+
+
+@app.get("/v2/user/tag/medias")
+def get_user_tagged_medias(user_id: str, page_id: str = None):
+    logging.info(f"Fetching tagged medias for user_id={user_id} page_id={page_id}")
+    try:
+        # Placeholder for now
+        items, next_page_id = [], None
+        return {"response": {"items": items}, "next_page_id": next_page_id}
+    except Exception as e:
+        logging.warning(f"Tagged media not found for {user_id}: {e}")
+        raise HTTPException(status_code=404, detail="Tagged media not found") from e
+
+
+@app.get("/v1/user/highlights")
+def get_user_highlights(user_id: str):
+    logging.info(f"Fetching highlights list for user_id={user_id}")
+    try:
+        highlights = cl.user_highlights(user_id)
+        return [h.model_dump() for h in highlights]
+    except Exception as e:
+        logging.warning(f"Highlights not found for {user_id}: {e}")
+        raise HTTPException(status_code=404, detail="No highlights found") from e
+
+
+@app.get("/v2/highlight/by/id")
+def get_highlight_by_id(id: str):
+    logging.info(f"Fetching highlight details for id={id}")
+    try:
+        highlight = cl.highlight_info(id)
+        return {"response": {"reels": {f"highlight:{id}": highlight.model_dump()}}}
+    except Exception as e:
+        logging.warning(f"Highlight not found for id {id}: {e}")
+        raise HTTPException(status_code=404, detail="Highlight not found") from e
+
+
+@app.get("/v1/user/stories/by/username")
+def get_stories(username: str):
+    logging.info(f"Fetching stories for username={username}")
+    try:
+        user_id = cl.user_id_from_username(username)
+        stories = cl.user_stories(user_id)
+        return [story.model_dump() for story in stories]
+    except Exception as e:
+        logging.warning(f"Stories not found for {username}: {e}")
+        raise HTTPException(status_code=404, detail="Stories not found") from e
+
+
+@app.get("/v2/user/by/username")
+def get_user_by_username(username: str):
+    logging.info(f"Fetching user profile for username={username}")
+    try:
+        user = cl.user_info_by_username(username)
+        return {"user": user.model_dump()}
+    except Exception as e:
+        logging.warning(f"User not found: {username}: {e}")
+        raise HTTPException(status_code=404, detail="User not found") from e
+
+
+@app.get("/v1/user/medias/chunk")
+def get_user_medias(user_id: str, end_cursor: str = None):
+    logging.info(f"Fetching paginated medias for user_id={user_id}, end_cursor={end_cursor}")
+    try:
+        posts, next_cursor = cl.user_medias_paginated(user_id, end_cursor=end_cursor)
+        return [[post.model_dump() for post in posts], next_cursor]
+    except Exception as e:
+        logging.warning(f"No posts found for user_id={user_id}: {e}")
+        raise HTTPException(status_code=404, detail="No posts found") from e
+
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run(app, host="0.0.0.0", port=8000)

scripts/settings/.gitignore

@@ -0,0 +1,24 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+dist
+dist-ssr
+*.local
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?

scripts/settings/index.html

@@ -0,0 +1,3 @@
+
+    <div id="root"></div>
+    <script type="module" src="/src/main.tsx"></script>

scripts/settings/package.json

@@ -0,0 +1,31 @@
+{
+  "name": "material-ui-vite-ts",
+  "private": true,
+  "version": "5.0.0",
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "preview": "vite preview"
+  },
+  "dependencies": {
+    "@dnd-kit/core": "^6.3.1",
+    "@dnd-kit/sortable": "^10.0.0",
+    "@emotion/react": "latest",
+    "@emotion/styled": "latest",
+    "@mui/icons-material": "^7.1.1",
+    "@mui/material": "^7.1.1",
+    "react": "19.1.0",
+    "react-dom": "19.1.0",
+    "react-markdown": "^10.0.0",
+    "yaml": "^2.9.0"
+  },
+  "devDependencies": {
+    "@types/react": "latest",
+    "@types/react-dom": "latest",
+    "@vitejs/plugin-react": "latest",
+    "typescript": "latest",
+    "vite": "latest",
+    "vite-plugin-singlefile": "^2.1.0"
+  }
+}

scripts/settings/src/App.tsx

@@ -0,0 +1,450 @@
+import * as React from 'react';
+import { useEffect, useState, useRef } from 'react';
+import Container from '@mui/material/Container';
+import Typography from '@mui/material/Typography';
+import Box from '@mui/material/Box';
+import FileUploadIcon from '@mui/icons-material/FileUpload';
+
+import {
+  DndContext,
+  closestCenter,
+  KeyboardSensor,
+  PointerSensor,
+  useSensor,
+  useSensors,
+  DragOverlay
+} from "@dnd-kit/core";
+import {
+  arrayMove,
+  SortableContext,
+  sortableKeyboardCoordinates,
+  rectSortingStrategy
+} from "@dnd-kit/sortable";
+
+import type { DragStartEvent, DragEndEvent, UniqueIdentifier } from "@dnd-kit/core";
+
+
+import { Module } from './types';
+
+import { modules, steps, module_types, empty_config } from './schema.json';
+import {
+  Stack,
+  Button,
+} from '@mui/material';
+import Grid from '@mui/material/Grid';
+
+import { parseDocument, Document, YAMLSeq, YAMLMap, Scalar } from 'yaml'
+import StepCard from './StepCard';
+
+
+function FileDrop({ setYamlFile }: { setYamlFile: React.Dispatch<React.SetStateAction<Document>> }) {
+
+  const [showError, setShowError] = useState(false);
+  const [label, setLabel] = useState(<>Drag and drop your orchestration.yaml file here, or click to select a file.</>);
+  const wrapperRef = useRef(null);
+
+  function openYAMLFile(event: any) {
+    let file = event.target.files[0];
+    if (file.type.indexOf('yaml') === -1) {
+      setShowError(true);
+      setLabel(<>Invalid type, only YAML files are accepted.</>)
+      return;
+    }
+    let reader = new FileReader();
+    reader.onload = function (e) {
+      let contents = e.target ? e.target.result : '';
+      try {
+        let document = parseDocument(contents as string);
+        if (document.errors.length > 0) {
+          // not a valid yaml file
+          setShowError(true);
+          setLabel(<>Invalid file. Make sure your Orchestration is a valid YAML file with a 'steps' section in it.</>)
+          return;
+        } else {
+          setShowError(false);
+          setLabel(<>File loaded successfully.</>)
+        }
+        // do some basic validation of 'steps'
+        let steps = document.get('steps');
+        if (!steps) {
+          setShowError(true);
+          setLabel(<>Invalid file. Your orchestration file must have a 'steps' section in it.</>)
+          return;
+        }
+        const replacements = {
+          feeder: 'feeders',
+          formatter: 'formatters',
+          archivers: 'extractors',
+        };
+
+        let error = false;
+        for (let stepType of Object.keys(replacements)) {
+          if (steps.get(stepType) !== undefined) {
+            setShowError(true);
+            setLabel(<>Invalid file. Your orchestration file appears to be in the old (v0.12) format with a '{stepType}' section.<br/>You should manually update your orchestration file first (hint: {stepType} → {replacements[stepType]})</>);
+            error = true;
+            return;
+          }
+        };
+        setYamlFile(document);
+      } catch (e) {
+        console.error(e);
+      }
+    }
+    reader.readAsText(file);
+  }
+  return (
+    <>
+      <div
+      style={{
+        position: 'relative',
+        width: '100%',
+        border: 'dashed',
+        borderRadius:'5px',
+        textAlign: 'center',
+        borderWidth: '1px',
+        padding: '20px' }}
+      onDragEnter={(e) => {
+        e.currentTarget.style.backgroundColor = 'var(--mui-palette-LinearProgress-infoBg)';
+      }}
+      onDragLeave={(e) => {
+        e.currentTarget.style.backgroundColor = '';
+      }}
+      onDrop={(e) => {
+        e.currentTarget.style.backgroundColor = '';
+      }}
+      >
+        <FileUploadIcon style={{ fontSize: 50 }} />
+        <input style={{
+          opacity: 0,
+          position: 'absolute',
+          top: 0,
+          left: 0,
+          width: '100%',
+          height: '100%',
+          cursor: 'pointer',
+        }}
+        type="file" id="file"
+        accept=".yaml"
+        onChange={openYAMLFile} />
+        <Typography variant="body1" color={showError ? 'error' : ''} >
+          {label}
+        </Typography>
+      </div>
+    </>
+  );
+}
+
+function ModuleTypes({ stepType, setEnabledModules, enabledModules, configValues }: { stepType: string, setEnabledModules: any, enabledModules: any, configValues: any }) {
+  const [showError, setShowError] = useState<boolean>(false);
+  const [activeId, setActiveId] = useState<UniqueIdentifier>();
+  const [items, setItems] = useState<string[]>([]);
+
+  useEffect(() => {
+    setItems(enabledModules[stepType].map(([name, enabled]: [string, boolean]) => name));
+  }
+    , [enabledModules]);
+
+  const toggleModule = (event: any) => {
+    // make sure that 'feeder' and 'formatter' types only have one value
+    let name = event.target.id;
+    let checked = event.target.checked;
+    if (stepType === 'feeders' || stepType === 'formatters') {
+      // check how many modules of this type are enabled
+      const checkedModules = enabledModules[stepType].filter(([m, enabled]: [string, boolean]) => {
+        return (m !== name && enabled) || (checked && m === name)
+      });
+      if (checkedModules.length > 1) {
+        setShowError(true);
+      } else {
+        setShowError(false);
+      }
+    } else {
+      setShowError(false);
+    }
+    let newEnabledModules = { ...enabledModules };
+    newEnabledModules[stepType] = enabledModules[stepType].map(([m, enabled]: [string, boolean]) => {
+        return (m === name) ? [m, checked] : [m, enabled];
+      });
+    setEnabledModules(newEnabledModules);
+  }
+
+  const sensors = useSensors(
+    useSensor(PointerSensor),
+    useSensor(KeyboardSensor, {
+      coordinateGetter: sortableKeyboardCoordinates
+    })
+  );
+
+  const handleDragStart = (event: DragStartEvent) => {
+    setActiveId(event.active.id);
+  };
+
+  const handleDragEnd = (event: DragEndEvent) => {
+    setActiveId(undefined);
+    const { active, over } = event;
+
+    if (active.id !== over?.id) {
+      const oldIndex = items.indexOf(active.id as string);
+      const newIndex = items.indexOf(over?.id as string);
+
+      let newArray = arrayMove(items, oldIndex, newIndex);
+      // set it also on steps
+      let newEnabledModules = { ...enabledModules };
+      newEnabledModules[stepType] = enabledModules[stepType].sort((a, b) => {
+        return newArray.indexOf(a[0]) - newArray.indexOf(b[0]);
+      })
+      setEnabledModules(newEnabledModules);
+    }
+  };
+  return (
+    <>
+      <Box sx={{ my: 4 }}>
+        <Typography id={stepType} variant="h6" style={{ textTransform: 'capitalize' }} >
+          {stepType}
+        </Typography>
+        <Typography variant="body1" >
+          Select the <a href={`https://auto-archiver.readthedocs.io/en/latest/modules/${stepType.slice(0,-1)}.html`} target="_blank">{stepType}</a> you wish to enable. Drag to reorder.
+        </Typography>
+      </Box>
+      {showError ? <Typography variant="body1" color="error" >Only one {stepType.slice(0,-1)} can be enabled at a time.</Typography> : null}
+
+      <DndContext
+        sensors={sensors}
+        collisionDetection={closestCenter}
+        onDragEnd={handleDragEnd}
+        onDragStart={handleDragStart}
+      >
+        <Grid container spacing={1} key={stepType}>
+          <SortableContext items={items} strategy={rectSortingStrategy}>
+            {items.map((name: string) => {
+              let m: Module = modules[name];
+              return (
+                <StepCard key={name} type={stepType} module={m} toggleModule={toggleModule} enabledModules={enabledModules} configValues={configValues} />
+              );
+            })}
+            <DragOverlay>
+              {activeId ? (
+                <div
+                  style={{
+                    width: "100%",
+                    height: "100%",
+                    backgroundColor: "grey",
+                    opacity: 0.1,
+                  }}
+                ></div>
+
+              ) : null}
+            </DragOverlay>
+          </SortableContext>
+        </Grid>
+      </DndContext>
+    </>
+  );
+}
+
+
+export default function App() {
+  const [yamlFile, setYamlFile] = useState<Document>(new Document());
+  const [enabledModules, setEnabledModules] = useState<{}>(Object.fromEntries(Object.keys(steps).map(type => [type, steps[type].map((name: string) => [name, false])])));
+  const [configValues, setConfigValues] = useState<{
+    [key: string]: {
+      [key: string
+      ]: any
+    }
+  }>(
+    Object.keys(modules).reduce((acc, module) => {
+      acc[module] = {};
+      return acc;
+    }, {})
+  );
+
+  const saveSettings = function (copy: boolean = false) {
+    // edit the yamlFile
+
+    // generate the steps config
+    let stepsConfig = enabledModules;
+
+    let finalYamlFile: Document = null;
+    if (!yamlFile || yamlFile.contents == null) {
+      // create the yaml file from 
+      finalYamlFile = parseDocument(empty_config as string);
+    } else {
+      finalYamlFile = yamlFile;
+    }
+
+    // set the steps
+    module_types.forEach((type: string) => {
+      let stepType = type + 's';
+      let existingSteps  = finalYamlFile.getIn(['steps', stepType]) as YAMLSeq;
+      stepsConfig[stepType].forEach(([name, enabled]: [string, boolean]) => {
+        let index = existingSteps.items.findIndex((item) => { 
+          return (item.value || item) === name
+        });
+        let stepItem = finalYamlFile.getIn(['steps', stepType], true) as YAMLSeq;
+
+        if (enabled && index === -1) {
+            finalYamlFile.addIn(['steps', stepType], name);
+            stepItem.commentBefore = stepItem.commentBefore?.replace("\n - " + name, '');
+            stepItem.comment = stepItem.comment?.replace("\n - " + name, '');
+        } else if (!enabled && index !== -1) {
+            // set the value to empty and add a comment before with the commented value
+            finalYamlFile.deleteIn(['steps', stepType, index]);
+            stepItem.commentBefore += "\n - " + name;
+            finalYamlFile.setIn(['steps', stepType], stepItem);
+        }
+      });
+      // sort the items
+      existingSteps.items.sort((a: Scalar | string, b: Scalar | string) => {
+        return (stepsConfig[stepType].findIndex((val: [string, boolean]) => {return val[0] === (a.value || a)}) -
+                stepsConfig[stepType].findIndex((val: [string, boolean]) => {return val[0] === (b.value || b)}))
+      });
+      existingSteps.flow = existingSteps.items.length ? false : true;
+    });
+
+    // set all other settings
+    // loop through each item that isn't 'steps' in the finalYamlFile and check if it exists in configValues
+
+        Object.keys(configValues).forEach((module_name: string) => {
+          // get an existing key
+          let existingConfig = finalYamlFile.get(module_name, true) as YAMLMap;
+          if (existingConfig) {
+            Object.keys(configValues[module_name]).forEach((config_name: string) => {
+              let existingConfigYAML = existingConfig.get(config_name, true) as Scalar;
+              if (existingConfigYAML) {
+                existingConfigYAML.value = configValues[module_name][config_name];
+                existingConfig.set(config_name, existingConfigYAML);
+              } else {
+                existingConfig.set(config_name, configValues[module_name][config_name]);
+              }
+            });
+            finalYamlFile.set(module_name, existingConfig);
+          } else {
+            if (configValues[module_name] && Object.keys(configValues[module_name]).length > 0) {
+              finalYamlFile.set(module_name, configValues[module_name]);
+            }
+          }
+        });
+
+    if (copy) {
+      navigator.clipboard.writeText(String(finalYamlFile)).then(() => {
+        alert("Settings copied to clipboard.");
+      });
+    } else {
+      // offer the file for download
+      const blob = new Blob([String(finalYamlFile)], { type: 'application/x-yaml' });
+      const url = URL.createObjectURL(blob);
+      const a = document.createElement('a');
+      a.href = url;
+      a.download = 'orchestration.yaml';
+      a.click();
+    }
+  }
+
+  useEffect(() => {
+    // load the configs, and set the default values if they exist
+    let newConfigValues = {};
+    Object.keys(modules).map((module: string) => {
+      let m = modules[module];
+      let configs = m.configs;
+      if (!configs) {
+        return;
+      }
+      newConfigValues[module] = {};
+      Object.keys(configs).map((config: string) => {
+        let config_args = configs[config];
+        if (config_args.default !== undefined) {
+          newConfigValues[module][config] = config_args.default;
+        }
+      });
+    })
+    setConfigValues(newConfigValues);
+  }, []);
+
+  useEffect(() => {
+    if (!yamlFile || yamlFile.contents == null) {
+      return;
+    }
+
+    let settings = yamlFile.toJS();
+    // make a deep copy of settings
+    let stepSettings = settings['steps'];
+   
+    let newEnabledModules = Object.fromEntries(Object.keys(steps).map((type: string) => {
+      return [type, steps[type].map((name: string) => {
+        return [name, stepSettings[type].indexOf(name) !== -1];
+      }).sort((a, b) => {
+        let aIndex = stepSettings[type].indexOf(a[0]);
+        let bIndex = stepSettings[type].indexOf(b[0]);
+        if (aIndex === -1 && bIndex === -1) {
+          return a - b;
+        }
+        if (bIndex === -1) {
+          return -1;
+        }
+        if (aIndex === -1) {
+          return 1;
+        }
+        return aIndex - bIndex;
+      })];
+    }).sort((a, b) => {
+      return module_types.indexOf(a[0]) - module_types.indexOf(b[0]);
+    }));
+    setEnabledModules(newEnabledModules);
+
+    // set the config values
+    let newConfigValues = settings;
+    delete newConfigValues['steps'];
+
+
+    setConfigValues(Object.keys(modules).reduce((acc, module) => {
+      acc[module] = newConfigValues[module] || {};
+      return acc;
+    }, {}));
+  }, [yamlFile]);
+
+
+
+  return (
+    <Container maxWidth="lg">
+      <Box sx={{ my: 4 }}>
+        <Box sx={{ my: 4 }}>
+          <Typography variant="h5" >
+            1. Select your orchestration.yaml settings file.
+          </Typography>
+          <Typography variant="body1">Or skip this step to start from scratch</Typography>
+          <FileDrop setYamlFile={setYamlFile} />
+        </Box>
+        <Box sx={{ my: 4 }}>
+          <Typography variant="h5" >
+            2. Choose the Modules you wish to enable/disable
+          </Typography>
+          {Object.keys(steps).map((stepType: string) => {
+            return (
+              <Box key={stepType} sx={{ my: 4 }}>
+                <ModuleTypes stepType={stepType} setEnabledModules={setEnabledModules} enabledModules={enabledModules} configValues={configValues} />
+              </Box>
+            );
+          })}
+        </Box>
+        <Box sx={{ my: 4 }}>
+          <Typography variant="h5" >
+            3. Configure your Enabled Modules
+          </Typography>
+          <Typography variant="body1" >
+            Next to each module you've enabled, you can click 'Configure' to set the module's settings.
+          </Typography>
+        </Box>
+        <Box sx={{ my: 4 }}>
+          <Typography variant="h5" >
+            4. Save your settings
+          </Typography>
+          <Stack direction="row" spacing={2} sx={{ my: 2 }}>
+            <Button variant="contained" color="primary" onClick={() => saveSettings(true)}>Copy Settings to Clipboard</Button>
+            <Button variant="contained" color="primary" onClick={() => saveSettings()}>Save Settings to File</Button>
+          </Stack>
+        </Box>
+      </Box>
+    </Container>
+  );
+}

scripts/settings/src/StepCard.tsx

@@ -0,0 +1,258 @@
+import { useState } from "react";
+import { useSortable } from "@dnd-kit/sortable";
+import ReactMarkdown from 'react-markdown';
+
+import { CSS } from "@dnd-kit/utilities";
+
+import {
+    Card,
+    CardActions,
+    CardHeader,
+    Button,
+    Dialog,
+    DialogTitle,
+    DialogContent,
+    Box,
+    IconButton,
+    Checkbox,
+    Select,
+    MenuItem,
+    FormControl,
+    FormControlLabel,
+    FormHelperText,
+    TextField,
+    Stack,
+    Typography,
+    InputAdornment,
+} from '@mui/material';
+import Grid from '@mui/material/Grid';
+import DragIndicatorIcon from '@mui/icons-material/DragIndicator';
+import Visibility from '@mui/icons-material/Visibility';
+import VisibilityOff from '@mui/icons-material/VisibilityOff';
+import HelpIconOutlined from '@mui/icons-material/HelpOutline';
+import { Module, Config } from "./types";
+
+
+// adds 'capitalize' method to String prototype
+declare global {
+    interface String {
+        capitalize(): string;
+    }
+}
+String.prototype.capitalize = function (this: string) {
+    return this.charAt(0).toUpperCase() + this.slice(1);
+};
+
+const StepCard = ({
+    type,
+    module,
+    toggleModule,
+    enabledModules,
+    configValues
+}: {
+    type: string,
+    module: Module,
+    toggleModule: any,
+    enabledModules: any,
+    configValues: any
+}) => {
+    const {
+        attributes,
+        listeners,
+        setNodeRef,
+        transform,
+        transition,
+        isDragging
+    } = useSortable({ id: module.name });
+
+
+    const style = {
+        ...Card.style,
+        transform: CSS.Transform.toString(transform),
+        transition,
+        zIndex: isDragging ? "100" : "auto",
+        opacity: isDragging ? 0.3 : 1
+    };
+
+    let name = module.name;
+    const [helpOpen, setHelpOpen] = useState(false);
+    const [configOpen, setConfigOpen] = useState(false);
+    const enabled = enabledModules[type].find((m: any) => m[0] === name)[1];
+
+    return (
+        <Grid ref={setNodeRef} size={{ xs: 6, sm: 4, md: 3 }} style={style}>
+            <Card >
+                <CardHeader
+                    title={
+                        <FormControlLabel
+                            style={{paddingRight: '0 !important'}}
+                            control={<Checkbox title="Check to enable this module" sx={{paddingTop:0, paddingBottom:0}} id={name} onClick={toggleModule} checked={enabled} />}
+                            label={module.display_name} />
+                    }
+                />
+                <CardActions>
+                    <Box sx={{ justifyContent: 'space-between', display: 'flex', width: '100%' }}>
+                        <Box>
+                    <IconButton title="Module information" size="small" onClick={() => setHelpOpen(true)}>
+                        <HelpIconOutlined />
+                    </IconButton>
+                    {enabled && module.configs && name != 'cli_feeder' ? (
+                        <Button size="small" onClick={() => setConfigOpen(true)}>Configure</Button>
+                    ) : null}
+                    </Box>
+                    <IconButton size="small" title="Drag to reorder" sx={{ cursor: 'grab' }} {...listeners} {...attributes}>
+                        <DragIndicatorIcon/>
+                    </IconButton>
+                    </Box>
+                </CardActions>
+            </Card>
+            <Dialog
+                open={helpOpen}
+                onClose={() => setHelpOpen(false)}
+                maxWidth="lg"
+            >
+                <DialogTitle>
+                    {module.display_name}
+                </DialogTitle>
+                <DialogContent>
+                    <ReactMarkdown>
+                        {module.manifest.description.split("\n").map((line: string) => line.trim()).join("\n")}
+                    </ReactMarkdown>
+                </DialogContent>
+            </Dialog>
+            {module.configs && name != 'cli_feeder' && <ConfigPanel module={module} open={configOpen} setOpen={setConfigOpen} configValues={configValues} />}
+        </Grid>
+    )
+}
+
+function ConfigField({ config_value, module, configValues }: { config_value: any, module: Module, configValues: any }) {
+    const [showPassword, setShowPassword] = useState(false);
+    const handleClickShowPassword = () => setShowPassword((show) => !show);
+
+    const handleMouseDownPassword = (event: React.MouseEvent<HTMLButtonElement>) => {
+      event.preventDefault();
+    };
+  
+    const handleMouseUpPassword = (event: React.MouseEvent<HTMLButtonElement>) => {
+      event.preventDefault();
+    };
+
+    function setConfigValue(config: any, value: any) {
+        configValues[module.name][config] = value;
+    }
+    const config_args: Config = module.configs[config_value];
+    const config_name: string = config_value.replace(/_/g, " ");
+    const config_display_name = config_name.capitalize();
+    const value = configValues[module.name][config_value] || config_args.default;
+    
+
+    const config_value_lower = config_value.toLowerCase();
+    const is_password = config_value_lower.includes('password') ||
+                        config_value_lower.includes('secret') ||
+                        config_value_lower.includes('token') ||
+                        config_value_lower.includes('key') ||
+                        config_value_lower.includes('api_hash') ||
+                        config_args.type === 'password';
+
+    const text_input_type = is_password ? 'password' : (config_args.type === 'int' ? 'number' : 'text');
+
+    return (
+        <Box>
+            <Typography variant='body1' style={{ fontWeight: 'bold' }}>{config_display_name} {config_args.required && (`(required)`)} </Typography>
+            <FormControl size="small">
+                {config_args.type === 'bool' ?
+                    <FormControlLabel control={
+                        <Checkbox defaultChecked={value} size="small" id={`${module}.${config_value}`}
+                            onChange={(e) => {
+                                setConfigValue(config_value, e.target.checked);
+                            }}
+                        />} label={config_args.help.capitalize()}
+                    />
+                    :
+                    (
+                        config_args.choices !== undefined ?
+                            <Select size="small" id={`${module}.${config_value}`}
+                                defaultValue={config_args.default}
+                                value={value}
+                                onChange={(e) => {
+                                    setConfigValue(config_value, e.target.value);
+                                }}
+                            >
+                                {config_args.choices.map((choice: any) => {
+                                    return (
+                                        <MenuItem key={`${module}.${config_value}.${choice}`}
+                                            value={choice}>{choice}</MenuItem>
+                                    );
+                                })}
+                            </Select>
+                            :
+                            (config_args.type === 'json_loader' ?
+                                <TextField multiline size="small" id={`${module}.${config_value}`} defaultValue={JSON.stringify(value, null, 2)} rows={6} onChange={
+                                    (e) => {
+                                        try {
+                                            let val = JSON.parse(e.target.value);
+                                            setConfigValue(config_value, val);
+                                        } catch (e) {
+                                            console.log(e);
+                                        }
+                                    }
+                                } />
+                                :
+                                <TextField size="small" id={`${module}.${config_value}`} defaultValue={value} type={showPassword ? 'text' : text_input_type}
+                                    onChange={(e) => {
+                                        setConfigValue(config_value, e.target.value);
+                                    }}
+                                    required={config_args.required}
+                                    slotProps={ is_password ? {
+                                        input: { endAdornment: (
+                                            <InputAdornment position="end">
+                                                <IconButton
+                                                    aria-label="toggle password visibility"
+                                                    onClick={handleClickShowPassword}
+                                                    onMouseDown={handleMouseDownPassword}
+                                                    onMouseUp={handleMouseUpPassword}
+                                                >
+                                                    {showPassword ? <VisibilityOff /> : <Visibility />}
+                                                </IconButton>
+                                            </InputAdornment>
+                                        )}
+                                    } : {}}
+                                />
+                            )
+                    )
+                }
+                {config_args.type !== 'bool' && (
+                    <FormHelperText >{config_args.help.capitalize()}</FormHelperText>
+                )}
+            </FormControl>
+        </Box>
+    )
+}
+
+function ConfigPanel({ module, open, setOpen, configValues }: { module: Module, open: boolean, setOpen: any, configValues: any }) {
+
+    return (
+        <>
+            <Dialog
+                open={open}
+                onClose={() => setOpen(false)}
+                maxWidth="lg"
+            >
+                <DialogTitle>
+                    {module.display_name}
+                </DialogTitle>
+                <DialogContent>
+                    <Stack direction="column" spacing={1}>
+                        {Object.keys(module.configs).map((config_value: any) => {
+                            return (
+                                <ConfigField key={config_value} config_value={config_value} module={module} configValues={configValues} />
+                            );
+                        })}
+                    </Stack>
+                </DialogContent>
+            </Dialog>
+        </>
+    );
+}
+
+export default StepCard;

scripts/settings/src/main.tsx

@@ -0,0 +1,44 @@
+import * as React from 'react';
+import * as ReactDOM from 'react-dom/client';
+import { ThemeProvider } from '@mui/material/styles';
+import { CssBaseline } from '@mui/material';
+import App from './App';
+import { createTheme } from '@mui/material/styles';
+import { red } from '@mui/material/colors';
+import { useState, useEffect } from 'react';
+
+function RootApp() {
+  const [mode, setMode] = useState('light');
+
+useEffect(() => {
+    setMode(window.localStorage.getItem('theme') || 'light');
+}, []);
+
+var observer = new MutationObserver(function(mutations) {
+  setMode(window.localStorage.getItem('theme') || 'light');
+  
+})
+observer.observe(document.documentElement, {attributes: true, attributeFilter: ['data-theme']});
+
+// A custom theme for this app
+const theme = createTheme({
+  palette: {
+    mode: mode == 'light' ? 'light' : 'dark',
+  },
+  cssVariables: true
+});
+
+  return (
+    <ThemeProvider theme={theme}>
+    <CssBaseline />
+    <App  />
+  </ThemeProvider>
+  );
+}
+
+ReactDOM.createRoot(document.getElementById('root')!).render(
+  <React.StrictMode>
+    <RootApp />
+  </React.StrictMode>,
+);
+

scripts/settings/src/types.d.ts

@@ -0,0 +1,21 @@
+export interface Config {
+  name: string;
+  description: string;
+  type: string?;
+  default: any;
+  help: string;
+  choices: string[];
+  required: boolean;
+}
+
+interface Manifest {
+  description: string;
+}
+
+export interface Module {
+  name: string;
+  description: string;
+  configs: { [key: string]: Config };
+  manifest: Manifest;
+  display_name: string;
+}

scripts/settings/tsconfig.json

@@ -0,0 +1,21 @@
+{
+  "compilerOptions": {
+    "target": "ESNext",
+    "useDefineForClassFields": true,
+    "lib": ["DOM", "DOM.Iterable", "ESNext"],
+    "allowJs": false,
+    "skipLibCheck": true,
+    "esModuleInterop": false,
+    "allowSyntheticDefaultImports": true,
+    "strict": true,
+    "forceConsistentCasingInFileNames": true,
+    "module": "ESNext",
+    "moduleResolution": "Node",
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "noEmit": true,
+    "jsx": "react-jsx"
+  },
+  "include": ["src"],
+  "references": [{ "path": "./tsconfig.node.json" }]
+}

scripts/settings/tsconfig.node.json

@@ -0,0 +1,9 @@
+{
+  "compilerOptions": {
+    "composite": true,
+    "module": "ESNext",
+    "moduleResolution": "Node",
+    "allowSyntheticDefaultImports": true
+  },
+  "include": ["vite.config.ts"]
+}

scripts/settings/vite.config.ts

@@ -0,0 +1,12 @@
+import { defineConfig } from 'vite';
+import react from '@vitejs/plugin-react';
+import { viteSingleFile } from "vite-plugin-singlefile"
+
+// https://vite.dev/config/
+export default defineConfig({
+  plugins: [react(), viteSingleFile()],
+  build: {
+    // minify: false,
+    // sourcemap: true,
+  }
+});

scripts/telegram_setup.py

@@ -12,10 +12,9 @@ Then run this script to create a new session file.
 You will need to provide your phone number and a 2FA code the first time you run this script.
 """
 
-
 import os
 from telethon.sync import TelegramClient
-from loguru import logger
+from auto_archiver.utils.custom_logger import logger
 
 
 # Create a
@@ -25,5 +24,4 @@ SESSION_FILE = "secrets/anon-insta"
 
 os.makedirs("secrets", exist_ok=True)
 with TelegramClient(SESSION_FILE, API_ID, API_HASH) as client:
-    logger.success(f"New session file created: {SESSION_FILE}.session")
-
+    logger.success(f"new session file created: {SESSION_FILE}.session")

src/auto_archiver/__main__.py

@@ -1,9 +1,13 @@
-""" Entry point for the auto_archiver package. """
+"""Entry point for the auto_archiver package."""
+
 from auto_archiver.core.orchestrator import ArchivingOrchestrator
 import sys
 
+
 def main():
-    ArchivingOrchestrator().run(sys.argv[1:])
+    for _ in ArchivingOrchestrator()._command_line_run(sys.argv[1:]):
+        pass
+
 
 if __name__ == "__main__":
     main()

src/auto_archiver/core/__init__.py

@@ -1,9 +1,8 @@
-""" Core modules to handle things such as orchestration, metadata and configs..
+"""Core modules to handle things such as orchestration, metadata and configs.."""
 
-"""
 from .metadata import Metadata
 from .media import Media
-from .module import BaseModule
+from .base_module import BaseModule
 
 # cannot import ArchivingOrchestrator/Config to avoid circular dep
 # from .orchestrator import ArchivingOrchestrator
@@ -14,4 +13,4 @@ from .enricher import Enricher
 from .feeder import Feeder
 from .storage import Storage
 from .extractor import Extractor
-from .formatter import Formatter
+from .formatter import Formatter

src/auto_archiver/core/base_module.py

@@ -1,15 +1,19 @@
+from __future__ import annotations
 
-from urllib.parse import urlparse
-from typing import  Mapping, Any
+from typing import Mapping, Any, TYPE_CHECKING
 from abc import ABC
-from copy import deepcopy, copy
+from copy import deepcopy
 from tempfile import TemporaryDirectory
 from auto_archiver.utils import url as UrlUtil
+from auto_archiver.core.consts import MODULE_TYPES as CONF_MODULE_TYPES
+
+from auto_archiver.utils.custom_logger import logger
+
+if TYPE_CHECKING:
+    from .module import ModuleFactory
 
-from loguru import logger
 
 class BaseModule(ABC):
-
     """
     Base module class. All modules should inherit from this class.
 
@@ -17,63 +21,37 @@ class BaseModule(ABC):
     however modules can have a .setup() method to run any setup code
     (e.g. logging in to a site, spinning up a browser etc.)
 
-    See BaseModule.MODULE_TYPES for the types of modules you can create, noting that
+    See consts.MODULE_TYPES for the types of modules you can create, noting that
     a subclass can be of multiple types. For example, a module that extracts data from
     a website and stores it in a database would be both an 'extractor' and a 'database' module.
 
     Each module is a python package, and should have a __manifest__.py file in the
     same directory as the module file. The __manifest__.py specifies the module information
-    like name, author, version, dependencies etc. See BaseModule._DEFAULT_MANIFEST for the
+    like name, author, version, dependencies etc. See DEFAULT_MANIFEST for the
     default manifest structure.
 
     """
 
-    MODULE_TYPES = [
-        'feeder',
-        'extractor',
-        'enricher',
-        'database',
-        'storage',
-        'formatter'
-    ]
-
-    _DEFAULT_MANIFEST = {
-    'name': '', # the display name of the module
-    'author': 'Bellingcat', # creator of the module, leave this as Bellingcat or set your own name!
-    'type': [], # the type of the module, can be one or more of BaseModule.MODULE_TYPES
-    'requires_setup': True, # whether or not this module requires additional setup such as setting API Keys or installing additional softare
-    'description': '', # a description of the module
-    'dependencies': {}, # external dependencies, e.g. python packages or binaries, in dictionary format
-    'entry_point': '', # the entry point for the module, in the format 'module_name::ClassName'. This can be left blank to use the default entry point of module_name::ModuleName
-    'version': '1.0', # the version of the module
-    'configs': {} # any configuration options this module has, these will be exposed to the user in the config file or via the command line
-}
+    MODULE_TYPES = CONF_MODULE_TYPES
 
+    # NOTE: these here are declard as class variables, but they are overridden by the instance variables in the __init__ method
     config: Mapping[str, Any]
     authentication: Mapping[str, Mapping[str, str]]
     name: str
+    module_factory: ModuleFactory
 
     # this is set by the orchestrator prior to archiving
     tmp_dir: TemporaryDirectory = None
 
     @property
     def storages(self) -> list:
-        return self.config.get('storages', [])
+        return self.config.get("storages", [])
 
     def config_setup(self, config: dict):
-
-        authentication = config.get('authentication', {})
-        # extract out concatenated sites
-        for key, val in copy(authentication).items():
-            if "," in key:
-                for site in key.split(","):
-                    authentication[site] = val
-                del authentication[key]
-
         # this is important. Each instance is given its own deepcopied config, so modules cannot
         # change values to affect other modules
         config = deepcopy(config)
-        authentication = deepcopy(config.pop('authentication', {}))
+        authentication = deepcopy(config.pop("authentication", {}))
 
         self.authentication = authentication
         self.config = config
@@ -81,34 +59,50 @@ class BaseModule(ABC):
             setattr(self, key, val)
 
     def setup(self):
-        # For any additional setup required by modules, e.g. autehntication
+        # For any additional setup required by modules outside of the configs in the manifesst,
+        # e.g. authentication
         pass
 
     def auth_for_site(self, site: str, extract_cookies=True) -> Mapping[str, Any]:
         """
         Returns the authentication information for a given site. This is used to authenticate
         with a site before extracting data. The site should be the domain of the site, e.g. 'twitter.com'
-        
-        extract_cookies: bool - whether or not to extract cookies from the given browser and return the 
-        cookie jar (disabling can speed up) processing if you don't actually need the cookies jar
 
-        Currently, the dict can have keys of the following types:
-        - username: str - the username to use for login
-        - password: str - the password to use for login
-        - api_key: str - the API key to use for login
-        - api_secret: str - the API secret to use for login
-        - cookie: str - a cookie string to use for login (specific to this site)
-        - cookies_jar: YoutubeDLCookieJar | http.cookiejar.MozillaCookieJar - a cookie jar compatible with requests (e.g. `requests.get(cookies=cookie_jar)`)
+        :param site: the domain of the site to get authentication information for
+        :param extract_cookies: whether or not to extract cookies from the given browser/file and return the cookie jar (disabling can speed up processing if you don't actually need the cookies jar).
+
+        :returns: authdict dict -> {
+            "username": str,
+            "password": str,
+            "api_key": str,
+            "api_secret": str,
+            "cookie": str,
+            "cookies_file": str,
+            "cookies_from_browser": str,
+            "cookies_jar": CookieJar
+        }
+
+        **Global options:**\n
+        * cookies_from_browser: str - the name of the browser to extract cookies from (e.g. 'chrome', 'firefox' - uses ytdlp under the hood to extract\n
+        * cookies_file: str - the path to a cookies file to use for login\n
+
+        **Currently, the sites dict can have keys of the following types:**\n
+        * username: str - the username to use for login\n
+        * password: str - the password to use for login\n
+        * api_key: str - the API key to use for login\n
+        * api_secret: str - the API secret to use for login\n
+        * cookie: str - a cookie string to use for login (specific to this site)\n
+        * cookies_file: str - the path to a cookies file to use for login (specific to this site)\n
+        * cookies_from_browser: str - the name of the browser to extract cookies from (specitic for this site)\n
+
         """
         # TODO: think about if/how we can deal with sites that have multiple domains (main one is x.com/twitter.com)
         # for now the user must enter them both, like "x.com,twitter.com" in their config. Maybe we just hard-code?
-
-        site = UrlUtil.domain_for_url(site)
+        domain = UrlUtil.domain_for_url(site).removeprefix("www.")
         # add the 'www' version of the site to the list of sites to check
         authdict = {}
 
-
-        for to_try in [site, f"www.{site}"]:
+        for to_try in [site, domain, f"www.{domain}"]:
             if to_try in self.authentication:
                 authdict.update(self.authentication[to_try])
                 break
@@ -116,31 +110,46 @@ class BaseModule(ABC):
         # do a fuzzy string match just to print a warning - don't use it since it's insecure
         if not authdict:
             for key in self.authentication.keys():
-                if key in site or site in key:
-                    logger.debug(f"Could not find exact authentication information for site '{site}'. \
-                                    did find information for '{key}' which is close, is this what you meant? \
-                                    If so, edit your authentication settings to make sure it exactly matches.")
+                if key in domain or domain in key:
+                    logger.debug(
+                        f"Could not find exact authentication information for '{domain}'. \
+did find information for '{key}' which is close, is this what you meant? \
+If so, edit your authentication settings to make sure it exactly matches."
+                    )
 
         def get_ytdlp_cookiejar(args):
             import yt_dlp
             from yt_dlp import parse_options
+
             logger.debug(f"Extracting cookies from settings: {args[1]}")
             # parse_options returns a named tuple as follows, we only need the ydl_options part
             # collections.namedtuple('ParsedOptions', ('parser', 'options', 'urls', 'ydl_opts'))
-            ytdlp_opts = getattr(parse_options(args), 'ydl_opts')
+            ytdlp_opts = getattr(parse_options(args), "ydl_opts")
             return yt_dlp.YoutubeDL(ytdlp_opts).cookiejar
 
-        # get the cookies jar, prefer the browser cookies than the file
-        if 'cookies_from_browser' in self.authentication:
-            authdict['cookies_from_browser'] = self.authentication['cookies_from_browser']
-            if extract_cookies:
-                authdict['cookies_jar'] = get_ytdlp_cookiejar(['--cookies-from-browser', self.authentication['cookies_from_browser']])
-        elif 'cookies_file' in self.authentication:
-            authdict['cookies_file'] = self.authentication['cookies_file']
-            if extract_cookies:
-                authdict['cookies_jar'] = get_ytdlp_cookiejar(['--cookies', self.authentication['cookies_file']])
-        
+        get_cookiejar_options = None
+
+        # order of priority:
+        # 1. cookies_from_browser setting in site config
+        # 2. cookies_file setting in site config
+        # 3. cookies_from_browser setting in global config
+        # 4. cookies_file setting in global config
+
+        if "cookies_from_browser" in authdict:
+            get_cookiejar_options = ["--cookies-from-browser", authdict["cookies_from_browser"]]
+        elif "cookies_file" in authdict:
+            get_cookiejar_options = ["--cookies", authdict["cookies_file"]]
+        elif "cookies_from_browser" in self.authentication:
+            authdict["cookies_from_browser"] = self.authentication["cookies_from_browser"]
+            get_cookiejar_options = ["--cookies-from-browser", self.authentication["cookies_from_browser"]]
+        elif "cookies_file" in self.authentication:
+            authdict["cookies_file"] = self.authentication["cookies_file"]
+            get_cookiejar_options = ["--cookies", self.authentication["cookies_file"]]
+
+        if get_cookiejar_options:
+            authdict["cookies_jar"] = get_ytdlp_cookiejar(get_cookiejar_options)
+
         return authdict
-    
+
     def repr(self):
-        return f"Module<'{self.display_name}' (config: {self.config[self.name]})>"
+        return f"Module<'{self.display_name}' (config: {self.config[self.name]})>"

src/auto_archiver/core/config.py

@@ -6,23 +6,28 @@ flexible setup in various environments.
 """
 
 import argparse
-from ruamel.yaml import YAML, CommentedMap, add_representer
+from ruamel.yaml import YAML, CommentedMap
+import json
+import os
 
-from loguru import logger
+from auto_archiver.utils.custom_logger import logger
 
 from copy import deepcopy
-from .module import BaseModule
+from auto_archiver.core.consts import MODULE_TYPES
 
-from typing import Any, List, Type, Tuple
 
 _yaml: YAML = YAML()
 
-EMPTY_CONFIG = _yaml.load("""
-# Auto Archiver Configuration
-# Steps are the modules that will be run in the order they are defined
+DEFAULT_CONFIG_FILE = "secrets/orchestration.yaml"
 
-steps:""" + "".join([f"\n   {module}s: []" for module in BaseModule.MODULE_TYPES]) + \
-"""
+EMPTY_CONFIG = _yaml.load(
+    """
+# Auto Archiver Configuration
+
+# Steps are the modules that will be run in the order they are defined
+steps:"""
+    + "".join([f"\n   {module}s: []" for module in MODULE_TYPES])
+    + """
 
 # Global configuration
 
@@ -49,17 +54,71 @@ authentication: {}
 logging:
   level: INFO
 
-""")
+"""
+)
 # note: 'logging' is explicitly added above in order to better format the config file
 
-class DefaultValidatingParser(argparse.ArgumentParser):
 
+# Arg Parse Actions/Classes
+class AuthenticationJsonParseAction(argparse.Action):
+    def __call__(self, parser, namespace, values, option_string=None):
+        try:
+            auth_dict = json.loads(values)
+            setattr(namespace, self.dest, auth_dict)
+        except json.JSONDecodeError as e:
+            raise argparse.ArgumentTypeError(f"Invalid JSON input for argument '{self.dest}': {e}") from e
+
+        def load_from_file(path):
+            try:
+                with open(path, "r") as f:
+                    try:
+                        auth_dict = json.load(f)
+                    except json.JSONDecodeError:
+                        f.seek(0)
+                        # maybe it's yaml, try that
+                        auth_dict = _yaml.load(f)
+                    if auth_dict.get("authentication"):
+                        auth_dict = auth_dict["authentication"]
+                    auth_dict["load_from_file"] = path
+                    return auth_dict
+            except Exception:
+                return None
+
+        if isinstance(auth_dict, dict) and auth_dict.get("from_file"):
+            auth_dict = load_from_file(auth_dict["from_file"])
+        elif isinstance(auth_dict, str):
+            # if it's a string
+            auth_dict = load_from_file(auth_dict)
+
+        if not isinstance(auth_dict, dict):
+            raise argparse.ArgumentTypeError(
+                "Authentication must be a dictionary of site names and their authentication methods"
+            )
+        global_options = ["cookies_from_browser", "cookies_file", "load_from_file"]
+        for key, auth in auth_dict.items():
+            if key in global_options:
+                continue
+            if not isinstance(key, str) or not isinstance(auth, dict):
+                raise argparse.ArgumentTypeError(
+                    f"Authentication must be a dictionary of site names and their authentication methods. Valid global configs are {global_options}"
+                )
+
+        setattr(namespace, self.dest, auth_dict)
+
+
+class UniqueAppendAction(argparse.Action):
+    def __call__(self, parser, namespace, values, option_string=None):
+        for value in values:
+            if value not in getattr(namespace, self.dest):
+                getattr(namespace, self.dest).append(value)
+
+
+class DefaultValidatingParser(argparse.ArgumentParser):
     def error(self, message):
         """
         Override of error to format a nicer looking error message using logger
         """
-        logger.error("Problem with configuration file (tip: use --help to see the available options):")
-        logger.error(message)
+        logger.error(f"Problem with configuration file (tip: use --help to see the available options): \n{message}")
         self.exit(2)
 
     def parse_known_args(self, args=None, namespace=None):
@@ -76,13 +135,15 @@ class DefaultValidatingParser(argparse.ArgumentParser):
                     try:
                         self._check_value(action, action.default)
                     except argparse.ArgumentError as e:
-                        logger.error(f"You have an invalid setting in your configuration file ({action.dest}):")
-                        logger.error(e)
+                        logger.error(f"You have an invalid setting in your configuration file ({action.dest}):\n {e}")
                         exit()
 
         return super().parse_known_args(args, namespace)
 
 
+# Config Utils
+
+
 def to_dot_notation(yaml_conf: CommentedMap | dict) -> dict:
     dotdict = {}
 
@@ -96,6 +157,7 @@ def to_dot_notation(yaml_conf: CommentedMap | dict) -> dict:
     process_subdict(yaml_conf)
     return dotdict
 
+
 def from_dot_notation(dotdict: dict) -> dict:
     normal_dict = {}
 
@@ -116,9 +178,11 @@ def from_dot_notation(dotdict: dict) -> dict:
 def is_list_type(value):
     return isinstance(value, list) or isinstance(value, tuple) or isinstance(value, set)
 
+
 def is_dict_type(value):
     return isinstance(value, dict) or isinstance(value, CommentedMap)
 
+
 def merge_dicts(dotdict: dict, yaml_dict: CommentedMap) -> CommentedMap:
     yaml_dict: CommentedMap = deepcopy(yaml_dict)
 
@@ -129,6 +193,11 @@ def merge_dicts(dotdict: dict, yaml_dict: CommentedMap) -> CommentedMap:
                 yaml_subdict[key] = value
                 continue
 
+            if key == "steps":
+                for module_type, modules in value.items():
+                    # overwrite the 'steps' from the config file with the ones from the CLI
+                    yaml_subdict[key][module_type] = modules
+
             if is_dict_type(value):
                 update_dict(value, yaml_subdict[key])
             elif is_list_type(value):
@@ -137,9 +206,9 @@ def merge_dicts(dotdict: dict, yaml_dict: CommentedMap) -> CommentedMap:
                 yaml_subdict[key] = value
 
     update_dict(from_dot_notation(dotdict), yaml_dict)
-
     return yaml_dict
 
+
 def read_yaml(yaml_filename: str) -> CommentedMap:
     config = None
     try:
@@ -149,16 +218,30 @@ def read_yaml(yaml_filename: str) -> CommentedMap:
         pass
 
     if not config:
-        config = EMPTY_CONFIG
-    
+        config = deepcopy(EMPTY_CONFIG)
+
     return config
 
+
 # TODO: make this tidier/find a way to notify of which keys should not be stored
 
 
 def store_yaml(config: CommentedMap, yaml_filename: str) -> None:
     config_to_save = deepcopy(config)
 
-    config_to_save.pop('urls', None)
+    ## if the save path is the default location (secrets) then create the 'secrets' folder
+    if os.path.dirname(yaml_filename) == "secrets":
+        os.makedirs("secrets", exist_ok=True)
+
+    auth_dict = config_to_save.get("authentication", {})
+    if auth_dict and auth_dict.get("load_from_file"):
+        # remove all other values from the config, don't want to store it in the config file
+        auth_dict = {"load_from_file": auth_dict["load_from_file"]}
+
+    config_to_save.pop("urls", None)
     with open(yaml_filename, "w", encoding="utf-8") as outf:
-        _yaml.dump(config_to_save, outf)
+        _yaml.dump(config_to_save, outf)
+
+
+def is_valid_config(config: CommentedMap) -> bool:
+    return config and config != EMPTY_CONFIG

src/auto_archiver/core/consts.py

@@ -0,0 +1,19 @@
+class SetupError(ValueError):
+    pass
+
+
+MODULE_TYPES = ["feeder", "extractor", "enricher", "database", "storage", "formatter"]
+
+MANIFEST_FILE = "__manifest__.py"
+
+DEFAULT_MANIFEST = {
+    "name": "",  # the display name of the module
+    "author": "Bellingcat",  # creator of the module, leave this as Bellingcat or set your own name!
+    "type": [],  # the type of the module, can be one or more of MODULE_TYPES
+    "requires_setup": True,  # whether or not this module requires additional setup such as setting API Keys or installing additional software
+    "description": "",  # a description of the module
+    "dependencies": {},  # external dependencies, e.g. python packages or binaries, in dictionary format
+    "entry_point": "",  # the entry point for the module, in the format 'module_name::ClassName'. This can be left blank to use the default entry point of module_name::ModuleName
+    "version": "1.0",  # the version of the module
+    "configs": {},  # any configuration options this module has, these will be exposed to the user in the config file or via the command line
+}

src/auto_archiver/core/database.py

@@ -1,6 +1,6 @@
 """
 Database module for the auto-archiver that defines the interface for implementing database modules
-in the media archiving framework. 
+in the media archiving framework.
 """
 
 from __future__ import annotations
@@ -9,6 +9,7 @@ from typing import Union
 
 from auto_archiver.core import Metadata, BaseModule
 
+
 class Database(BaseModule):
     """
     Base class for implementing database modules in the media archiving framework.
@@ -20,7 +21,7 @@ class Database(BaseModule):
         """signals the DB that the given item archival has started"""
         pass
 
-    def failed(self, item: Metadata, reason:str) -> None:
+    def failed(self, item: Metadata, reason: str) -> None:
         """update DB accordingly for failure"""
         pass
 
@@ -34,6 +35,6 @@ class Database(BaseModule):
         return False
 
     @abstractmethod
-    def done(self, item: Metadata, cached: bool=False) -> None:
+    def done(self, item: Metadata, cached: bool = False) -> None:
         """archival result ready - should be saved to DB"""
         pass

src/auto_archiver/core/enricher.py

@@ -8,13 +8,15 @@ the archiving step and before storage or formatting.
 
 Enrichers are optional but highly useful for making the archived data more powerful.
 """
+
 from __future__ import annotations
 from abc import abstractmethod
 from auto_archiver.core import Metadata, BaseModule
 
+
 class Enricher(BaseModule):
-    """Base classes and utilities for enrichers in the Auto-Archiver system.
-    
+    """Base classes and utilities for enrichers in the Auto Archiver system.
+
     Enricher modules must implement the `enrich` method to define their behavior.
     """
 

src/auto_archiver/core/extractor.py

@@ -1,23 +1,23 @@
-""" The `extractor` module defines the base functionality for implementing extractors in the media archiving framework.
-    This class provides common utility methods and a standard interface for extractors.
+"""The `extractor` module defines the base functionality for implementing extractors in the media archiving framework.
+This class provides common utility methods and a standard interface for extractors.
 
-    Factory method to initialize an extractor instance based on its name.
+Factory method to initialize an extractor instance based on its name.
 
 
 """
+
 from __future__ import annotations
-from pathlib import Path
 from abc import abstractmethod
-from dataclasses import dataclass
+from contextlib import suppress
 import mimetypes
 import os
-import mimetypes
 import requests
-from loguru import logger
+from auto_archiver.utils.custom_logger import logger
 from retrying import retry
 import re
 
 from auto_archiver.core import Metadata, BaseModule
+from auto_archiver.utils.url import get_media_url_best_quality
 
 
 class Extractor(BaseModule):
@@ -39,7 +39,7 @@ class Extractor(BaseModule):
         Used to clean unnecessary URL parameters OR unfurl redirect links
         """
         return url
-    
+
     def match_link(self, url: str) -> re.Match:
         """
         Returns a match object if the given URL matches the valid_url pattern or False/None if not.
@@ -58,7 +58,7 @@ class Extractor(BaseModule):
         """
         if self.valid_url:
             return self.match_link(url) is not None
-        
+
         return True
 
     def _guess_file_type(self, path: str) -> str:
@@ -72,18 +72,31 @@ class Extractor(BaseModule):
         return ""
 
     @retry(wait_random_min=500, wait_random_max=3500, stop_max_attempt_number=5)
-    def download_from_url(self, url: str, to_filename: str = None, verbose=True) -> str:
+    def download_from_url(self, url: str, to_filename: str = None, verbose=True, try_best_quality=False) -> str:
         """
-            downloads a URL to provided filename, or inferred from URL, returns local filename
+        downloads a URL to provided filename, or inferred from URL, returns local filename
+        Warning: if try_best_quality is True, it will return a tuple of (filename, best_quality_url) if the download was successful.
         """
+        if any(url.startswith(x) for x in ["blob:", "data:"]):
+            return None, url if try_best_quality else None
+
+        if try_best_quality:
+            with suppress(Exception):
+                # Attempt to download the original URL
+                best_quality_url = get_media_url_best_quality(url)
+                orig_download = self.download_from_url(best_quality_url, to_filename, verbose)
+                if orig_download:
+                    return orig_download, best_quality_url
+
         if not to_filename:
-            to_filename = url.split('/')[-1].split('?')[0]
+            to_filename = url.split("/")[-1].split("?")[0]
             if len(to_filename) > 64:
                 to_filename = to_filename[-64:]
         to_filename = os.path.join(self.tmp_dir, to_filename)
-        if verbose: logger.debug(f"downloading {url[0:50]=} {to_filename=}")
+        if verbose:
+            logger.debug(f"Downloading {to_filename=}")
         headers = {
-            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36'
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36"
         }
         try:
             d = requests.get(url, stream=True, headers=headers, timeout=30)
@@ -91,25 +104,29 @@ class Extractor(BaseModule):
 
             # get mimetype from the response headers
             if not mimetypes.guess_type(to_filename)[0]:
-                content_type = d.headers.get('Content-Type') or self._guess_file_type(url)
+                content_type = d.headers.get("Content-Type") or self._guess_file_type(url)
                 extension = mimetypes.guess_extension(content_type)
                 if extension:
                     to_filename += extension
 
-            with open(to_filename, 'wb') as f:
+            with open(to_filename, "wb") as f:
                 for chunk in d.iter_content(chunk_size=8192):
                     f.write(chunk)
+            if try_best_quality:
+                return to_filename, url
             return to_filename
 
         except requests.RequestException as e:
             logger.warning(f"Failed to fetch the Media URL: {e}")
+        if try_best_quality:
+            return None, url
 
     @abstractmethod
     def download(self, item: Metadata) -> Metadata | False:
         """
         Downloads the media from the given URL and returns a Metadata object with the downloaded media.
-        
+
         If the URL is not supported or the download fails, this method should return False.
 
         """
-        pass
+        pass

src/auto_archiver/core/feeder.py

@@ -1,5 +1,5 @@
 """
-The feeder base module defines the interface for implementing feeders in the media archiving framework. 
+The feeder base module defines the interface for implementing feeders in the media archiving framework.
 """
 
 from __future__ import annotations
@@ -7,8 +7,8 @@ from abc import abstractmethod
 from auto_archiver.core import Metadata
 from auto_archiver.core import BaseModule
 
-class Feeder(BaseModule):
 
+class Feeder(BaseModule):
     """
     Base class for implementing feeders in the media archiving framework.
 
@@ -19,7 +19,7 @@ class Feeder(BaseModule):
     def __iter__(self) -> Metadata:
         """
         Returns an iterator (use `yield`) over the items to be archived.
-        
+
         These should be instances of Metadata, typically created with Metadata().set_url(url).
         """
-        return None
+        return None

src/auto_archiver/core/formatter.py

@@ -12,7 +12,7 @@ from auto_archiver.core import Metadata, Media, BaseModule
 class Formatter(BaseModule):
     """
     Base class for implementing formatters in the media archiving framework.
-    
+
     Subclasses must implement the `format` method to define their behavior.
     """
 
@@ -21,4 +21,4 @@ class Formatter(BaseModule):
         """
         Formats a Metadata object into a user-viewable format (e.g. HTML) and stores it if needed.
         """
-        return None
+        return None

src/auto_archiver/core/media.py

@@ -6,12 +6,12 @@ nested media retrieval, and type validation.
 from __future__ import annotations
 import os
 import traceback
-from typing import Any, List
+from typing import Any, List, Iterator
 from dataclasses import dataclass, field
 from dataclasses_json import dataclass_json, config
 import mimetypes
 
-from loguru import logger
+from auto_archiver.utils.custom_logger import logger
 
 
 @dataclass_json  # annotation order matters
@@ -21,14 +21,14 @@ class Media:
     Represents a media file with associated properties and storage details.
 
     Attributes:
-    - filename: The file path of the media.
-    - key: An optional identifier for the media.
+    - filename: The file path of the media as saved locally (temporarily, before uploading to the storage).
     - urls: A list of URLs where the media is stored or accessible.
     - properties: Additional metadata or transformations for the media.
     - _mimetype: The media's mimetype (e.g., image/jpeg, video/mp4).
     """
+
     filename: str
-    key: str = None
+    _key: str = None
     urls: List[str] = field(default_factory=list)
     properties: dict = field(default_factory=dict)
     _mimetype: str = None  # eg: image/jpeg
@@ -47,19 +47,20 @@ class Media:
             for any_media in self.all_inner_media(include_self=True):
                 s.store(any_media, url, metadata=metadata)
 
-    def all_inner_media(self, include_self=False):
+    def all_inner_media(self, include_self=False) -> Iterator[Media]:
         """Retrieves all media, including nested media within properties or transformations on original media.
         This function returns a generator for all the inner media.
 
         """
-        if include_self: yield self
+        if include_self:
+            yield self
         for prop in self.properties.values():
-            if isinstance(prop, Media): 
+            if isinstance(prop, Media):
                 for inner_media in prop.all_inner_media(include_self=True):
                     yield inner_media
             if isinstance(prop, list):
                 for prop_media in prop:
-                    if isinstance(prop_media, Media): 
+                    if isinstance(prop_media, Media):
                         for inner_media in prop_media.all_inner_media(include_self=True):
                             yield inner_media
 
@@ -67,6 +68,10 @@ class Media:
         # checks if the media is already stored in the given storage
         return len(self.urls) > 0 and len(self.urls) == len(in_storage.config["steps"]["storages"])
 
+    @property
+    def key(self) -> str:
+        return self._key
+
     def set(self, key: str, value: Any) -> Media:
         self.properties[key] = value
         return self
@@ -81,7 +86,7 @@ class Media:
     @property  # getter .mimetype
     def mimetype(self) -> str:
         if not self.filename or len(self.filename) == 0:
-            logger.warning(f"cannot get mimetype from media without filename: {self}")
+            logger.warning(f"Cannot get mimetype from media without filename: {self}")
             return ""
         if not self._mimetype:
             self._mimetype = mimetypes.guess_type(self.filename)[0]
@@ -110,15 +115,16 @@ class Media:
         # checks for video streams with ffmpeg, or min file size for a video
         # self.is_video() should be used together with this method
         try:
-            streams = ffmpeg.probe(self.filename, select_streams='v')['streams']
-            logger.warning(f"STREAMS FOR {self.filename} {streams}")
+            streams = ffmpeg.probe(self.filename, select_streams="v")["streams"]
+            logger.debug(f"Streams for {self.filename}: {streams}")
             return any(s.get("duration_ts", 0) > 0 for s in streams)
-        except Error: return False # ffmpeg errors when reading bad files
+        except Error:
+            return False  # ffmpeg errors when reading bad files
         except Exception as e:
-            logger.error(e)
-            logger.error(traceback.format_exc())
+            logger.error(f"{e}: {traceback.format_exc()}")
             try:
                 fsize = os.path.getsize(self.filename)
                 return fsize > 20_000
-            except: pass
+            except Exception as e:
+                pass
         return True

src/auto_archiver/core/metadata.py

@@ -11,16 +11,18 @@ Key Functionalities:
 
 from __future__ import annotations
 import hashlib
+import os
 from typing import Any, List, Union, Dict
 from dataclasses import dataclass, field
-from dataclasses_json import dataclass_json, config
+from dataclasses_json import dataclass_json
 import datetime
 from urllib.parse import urlparse
 from dateutil.parser import parse as parse_dt
-from loguru import logger
+from auto_archiver.utils.custom_logger import logger
 
 from .media import Media
 
+
 @dataclass_json  # annotation order matters
 @dataclass
 class Metadata:
@@ -40,19 +42,23 @@ class Metadata:
         - If `True`, this instance's values are overwritten by `right`.
         - If `False`, the inverse applies.
         """
-        if not right: return self
+        if not right:
+            return self
         if overwrite_left:
             if right.status and len(right.status):
                 self.status = right.status
             self._context.update(right._context)
             for k, v in right.metadata.items():
-                assert k not in self.metadata or type(v) == type(self.get(k))
-                if type(v) not in [dict, list, set] or k not in self.metadata:
+                assert k not in self.metadata or type(v) is type(self.get(k))
+                if not isinstance(v, (dict, list, set)) or k not in self.metadata:
                     self.set(k, v)
                 else:  # key conflict
-                    if type(v) in [dict, set]: self.set(k, self.get(k) | v)
-                    elif type(v) == list: self.set(k, self.get(k) + v)
+                    if isinstance(v, (dict, set)):
+                        self.set(k, self.get(k) | v)
+                    elif type(v) is list:
+                        self.set(k, self.get(k) + v)
             self.media.extend(right.media)
+
         else:  # invert and do same logic
             return right.merge(self)
         return self
@@ -69,7 +75,7 @@ class Metadata:
 
     def append(self, key: str, val: Any) -> Metadata:
         if key not in self.metadata:
-            self.metadata[key] = []    
+            self.metadata[key] = []
         self.metadata[key] = val
         return self
 
@@ -80,24 +86,26 @@ class Metadata:
         return self.metadata.get(key, default)
 
     def success(self, context: str = None) -> Metadata:
-        if context: self.status = f"{context}: success"
-        else: self.status = "success"
+        if context:
+            self.status = f"{context}: success"
+        else:
+            self.status = "success"
         return self
 
     def is_success(self) -> bool:
         return "success" in self.status
 
     def is_empty(self) -> bool:
-        meaningfull_ids = set(self.metadata.keys()) - set(["_processed_at", "url", "total_bytes", "total_size", "archive_duration_seconds"])
+        meaningfull_ids = set(self.metadata.keys()) - set(
+            ["_processed_at", "url", "original_url", "total_bytes", "total_size", "archive_duration_seconds"]
+        )
         return not self.is_success() and len(self.media) == 0 and len(meaningfull_ids) == 0
 
     @property  # getter .netloc
     def netloc(self) -> str:
         return urlparse(self.get_url()).netloc
 
-
-# custom getter/setters
-
+    # custom getter/setters
 
     def set_url(self, url: str) -> Metadata:
         assert type(url) is str and len(url) > 0, "invalid URL"
@@ -120,36 +128,43 @@ class Metadata:
         return self.get("title")
 
     def set_timestamp(self, timestamp: datetime.datetime) -> Metadata:
-        if type(timestamp) == str:
+        if isinstance(timestamp, str):
             timestamp = parse_dt(timestamp)
-        assert type(timestamp) == datetime.datetime, "set_timestamp expects a datetime instance"
+        assert isinstance(timestamp, datetime.datetime), "set_timestamp expects a datetime instance"
         return self.set("timestamp", timestamp)
 
-    def get_timestamp(self, utc=True, iso=True) -> datetime.datetime:
+    def get_timestamp(self, utc=True, iso=True) -> datetime.datetime | str | None:
         ts = self.get("timestamp")
-        if not ts: return 
+        if not ts:
+            return None
         try:
-            if type(ts) == str: ts = datetime.datetime.fromisoformat(ts)
-            if type(ts) == float: ts = datetime.datetime.fromtimestamp(ts)
-            if utc: ts = ts.replace(tzinfo=datetime.timezone.utc)
-            if iso: return ts.isoformat()
-            return ts
+            if isinstance(ts, str):
+                ts = datetime.datetime.fromisoformat(ts)
+            elif isinstance(ts, float):
+                ts = datetime.datetime.fromtimestamp(ts)
+            if utc:
+                ts = ts.replace(tzinfo=datetime.timezone.utc)
+            return ts.isoformat() if iso else ts
         except Exception as e:
             logger.error(f"Unable to parse timestamp {ts}: {e}")
-            return
+            return None
 
     def add_media(self, media: Media, id: str = None) -> Metadata:
         # adds a new media, optionally including an id
-        if media is None: return
+        if media is None:
+            return
         if id is not None:
-            assert not len([1 for m in self.media if m.get("id") == id]), f"cannot add 2 pieces of media with the same id {id}"
+            assert not len([1 for m in self.media if m.get("id") == id]), (
+                f"cannot add 2 pieces of media with the same id {id}"
+            )
             media.set("id", id)
         self.media.append(media)
         return media
 
     def get_media_by_id(self, id: str, default=None) -> Media:
         for m in self.media:
-            if m.get("id") == id: return m
+            if m.get("id") == id:
+                return m
         return default
 
     def remove_duplicate_media_by_hash(self) -> None:
@@ -159,23 +174,33 @@ class Metadata:
             with open(filename, "rb") as f:
                 while True:
                     buf = f.read(chunksize)
-                    if not buf: break
+                    if not buf:
+                        break
                     hash_algo.update(buf)
             return hash_algo.hexdigest()
 
         media_hashes = set()
         new_media = []
         for m in self.media:
+            if not m.filename:
+                new_media.append(m)
+                continue
             h = m.get("hash")
-            if not h: h = calculate_hash_in_chunks(hashlib.sha256(), int(1.6e7), m.filename)
-            if len(h) and h in media_hashes: continue
+            if not h:
+                if not os.path.exists(m.filename):
+                    logger.warning(f"Skipping missing media file: {m.filename}")
+                    continue
+                h = calculate_hash_in_chunks(hashlib.sha256(), int(1.6e7), m.filename)
+            if len(h) and h in media_hashes:
+                continue
             media_hashes.add(h)
             new_media.append(m)
         self.media = new_media
 
     def get_first_image(self, default=None) -> Media:
         for m in self.media:
-            if "image" in m.mimetype: return m
+            if "image" in m.mimetype:
+                return m
         return default
 
     def set_final_media(self, final: Media) -> Metadata:
@@ -193,22 +218,25 @@ class Metadata:
     def __str__(self) -> str:
         return self.__repr__()
 
-
     @staticmethod
     def choose_most_complete(results: List[Metadata]) -> Metadata:
         # returns the most complete result from a list of results
         # prioritizes results with more media, then more metadata
-        if len(results) == 0: return None
-        if len(results) == 1: return results[0]
+        if len(results) == 0:
+            return None
+        if len(results) == 1:
+            return results[0]
         most_complete = results[0]
         for r in results[1:]:
-            if len(r.media) > len(most_complete.media): most_complete = r
-            elif len(r.media) == len(most_complete.media) and len(r.metadata) > len(most_complete.metadata): most_complete = r
+            if len(r.media) > len(most_complete.media):
+                most_complete = r
+            elif len(r.media) == len(most_complete.media) and len(r.metadata) > len(most_complete.metadata):
+                most_complete = r
         return most_complete
 
     def set_context(self, key: str, val: Any) -> Metadata:
         self._context[key] = val
         return self
-    
+
     def get_context(self, key: str, default: Any = None) -> Any:
-        return self._context.get(key, default)
+        return self._context.get(key, default)

src/auto_archiver/core/module.py

@@ -3,10 +3,12 @@ Defines the Step abstract base class, which acts as a blueprint for steps in the
 by handling user configuration, validating the steps properties, and implementing dynamic instantiation.
 
 """
+
 from __future__ import annotations
+import subprocess
 
 from dataclasses import dataclass
-from typing import List
+from typing import List, TYPE_CHECKING, Type
 import shutil
 import ast
 import copy
@@ -14,146 +16,176 @@ import sys
 from importlib.util import find_spec
 import os
 from os.path import join
-from loguru import logger
+from auto_archiver.utils.custom_logger import logger
 import auto_archiver
-from .base_module import BaseModule
+from auto_archiver.core.consts import DEFAULT_MANIFEST, MANIFEST_FILE, SetupError
 
-_LAZY_LOADED_MODULES = {}
-
-MANIFEST_FILE = "__manifest__.py"
+if TYPE_CHECKING:
+    from .base_module import BaseModule
 
 
-def setup_paths(paths: list[str]) -> None:
-    """
-    Sets up the paths for the modules to be loaded from
-    
-    This is necessary for the modules to be imported correctly
-    
-    """
-    for path in paths:
-        # check path exists, if it doesn't, log a warning
-        if not os.path.exists(path):
-            logger.warning(f"Path '{path}' does not exist. Skipping...")
-            continue
+HAS_SETUP_PATHS = False
 
-        # see odoo/module/module.py -> initialize_sys_path
-        if path not in auto_archiver.modules.__path__:
+
+class ModuleFactory:
+    def __init__(self):
+        self._lazy_modules = {}
+
+    def setup_paths(self, paths: list[str]) -> None:
+        """
+        Sets up the paths for the modules to be loaded from
+
+        This is necessary for the modules to be imported correctly
+
+        """
+        global HAS_SETUP_PATHS
+
+        for path in paths:
+            # check path exists, if it doesn't, log a warning
+            if not os.path.exists(path):
+                logger.warning(f"Path '{path}' does not exist. Skipping...")
+                continue
+
+            # see odoo/module/module.py -> initialize_sys_path
+            if path not in auto_archiver.modules.__path__:
+                if HAS_SETUP_PATHS:
+                    logger.warning(
+                        f"You are attempting to re-initialise the module paths with: '{path}' for a 2nd time. \
+                                       This could lead to unexpected behaviour. It is recommended to only use a single modules path. \
+                                       If you wish to load modules from different paths then load a 2nd python interpreter (e.g. using multiprocessing)."
+                    )
                 auto_archiver.modules.__path__.append(path)
 
-    # sort based on the length of the path, so that the longest path is last in the list
-    auto_archiver.modules.__path__ = sorted(auto_archiver.modules.__path__, key=len, reverse=True)
+        # sort based on the length of the path, so that the longest path is last in the list
+        auto_archiver.modules.__path__ = sorted(auto_archiver.modules.__path__, key=len, reverse=True)
 
-def get_module(module_name: str, config: dict) -> BaseModule:
-    """
-    Gets and sets up a module using the provided config
-    
-    This will actually load and instantiate the module, and load all its dependencies (i.e. not lazy)
-    
-    """
-    return get_module_lazy(module_name).load(config)
+        HAS_SETUP_PATHS = True
 
-def get_module_lazy(module_name: str, suppress_warnings: bool = False) -> LazyBaseModule:
-    """
-    Lazily loads a module, returning a LazyBaseModule
-    
-    This has all the information about the module, but does not load the module itself or its dependencies
-    
-    To load an actual module, call .setup() on a lazy module
-    
-    """
-    if module_name in _LAZY_LOADED_MODULES:
-        return _LAZY_LOADED_MODULES[module_name]
+    def get_module(self, module_name: str, config: dict) -> Type[BaseModule]:
+        """
+        Gets and sets up a module using the provided config
 
-    available = available_modules(limit_to_modules=[module_name], suppress_warnings=suppress_warnings)
-    if not available:
-        raise IndexError(f"Module '{module_name}' not found. Are you sure it's installed/exists?")
-    return available[0]
+        This will actually load and instantiate the module, and load all its dependencies (i.e. not lazy)
 
-def available_modules(with_manifest: bool=False, limit_to_modules: List[str]= [], suppress_warnings: bool = False) -> List[LazyBaseModule]:
-    
-    # search through all valid 'modules' paths. Default is 'modules' in the current directory
+        """
+        return self.get_module_lazy(module_name).load(config)
 
-    # see odoo/modules/module.py -> get_modules
-    def is_really_module(module_path):
-        if os.path.isfile(join(module_path, MANIFEST_FILE)):
-            return True
+    def get_module_lazy(self, module_name: str, suppress_warnings: bool = False) -> LazyBaseModule:
+        """
+        Lazily loads a module, returning a LazyBaseModule
 
-    all_modules = []
+        This has all the information about the module, but does not load the module itself or its dependencies
 
-    for module_folder in auto_archiver.modules.__path__:
-        # walk through each module in module_folder and check if it has a valid manifest
-        try:
-            possible_modules = os.listdir(module_folder)
-        except FileNotFoundError:
-            logger.warning(f"Module folder {module_folder} does not exist")
-            continue
+        To load an actual module, call .setup() on a lazy module
 
-        for possible_module in possible_modules:
-            if limit_to_modules and possible_module not in limit_to_modules:
+        """
+        if module_name in self._lazy_modules:
+            return self._lazy_modules[module_name]
+
+        available = self.available_modules(limit_to_modules=[module_name], suppress_warnings=suppress_warnings)
+        if not available:
+            message = f"Module '{module_name}' not found. Are you sure it's installed/exists?"
+            if "archiver" in module_name:
+                message += f" Did you mean '{module_name.replace('archiver', 'extractor')}'?"
+            elif "gsheet" in module_name:
+                message += " Did you mean 'gsheet_feeder_db'?"
+            elif "atlos" in module_name:
+                message += " Did you mean 'atlos_feeder_db_storage'?"
+            raise IndexError(message)
+        return available[0]
+
+    def available_modules(
+        self, limit_to_modules: List[str] = [], suppress_warnings: bool = False
+    ) -> List[LazyBaseModule]:
+        # search through all valid 'modules' paths. Default is 'modules' in the current directory
+
+        # see odoo/modules/module.py -> get_modules
+        def is_really_module(module_path):
+            if os.path.isfile(join(module_path, MANIFEST_FILE)):
+                return True
+
+        all_modules = []
+
+        for module_folder in auto_archiver.modules.__path__:
+            # walk through each module in module_folder and check if it has a valid manifest
+            try:
+                possible_modules = os.listdir(module_folder)
+            except FileNotFoundError:
+                logger.warning(f"Module folder {module_folder} does not exist")
                 continue
 
-            possible_module_path = join(module_folder, possible_module)
-            if not is_really_module(possible_module_path):
-                continue
-            if _LAZY_LOADED_MODULES.get(possible_module):
-                continue
-            lazy_module = LazyBaseModule(possible_module, possible_module_path)
+            for possible_module in possible_modules:
+                if limit_to_modules and possible_module not in limit_to_modules:
+                    continue
 
-            _LAZY_LOADED_MODULES[possible_module] = lazy_module
+                possible_module_path = join(module_folder, possible_module)
+                if not is_really_module(possible_module_path):
+                    continue
+                if self._lazy_modules.get(possible_module):
+                    continue
+                lazy_module = LazyBaseModule(possible_module, possible_module_path, factory=self)
 
-            all_modules.append(lazy_module)
-    
-    if not suppress_warnings:
-        for module in limit_to_modules:
-            if not any(module == m.name for m in all_modules):
-                logger.warning(f"Module '{module}' not found. Are you sure it's installed?")
+                self._lazy_modules[possible_module] = lazy_module
+
+                all_modules.append(lazy_module)
+
+        if not suppress_warnings:
+            for module in limit_to_modules:
+                if not any(module == m.name for m in all_modules):
+                    logger.warning(f"Module '{module}' not found. Are you sure it's installed?")
+
+        return all_modules
 
-    return all_modules
 
 @dataclass
 class LazyBaseModule:
-
     """
     A lazy module class, which only loads the manifest and does not load the module itself.
 
     This is useful for getting information about a module without actually loading it.
 
     """
+
     name: str
-    type: list
     description: str
     path: str
+    module_factory: ModuleFactory
 
     _manifest: dict = None
     _instance: BaseModule = None
     _entry_point: str = None
 
-    def __init__(self, module_name, path):
+    def __init__(self, module_name, path, factory: ModuleFactory):
         self.name = module_name
         self.path = path
+        self.module_factory = factory
+
+    @property
+    def type(self):
+        return self.manifest["type"]
 
     @property
     def entry_point(self):
-        if not self._entry_point and not self.manifest['entry_point']:
+        if not self._entry_point and not self.manifest["entry_point"]:
             # try to create the entry point from the module name
             self._entry_point = f"{self.name}::{self.name.replace('_', ' ').title().replace(' ', '')}"
         return self._entry_point
 
     @property
     def dependencies(self) -> dict:
-        return self.manifest['dependencies']
-    
+        return self.manifest["dependencies"]
+
     @property
     def configs(self) -> dict:
-        return self.manifest['configs']
-    
+        return self.manifest["configs"]
+
     @property
     def requires_setup(self) -> bool:
-        return self.manifest['requires_setup']
-    
+        return self.manifest["requires_setup"]
+
     @property
     def display_name(self) -> str:
-        return self.manifest['name']
+        return self.manifest["name"]
 
     @property
     def manifest(self) -> dict:
@@ -161,46 +193,44 @@ class LazyBaseModule:
             return self._manifest
         # print(f"Loading manifest for module {module_path}")
         # load the manifest file
-        manifest = copy.deepcopy(BaseModule._DEFAULT_MANIFEST)
+        manifest = copy.deepcopy(DEFAULT_MANIFEST)
 
         with open(join(self.path, MANIFEST_FILE)) as f:
             try:
                 manifest.update(ast.literal_eval(f.read()))
             except (ValueError, TypeError, SyntaxError, MemoryError, RecursionError) as e:
-                logger.error(f"Error loading manifest from file {self.path}/{MANIFEST_FILE}: {e}")
-            
+                raise ValueError(f"Error loading manifest from file {self.path}/{MANIFEST_FILE}: {e}") from e
+
         self._manifest = manifest
-        self.type = manifest['type']
-        self._entry_point = manifest['entry_point']
-        self.description = manifest['description']
-        self.version = manifest['version']
+        self._entry_point = manifest["entry_point"]
+        self.description = manifest["description"]
+        self.version = manifest["version"]
 
         return manifest
 
     def load(self, config) -> BaseModule:
-
         if self._instance:
             return self._instance
 
         # check external dependencies are installed
         def check_deps(deps, check):
-            for dep in deps:
-                if not len(dep):
-                    # clear out any empty strings that a user may have erroneously added
-                    continue
-                if not check(dep):
-                    logger.error(f"Module '{self.name}' requires external dependency '{dep}' which is not available/setup. Have you installed the required dependencies for the '{self.name}' module? See the README for more information.")
-                    exit(1)
+            for dep in filter(lambda d: len(d.strip()) > 0, deps):
+                if not check(dep.strip()):
+                    logger.error(
+                        f"Module '{self.name}' requires external dependency '{dep}' which is not available/setup. \
+                                 Have you installed the required dependencies for the '{self.name}' module? See the documentation for more information."
+                    )
+                    raise SetupError()
 
         def check_python_dep(dep):
             # first check if it's a module:
             try:
-                m = get_module_lazy(dep, suppress_warnings=True)
+                m = self.module_factory.get_module_lazy(dep, suppress_warnings=True)
                 try:
-                # we must now load this module and set it up with the config
+                    # we must now load this module and set it up with the config
                     m.load(config)
                     return True
-                except:
+                except Exception:
                     logger.error(f"Unable to setup module '{dep}' for use in module '{self.name}'")
                     return False
             except IndexError:
@@ -209,13 +239,26 @@ class LazyBaseModule:
 
             return find_spec(dep)
 
-        check_deps(self.dependencies.get('python', []), check_python_dep)
-        check_deps(self.dependencies.get('bin', []), lambda dep: shutil.which(dep))
+        def check_bin_dep(dep):
+            dep_exists = shutil.which(dep)
 
+            if dep == "docker":
+                if os.environ.get("RUNNING_IN_DOCKER"):
+                    # this is only for the WACZ enricher, which requires docker
+                    # if we're already running in docker then we don't need docker
+                    return True
+
+                # check if docker daemon is running
+                return dep_exists and subprocess.run(["docker", "ps", "-q"]).returncode == 0
+
+            return dep_exists
+
+        check_deps(self.dependencies.get("python", []), check_python_dep)
+        check_deps(self.dependencies.get("bin", []), check_bin_dep)
 
         logger.debug(f"Loading module '{self.display_name}'...")
 
-        for qualname in [self.name, f'auto_archiver.modules.{self.name}']:
+        for qualname in [self.name, f"auto_archiver.modules.{self.name}"]:
             try:
                 # first import the whole module, to make sure it's working properly
                 __import__(qualname)
@@ -224,26 +267,29 @@ class LazyBaseModule:
                 pass
 
         # then import the file for the entry point
-        file_name, class_name = self.entry_point.split('::')
-        sub_qualname = f'{qualname}.{file_name}'
+        file_name, class_name = self.entry_point.split("::")
+        sub_qualname = f"{qualname}.{file_name}"
 
-        __import__(f'{qualname}.{file_name}', fromlist=[self.entry_point])
+        __import__(f"{qualname}.{file_name}", fromlist=[self.entry_point])
         # finally, get the class instance
         instance: BaseModule = getattr(sys.modules[sub_qualname], class_name)()
-        if not getattr(instance, 'name', None):
-            instance.name = self.name
-
-        if not getattr(instance, 'display_name', None):
-            instance.display_name = self.display_name
 
+        # save the instance for future easy loading
         self._instance = instance
 
+        # set the name, display name and module factory
+        instance.name = self.name
+        instance.display_name = self.display_name
+        instance.module_factory = self.module_factory
+
         # merge the default config with the user config
-        default_config = dict((k, v['default']) for k, v in self.configs.items() if v.get('default'))
-        config[self.name] = default_config  | config.get(self.name, {})
+        default_config = dict((k, v["default"]) for k, v in self.configs.items() if "default" in v)
+
+        config[self.name] = default_config | config.get(self.name, {})
         instance.config_setup(config)
         instance.setup()
+
         return instance
 
     def __repr__(self):
-        return f"Module<'{self.display_name}' ({self.name})>"
+        return f"Module<'{self.display_name}' ({self.name})>"

src/auto_archiver/core/orchestrator.py

@@ -1,201 +1,295 @@
-""" Orchestrates all archiving steps, including feeding items,
-    archiving them with specific archivers, enrichment, storage,
-    formatting, database operations and clean up.
+"""Orchestrates all archiving steps, including feeding items,
+archiving them with specific archivers, enrichment, storage,
+formatting, database operations and clean up.
 
 """
 
 from __future__ import annotations
-from typing import Generator, Union, List, Type
-from urllib.parse import urlparse
-from ipaddress import ip_address
+from packaging import version
+from typing import Generator, Union, List, Type, TYPE_CHECKING
 import argparse
 import os
 import sys
-import json
 from tempfile import TemporaryDirectory
 import traceback
+from copy import copy
 
 from rich_argparse import RichHelpFormatter
+from auto_archiver.utils.custom_logger import format_for_human_readable_console, logger
+import requests
 
+from auto_archiver.utils.misc import random_str
 
 from .metadata import Metadata, Media
 from auto_archiver.version import __version__
-from .config import _yaml, read_yaml, store_yaml, to_dot_notation, merge_dicts, EMPTY_CONFIG, DefaultValidatingParser
-from .module import available_modules, LazyBaseModule, get_module, setup_paths
+from .config import (
+    read_yaml,
+    store_yaml,
+    to_dot_notation,
+    merge_dicts,
+    is_valid_config,
+    DefaultValidatingParser,
+    UniqueAppendAction,
+    AuthenticationJsonParseAction,
+    DEFAULT_CONFIG_FILE,
+)
+from .module import ModuleFactory, LazyBaseModule
 from . import validators, Feeder, Extractor, Database, Storage, Formatter, Enricher
-from .module import BaseModule
+from .consts import MODULE_TYPES, SetupError
+from auto_archiver.utils.url import check_url_or_raise, clean
 
-from loguru import logger
+if TYPE_CHECKING:
+    from .base_module import BaseModule
+    from .module import LazyBaseModule
 
 
-DEFAULT_CONFIG_FILE = "orchestration.yaml"
-
-class JsonParseAction(argparse.Action):
-    def __call__(self, parser, namespace, values, option_string=None):
-        try:
-            setattr(namespace, self.dest, json.loads(values))
-        except json.JSONDecodeError as e:
-            raise argparse.ArgumentTypeError(f"Invalid JSON input for argument '{self.dest}': {e}")
-
-
-class AuthenticationJsonParseAction(JsonParseAction):
-    def __call__(self, parser, namespace, values, option_string=None):
-        super().__call__(parser, namespace, values, option_string)
-        auth_dict = getattr(namespace, self.dest)
-        if isinstance(auth_dict, str):
-            # if it's a string
-            try:
-                with open(auth_dict, 'r') as f:
-                    try:
-                        auth_dict = json.load(f)
-                    except json.JSONDecodeError:
-                        # maybe it's yaml, try that
-                        auth_dict = _yaml.load(f)
-            except:
-                pass
-
-        if not isinstance(auth_dict, dict):
-            raise argparse.ArgumentTypeError("Authentication must be a dictionary of site names and their authentication methods")
-        for site, auth in auth_dict.items():
-            if not isinstance(site, str) or not isinstance(auth, dict):
-                raise argparse.ArgumentTypeError("Authentication must be a dictionary of site names and their authentication methods")
-        setattr(namespace, self.dest, auth_dict)
-class UniqueAppendAction(argparse.Action):
-    def __call__(self, parser, namespace, values, option_string=None):
-        if not hasattr(namespace, self.dest):
-            setattr(namespace, self.dest, [])
-        for value in values:
-            if value not in getattr(namespace, self.dest):
-                getattr(namespace, self.dest).append(value)
-
 class ArchivingOrchestrator:
+    # instance variables
+    module_factory: ModuleFactory
+    setup_finished: bool
+    logger_id: int
 
+    # instance variables, used for convenience to access modules by step
     feeders: List[Type[Feeder]]
     extractors: List[Type[Extractor]]
     enrichers: List[Type[Enricher]]
     databases: List[Type[Database]]
     storages: List[Type[Storage]]
     formatters: List[Type[Formatter]]
-    
+
+    def __init__(self):
+        self.module_factory = ModuleFactory()
+        self.setup_finished = False
+        self.logger_id = None
+
     def setup_basic_parser(self):
         parser = argparse.ArgumentParser(
-                prog="auto-archiver",
-                add_help=False,
-                description="""
+            prog="auto-archiver",
+            add_help=False,
+            description="""
                 Auto Archiver is a CLI tool to archive media/metadata from online URLs;
                 it can read URLs from many sources (Google Sheets, Command Line, ...); and write results to many destinations too (CSV, Google Sheets, MongoDB, ...)!
                 """,
-                epilog="Check the code at https://github.com/bellingcat/auto-archiver",
-                formatter_class=RichHelpFormatter,
+            epilog="Check the code at https://github.com/bellingcat/auto-archiver",
+            formatter_class=RichHelpFormatter,
+        )
+        parser.add_argument("--help", "-h", action="store_true", dest="help", help="show a full help message and exit")
+        parser.add_argument("--version", action="version", version=__version__)
+        parser.add_argument(
+            "--config",
+            action="store",
+            dest="config_file",
+            help="the filename of the YAML configuration file (defaults to 'config.yaml')",
+            default=DEFAULT_CONFIG_FILE,
+        )
+        parser.add_argument(
+            "--mode",
+            action="store",
+            dest="mode",
+            type=str,
+            choices=["simple", "full"],
+            help="the mode to run the archiver in",
+            default="simple",
         )
-        parser.add_argument('--help', '-h', action='store_true', dest='help', help='show a full help message and exit')
-        parser.add_argument('--version', action='version', version=__version__)
-        parser.add_argument('--config', action='store', dest="config_file", help='the filename of the YAML configuration file (defaults to \'config.yaml\')', default=DEFAULT_CONFIG_FILE)
-        parser.add_argument('--mode', action='store', dest='mode', type=str, choices=['simple', 'full'], help='the mode to run the archiver in', default='simple')
         # override the default 'help' so we can inject all the configs and show those
-        parser.add_argument('-s', '--store', dest='store', default=False, help='Store the created config in the config file', action=argparse.BooleanOptionalAction)
-        parser.add_argument('--module_paths', dest='module_paths', nargs='+', default=[], help='additional paths to search for modules', action=UniqueAppendAction)
+        parser.add_argument(
+            "-s",
+            "--store",
+            dest="store",
+            default=False,
+            help="Store the created config in the config file",
+            action=argparse.BooleanOptionalAction,
+        )
+        parser.add_argument(
+            "--module_paths",
+            dest="module_paths",
+            nargs="+",
+            default=[],
+            help="additional paths to search for modules",
+            action=UniqueAppendAction,
+        )
 
         self.basic_parser = parser
         return parser
 
+    def check_steps(self, config):
+        for module_type in MODULE_TYPES:
+            if not config["steps"].get(f"{module_type}s", []):
+                if (module_type == "feeder" or module_type == "formatter") and config["steps"].get(f"{module_type}"):
+                    raise SetupError(
+                        f"It appears you have '{module_type}' set under 'steps' in your configuration file, but as of version 0.13.0 of Auto Archiver, you must use '{module_type}s'. Change this in your configuration file and try again. \
+Here's how that would look: \n\nsteps:\n  {module_type}s:\n  - [your_{module_type}_name_here]\n  {'extractors:...' if module_type == 'feeder' else '...'}\n"
+                    )
+                if module_type == "extractor" and config["steps"].get("archivers"):
+                    raise SetupError(
+                        "As of version 0.13.0 of Auto Archiver, the 'archivers' step name has been changed to 'extractors'. Change this in your configuration file and try again. \
+Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_here]\n  enrichers:...\n"
+                    )
+                raise SetupError(
+                    f"No {module_type}s were configured. Make sure to set at least one {module_type} in your configuration file or on the command line (using --{module_type}s)"
+                )
+
     def setup_complete_parser(self, basic_config: dict, yaml_config: dict, unused_args: list[str]) -> None:
+        # modules parser to get the overridden 'steps' values
+        modules_parser = argparse.ArgumentParser(
+            add_help=False,
+        )
+        self.add_modules_args(modules_parser)
+        cli_modules, unused_args = modules_parser.parse_known_args(unused_args)
+        for module_type in MODULE_TYPES:
+            yaml_config["steps"][f"{module_type}s"] = getattr(cli_modules, f"{module_type}s", []) or yaml_config[
+                "steps"
+            ].get(f"{module_type}s", [])
+
         parser = DefaultValidatingParser(
             add_help=False,
         )
         self.add_additional_args(parser)
 
+        # merge command line module args (--feeders, --enrichers etc.) and add them to the config
+
         # check what mode we're in
         # if we have a config file, use that to decide which modules to load
         # if simple, we'll load just the modules that has requires_setup = False
         # if full, we'll load all modules
         # TODO: BUG** - basic_config won't have steps in it, since these args aren't added to 'basic_parser'
         # but should we add them? Or should we just add them to the 'complete' parser?
-        if yaml_config != EMPTY_CONFIG:
+
+        if is_valid_config(yaml_config):
+            self.check_steps(yaml_config)
             # only load the modules enabled in config
             # TODO: if some steps are empty (e.g. 'feeders' is empty), should we default to the 'simple' ones? Or only if they are ALL empty?
             enabled_modules = []
             # first loads the modules from the config file, then from the command line
-            for config in [yaml_config['steps'], basic_config.__dict__]:
-                for module_type in BaseModule.MODULE_TYPES:
-                    enabled_modules.extend(config.get(f"{module_type}s", []))
+            for module_type in MODULE_TYPES:
+                enabled_modules.extend(yaml_config["steps"].get(f"{module_type}s", []))
 
             # clear out duplicates, but keep the order
             enabled_modules = list(dict.fromkeys(enabled_modules))
-            avail_modules = available_modules(with_manifest=True, limit_to_modules=enabled_modules, suppress_warnings=True)
-            self.add_module_args(avail_modules, parser)
-        elif basic_config.mode == 'simple':
-            simple_modules = [module for module in available_modules(with_manifest=True) if not module.requires_setup]
-            self.add_module_args(simple_modules, parser)
+            avail_modules = self.module_factory.available_modules(
+                limit_to_modules=enabled_modules, suppress_warnings=True
+            )
+            self.add_individual_module_args(avail_modules, parser)
+        elif basic_config.mode == "simple":
+            simple_modules = [module for module in self.module_factory.available_modules() if not module.requires_setup]
+            self.add_individual_module_args(simple_modules, parser)
 
-            # for simple mode, we use the cli_feeder and any modules that don't require setup
-            yaml_config['steps']['feeders'] = ['cli_feeder']
-            
             # add them to the config
             for module in simple_modules:
                 for module_type in module.type:
-                    yaml_config['steps'].setdefault(f"{module_type}s", []).append(module.name)
+                    yaml_config["steps"].setdefault(f"{module_type}s", []).append(module.name)
         else:
             # load all modules, they're not using the 'simple' mode
-            self.add_module_args(available_modules(with_manifest=True), parser)
+            all_modules = self.module_factory.available_modules()
+            # add all the modules to the steps
+            for module in all_modules:
+                for module_type in module.type:
+                    yaml_config["steps"].setdefault(f"{module_type}s", []).append(module.name)
+
+            self.add_individual_module_args(all_modules, parser)
 
         parser.set_defaults(**to_dot_notation(yaml_config))
 
         # reload the parser with the new arguments, now that we have them
         parsed, unknown = parser.parse_known_args(unused_args)
-
         # merge the new config with the old one
-        self.config = merge_dicts(vars(parsed), yaml_config)
+        config = merge_dicts(vars(parsed), yaml_config)
+
+        # set up the authentication dict as needed
+        config = self.setup_authentication(config)
+
         # clean out args from the base_parser that we don't want in the config
         for key in vars(basic_config):
-            self.config.pop(key, None)
+            config.pop(key, None)
 
         # setup the logging
-        self.setup_logging()
+        self.setup_logging(config)
 
         if unknown:
             logger.warning(f"Ignoring unknown/unused arguments: {unknown}\nPerhaps you don't have this module enabled?")
-        
-        if (self.config != yaml_config and basic_config.store) or not os.path.isfile(basic_config.config_file):
+
+        if (config != yaml_config and basic_config.store) or not os.path.isfile(basic_config.config_file):
             logger.info(f"Storing configuration file to {basic_config.config_file}")
-            store_yaml(self.config, basic_config.config_file)
-        
-        return self.config
-    
+            store_yaml(config, basic_config.config_file)
+
+        return config
+
+    def add_modules_args(self, parser: argparse.ArgumentParser = None):
+        if not parser:
+            parser = self.parser
+
+        # Module loading from the command line
+        for module_type in MODULE_TYPES:
+            parser.add_argument(
+                f"--{module_type}s",
+                dest=f"{module_type}s",
+                nargs="+",
+                help=f"the {module_type}s to use",
+                default=[],
+                action=UniqueAppendAction,
+            )
+
     def add_additional_args(self, parser: argparse.ArgumentParser = None):
         if not parser:
             parser = self.parser
 
-
-        # allow passing URLs directly on the command line
-        parser.add_argument('urls', nargs='*', default=[], help='URL(s) to archive, either a single URL or a list of urls, should not come from config.yaml')
-
-        parser.add_argument('--feeders', dest='steps.feeders', nargs='+', default=['cli_feeder'], help='the feeders to use', action=UniqueAppendAction)
-        parser.add_argument('--enrichers', dest='steps.enrichers',  nargs='+', help='the enrichers to use', action=UniqueAppendAction)
-        parser.add_argument('--extractors', dest='steps.extractors', nargs='+', help='the extractors to use', action=UniqueAppendAction)
-        parser.add_argument('--databases', dest='steps.databases', nargs='+', help='the databases to use', action=UniqueAppendAction)
-        parser.add_argument('--storages', dest='steps.storages', nargs='+', help='the storages to use', action=UniqueAppendAction)
-        parser.add_argument('--formatters', dest='steps.formatters', nargs='+', help='the formatter to use', action=UniqueAppendAction)
-
-        parser.add_argument('--authentication', dest='authentication', help='A dictionary of sites and their authentication methods \
+        parser.add_argument(
+            "--authentication",
+            dest="authentication",
+            help="A dictionary of sites and their authentication methods \
                                                                             (token, username etc.) that extractors can use to log into \
                                                                             a website. If passing this on the command line, use a JSON string. \
-                                                                            You may also pass a path to a valid JSON/YAML file which will be parsed.',\
-                                                                            default={},
-                                                                            action=AuthenticationJsonParseAction)
+                                                                            You may also pass a path to a valid JSON/YAML file which will be parsed.",
+            default={},
+            nargs="?",
+            action=AuthenticationJsonParseAction,
+        )
+
         # logging arguments
-        parser.add_argument('--logging.level', action='store', dest='logging.level', choices=['INFO', 'DEBUG', 'ERROR', 'WARNING'], help='the logging level to use', default='INFO', type=str.upper)
-        parser.add_argument('--logging.file', action='store', dest='logging.file', help='the logging file to write to', default=None)
-        parser.add_argument('--logging.rotation', action='store', dest='logging.rotation', help='the logging rotation to use', default=None)
+        parser.add_argument(
+            "--logging.level",
+            action="store",
+            dest="logging.level",
+            choices=["INFO", "DEBUG", "ERROR", "WARNING"],
+            help="the logging level to use for the standard output and file logging",
+            default="INFO",
+            type=str.upper,
+        )
+        parser.add_argument(
+            "--logging.file", action="store", dest="logging.file", help="the logging file to write to", default=None
+        )
+        parser.add_argument(
+            "--logging.rotation",
+            action="store",
+            dest="logging.rotation",
+            help="the logging rotation to use",
+            default=None,
+        )
 
+        parser.add_argument(
+            "--logging.each_level_in_separate_file",
+            action="store",
+            dest="logging.each_level_in_separate_file",
+            help="if set, writes each logging level to a separate file (ignores --logging.level), you must also set --logging.file. Each level will have a dedicate logs file matching your <file>.debug, <file>.info, etc.",
+            default=False,
+        )
 
-    def add_module_args(self, modules: list[LazyBaseModule] = None, parser: argparse.ArgumentParser = None) -> None:
-
+    def add_individual_module_args(
+        self, modules: list[LazyBaseModule] = None, parser: argparse.ArgumentParser = None
+    ) -> None:
         if not modules:
-            modules = available_modules(with_manifest=True)
+            modules = self.module_factory.available_modules()
 
-        module: LazyBaseModule
         for module in modules:
+            if module.name == "cli_feeder":
+                # special case. For the CLI feeder, allow passing URLs directly on the command line without setting --cli_feeder.urls=
+                parser.add_argument(
+                    "urls",
+                    nargs="*",
+                    default=[],
+                    help="URL(s) to archive, either a single URL or a list of urls, should not come from config.yaml",
+                )
+                continue
 
             if not module.configs:
                 # this module has no configs, don't show anything in the help
@@ -205,100 +299,131 @@ class ArchivingOrchestrator:
             group = parser.add_argument_group(module.display_name or module.name, f"{module.description[:100]}...")
 
             for name, kwargs in module.configs.items():
-                if not kwargs.get('metavar', None):
+                if not kwargs.get("metavar", None):
                     # make a nicer metavar, metavar is what's used in the help, e.g. --cli_feeder.urls [METAVAR]
-                    kwargs['metavar'] = name.upper()
+                    kwargs["metavar"] = name.upper()
 
-                if kwargs.get('required', False):
+                if kwargs.get("required", False):
                     # required args shouldn't have a 'default' value, remove it
-                    kwargs.pop('default', None)
+                    kwargs.pop("default", None)
 
-                kwargs.pop('cli_set', None)
-                should_store = kwargs.pop('should_store', False)
-                kwargs['dest'] = f"{module.name}.{kwargs.pop('dest', name)}"
+                kwargs.pop("cli_set", None)
+                should_store = kwargs.pop("should_store", False)
+                kwargs["dest"] = f"{module.name}.{kwargs.pop('dest', name)}"
                 try:
-                    kwargs['type'] = getattr(validators, kwargs.get('type', '__invalid__'))
+                    kwargs["type"] = getattr(validators, kwargs.get("type", "__invalid__"))
                 except AttributeError:
-                    kwargs['type'] = __builtins__.get(kwargs.get('type'), str)
+                    kwargs["type"] = __builtins__.get(kwargs.get("type"), str)
                 arg = group.add_argument(f"--{module.name}.{name}", **kwargs)
                 arg.should_store = should_store
 
     def show_help(self, basic_config: dict):
-        # for the help message, we want to load *all* possible modules and show the help
-            # add configs as arg parser arguments
-        
+        # for the help message, we want to load manifests from *all* possible modules and show their help/settings
+        # add configs as arg parser arguments
+
+        self.add_modules_args(self.basic_parser)
         self.add_additional_args(self.basic_parser)
-        self.add_module_args(parser=self.basic_parser)
+        self.add_individual_module_args(parser=self.basic_parser)
         self.basic_parser.print_help()
         self.basic_parser.exit()
-    
-    def setup_logging(self):
+
+    def setup_logging(self, config):
+        logging_config = config["logging"]
+
+        if logging_config.get("enabled", True) is False:
+            # disabled logging settings, they're set on a higher level
+            logger.disable("auto_archiver")
+            return
+
         # setup loguru logging
-        logger.remove(0) # remove the default logger
-        logging_config = self.config['logging']
-        logger.add(sys.stderr, level=logging_config['level'])
-        if log_file := logging_config['file']:
-            logger.add(log_file) if not logging_config['rotation'] else logger.add(log_file, rotation=logging_config['rotation'])
+        try:
+            logger.remove(0)  # remove the default logger
+        except ValueError:
+            pass
+
+        # add other logging info
+        if self.logger_id is None:  # note - need direct comparison to None since need to consider falsy value 0
+            use_level = logging_config["level"]
+            self.logger_id = logger.add(
+                sys.stderr,
+                level=use_level,
+                catch=True,
+                format="<level>{extra[serialized]}</level>"
+                if logging_config.get("format", "").lower() == "json"
+                else format_for_human_readable_console(),
+            )
+
+            rotation = logging_config["rotation"]
+            log_file = logging_config["file"]
+
+            if logging_config.get("each_level_in_separate_file"):
+                assert logging_config["file"], (
+                    "You must set --logging.file if you want to use --logging.each_level_in_separate_file"
+                )
+                for i, level in enumerate(["DEBUG", "INFO", "SUCCESS", "WARNING", "ERROR"], start=1):
+                    logger.add(
+                        f"{log_file}.{i}_{level.lower()}",
+                        filter=lambda rec, lvl=level: rec["level"].name == lvl,
+                        rotation=rotation,
+                        format="{extra[serialized]}",
+                    )
+            elif log_file:
+                logger.add(log_file, rotation=rotation, level=use_level, format="{extra[serialized]}")
 
     def install_modules(self, modules_by_type):
         """
-        Traverses all modules in 'steps' and loads them into the orchestrator, storing them in the 
+        Traverses all modules in 'steps' and loads them into the orchestrator, storing them in the
         orchestrator's attributes (self.feeders, self.extractors etc.). If no modules of a certain type
         are loaded, the program will exit with an error message.
         """
-        
-        invalid_modules = []
-        for module_type in BaseModule.MODULE_TYPES:
 
+        invalid_modules = []
+        for module_type in MODULE_TYPES:
             step_items = []
             modules_to_load = modules_by_type[f"{module_type}s"]
-            assert modules_to_load, f"No {module_type}s were configured. Make sure to set at least one {module_type} in your configuration file or on the command line (using --{module_type}s)"
+            if not modules_to_load:
+                raise SetupError(
+                    f"No {module_type}s were configured. Make sure to set at least one {module_type} in your configuration file or on the command line (using --{module_type}s)"
+                )
 
             def check_steps_ok():
                 if not len(step_items):
-                    logger.error(f"NO {module_type.upper()}S LOADED. Please check your configuration and try again.")
                     if len(modules_to_load):
-                        logger.error(f"Tried to load the following modules, but none were available: {modules_to_load}")
-                    exit()
+                        logger.error(
+                            f"Unable to load any {module_type}s. Tried the following, but none were available: {modules_to_load}"
+                        )
+                    raise SetupError(
+                        f"NO {module_type.upper()}S LOADED. Please check your configuration and try again."
+                    )
 
-                if (module_type == 'feeder' or module_type == 'formatter') and len(step_items) > 1:
-                    logger.error(f"Only one {module_type} is allowed, found {len(step_items)} {module_type}s. Please remove one of the following from your configuration file: {modules_to_load}")
-                    exit()
+                if (module_type == "feeder" or module_type == "formatter") and len(step_items) > 1:
+                    raise SetupError(
+                        f"Only one {module_type} is allowed, found {len(step_items)} {module_type}s. Please remove one of the following from your configuration file: {modules_to_load}"
+                    )
 
             for module in modules_to_load:
-                if module == 'cli_feeder':
-                    # pseudo module, don't load it
-                    urls = self.config['urls']
-                    if not urls:
-                        logger.error("No URLs provided. Please provide at least one URL via the command line, or set up an alternative feeder. Use --help for more information.")
-                        exit()
-                    # cli_feeder is a pseudo module, it just takes the command line args
-                    def feed(self) -> Generator[Metadata]:
-                        for url in urls:
-                            logger.debug(f"Processing URL: '{url}'")
-                            yield Metadata().set_url(url)
-
-                    pseudo_module = type('CLIFeeder', (Feeder,), {
-                        'name': 'cli_feeder',
-                        'display_name': 'CLI Feeder',
-                        '__iter__': feed
-
-                    })()
-  
-
-                    pseudo_module.__iter__ = feed
-                    step_items.append(pseudo_module)
-                    continue
-
                 if module in invalid_modules:
                     continue
+
+                # check to make sure that we're trying to load it as the correct type - i.e. make sure the user hasn't put it under the wrong 'step'
+                lazy_module: LazyBaseModule = self.module_factory.get_module_lazy(module)
+                if module_type not in lazy_module.type:
+                    types = ",".join(f"'{t}'" for t in lazy_module.type)
+                    raise SetupError(
+                        f"Configuration Error: Module '{module}' is not a {module_type}, but has the types: {types}. Please check you set this module up under the right step in your orchestration file."
+                    )
+
+                loaded_module = None
                 try:
-                    loaded_module: BaseModule = get_module(module, self.config)
+                    loaded_module: BaseModule = lazy_module.load(self.config)
                 except (KeyboardInterrupt, Exception) as e:
-                    logger.error(f"Error during setup of modules: {e}\n{traceback.format_exc()}")
-                    if module_type == 'extractor' and loaded_module.name == module:
-                        loaded_module.cleanup()
-                    exit()
+                    if not isinstance(e, KeyboardInterrupt) and not isinstance(e, SetupError):
+                        logger.error(f"Error during setup of modules: {e}\n{traceback.format_exc()}")
+
+                    # access the _instance here because loaded_module may not return if there's an error
+                    if lazy_module._instance and module_type == "extractor":
+                        lazy_module._instance.cleanup()
+                    raise e
 
                 if not loaded_module:
                     invalid_modules.append(module)
@@ -308,55 +433,123 @@ class ArchivingOrchestrator:
 
             check_steps_ok()
             setattr(self, f"{module_type}s", step_items)
-    
+
     def load_config(self, config_file: str) -> dict:
         if not os.path.exists(config_file) and config_file != DEFAULT_CONFIG_FILE:
-            logger.error(f"The configuration file {config_file} was  not found. Make sure the file exists and try again, or run without the --config file to use the default settings.")
-            exit()
+            logger.error(
+                f"The configuration file {config_file} was  not found. Make sure the file exists and try again, or run without the --config file to use the default settings."
+            )
+            raise FileNotFoundError(f"Configuration file {config_file} not found")
 
         return read_yaml(config_file)
 
-    def run(self, args: list) -> None:
-        
+    def setup_config(self, args: list) -> dict:
+        """
+        Sets up the configuration file, merging the default config with the user's config
+
+        This function should only ever be run once.
+        """
+
         self.setup_basic_parser()
 
         # parse the known arguments for now (basically, we want the config file)
         basic_config, unused_args = self.basic_parser.parse_known_args(args)
 
         # setup any custom module paths, so they'll show in the help and for arg parsing
-        setup_paths(basic_config.module_paths)
+        self.module_factory.setup_paths(basic_config.module_paths)
 
         # if help flag was called, then show the help
         if basic_config.help:
             self.show_help(basic_config)
-
+        # merge command line --feeder etc. args with what's in the yaml config
         yaml_config = self.load_config(basic_config.config_file)
-        self.setup_complete_parser(basic_config, yaml_config, unused_args)
+
+        return self.setup_complete_parser(basic_config, yaml_config, unused_args)
+
+    def check_for_updates(self):
+        try:
+            response = requests.get("https://pypi.org/pypi/auto-archiver/json", timeout=10).json()
+        except Exception as e:
+            logger.debug(f"Unable to check for updates: {e}")
+            return
+        latest_version = version.parse(response["info"]["version"])
+        current_version = version.parse(__version__)
+        # check version compared to current version
+        if latest_version > current_version:
+            if os.environ.get("RUNNING_IN_DOCKER"):
+                update_cmd = "`docker pull bellingcat/auto-archiver:latest`"
+            else:
+                update_cmd = "`pip install --upgrade auto-archiver`"
+            logger.warning(
+                f"\n********* IMPORTANT: UPDATE AVAILABLE ********\nA new version of auto-archiver is available (v{latest_version}, you have v{current_version})\nMake sure to update to the latest version using: {update_cmd}\n"
+            )
+
+    def setup(self, args: list):
+        """
+        Function to configure all setup of the orchestrator: setup configs and load modules.
+
+        This method should only ever be called once
+        """
+
+        self.check_for_updates()
+
+        if self.setup_finished:
+            logger.warning(
+                "The `setup_config()` function should only ever be run once. \
+                           If you need to re-run the setup, please re-instantiate a new instance of the orchestrator. \
+                           For code implementatations, you should call .setup_config() once then you may call .feed() \
+                           multiple times to archive multiple URLs."
+            )
+            return
+
+        self.setup_basic_parser()
+        self.config = self.setup_config(args)
 
         logger.info(f"======== Welcome to the AUTO ARCHIVER ({__version__}) ==========")
-        self.install_modules(self.config['steps'])
+        self.install_modules(self.config["steps"])
 
         # log out the modules that were loaded
-        for module_type in BaseModule.MODULE_TYPES:
-            logger.info(f"{module_type.upper()}S: " + ", ".join(m.display_name for m in getattr(self, f"{module_type}s")))
+        for module_type in MODULE_TYPES:
+            logger.info(
+                f"{module_type.upper()}S: " + ", ".join(m.display_name for m in getattr(self, f"{module_type}s"))
+            )
 
-        for _ in self.feed():
-            pass
+        self.setup_finished = True
 
-    def cleanup(self)->None:
+    def _command_line_run(self, args: list) -> Generator[Metadata]:
+        """
+        This is the main entry point for the orchestrator, when run from the command line.
+
+        :param args: list of arguments to pass to the orchestrator - these are the command line args
+
+        You should not call this method from code implementations.
+
+        This method sets up the configuration, loads the modules, and runs the feed.
+        If you wish to make code invocations yourself, you should use the 'setup' and 'feed' methods separately.
+        To test configurations, without loading any modules you can also first call 'setup_configs'
+        """
+        try:
+            self.setup(args)
+            return self.feed()
+        except Exception as e:
+            logger.error(f"{e}: {traceback.format_exc()}")
+            exit(1)
+
+    def cleanup(self) -> None:
         logger.info("Cleaning up")
         for e in self.extractors:
             e.cleanup()
 
     def feed(self) -> Generator[Metadata]:
-
         url_count = 0
         for feeder in self.feeders:
             for item in feeder:
-                yield self.feed_item(item)
-                url_count += 1
+                with logger.contextualize(url=item.get_url(), trace=random_str(12)):
+                    logger.info("Started processing")
+                    yield self.feed_item(item)
+                    url_count += 1
 
-        logger.success(f"Processed {url_count} URL(s)")
+        logger.info(f"Processed {url_count} URL(s)")
         self.cleanup()
 
     def feed_item(self, item: Metadata) -> Metadata:
@@ -374,15 +567,15 @@ class ArchivingOrchestrator:
             return self.archive(item)
         except KeyboardInterrupt:
             # catches keyboard interruptions to do a clean exit
-            logger.warning(f"caught interrupt on {item=}")
+            logger.warning("Caught interrupt")
             for d in self.databases:
                 d.aborted(item)
             self.cleanup()
             exit()
         except Exception as e:
-            logger.error(f'Got unexpected error on item {item}: {e}\n{traceback.format_exc()}')
+            logger.error(f"Got unexpected error: {e}\n{traceback.format_exc()}")
             for d in self.databases:
-                if type(e) == AssertionError:
+                if isinstance(e, AssertionError):
                     d.failed(item, str(e))
                 else:
                     d.failed(item, reason="unexpected error")
@@ -393,32 +586,33 @@ class ArchivingOrchestrator:
                     m.tmp_dir = None
                 tmp_dir.cleanup()
 
-
     def archive(self, result: Metadata) -> Union[Metadata, None]:
         """
-            Runs the archiving process for a single URL
-            1. Each archiver can sanitize its own URLs
-            2. Check for cached results in Databases, and signal start to the databases
-            3. Call Archivers until one succeeds
-            4. Call Enrichers
-            5. Store all downloaded/generated media
-            6. Call selected Formatter and store formatted if needed
+        Runs the archiving process for a single URL
+        1. Each archiver can sanitize its own URLs
+        2. Check for cached results in Databases, and signal start to the databases
+        3. Call Archivers until one succeeds
+        4. Call Enrichers
+        5. Store all downloaded/generated media
+        6. Call selected Formatter and store formatted if needed
         """
 
         original_url = result.get_url().strip()
         try:
-            self.assert_valid_url(original_url)
-        except AssertionError as e:
-            logger.error(f"Error archiving URL {original_url}: {e}")
+            check_url_or_raise(original_url)
+        except ValueError as e:
+            logger.error(f"Error archiving: {e}")
             raise e
 
         # 1 - sanitize - each archiver is responsible for cleaning/expanding its own URLs
-        url = original_url
+        url = clean(original_url)
         for a in self.extractors:
             url = a.sanitize_url(url)
 
         result.set_url(url)
-        if original_url != url: result.set("original_url", original_url)
+        if original_url != url:
+            logger.debug(f"Sanitized URL to {url}")
+            result.set("original_url", original_url)
 
         # 2 - notify start to DBs, propagate already archived if feature enabled in DBs
         cached_result = None
@@ -429,25 +623,28 @@ class ArchivingOrchestrator:
         if cached_result:
             logger.debug("Found previously archived entry")
             for d in self.databases:
-                try: d.done(cached_result, cached=True)
+                try:
+                    d.done(cached_result, cached=True)
                 except Exception as e:
-                    logger.error(f"ERROR database {d.name}: {e}: {traceback.format_exc()}")
+                    logger.error(f"Database {d.name}: {e}: {traceback.format_exc()}")
             return cached_result
 
         # 3 - call extractors until one succeeds
         for a in self.extractors:
-            logger.info(f"Trying extractor {a.name} for {url}")
+            logger.info(f"Trying extractor {a.name}")
             try:
                 result.merge(a.download(result))
-                if result.is_success(): break
-            except Exception as e: 
-                logger.error(f"ERROR archiver {a.name}: {e}: {traceback.format_exc()}")
+                if result.is_success():
+                    break
+            except Exception as e:
+                logger.error(f"Extractor {a.name}: {e}: {traceback.format_exc()}")
 
         # 4 - call enrichers to work with archived content
         for e in self.enrichers:
-            try: e.enrich(result)
-            except Exception as exc: 
-                logger.error(f"ERROR enricher {e.name}: {exc}: {traceback.format_exc()}")
+            try:
+                e.enrich(result)
+            except Exception as exc:
+                logger.error(f"Enricher {e.name}: {exc}: {traceback.format_exc()}")
 
         # 5 - store all downloaded/generated media
         result.store(storages=self.storages)
@@ -463,35 +660,35 @@ class ArchivingOrchestrator:
 
         # signal completion to databases and archivers
         for d in self.databases:
-            try: d.done(result)
+            try:
+                d.done(result)
             except Exception as e:
-                logger.error(f"ERROR database {d.name}: {e}: {traceback.format_exc()}")
+                logger.error(f"Database {d.name}: {e}: {traceback.format_exc()}")
 
         return result
 
-    def assert_valid_url(self, url: str) -> bool:
+    def setup_authentication(self, config: dict) -> dict:
         """
-        Blocks localhost, private, reserved, and link-local IPs and all non-http/https schemes.
+        Setup authentication for all modules that require it
+
+        Split up strings into multiple sites if they are comma separated
         """
-        assert url.startswith("http://") or url.startswith("https://"), f"Invalid URL scheme"
-        
-        parsed = urlparse(url)
-        assert parsed.scheme in ["http", "https"], f"Invalid URL scheme"
-        assert parsed.hostname, f"Invalid URL hostname"
-        assert parsed.hostname != "localhost", f"Invalid URL"
 
-        try: # special rules for IP addresses
-            ip = ip_address(parsed.hostname)
-        except ValueError: pass
-        else:
-            assert ip.is_global, f"Invalid IP used"
-            assert not ip.is_reserved, f"Invalid IP used"
-            assert not ip.is_link_local, f"Invalid IP used"
-            assert not ip.is_private, f"Invalid IP used"
+        authentication = config.get("authentication", {})
 
+        # extract out concatenated sites
+        for key, val in copy(authentication).items():
+            if "," in key:
+                for site in key.split(","):
+                    site = site.strip()
+                    authentication[site] = val
+                del authentication[key]
+
+        config["authentication"] = authentication
+        return config
 
     # Helper Properties
-    
+
     @property
     def all_modules(self) -> List[Type[BaseModule]]:
-        return self.feeders + self.extractors + self.enrichers + self.databases + self.storages + self.formatters
+        return self.feeders + self.extractors + self.enrichers + self.databases + self.storages + self.formatters

src/auto_archiver/core/storage.py

@@ -1,5 +1,22 @@
 """
 Base module for Storage modules – modular components that store media objects in various locations.
+
+If you are looking to implement a new storage module, you should subclass the `Storage` class and
+implement the `get_cdn_url` and `uploadf` methods.
+
+Your module **must** also have two config variables 'path_generator' and 'filename_generator' which
+determine how the key is generated for the media object. The 'path_generator' and 'filename_generator'
+variables can be set to one of the following values:
+- 'flat': A flat structure with no subfolders
+- 'url': A structure based on the URL of the media object
+- 'random': A random structure
+
+The 'filename_generator' variable can be set to one of the following values:
+- 'random': A random string
+- 'static': A replicable strategy such as a hash
+
+If you don't want to use this naming convention, you can override the `set_key` method in your subclass.
+
 """
 
 from __future__ import annotations
@@ -7,26 +24,27 @@ from abc import abstractmethod
 from typing import IO
 import os
 
-from loguru import logger
+from auto_archiver.utils.custom_logger import logger
 from slugify import slugify
 
 from auto_archiver.utils.misc import random_str
 
 from auto_archiver.core import Media, BaseModule, Metadata
 from auto_archiver.modules.hash_enricher.hash_enricher import HashEnricher
-from auto_archiver.core.module import get_module
+
+
 class Storage(BaseModule):
-    
     """
     Base class for implementing storage modules in the media archiving framework.
 
     Subclasses must implement the `get_cdn_url` and `uploadf` methods to define their behavior.
     """
 
-    def store(self, media: Media, url: str, metadata: Metadata=None) -> None:
-        if media.is_stored(in_storage=self): 
+    def store(self, media: Media, url: str, metadata: Metadata = None) -> None:
+        if media.is_stored(in_storage=self):
             logger.debug(f"{media.key} already stored, skipping")
             return
+
         self.set_key(media, url, metadata)
         self.upload(media, metadata=metadata)
         media.add_url(self.get_cdn_url(media))
@@ -42,42 +60,55 @@ class Storage(BaseModule):
     def uploadf(self, file: IO[bytes], key: str, **kwargs: dict) -> bool:
         """
         Uploads (or saves) a file to the storage service/location.
+
+        This method should not be called directly, but instead through the 'store' method,
+        which sets up the media for storage.
         """
         pass
 
     def upload(self, media: Media, **kwargs) -> bool:
-        logger.debug(f'[{self.__class__.__name__}] storing file {media.filename} with key {media.key}')
-        with open(media.filename, 'rb') as f:
+        """
+        Uploads a media object to the storage service.
+
+        This method should not be called directly, but instead be called through the 'store' method,
+        which sets up the media for storage.
+        """
+        logger.debug(f"[{self.__class__.__name__}] storing file {media.filename} with key {media.key}")
+        with open(media.filename, "rb") as f:
             return self.uploadf(f, media, **kwargs)
 
-    def set_key(self, media: Media, url, metadata: Metadata) -> None:
+    def set_key(self, media: Media, url: str, metadata: Metadata) -> None:
         """takes the media and optionally item info and generates a key"""
-        if media.key is not None and len(media.key) > 0: return
-        folder = metadata.get_context('folder', '')
+
+        if media.key is not None and len(media.key) > 0:
+            # media key is already set
+            return
+
+        folder = metadata.get_context("folder", "")
         filename, ext = os.path.splitext(media.filename)
 
         # Handle path_generator logic
-        path_generator = self.config.get("path_generator", "url")
+        path_generator = self.path_generator
         if path_generator == "flat":
             path = ""
-            filename = slugify(filename)  # Ensure filename is slugified
         elif path_generator == "url":
-            path = slugify(url)
+            path = slugify(url)[:70]
         elif path_generator == "random":
-            path = self.config.get("random_path", random_str(24), True)
+            path = random_str(24)
         else:
             raise ValueError(f"Invalid path_generator: {path_generator}")
 
         # Handle filename_generator logic
-        filename_generator = self.config.get("filename_generator", "random")
+        filename_generator = self.filename_generator
         if filename_generator == "random":
             filename = random_str(24)
         elif filename_generator == "static":
             # load the hash_enricher module
-            he = get_module(HashEnricher, self.config)
+            he: HashEnricher = self.module_factory.get_module("hash_enricher", self.config)
             hd = he.calculate_hash(media.filename)
             filename = hd[:24]
         else:
             raise ValueError(f"Invalid filename_generator: {filename_generator}")
 
-        media.key = os.path.join(folder, path, f"{filename}{ext}")
+        key = os.path.join(folder, path, f"{filename}{ext}")
+        media._key = key

src/auto_archiver/core/validators.py

@@ -1,11 +1,8 @@
 # used as validators for config values. Should raise an exception if the value is invalid.
 from pathlib import Path
 import argparse
+import json
 
-def example_validator(value):
-    if "example" not in value:
-        raise argparse.ArgumentTypeError(f"{value} is not a valid value for this argument")
-    return value
 
 def positive_number(value):
     if value < 0:
@@ -16,4 +13,8 @@ def positive_number(value):
 def valid_file(value):
     if not Path(value).is_file():
         raise argparse.ArgumentTypeError(f"File '{value}' does not exist.")
-    return value
+    return value
+
+
+def json_loader(cli_val):
+    return json.loads(cli_val)

src/auto_archiver/modules/antibot_extractor_enricher/__manifest__.py

@@ -0,0 +1,53 @@
+{
+    "name": "Antibot Extractor/Enricher",
+    "type": ["extractor", "enricher"],
+    "requires_setup": False,
+    "dependencies": {"python": ["loguru", "seleniumbase", "yt_dlp"], "bin": ["ffmpeg"]},
+    "configs": {
+        "save_to_pdf": {
+            "default": False,
+            "type": "bool",
+            "help": "save a PDF snapshot of the page.",
+        },
+        "max_download_images": {
+            "default": 50,
+            "help": "maximum number of images to download from the page (0 = no download, inf = no limit).",
+        },
+        "max_download_videos": {
+            "default": 50,
+            "help": "maximum number of videos to download from the page (0 = no download, inf = no limit).",
+        },
+        "user_data_dir": {
+            "default": "secrets/antibot_user_data",
+            "help": "Path to the user data directory for the webdriver. This is used to persist browser state, such as cookies and local storage. If you use the docker deployment, this path will be appended with `_docker` that is because the folder cannot be shared between the host and the container due to user permissions.",
+        },
+        "detect_auth_wall": {
+            "default": True,
+            "type": "bool",
+            "help": "detect if the page is behind an authentication wall (e.g. login required) and skip it. disable if you want to archive pages where logins are required.",
+        },
+        "proxy": {
+            "default": None,
+            "help": "proxy to use for the webdriver, Format: 'SERVER:PORT' or 'USER:PASS@SERVER:PORT'",
+        },
+    },
+    "autodoc_dropins": True,
+    "description": """
+    Uses a browser controlled by SeleniumBase to capture HTML, media, and screenshots/PDFs of a web page, by bypassing anti-bot measures like Cloudflare's Turnstile or Google Recaptcha.
+	
+	> ⚠️ Still in trial development, please report any issues or suggestions via [GitHub Issues](https://github.com/bellingcat/auto-archiver/issues).
+	
+    ### Features
+	- Extracts the HTML source code of the page.
+    - Takes full-page screenshots of web pages.
+	- Takes full-page PDF snapshots of web pages.
+	- Downloads images and videos from the page, excluding specified file extensions.
+
+    ### Notes
+	- Using a proxy affects Cloudflare Turnstile captcha handling, so it is recommended to use a proxy only if necessary.
+
+	### Dropins
+	This module uses sub-modules called Dropins for specific sites that allow it to handle anti-bot measures and custom Login flows. You don't need to include the dropins in your configuration, but you do need to add authentication credentials if you want to overcome login walls on those sites, see detailed instructions for each Dropin below.
+
+    """,
+}

src/auto_archiver/modules/antibot_extractor_enricher/antibot_extractor_enricher.py

@@ -0,0 +1,330 @@
+import base64
+import math
+import os
+import sys
+import traceback
+from urllib.parse import urljoin
+import glob
+import importlib.util
+
+from auto_archiver.utils.custom_logger import logger
+import selenium
+from seleniumbase import SB
+
+from auto_archiver.core import Extractor, Enricher, Metadata, Media
+from auto_archiver.modules.antibot_extractor_enricher.dropin import Dropin
+from auto_archiver.modules.antibot_extractor_enricher.dropins.default import DefaultDropin
+from auto_archiver.utils.misc import random_str
+from auto_archiver.utils.url import is_relevant_url
+from auto_archiver.utils.deletion_detection import detect_deletion, flag_as_deleted
+
+
+class AntibotExtractorEnricher(Extractor, Enricher):
+    def setup(self) -> None:
+        self.agent = "cool"
+        if "linux" in sys.platform or "win32" in sys.platform:
+            self.agent = None  # Use the default UserAgent
+
+        # parse configuration options
+        if self.max_download_images == "inf":
+            self.max_download_images = math.inf
+        else:
+            self.max_download_images = int(self.max_download_images)
+
+        if self.max_download_videos == "inf":
+            self.max_download_videos = math.inf
+        else:
+            self.max_download_videos = int(self.max_download_videos)
+
+        self._prepare_user_data_dir()
+
+        self.dropins = self.load_dropins()
+
+    def load_dropins(self):
+        dropins = []
+
+        # TODO: add user-configurable drop-ins via config like generic_extractor
+        dropins_dir = os.path.join(os.path.dirname(__file__), "dropins")
+        for file_path in glob.glob(os.path.join(dropins_dir, "*.py")):
+            if os.path.basename(file_path).startswith("_"):
+                continue  # skip __init__.py or private modules
+            module_name = f"auto_archiver.modules.antibot_extractor_enricher.dropins.{os.path.splitext(os.path.basename(file_path))[0]}"
+            spec = importlib.util.spec_from_file_location(module_name, file_path)
+            module = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(module)
+            for attr in dir(module):
+                obj = getattr(module, attr)
+                if getattr(obj, "__module__", None) != module.__name__:
+                    continue  # Skip imported modules/classes/functions
+                if isinstance(obj, type) and issubclass(obj, Dropin):
+                    dropins.append(obj)
+        logger.debug(f"Loaded drop-in classes: {', '.join([d.__name__ for d in dropins])}")
+        return dropins
+
+    def sanitize_url(self, url: str) -> str:
+        for dropin in self.dropins:
+            if dropin.suitable(url):
+                return dropin.sanitize_url(url)
+        return url
+
+    def download(self, item: Metadata) -> Metadata:
+        result = Metadata()
+        result.merge(item)
+        if self.enrich(result):
+            result.status = "antibot"
+            return result
+        return False
+
+    def _prepare_user_data_dir(self):
+        if self.user_data_dir:
+            in_docker = os.environ.get("RUNNING_IN_DOCKER")
+            if in_docker:
+                self.user_data_dir = self.user_data_dir.rstrip(os.path.sep) + "_docker"
+            os.makedirs(self.user_data_dir, exist_ok=True)
+
+    def enrich(self, to_enrich: Metadata, custom_data_dir: bool = True) -> bool:
+        if to_enrich.get_media_by_id("html_source_code"):
+            logger.info("Antibot has already been executed, skipping.")
+            return True
+        using_user_data_dir = self.user_data_dir if custom_data_dir else None
+        url = to_enrich.get_url()
+
+        # Use xvfb in Docker environments where no display is available
+        use_xvfb = bool(os.environ.get("RUNNING_IN_DOCKER"))
+
+        try:
+            with SB(
+                uc=True,
+                agent=self.agent,
+                headed=None,
+                user_data_dir=using_user_data_dir,
+                proxy=self.proxy,
+                xvfb=use_xvfb,
+            ) as sb:
+                logger.info(f"Selenium browser is up with agent {self.agent}, opening url...")
+                sb.uc_open_with_reconnect(url, 4)
+
+                logger.debug("Handling CAPTCHAs for...")
+                sb.uc_gui_handle_cf()
+                sb.uc_gui_click_rc()  # NB: using handle instead of click breaks some sites like reddit, for now we separate here but can have dropins deciding this in the future
+
+                dropin = self._get_suitable_dropin(url, sb)
+                if not dropin.open_page(url):
+                    # Check for deletion indicators
+                    page_title = sb.get_title()
+                    html_source = sb.get_page_source()
+                    deletion_info = detect_deletion(html_content=html_source, page_title=page_title, url=url)
+                    if deletion_info:
+                        flag_as_deleted(to_enrich, deletion_info)
+                        return to_enrich
+                    logger.warning("Failed to open drop-in page (not detected as deleted)")
+                    return False
+
+                if self.detect_auth_wall and (dropin.hit_auth_wall() and self._hit_auth_wall(sb)):
+                    logger.warning("Skipping since auth wall or CAPTCHA was detected")
+                    return False
+
+                sb.wait_for_ready_state_complete()
+                sb.sleep(1)  # margin for the page to load completely
+
+                page_title = sb.get_title()
+                html_source = sb.get_page_source()
+
+                # Check if the page indicates content was deleted
+                deletion_info = detect_deletion(html_content=html_source, page_title=page_title, url=url)
+                if deletion_info:
+                    flag_as_deleted(to_enrich, deletion_info)
+
+                to_enrich.set_title(page_title)
+                self._enrich_html_source_code(sb, to_enrich)
+
+                self._enrich_full_page_screenshot(sb, to_enrich)
+                if self.save_to_pdf:
+                    self._enrich_full_page_pdf(sb, to_enrich)
+
+                downloaded_images, downloaded_videos = dropin.add_extra_media(to_enrich)
+
+                self._enrich_download_media(
+                    sb,
+                    to_enrich,
+                    js_css_selector=dropin.js_for_image_css_selectors(),
+                    max_media=self.max_download_images - downloaded_images,
+                )
+                self._enrich_download_media(
+                    sb,
+                    to_enrich,
+                    js_css_selector=dropin.js_for_video_css_selectors(),
+                    max_media=self.max_download_videos - downloaded_videos,
+                )
+                logger.info("Completed")
+
+            return to_enrich
+        except selenium.common.exceptions.SessionNotCreatedException as e:
+            if custom_data_dir:  # the retry logic only works once
+                logger.error(
+                    f"Session not created error: {e}. Please remove the user_data_dir {self.user_data_dir} and try again, will retry without user data dir though."
+                )
+                return self.enrich(to_enrich, custom_data_dir=False)
+            raise e  # re-raise
+        except Exception as e:
+            logger.error(f"Runtime error: {e}: {traceback.format_exc()}")
+            return False
+
+    def _get_suitable_dropin(self, url: str, sb: SB):
+        """
+        Returns a suitable drop-in for the given URL.
+        This method checks if the URL is suitable for any of the registered drop-ins.
+        """
+        for dropin in self.dropins:
+            if dropin.suitable(url):
+                logger.debug(f"Using drop-in {dropin.__name__}")
+                return dropin(sb, self)
+
+        return DefaultDropin(sb, self)
+
+    def _hit_auth_wall(self, sb: SB) -> bool:
+        """
+        Tries to detect if the currently loaded page is an auth/login wall.
+        Returns True if login is likely required.
+        """
+        # TODO: improve this detection logic, currently it is very basic and may not cover all cases
+
+        # Common URL patterns
+        current_url = sb.get_current_url().lower()
+        if any(kw in current_url for kw in ["login", "signin", "signup", "register", "captcha"]):
+            return True
+
+        # Common visible text markers
+        login_keywords = [
+            "sign up or log in",
+            "log in to continue",
+            "sign in to continue",
+            "login required",
+            "please log in",
+            "please sign up",
+            "please sign in",
+            "login to access",
+            "sign up to access",
+            "register to access",
+            "captcha verification",
+        ]
+        for word in login_keywords + [w.capitalize() for w in login_keywords]:
+            if sb.is_text_visible(word):
+                return True
+
+        # Common title markers
+        title = sb.get_title().lower()
+        if any(
+            kw in title
+            for kw in [
+                "just a moment...",
+                "tiktok - make your day",
+                "um momento...",
+                "log in",
+                "sign in",
+                "sign up",
+                "register",
+                "captcha",
+                "verification required",
+                "access denied",
+            ]
+        ):
+            return True
+
+        # Common form fields
+        elements = [
+            "input[type='password']",
+            "input[type='email']",
+            "input[type='username']",
+            "input[type='phone']",
+            "input[name='username']",
+            "input[name='email']",
+            "input[name='password']",
+            "input[name='login']",
+        ]
+        if any(sb.is_element_visible(el) for el in elements):
+            return True
+
+        return False
+
+    @logger.catch
+    def _enrich_html_source_code(self, sb: SB, to_enrich: Metadata):
+        """
+        Enriches the HTML source code of the Metadata object.
+        This method is called by the enrich method.
+        """
+        source = sb.get_page_source()
+
+        html_filename = os.path.join(self.tmp_dir, f"source{random_str(6)}.html")
+        with open(html_filename, "w", encoding="utf-8") as f:
+            f.write(source)
+
+        to_enrich.add_media(Media(filename=html_filename), id="html_source_code")
+
+    @logger.catch
+    def _enrich_full_page_screenshot(self, sb: SB, to_enrich: Metadata):
+        """
+        Enriches the full page screenshot of the Metadata object.
+        This method is called by the enrich method.
+        """
+        start_size = sb.get_window_size()
+        w, h = start_size["width"], start_size["height"]
+
+        x = max(sb.execute_script("return document.documentElement.scrollWidth"), w)
+        y = min(max(sb.execute_script("return document.documentElement.scrollHeight"), h), 25_000)
+        logger.debug(f"Setting window size to {x}x{y} for full page screenshot.")
+        sb.set_window_size(x, y)
+
+        screen_filename = os.path.join(self.tmp_dir, f"screenshot{random_str(6)}.png")
+        sb.save_screenshot(screen_filename)
+
+        to_enrich.add_media(Media(filename=screen_filename), id="screenshot")
+
+    @logger.catch
+    def _enrich_full_page_pdf(self, sb: SB, to_enrich: Metadata):
+        """
+        Enriches the full page PDF of the Metadata object.
+        This method is called by the enrich method.
+        """
+        result = sb.driver.execute_cdp_cmd("Page.printToPDF", {"printBackground": True, "landscape": False})
+
+        pdf_data = base64.b64decode(result["data"])
+
+        pdf_filename = os.path.join(self.tmp_dir, f"pdf{random_str(6)}.pdf")
+        with open(pdf_filename, "wb") as f:
+            f.write(pdf_data)
+
+        to_enrich.add_media(Media(filename=pdf_filename), id="pdf")
+
+    @logger.catch
+    def _enrich_download_media(self, sb: SB, to_enrich: Metadata, js_css_selector: str, max_media: int):
+        """
+        Downloads media from the page and adds them to the Metadata object.
+        This method is called by the enrich method.
+        """
+        if max_media == 0:
+            return
+        url = to_enrich.get_url()
+        all_urls = set()
+        logger.debug(f"Extracting media for {js_css_selector=}")
+
+        try:
+            sources = sb.execute_script(js_css_selector)
+        except selenium.common.exceptions.JavascriptException as e:
+            logger.error(f"Error executing JavaScript selector {js_css_selector}: {e}")
+            return
+
+        # js_for_css_selectors
+        for src in sources:
+            if len(all_urls) >= max_media:
+                logger.debug(f"Reached max download limit of {max_media} images/videos.")
+                break
+            if not is_relevant_url(src):
+                continue
+            full_src = urljoin(url, src)
+            if full_src not in all_urls:
+                filename, full_src = self.download_from_url(full_src, try_best_quality=True)
+                if not filename:
+                    continue
+                all_urls.add(full_src)
+                to_enrich.add_media(Media(filename=filename, properties={"url": full_src}))

src/auto_archiver/modules/antibot_extractor_enricher/captcha_services/.gitignore

@@ -0,0 +1 @@
+*.py

src/auto_archiver/modules/antibot_extractor_enricher/dropin.py

@@ -0,0 +1,173 @@
+import json
+import os
+import traceback
+from typing import Mapping
+from auto_archiver.utils.custom_logger import logger
+from seleniumbase import SB
+import yt_dlp
+
+from auto_archiver.core import Extractor, Media, Metadata
+from auto_archiver.utils.misc import ydl_entry_to_filename
+
+
+class Dropin:
+    """
+    A class to handle drop-in functionality for the antibot extractor enricher module.
+    This class is designed to be a base class for drop-ins that can handle specific websites.
+    """
+
+    @staticmethod
+    def documentation() -> Mapping[str, str]:
+        """
+        Each Dropin should auto-document itself with this method.
+        Return dictionary can include:
+        - 'name': A string representing the name of the dropin.
+        - 'description': A string describing the functionality of the dropin.
+        - 'site': A string representing the site this dropin is for.
+        - 'authentication': A dictionary with authentication example for the site.
+
+        """
+        return {}
+
+    def __init__(self, sb: SB, extractor: Extractor):
+        """
+        Initialize the Dropin with the given SeleniumBase instance.
+
+        :param sb: An instance of the SeleniumBase class that this drop-in will use.
+        :param extractor: An instance of the Extractor class that this drop-in will use.
+        """
+        self.sb: SB = sb
+        self.extractor: Extractor = extractor
+
+    @staticmethod
+    def suitable(url: str) -> bool:
+        """
+        Check if the URL is suitable for processing with this dropin.
+        :param url: The URL to check.
+        :return: True if the URL is suitable for processing, False otherwise.
+        """
+        raise NotImplementedError("This method should be implemented in the subclass")
+
+    @staticmethod
+    def sanitize_url(url: str) -> str:
+        """
+        Used to clean URLs before processing them.
+        """
+        return url
+
+    @staticmethod
+    def images_selectors() -> str:
+        """
+        CSS selector to find images in the HTML page
+        """
+        return "img"
+
+    @staticmethod
+    def video_selectors() -> str:
+        """
+        CSS selector to find videos in the HTML page.
+        """
+        return "video, source"
+
+    def js_for_image_css_selectors(self) -> str:
+        """
+        A configurable JS script that receives a css selector from the dropin itself and returns an array of Image elements according to the selection.
+
+        You can overwrite this instead of `images_selector` for more control over scraped images.
+        """
+        if not self.images_selectors():
+            return "return [];"
+        safe_selector = json.dumps(self.images_selectors())
+        return f"""
+            return Array.from(document.querySelectorAll({safe_selector})).map(el => el.src || el.href).filter(Boolean);
+        """
+
+    def js_for_video_css_selectors(self) -> str:
+        """
+        A configurable JS script that receives a css selector from the dropin itself and returns an array of Video elements according to the selection.
+
+        You can overwrite this instead of `video_selector` for more control over scraped videos.
+        """
+        if not self.video_selectors():
+            return "return [];"
+        safe_selector = json.dumps(self.video_selectors())
+        return f"""
+            return Array.from(document.querySelectorAll({safe_selector})).map(el => el.src || el.href).filter(Boolean);
+        """
+
+    def open_page(self, url) -> bool:
+        """
+        Make sure the page is opened, even if it requires authentication, captcha solving, etc.
+        :param url: The URL to open.
+        :return: True if success, False otherwise.
+        """
+        raise NotImplementedError("This method should be implemented in the subclass")
+
+    def add_extra_media(self, to_enrich: Metadata) -> tuple[int, int]:
+        """
+        Extract image and/or video data from the currently open post with SeleniumBase. Media is added to the `to_enrich` Metadata object.
+        :return: A tuple (number of Images added, number of Videos added).
+        """
+        return 0, 0
+
+    def hit_auth_wall(self) -> bool:
+        """
+        Custom check to see if the current page is behind an authentication wall, if True is returned the default global auth wall detector is used instead. If false, no auth wall is detected and the page is considered open.
+        """
+        return True
+
+    def _get_username_password(self, site) -> tuple[str, str]:
+        """
+        Get the username and password for the site from the extractor's auth data.
+        :return: A tuple (username, password).
+        """
+        auth = self.extractor.auth_for_site(site)
+        username = auth.get("username", "")
+        password = auth.get("password", "")
+        if not username or not password:
+            raise ValueError(f"{site} authentication requires a username and password.")
+        return username, password
+
+    def _download_videos_with_ytdlp(self, video_urls: list[str], to_enrich: Metadata) -> int:
+        """
+        Download videos using yt-dlp.
+        :param video_urls: List of video URLs to download.
+        :return: The number of videos downloaded.
+        """
+        if type(self.extractor.max_download_videos) is int:
+            video_urls = video_urls[: self.extractor.max_download_videos]
+
+        if not video_urls:
+            return 0
+
+        ydl_options = [
+            "-o",
+            os.path.join(self.extractor.tmp_dir, "%(id)s.%(ext)s"),
+            "--quiet",
+            "--no-playlist",
+            "--no-write-subs",
+            "--no-write-auto-subs",
+            "--postprocessor-args",
+            "ffmpeg:-bitexact",
+            "--max-filesize",
+            "1000M",  # Limit to 1GB per video
+        ]
+        *_, validated_options = yt_dlp.parse_options(ydl_options)
+        downloaded = 0
+        with yt_dlp.YoutubeDL(validated_options) as ydl:
+            for url in video_urls:
+                try:
+                    logger.debug(f"Downloading video from url: {url}")
+                    info = ydl.extract_info(url, download=True)
+                    filename = ydl_entry_to_filename(ydl, info)
+                    if not filename:  # Failed to download video.
+                        continue
+                    media = Media(filename)
+                    for x in ["duration", "original_url", "fulltitle", "description", "upload_date"]:
+                        if x in info:
+                            media.set(x, info[x])
+                    to_enrich.add_media(media)
+                    downloaded += 1
+                except Exception as e:
+                    logger.error(f"Download failed: {e} {traceback.format_exc()}")
+        return downloaded

src/auto_archiver/modules/antibot_extractor_enricher/dropins/default.py

@@ -0,0 +1,14 @@
+from auto_archiver.modules.antibot_extractor_enricher.dropin import Dropin
+
+
+class DefaultDropin(Dropin):
+    """
+    A default fallback drop-in class for handling generic cases in the antibot extractor enricher module.
+    """
+
+    @staticmethod
+    def suitable(url: str) -> bool:
+        return False
+
+    def open_page(self, url) -> bool:
+        return True

src/auto_archiver/modules/antibot_extractor_enricher/dropins/linkedin.py

@@ -0,0 +1,74 @@
+from typing import Mapping
+from auto_archiver.utils.custom_logger import logger
+from auto_archiver.modules.antibot_extractor_enricher.dropin import Dropin
+
+
+class LinkedinDropin(Dropin):
+    """
+    A class to handle LinkedIn drop-in functionality for the antibot extractor enricher module.
+    """
+
+    @staticmethod
+    def documentation() -> Mapping[str, str]:
+        return {
+            "name": "Linkedin Dropin",
+            "description": "Handles LinkedIn pages/posts and requires authentication to access most content but will still be useful without it. The first time you login to a new IP, LinkedIn may require an email verification code, you can do a manual login first and then it won't ask for it again.",
+            "site": "linkedin.com",
+            "authentication": {
+                "linkedin.com": {
+                    "username": "email address or phone number",
+                    "password": "password",
+                }
+            },
+        }
+
+    notifications_css_selector = 'a[href*="linkedin.com/notifications"]'
+
+    @staticmethod
+    def suitable(url: str) -> bool:
+        return "linkedin.com" in url
+
+    def js_for_image_css_selectors(self) -> str:
+        get_all_css = "main img:not([src*='profile-displayphoto']):not([src*='profile-framedphoto'])"
+        get_first_css = (
+            "main img[src*='profile-framedphoto'], main img[src*='profile-displayphoto'], main img[src*='company-logo']"
+        )
+
+        return f"""
+            const all = Array.from(document.querySelectorAll("{get_all_css}")).map(el => el.src || el.href).filter(Boolean);
+            const profile = document.querySelector("{get_first_css}");
+            return all.concat(profile?.src || profile?.href || []).filter(Boolean);
+        """
+
+    @staticmethod
+    def video_selectors() -> str:
+        # usually videos are from blob: but running the generic extractor should handle that
+        return "main video"
+
+    def open_page(self, url) -> bool:
+        if not self.sb.is_element_present(self.notifications_css_selector):
+            self._login()
+            if url != self.sb.get_current_url():
+                self.sb.open(url)
+        return True
+
+    @logger.catch
+    def _login(self) -> bool:
+        if self.sb.is_text_visible("Sign in to view more content"):
+            self.sb.click_link_text("Sign in", timeout=2)
+            self.sb.wait_for_ready_state_complete()
+        else:
+            self.sb.open("https://www.linkedin.com/login")
+            self.sb.wait_for_ready_state_complete()
+
+        username, password = self._get_username_password("linkedin.com")
+        logger.debug("Logging in to Linkedin with username: {}", username)
+        self.sb.type("#username", username)
+        self.sb.type("#password", password)
+        self.sb.click_if_visible("#password-visibility-toggle", timeout=0.5)
+        self.sb.click("button[type='submit']")
+        self.sb.wait_for_ready_state_complete()
+        # TODO: on suspicious login, LinkedIn may require an email verification code
+
+        if not self.sb.is_element_present(self.notifications_css_selector):
+            self.sb.click_if_visible('button[aria-label="Dismiss"]', timeout=0.5)

src/auto_archiver/modules/antibot_extractor_enricher/dropins/reddit.py

@@ -0,0 +1,92 @@
+from contextlib import suppress
+from typing import Mapping
+from auto_archiver.core.metadata import Metadata
+from auto_archiver.modules.antibot_extractor_enricher.dropin import Dropin
+
+from auto_archiver.utils.custom_logger import logger
+
+
+class RedditDropin(Dropin):
+    """
+    A class to handle Reddit drop-in functionality for the antibot extractor enricher module.
+    """
+
+    def documentation() -> Mapping[str, str]:
+        return {
+            "name": "Reddit Dropin",
+            "description": "Handles Reddit posts and works without authentication until Reddit flags your IP, so authentication is advised.",
+            "site": "reddit.com",
+            "authentication": {
+                "reddit.com": {
+                    "username": "email address or username",
+                    "password": "password",
+                }
+            },
+        }
+
+    @staticmethod
+    def suitable(url: str) -> bool:
+        return "reddit.com" in url
+
+    @staticmethod
+    def images_selectors() -> str:
+        return "shreddit-post img"
+
+    @staticmethod
+    def video_selectors() -> str:
+        return "shreddit-post video, shreddit-post source"
+
+    def open_page(self, url) -> bool:
+        if self.sb.is_text_visible("You've been blocked by network security."):
+            self._login()
+            if url != self.sb.get_current_url():
+                self.sb.open(url)
+        return True
+
+    @logger.catch
+    def _login(self):
+        self.sb.click_link_text("Log in")
+        self.sb.wait_for_ready_state_complete()
+        self._close_cookies_banner()
+
+        username, password = self._get_username_password("reddit.com")
+        logger.debug("Logging in to Reddit with username: {}", username)
+
+        self.sb.type("#login-username", username)
+        self.sb.type("#login-password", password)
+
+        elem = self.sb.find_element("button.login")
+        self.sb.execute_script("arguments[0].scrollIntoView(true);", elem)
+        self.sb.slow_click("button.login")
+        self.sb.wait_for_ready_state_complete()
+
+        if "https://www.reddit.com/login/" in self.sb.get_current_url():
+            self.sb.sleep(5)
+            self.sb.wait_for_ready_state_complete()
+
+        if self.sb.is_text_visible("You've been blocked by network security."):
+            self.sb.click_link_text("Log in")
+            self.sb.wait_for_ready_state_complete()
+            if self.sb.is_text_visible("Welcome back"):
+                logger.debug("Login successful")
+                self.sb.click_if_visible("this link")
+
+    def _close_cookies_banner(self):
+        with suppress(Exception):  # selenium.common.exceptions.JavascriptException
+            self.sb.execute_script("""
+                document
+                    .querySelector("reddit-cookie-banner")
+                    .shadowRoot.querySelector("faceplate-dialog")
+                    .querySelector("#accept-all-cookies-button button")
+                    .click()
+            """)
+
+    @logger.catch
+    def add_extra_media(self, to_enrich: Metadata) -> tuple[int, int]:
+        filtered_urls = self.sb.execute_script(rf"""
+            return [...document.querySelectorAll("{self.video_selectors()}")]
+            .map(el => el.src || el.href)
+            .filter(url => url && /\.(m3u8|mpd|ism)$/.test(url));
+        """)
+        logger.debug("Found {} video URLs", len(filtered_urls))
+        return 0, self._download_videos_with_ytdlp(filtered_urls, to_enrich)

src/auto_archiver/modules/antibot_extractor_enricher/dropins/tiktok.py

@@ -0,0 +1,56 @@
+from contextlib import suppress
+from typing import Mapping
+
+from auto_archiver.utils.custom_logger import logger
+from auto_archiver.modules.antibot_extractor_enricher.dropin import Dropin
+
+
+class TikTokDropin(Dropin):
+    """
+    A class to handle TikTok drop-in functionality for the antibot extractor enricher module.
+
+    """
+
+    def documentation() -> Mapping[str, str]:
+        return {
+            "name": "TikTok Dropin",
+            "description": "Handles TikTok posts and works without authentication.\nNOTE: This dropin is highly susceptible to TikTok's bot detection mechanisms and may not work reliably if you reuse the same IP. The GenericExtractor is recommended for TikTok posts, as it handles video/image download more reliable. In the future we plan to implement better anti captcha measures for this dropin.",
+            "site": "tiktok.com",
+        }
+
+    @staticmethod
+    def suitable(url: str) -> bool:
+        return "tiktok.com" in url
+
+    @staticmethod
+    def images_selectors() -> str:
+        return '[data-e2e="detail-photo"] img'
+
+    @staticmethod
+    def video_selectors() -> str:
+        return None  # TikTok videos should be handled by the generic extractor
+
+    def open_page(self, url) -> bool:
+        self.sb.wait_for_ready_state_complete()
+        self._close_cookies_banner()
+        # TODO: implement login logic
+        if url != self.sb.get_current_url():
+            return False
+        if self.sb.is_text_visible("Video currently unavailable"):
+            logger.debug("Video may have been removed or is private.")
+            return False
+        return True
+
+    def hit_auth_wall(self) -> bool:
+        return False  # TikTok does not require authentication for public posts
+
+    def _close_cookies_banner(self):
+        with suppress(Exception):  # selenium.common.exceptions.JavascriptException
+            self.sb.execute_script("""
+                document
+                    .querySelector("tiktok-cookie-banner")
+                    .shadowRoot.querySelector("faceplate-dialog")
+                    .querySelector("button")
+                    .click()
+            """)
+        self.sb.click_if_visible("Skip")