makes orchestrator.run return the results to allow for code integration (#196)

* disables tests on ubuntu-latest

* drops fixed oscrypto version for git commit

* version bump

.github/workflows/docker-publish.yaml

@@ -8,9 +8,6 @@ name: Docker
 on:
   release:
     types: [published]
-  push:
-    # branches: [ "main" ]
-    tags: [ "v*.*.*" ]
 
 env:
   # Use docker.io for Docker Hub if empty
@@ -28,22 +25,22 @@ jobs:
         uses: actions/checkout@v3
         
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
       # https://github.com/docker/setup-buildx-action
-      
+
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v1
-      
+        uses: docker/setup-buildx-action@v3
+
       - name: Log in to Docker Hub
-        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
-      
+
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96
         with:
           images: bellingcat/auto-archiver
       

.github/workflows/python-publish.yaml

@@ -1,4 +1,4 @@
-# This workflow will upload a Python Package using Twine when a release is created
+# This workflow uploads a Python Package to PyPI using Poetry when a release is created
 # For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
 
 # This workflow uses actions that are not certified by GitHub.
@@ -11,9 +11,6 @@ name: Pypi
 on:
   release:
     types: [published]
-  push:
-    # branches: [ "main" ]
-    tags: [ "v*.*.*" ]
 
 permissions:
   contents: read
@@ -24,30 +21,27 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
+    - name: Checkout Repository
+      uses: actions/checkout@v4
 
-    - name: Set up Python 3.10
-      uses: actions/setup-python@v4
+    - name: Set up Python
+      uses: actions/setup-python@v5
       with:
-        python-version: "3.10"
+        python-version-file: pyproject.toml
+
+    - name: Install Poetry
+      run: |
+        pipx install "poetry>=2.0.0,<3.0.0"
 
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade --upgrade-strategy=eager pip setuptools wheel twine pipenv
-        python -m pip install -e . --upgrade
-        python -m pipenv install --dev --python 3.10
-      env:
-        PIPENV_DEFAULT_PYTHON_VERSION: "3.10"
+        poetry install --no-interaction --no-root
 
-    - name: Build wheels
+    - name: Build the package
       run: |
-        python -m pipenv run python setup.py sdist bdist_wheel
+        poetry build
 
-    - name: Publish a Python distribution to PyPI
-      uses: pypa/gh-action-pypi-publish@release/v1
-      with:
-        user: __token__
-        verbose: true
-        skip_existing: true
-        password: ${{ secrets.PYPI_API_TOKEN }}
-        packages_dir: dist/
+    # Step 6: Publish to PyPI
+    - name: Publish to PyPI
+      run: |
+        poetry publish --username __token__ --password ${{ secrets.PYPI_API_TOKEN }}

.github/workflows/tests-core.yaml

@@ -0,0 +1,43 @@
+name: Core Tests
+
+on:
+  push:
+    branches: [ main ]
+    paths:
+      - src/**
+  pull_request:
+    paths:
+      - src/**
+
+jobs:
+  tests:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+        os: [ubuntu-22.04]
+        #TODO: re-enable ubuntu-latest, this is disabled as oscrypto cannot be pinned to github commit and pushed to pypi
+    defaults:
+      run:
+        working-directory: ./
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Poetry
+        run: pipx install poetry
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: 'poetry'
+
+      - name: Install dependencies
+        run: poetry install --no-interaction --with dev
+
+      - name: Run Core Tests
+        run: |
+          poetry run auto-archiver --version || true
+          poetry run pytest -ra -v -m "not download"

.github/workflows/tests-download.yaml

@@ -0,0 +1,40 @@
+name: Download Tests
+
+on:
+  schedule:
+    - cron: '35 14 * * 1'
+  pull_request:
+    branches: [ main ]
+    paths:
+      - src/**
+
+jobs:
+  tests:
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10"] # only run expensive downloads on one (lowest) python version
+    defaults:
+      run:
+        working-directory: ./
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install poetry
+        run: pipx install poetry
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: 'poetry'
+
+      - name: Install dependencies
+        run: poetry install --no-interaction --with dev
+
+      - name: Run Download Tests
+        run: poetry run pytest -ra -v -x -m "download"
+        env:
+          TWITTER_BEARER_TOKEN: ${{ secrets.TWITTER_BEARER_TOKEN }}

.gitignore

@@ -27,4 +27,9 @@ instaloader.session
 orchestration.yaml
 auto_archiver.egg-info*
 logs*
-*.csv
+*.csv
+archived/
+dist*
+docs/_build/
+docs/source/autoapi/
+docs/source/modules/autogen/

.pylintrc

@@ -0,0 +1,3 @@
+[MAIN]
+
+ignore-patterns=(.*tests.*.py, __manifest__.py)

.readthedocs.yaml

@@ -0,0 +1,22 @@
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.10"
+  jobs:
+    post_install:
+      - pip install poetry
+      # https://python-poetry.org/docs/managing-dependencies/#dependency-groups
+      # VIRTUAL_ENV needs to be set manually for now.
+      # See https://github.com/readthedocs/readthedocs.org/pull/11152/
+      - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH poetry install --with docs
+
+
+sphinx:
+  configuration: docs/source/conf.py

CONTRIBUTING.md

@@ -0,0 +1,49 @@
+# Contributing to Auto Archiver
+
+Thank you for your interest in contributing to Auto Archiver! Your contributions help improve the project and make it more useful for everyone. Please follow the guidelines below to ensure a smooth collaboration.
+
+### 1. Reporting a Bug
+
+If you encounter a bug, please create an issue on GitHub with the following details:
+
+* Describe the bug: Provide a clear and concise description of the issue.
+* Steps to reproduce: Include the steps needed to reproduce the bug.
+* Expected behavior: Describe what you expected to happen.
+* Actual behavior: Explain what actually happened.
+* Screenshots/logs: If applicable, attach screenshots or logs to help diagnose the problem.
+* Environment: Mention the OS, Ruby version, and any other relevant details.
+
+### 2. Writing a Patch/Fix and Submitting Pull Requests
+
+If you’d like to fix a bug or improve existing code:
+
+1. Open a pull request on GitHub and link it to the relevant issue.
+2. Make sure to document your pull request with a clear description of what changes were made and why.
+3. Wait for review and make any requested changes.
+
+### 3. Creating New Modules
+
+If you want to add a new module to Auto Archiver:
+
+1. Ensure your module follows the existing [coding style and project structure](https://auto-archiver.readthedocs.io/en/latest/development/creating_modules.html).
+2. Write clear documentation explaining what your module does and how to use it.
+3. Ideally, include unit tests for your module!
+4. Follow the steps in Section 2 to submit a pull request.
+
+### 4. Do You Have Questions About the Source Code?
+
+If you have any questions about how the source code works or need help using Auto Archiver
+
+📝 Check the [Auto Archiver](https://auto-archiver.readthedocs.io/en/latest/) documentation.
+
+👉 Ask your questions in the [Bellingcat Discord](https://www.bellingcat.com/follow-bellingcat-on-social-media/).
+
+### 5. Do You Want to Contribute to the Documentation?
+
+We welcome contributions to the documentation!
+
+📖 Please read [Contributing to the Auto Archiver Documentation](https://auto-archiver.readthedocs.io/en/latest/development/docs.html) to learn how you can help improve the project's documentation.
+
+------------------
+
+Thank you for contributing to Auto Archiver! 🚀

Dockerfile

@@ -1,31 +1,60 @@
-FROM webrecorder/browsertrix-crawler:latest
+FROM webrecorder/browsertrix-crawler:1.4.2 AS base
 
-ENV RUNNING_IN_DOCKER=1
+ENV RUNNING_IN_DOCKER=1 \
+    LANG=C.UTF-8 \
+    LC_ALL=C.UTF-8 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONFAULTHANDLER=1 \
+    PATH="/root/.local/bin:$PATH"
+
+# Installing system dependencies
+RUN add-apt-repository ppa:mozillateam/ppa && \
+	apt-get update && \
+    apt-get install -y --no-install-recommends gcc ffmpeg fonts-noto exiftool && \
+	apt-get install -y --no-install-recommends firefox-esr && \
+    ln -s /usr/bin/firefox-esr /usr/bin/firefox && \
+    wget https://github.com/mozilla/geckodriver/releases/download/v0.35.0/geckodriver-v0.35.0-linux64.tar.gz && \
+    tar -xvzf geckodriver* -C /usr/local/bin && \
+    chmod +x /usr/local/bin/geckodriver && \
+    rm geckodriver-v* && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+
+# Poetry and runtime
+FROM base AS runtime
+
+ENV POETRY_NO_INTERACTION=1 \
+    POETRY_VIRTUALENVS_IN_PROJECT=1 \
+    POETRY_VIRTUALENVS_CREATE=1
+
+
+# Create a virtual environment for poetry and install it
+RUN python3 -m venv /poetry-venv && \
+    /poetry-venv/bin/python -m pip install --upgrade pip && \
+    /poetry-venv/bin/python -m pip install "poetry>=2.0.0,<3.0.0"
 
 WORKDIR /app
 
-RUN pip install --upgrade pip && \
-	pip install pipenv && \
-	add-apt-repository ppa:mozillateam/ppa && \
-	apt-get update && \
-	apt-get install -y gcc ffmpeg fonts-noto exiftool && \
-	apt-get install -y --no-install-recommends firefox-esr && \
-	ln -s /usr/bin/firefox-esr /usr/bin/firefox && \
-	wget https://github.com/mozilla/geckodriver/releases/download/v0.33.0/geckodriver-v0.33.0-linux64.tar.gz && \
-	tar -xvzf geckodriver* -C /usr/local/bin && \
-	chmod +x /usr/local/bin/geckodriver && \
-	rm geckodriver-v*
+
+COPY pyproject.toml poetry.lock README.md ./
+# Copy dependency files and install dependencies (excluding the package itself)
+RUN /poetry-venv/bin/poetry install --only main --no-root --no-cache
 
 
-COPY Pipfile* ./
-# install from pipenv, with browsertrix-only requirements
-RUN pipenv install && \
-	pipenv install pywb uwsgi
-	
-# doing this at the end helps during development, builds are quick
-COPY ./src/ . 
+# Copy code: This is needed for poetry to install the package itself,
+# but the environment should be cached from the previous step if toml and lock files haven't changed
+COPY ./src/ .
+RUN /poetry-venv/bin/poetry install --only main --no-cache
 
-ENTRYPOINT ["pipenv", "run", "python3", "-m", "auto_archiver"]
+
+# Update PATH to include virtual environment binaries
+# Allowing entry point to run the application directly with Python
+ENV VIRTUAL_ENV=/app/.venv \
+    PATH="/app/.venv/bin:$PATH"
+
+ENTRYPOINT ["python3", "-m", "auto_archiver"]
 
 # should be executed with 2 volumes (3 if local_storage is used)
 # docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive aa pipenv run python3 -m auto_archiver --config secrets/orchestration.yaml
+

Pipfile

@@ -1,45 +0,0 @@
-[[source]]
-url = "https://pypi.org/simple"
-verify_ssl = true
-name = "pypi"
-
-[packages]
-gspread = "*"
-boto3 = "*"
-argparse = "*"
-beautifulsoup4 = "*"
-tiktok-downloader = "*"
-bs4 = "*"
-loguru = "*"
-ffmpeg-python = "*"
-selenium = "*"
-snscrape = "*"
-telethon = "*"
-google-api-python-client = "*"
-google-auth-httplib2 = "*"
-google-auth-oauthlib = "*"
-oauth2client = "*"
-pdqhash = "*"
-pillow = "*"
-python-slugify = "*"
-pyyaml = "*"
-dateparser = "*"
-python-twitter-v2 = "*"
-instaloader = "*"
-tqdm = "*"
-jinja2 = "*"
-cryptography = "*"
-dataclasses-json = "*"
-yt-dlp = "*"
-vk-url-scraper = "*"
-requests = {extras = ["socks"], version = "*"}
-numpy = "*"
-warcio = "*"
-jsonlines = "*"
-
-[dev-packages]
-autopep8 = "*"
-setuptools-pipfile = "*"
-
-[requires]
-python_version = "3.10"

README.md

@@ -2,243 +2,38 @@
 
 [![PyPI version](https://badge.fury.io/py/auto-archiver.svg)](https://badge.fury.io/py/auto-archiver)
 [![Docker Image Version (latest by date)](https://img.shields.io/docker/v/bellingcat/auto-archiver?label=version&logo=docker)](https://hub.docker.com/r/bellingcat/auto-archiver)
+[![Core Test Status](https://github.com/bellingcat/auto-archiver/workflows/Core%20Tests/badge.svg)](https://github.com/bellingcat/auto-archiver/actions/workflows/tests-core.yaml)
+[![Download Test Status](https://github.com/bellingcat/auto-archiver/workflows/Download%20Tests/badge.svg)](https://github.com/bellingcat/auto-archiver/actions/workflows/tests-download.yaml)
 <!-- ![Docker Pulls](https://img.shields.io/docker/pulls/bellingcat/auto-archiver) -->
 <!-- [![PyPI download month](https://img.shields.io/pypi/dm/auto-archiver.svg)](https://pypi.python.org/pypi/auto-archiver/) -->
 <!-- [![Documentation Status](https://readthedocs.org/projects/vk-url-scraper/badge/?version=latest)](https://vk-url-scraper.readthedocs.io/en/latest/?badge=latest) -->
 
 
+
+Auto Archiver is a Python tool to automatically archive content on the web in a secure and verifiable way. It takes URLs from different sources (e.g. a CSV file, Google Sheets, command line etc.) and archives the content of each one. It can archive social media posts, videos, images and webpages. Content can enriched, then saved either locally or remotely (S3 bucket, Google Drive). The status of the archiving process can be appended to a CSV report, or if using Google Sheets – back to the original sheet.
+
+<div class="hidden_rtd">
+  
+**[See the Auto Archiver documentation for more information.](https://auto-archiver.readthedocs.io/en/latest/)**
+
+</div>
+
 Read the [article about Auto Archiver on bellingcat.com](https://www.bellingcat.com/resources/2022/09/22/preserve-vital-online-content-with-bellingcats-auto-archiver-tool/).
 
 
-Python tool to automatically archive social media posts, videos, and images from a Google Sheets, the console, and more. Uses different archivers depending on the platform, and can save content to local storage, S3 bucket (Digital Ocean Spaces, AWS, ...), and Google Drive. If using Google Sheets as the source for links, it will be updated with information about the archived content. It can be run manually or on an automated basis.
+## Installation
 
-There are 3 ways to use the auto-archiver:
-1. (easiest installation) via docker
-2. (local python install) `pip install auto-archiver`
-3. (legacy/development) clone and manually install from repo (see legacy [tutorial video](https://youtu.be/VfAhcuV2tLQ))
+View the [Installation Guide](installation/installation.md) for full instructions
 
-But **you always need a configuration/orchestration file**, which is where you'll configure where/what/how to archive. Make sure you read [orchestration](#orchestration).
+To get started quickly using Docker:
 
+`docker pull bellingcat/auto-archiver && docker run`
 
-## How to install and run the auto-archiver
+Or pip:
 
-### Option 1 - docker
+`pip install auto-archiver && auto-archiver --help`
 
-[![dockeri.co](https://dockerico.blankenship.io/image/bellingcat/auto-archiver)](https://hub.docker.com/r/bellingcat/auto-archiver)
+## Contributing
 
-Docker works like a virtual machine running inside your computer, it isolates everything and makes installation simple. Since it is an isolated environment when you need to pass it your orchestration file or get downloaded media out of docker you will need to connect folders on your machine with folders inside docker with the `-v` volume flag.
+We welcome contributions to the Auto Archiver project! See the [Contributing Guide](https://auto-archiver.readthedocs.io/en/latest/contributing.html) for how to get involved!
 
-
-1. install [docker](https://docs.docker.com/get-docker/)
-2. pull the auto-archiver docker [image](https://hub.docker.com/r/bellingcat/auto-archiver) with `docker pull bellingcat/auto-archiver`
-3. run the docker image locally in a container: `docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver --config secrets/orchestration.yaml` breaking this command down:
-   1. `docker run` tells docker to start a new container (an instance of the image)
-   2. `--rm` makes sure this container is removed after execution (less garbage locally)
-   3. `-v $PWD/secrets:/app/secrets` - your secrets folder
-      1. `-v` is a volume flag which means a folder that you have on your computer will be connected to a folder inside the docker container
-      2. `$PWD/secrets` points to a `secrets/` folder in your current working directory (where your console points to), we use this folder as a best practice to hold all the secrets/tokens/passwords/... you use
-      3. `/app/secrets` points to the path the docker container where this image can be found
-   4.  `-v $PWD/local_archive:/app/local_archive` - (optional) if you use local_storage
-       1.  `-v` same as above, this is a volume instruction
-       2.  `$PWD/local_archive` is a folder `local_archive/` in case you want to archive locally and have the files accessible outside docker
-       3.  `/app/local_archive` is a folder inside docker that you can reference in your orchestration.yml file 
-
-### Option 2 - python package
-
-<details><summary><code>Python package instructions</code></summary>
-
-1. make sure you have python 3.8 or higher installed
-2. install the package `pip/pipenv/conda install auto-archiver`
-3. test it's installed with `auto-archiver --help`
-4. run it with your orchestration file and pass any flags you want in the command line `auto-archiver --config secrets/orchestration.yaml` if your orchestration file is inside a `secrets/`, which we advise
-   
-You will also need [ffmpeg](https://www.ffmpeg.org/), [firefox](https://www.mozilla.org/en-US/firefox/new/) and [geckodriver](https://github.com/mozilla/geckodriver/releases), and optionally [fonts-noto](https://fonts.google.com/noto). Similar to the local installation. 
-
-</details>
-
-
-### Option 3 - local installation
-This can also be used for development.
-
-<details><summary><code>Legacy instructions, only use if docker/package is not an option</code></summary>
-
-
-Install the following locally:
-1. [ffmpeg](https://www.ffmpeg.org/) must also be installed locally for this tool to work. 
-2. [firefox](https://www.mozilla.org/en-US/firefox/new/) and [geckodriver](https://github.com/mozilla/geckodriver/releases) on a path folder like `/usr/local/bin`. 
-3. (optional) [fonts-noto](https://fonts.google.com/noto) to deal with multiple unicode characters during selenium/geckodriver's screenshots: `sudo apt install fonts-noto -y`. 
-
-Clone and run:
-1. `git clone https://github.com/bellingcat/auto-archiver`
-2. `pipenv install`
-3. `pipenv run python -m src.auto_archiver --config secrets/orchestration.yaml`
-
-
-</details><br/>
-
-# Orchestration
-The archiver work is orchestrated by the following workflow (we call each a **step**): 
-1. **Feeder** gets the links (from a spreadsheet, from the console, ...)
-2. **Archiver** tries to archive the link (twitter, youtube, ...)
-3. **Enricher** adds more info to the content (hashes, thumbnails, ...)
-4. **Formatter** creates a report from all the archived content (HTML, PDF, ...)
-5. **Database** knows what's been archived and also stores the archive result (spreadsheet, CSV, or just the console)
-
-To setup an auto-archiver instance create an `orchestration.yaml` which contains the workflow you would like. We advise you put this file into a `secrets/` folder and do not share it with others because it will contain passwords and other secrets. 
-
-The structure of orchestration file is split into 2 parts: `steps` (what **steps** to use) and `configurations` (how those steps should behave), here's a simplification:
-```yaml
-# orchestration.yaml content
-steps:
-  feeder: gsheet_feeder
-  archivers: # order matters
-    - youtubedl_archiver
-  enrichers:
-    - thumbnail_enricher
-  formatter: html_formatter
-  storages:
-    - local_storage
-  databases:
-    - gsheet_db
-
-configurations:
-  gsheet_feeder:
-    sheet: "your google sheet name"
-    header: 2 # row with header for your sheet
-  # ... configurations for the other steps here ...
-```
-
-To see all available `steps` (which archivers, storages, databses, ...) exist check the [example.orchestration.yaml](example.orchestration.yaml).
-
-All the `configurations` in the `orchestration.yaml` file (you can name it differently but need to pass it in the `--config FILENAME` argument) can be seen in the console by using the `--help` flag. They can also be overwritten, for example if you are using the `cli_feeder` to archive from the command line and want to provide the URLs you should do:
-
-```bash
-auto-archiver --config secrets/orchestration.yaml --cli_feeder.urls="url1,url2,url3"
-```
-
-Here's the complete workflow that the auto-archiver goes through:
-```mermaid
-graph TD
-    s((start)) --> F(fa:fa-table Feeder)
-    F -->|get and clean URL| D1{fa:fa-database Database}
-    D1 -->|is already archived| e((end))
-    D1 -->|not yet archived| a(fa:fa-download Archivers)
-    a -->|got media| E(fa:fa-chart-line Enrichers)
-    E --> S[fa:fa-box-archive Storages]
-    E --> Fo(fa:fa-code Formatter)
-    Fo --> S
-    Fo -->|update database| D2(fa:fa-database Database)
-    D2 --> e
-```
-
-## Orchestration checklist
-Use this to make sure you help making sure you did all the required steps:
-* [ ] you have a `/secrets` folder with all your configuration files including
-  * [ ] a orchestration file eg: `orchestration.yaml` pointing to the correct location of other files
-  * [ ] (optional if you use GoogleSheets) you have a `service_account.json` (see [how-to](https://gspread.readthedocs.io/en/latest/oauth2.html#for-bots-using-service-account))
-  * [ ] (optional for telegram) a `anon.session` which appears after the 1st run where you login to telegram
-    * if you use private channels you need to add `channel_invites` and set `join_channels=true` at least once
-  * [ ] (optional for VK) a `vk_config.v2.json`
-  * [ ] (optional for using GoogleDrive storage) `gd-token.json` (see [help script](scripts/create_update_gdrive_oauth_token.py))
-  * [ ] (optional for instagram) `instaloader.session` file which appears after the 1st run and login in instagram
-  * [ ] (optional for browsertrix) `profile.tar.gz` file
-
-#### Example invocations
-The recommended way to run the auto-archiver is through Docker. The invocations below will run the auto-archiver Docker image using a configuration file that you have specified
-
-```bash
-# all the configurations come from ./secrets/orchestration.yaml
-docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver --config secrets/orchestration.yaml
-# uses the same configurations but for another google docs sheet 
-# with a header on row 2 and with some different column names
-# notice that columns is a dictionary so you need to pass it as JSON and it will override only the values provided
-docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver --config secrets/orchestration.yaml --gsheet_feeder.sheet="use it on another sheets doc" --gsheet_feeder.header=2 --gsheet_feeder.columns='{"url": "link"}'
-# all the configurations come from orchestration.yaml and specifies that s3 files should be private
-docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver --config secrets/orchestration.yaml --s3_storage.private=1
-```
-
-The auto-archiver can also be run locally, if pre-requisites are correctly configured. Equivalent invocations are below.
-
-```bash
-# all the configurations come from ./secrets/orchestration.yaml
-auto-archiver --config secrets/orchestration.yaml
-# uses the same configurations but for another google docs sheet 
-# with a header on row 2 and with some different column names
-# notice that columns is a dictionary so you need to pass it as JSON and it will override only the values provided
-auto-archiver --config secrets/orchestration.yaml --gsheet_feeder.sheet="use it on another sheets doc" --gsheet_feeder.header=2 --gsheet_feeder.columns='{"url": "link"}'
-# all the configurations come from orchestration.yaml and specifies that s3 files should be private
-auto-archiver --config secrets/orchestration.yaml --s3_storage.private=1
-```
-
-### Extra notes on configuration
-#### Google Drive
-To use Google Drive storage you need the id of the shared folder in the `config.yaml` file which must be shared with the service account eg `autoarchiverservice@auto-archiver-111111.iam.gserviceaccount.com` and then you can use `--storage=gd`
-
-#### Telethon + Instagram with telegram bot
-The first time you run, you will be prompted to do a authentication with the phone number associated, alternatively you can put your `anon.session` in the root.
-
-
-## Running on Google Sheets Feeder (gsheet_feeder)
-The `--gseets_feeder.sheet` property is the name of the Google Sheet to check for URLs. 
-This sheet must have been shared with the Google Service account used by `gspread`. 
-This sheet must also have specific columns (case-insensitive) in the `header` as specified in [Gsheet.configs](src/auto_archiver/utils/gsheet.py). The default names of these columns and their purpose is:
-
-Inputs:
-
-* **Link** *(required)*: the URL of the post to archive
-* **Destination folder**: custom folder for archived file (regardless of storage)
-
-Outputs:
-* **Archive status** *(required)*: Status of archive operation
-* **Archive location**: URL of archived post
-* **Archive date**: Date archived
-* **Thumbnail**: Embeds a thumbnail for the post in the spreadsheet
-* **Timestamp**: Timestamp of original post
-* **Title**: Post title
-* **Text**: Post text
-* **Screenshot**: Link to screenshot of post
-* **Hash**: Hash of archived HTML file (which contains hashes of post media) - for checksums/verification
-* **Perceptual Hash**: Perceptual hashes of found images - these can be used for de-duplication of content
-* **WACZ**: Link to a WACZ web archive of post
-* **ReplayWebpage**: Link to a ReplayWebpage viewer of the WACZ archive
-
-For example, this is a spreadsheet configured with all of the columns for the auto archiver and a few URLs to archive. (Note that the column names are not case sensitive.)
-
-![A screenshot of a Google Spreadsheet with column headers defined as above, and several Youtube and Twitter URLs in the "Link" column](docs/demo-before.png)
-
-Now the auto archiver can be invoked, with this command in this example: `docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver:dockerize --config secrets/orchestration-global.yaml --gsheet_feeder.sheet "Auto archive test 2023-2"`. Note that the sheet name has been overridden/specified in the command line invocation.
-
-When the auto archiver starts running, it updates the "Archive status" column.
-
-![A screenshot of a Google Spreadsheet with column headers defined as above, and several Youtube and Twitter URLs in the "Link" column. The auto archiver has added "archive in progress" to one of the status columns.](docs/demo-progress.png)
-
-The links are downloaded and archived, and the spreadsheet is updated to the following:
-
-![A screenshot of a Google Spreadsheet with videos archived and metadata added per the description of the columns above.](docs/demo-after.png)
-
-Note that the first row is skipped, as it is assumed to be a header row (`--gsheet_feeder.header=1` and you can change it if you use more rows above). Rows with an empty URL column, or a non-empty archive column are also skipped. All sheets in the document will be checked.
-
-The "archive location" link contains the path of the archived file, in local storage, S3, or in Google Drive.
-
-![The archive result for a link in the demo sheet.](docs/demo-archive.png)
-
----
-## Development
-Use `python -m src.auto_archiver --config secrets/orchestration.yaml` to run from the local development environment.
-
-#### Docker development
-working with docker locally:
-  * `docker build . -t auto-archiver` to build a local image
-  * `docker run --rm -v $PWD/secrets:/app/secrets auto-archiver  --config secrets/orchestration.yaml`
-    * to use local archive, also create a volume `-v` for it by adding `-v $PWD/local_archive:/app/local_archive`
-
-
-release to docker hub
-  * `docker image tag auto-archiver bellingcat/auto-archiver:latest`
-  * `docker push bellingcat/auto-archiver`
-
-#### RELEASE
-* update version in [version.py](src/auto_archiver/version.py)
-* run `bash ./scripts/release.sh` and confirm
-* package is automatically updated in pypi
-* docker image is automatically pushed to dockerhup

docker-compose.yaml

@@ -0,0 +1,16 @@
+version: '3.8'
+
+services:
+  auto-archiver:
+  # point to the local dockerfile
+    build:
+      context: .
+      dockerfile: Dockerfile
+    container_name: auto-archiver
+    volumes:
+      - ./secrets:/app/secrets
+      - ./local_archive:/app/local_archive
+    environment:
+      - WACZ_ENABLE_DOCKER=true
+      - RUNNING_IN_DOCKER=true
+    command: --config secrets/orchestration.yaml

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = source
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/_static/custom.css

@@ -0,0 +1,4 @@
+.hidden_rtd {
+    display:none;
+}
+

docs/_templates/autoapi/index.rst

@@ -0,0 +1,46 @@
+API Reference
+=============
+
+These pages are intended for developers of the `auto-archiver` package, 
+and include documentation on the core classes and functions used by 
+the auto-archiver
+
+
+Core Classes
+------------
+
+
+.. toctree::
+   :titlesonly:
+
+   {% for page in pages|selectattr("is_top_level_object") %}
+   {% if page.name == 'core' %}
+   {{ page.include_path }}
+   {% endif %}
+   {% endfor %}
+
+Util Functions
+--------------
+
+.. toctree::
+   :titlesonly:
+
+   {% for page in pages|selectattr("is_top_level_object") %}
+   {% if page.name == 'utils' %}
+   {{ page.include_path }}
+   {% endif %}
+   {% endfor %}
+
+
+Core Modules
+------------
+
+.. toctree::
+   :titlesonly:
+
+   {% for page in pages|selectattr("is_top_level_object") %}
+   {% if page.name != 'core' and page.name != 'utils' %}
+   {{ page.include_path }}
+   {% endif %}
+   {% endfor %}
+

docs/_templates/autoapi/python/attribute.rst

@@ -0,0 +1 @@
+{% extends "python/data.rst" %}

docs/_templates/autoapi/python/class.rst

@@ -0,0 +1,104 @@
+{% if obj.display %}
+   {% if is_own_page %}
+{{ obj.id }}
+{{ "=" * obj.id | length }}
+
+   {% endif %}
+   {% set visible_children = obj.children|selectattr("display")|list %}
+   {% set own_page_children = visible_children|selectattr("type", "in", own_page_types)|list %}
+   {% if is_own_page and own_page_children %}
+.. toctree::
+   :hidden:
+
+      {% for child in own_page_children %}
+   {{ child.include_path }}
+      {% endfor %}
+
+   {% endif %}
+.. py:{{ obj.type }}:: {% if is_own_page %}{{ obj.id }}{% else %}{{ obj.short_name }}{% endif %}{% if obj.args %}({{ obj.args }}){% endif %}
+
+   {% for (args, return_annotation) in obj.overloads %}
+      {{ " " * (obj.type | length) }}   {{ obj.short_name }}{% if args %}({{ args }}){% endif %}
+
+   {% endfor %}
+   {% if obj.bases %}
+      {% if "show-inheritance" in autoapi_options %}
+
+   Bases: {% for base in obj.bases %}{{ base|link_objs }}{% if not loop.last %}, {% endif %}{% endfor %}
+      {% endif %}
+
+
+      {% if "show-inheritance-diagram" in autoapi_options and obj.bases != ["object"] %}
+   .. autoapi-inheritance-diagram:: {{ obj.obj["full_name"] }}
+      :parts: 1
+         {% if "private-members" in autoapi_options %}
+      :private-bases:
+         {% endif %}
+
+      {% endif %}
+   {% endif %}
+   {% if obj.docstring %}
+
+   {{ obj.docstring|indent(3) }}
+   {% endif %}
+   {% for obj_item in visible_children %}
+      {% if obj_item.type not in own_page_types %}
+
+   {{ obj_item.render()|indent(3) }}
+      {% endif %}
+   {% endfor %}
+   {% if is_own_page and own_page_children %}
+      {% set visible_attributes = own_page_children|selectattr("type", "equalto", "attribute")|list %}
+      {% if visible_attributes %}
+Attributes
+----------
+
+.. autoapisummary::
+
+         {% for attribute in visible_attributes %}
+   {{ attribute.id }}
+         {% endfor %}
+
+
+      {% endif %}
+      {% set visible_exceptions = own_page_children|selectattr("type", "equalto", "exception")|list %}
+      {% if visible_exceptions %}
+Exceptions
+----------
+
+.. autoapisummary::
+
+         {% for exception in visible_exceptions %}
+   {{ exception.id }}
+         {% endfor %}
+
+
+      {% endif %}
+      {% set visible_classes = own_page_children|selectattr("type", "equalto", "class")|list %}
+      {% if visible_classes %}
+Classes
+-------
+
+.. autoapisummary::
+
+         {% for klass in visible_classes %}
+   {{ klass.id }}
+         {% endfor %}
+
+
+      {% endif %}
+      {% set visible_methods = own_page_children|selectattr("type", "equalto", "method")|list %}
+      {% if visible_methods %}
+Methods
+-------
+
+.. autoapisummary::
+
+            {% for method in visible_methods %}
+   {{ method.id }}
+            {% endfor %}
+
+
+      {% endif %}
+   {% endif %}
+{% endif %}

docs/_templates/autoapi/python/data.rst

@@ -0,0 +1,38 @@
+{% if obj.display %}
+   {% if is_own_page %}
+{{ obj.id }}
+{{ "=" * obj.id | length }}
+
+   {% endif %}
+.. py:{{ obj.type }}:: {% if is_own_page %}{{ obj.id }}{% else %}{{ obj.name }}{% endif %}
+   {% if obj.annotation is not none %}
+
+   :type: {% if obj.annotation %} {{ obj.annotation }}{% endif %}
+   {% endif %}
+   {% if obj.value is not none %}
+
+      {% if obj.value.splitlines()|count > 1 %}
+   :value: Multiline-String
+
+   .. raw:: html
+
+      <details><summary>Show Value</summary>
+
+   .. code-block:: python
+
+      {{ obj.value|indent(width=6,blank=true) }}
+
+   .. raw:: html
+
+      </details>
+
+      {% else %}
+   :value: {{ obj.value|truncate(100) }}
+      {% endif %}
+   {% endif %}
+
+   {% if obj.docstring %}
+
+   {{ obj.docstring|indent(3) }}
+   {% endif %}
+{% endif %}

docs/_templates/autoapi/python/exception.rst

@@ -0,0 +1 @@
+{% extends "python/class.rst" %}

docs/_templates/autoapi/python/function.rst

@@ -0,0 +1,21 @@
+{% if obj.display %}
+   {% if is_own_page %}
+{{ obj.id }}
+{{ "=" * obj.id | length }}
+
+   {% endif %}
+.. py:function:: {% if is_own_page %}{{ obj.id }}{% else %}{{ obj.short_name }}{% endif %}({{ obj.args }}){% if obj.return_annotation is not none %} -> {{ obj.return_annotation }}{% endif %}
+   {% for (args, return_annotation) in obj.overloads %}
+
+                 {%+ if is_own_page %}{{ obj.id }}{% else %}{{ obj.short_name }}{% endif %}({{ args }}){% if return_annotation is not none %} -> {{ return_annotation }}{% endif %}
+   {% endfor %}
+   {% for property in obj.properties %}
+
+   :{{ property }}:
+   {% endfor %}
+
+   {% if obj.docstring %}
+
+   {{ obj.docstring|indent(3) }}
+   {% endif %}
+{% endif %}

docs/_templates/autoapi/python/method.rst

@@ -0,0 +1,21 @@
+{% if obj.display %}
+   {% if is_own_page %}
+{{ obj.id }}
+{{ "=" * obj.id | length }}
+
+   {% endif %}
+.. py:method:: {% if is_own_page %}{{ obj.id }}{% else %}{{ obj.short_name }}{% endif %}({{ obj.args }}){% if obj.return_annotation is not none %} -> {{ obj.return_annotation }}{% endif %}
+   {% for (args, return_annotation) in obj.overloads %}
+
+               {%+ if is_own_page %}{{ obj.id }}{% else %}{{ obj.short_name }}{% endif %}({{ args }}){% if return_annotation is not none %} -> {{ return_annotation }}{% endif %}
+   {% endfor %}
+   {% for property in obj.properties %}
+
+   :{{ property }}:
+   {% endfor %}
+
+   {% if obj.docstring %}
+
+   {{ obj.docstring|indent(3) }}
+   {% endif %}
+{% endif %}

docs/_templates/autoapi/python/module.rst

@@ -0,0 +1,156 @@
+{% if obj.display %}
+   {% if is_own_page %}
+{{ obj.id }}
+{{ "=" * obj.id|length }}
+
+.. py:module:: {{ obj.name }}
+
+      {% if obj.docstring %}
+.. autoapi-nested-parse::
+
+   {{ obj.docstring|indent(3) }}
+
+      {% endif %}
+
+      {% block submodules %}
+         {% set visible_subpackages = obj.subpackages|selectattr("display")|list %}
+         {% set visible_submodules = obj.submodules|selectattr("display")|list %}
+         {% set visible_submodules = (visible_subpackages + visible_submodules)|sort %}
+         {% if visible_submodules %}
+Submodules
+----------
+
+.. toctree::
+   :maxdepth: 1
+
+            {% for submodule in visible_submodules %}
+   {{ submodule.include_path }}
+            {% endfor %}
+
+
+         {% endif %}
+      {% endblock %}
+      {% block content %}
+         {% set visible_children = obj.children|selectattr("display")|list %}
+         {% if visible_children %}
+            {% set visible_attributes = visible_children|selectattr("type", "equalto", "data")|list %}
+            {% if visible_attributes %}
+               {% if "attribute" in own_page_types or "show-module-summary" in autoapi_options %}
+Attributes
+----------
+
+                  {% if "attribute" in own_page_types %}
+.. toctree::
+   :hidden:
+
+                     {% for attribute in visible_attributes %}
+   {{ attribute.include_path }}
+                     {% endfor %}
+
+                  {% endif %}
+.. autoapisummary::
+
+                  {% for attribute in visible_attributes %}
+   {{ attribute.id }}
+                  {% endfor %}
+               {% endif %}
+
+
+            {% endif %}
+            {% set visible_exceptions = visible_children|selectattr("type", "equalto", "exception")|list %}
+            {% if visible_exceptions %}
+               {% if "exception" in own_page_types or "show-module-summary" in autoapi_options %}
+Exceptions
+----------
+
+                  {% if "exception" in own_page_types %}
+.. toctree::
+   :hidden:
+
+                     {% for exception in visible_exceptions %}
+   {{ exception.include_path }}
+                     {% endfor %}
+
+                  {% endif %}
+.. autoapisummary::
+
+                  {% for exception in visible_exceptions %}
+   {{ exception.id }}
+                  {% endfor %}
+               {% endif %}
+
+
+            {% endif %}
+            {% set visible_classes = visible_children|selectattr("type", "equalto", "class")|list %}
+            {% if visible_classes %}
+               {% if "class" in own_page_types or "show-module-summary" in autoapi_options %}
+Classes
+-------
+
+                  {% if "class" in own_page_types %}
+.. toctree::
+   :hidden:
+
+                     {% for klass in visible_classes %}
+   {{ klass.include_path }}
+                     {% endfor %}
+
+                  {% endif %}
+.. autoapisummary::
+
+                  {% for klass in visible_classes %}
+   {{ klass.id }}
+                  {% endfor %}
+               {% endif %}
+
+
+            {% endif %}
+            {% set visible_functions = visible_children|selectattr("type", "equalto", "function")|list %}
+            {% if visible_functions %}
+               {% if "function" in own_page_types or "show-module-summary" in autoapi_options %}
+Functions
+---------
+
+                  {% if "function" in own_page_types %}
+.. toctree::
+   :hidden:
+
+                     {% for function in visible_functions %}
+   {{ function.include_path }}
+                     {% endfor %}
+
+                  {% endif %}
+.. autoapisummary::
+
+                  {% for function in visible_functions %}
+   {{ function.id }}
+                  {% endfor %}
+               {% endif %}
+
+
+            {% endif %}
+            {% set this_page_children = visible_children|rejectattr("type", "in", own_page_types)|list %}
+            {% if this_page_children %}
+{{ obj.type|title }} Contents
+{{ "-" * obj.type|length }}---------
+
+               {% for obj_item in this_page_children %}
+{{ obj_item.render()|indent(0) }}
+               {% endfor %}
+            {% endif %}
+         {% endif %}
+      {% endblock %}
+   {% else %}
+.. py:module:: {{ obj.name }}
+
+      {% if obj.docstring %}
+   .. autoapi-nested-parse::
+
+      {{ obj.docstring|indent(6) }}
+
+      {% endif %}
+      {% for obj_item in visible_children %}
+   {{ obj_item.render()|indent(3) }}
+      {% endfor %}
+   {% endif %}
+{% endif %}

docs/_templates/autoapi/python/package.rst

@@ -0,0 +1 @@
+{% extends "python/module.rst" %}

docs/_templates/autoapi/python/property.rst

@@ -0,0 +1,21 @@
+{% if obj.display %}
+   {% if is_own_page %}
+{{ obj.id }}
+{{ "=" * obj.id | length }}
+
+   {% endif %}
+.. py:property:: {% if is_own_page %}{{ obj.id}}{% else %}{{ obj.short_name }}{% endif %}
+   {% if obj.annotation %}
+
+   :type: {{ obj.annotation }}
+   {% endif %}
+   {% for property in obj.properties %}
+
+   :{{ property }}:
+   {% endfor %}
+
+   {% if obj.docstring %}
+
+   {{ obj.docstring|indent(3) }}
+   {% endif %}
+{% endif %}

docs/scripts/__init__.py

@@ -0,0 +1 @@
+from scripts import generate_module_docs

docs/scripts/scripts.py

@@ -0,0 +1,105 @@
+# iterate through all the modules in auto_archiver.modules and turn the __manifest__.py file into a markdown table
+from pathlib import Path
+from auto_archiver.core.module import available_modules
+from auto_archiver.core.base_module import BaseModule
+from ruamel.yaml import YAML
+import io
+
+MODULES_FOLDER = Path(__file__).parent.parent.parent.parent / "src" / "auto_archiver" / "modules"
+SAVE_FOLDER = Path(__file__).parent.parent / "source" / "modules" / "autogen"
+
+type_color = {
+    'feeder': "<span style='color: #FFA500'>[feeder](/core_modules.md#feeder-modules)</a></span>",
+    'extractor': "<span style='color: #00FF00'>[extractor](/core_modules.md#extractor-modules)</a></span>",
+    'enricher': "<span style='color: #0000FF'>[enricher](/core_modules.md#enricher-modules)</a></span>",
+    'database': "<span style='color: #FF00FF'>[database](/core_modules.md#database-modules)</a></span>",
+    'storage': "<span style='color: #FFFF00'>[storage](/core_modules.md#storage-modules)</a></span>",
+    'formatter': "<span style='color: #00FFFF'>[formatter](/core_modules.md#formatter-modules)</a></span>",
+}
+
+TABLE_HEADER = ("Option", "Description", "Default", "Type")
+
+def generate_module_docs():
+    yaml = YAML()
+    SAVE_FOLDER.mkdir(exist_ok=True)
+    modules_by_type = {}
+
+    header_row = "| " + " | ".join(TABLE_HEADER) + "|\n" + "| --- " * len(TABLE_HEADER) + "|\n"
+    configs_cheatsheet = "\n## Configuration Options\n"
+    configs_cheatsheet += header_row
+
+    for module in sorted(available_modules(with_manifest=True), key=lambda x: (x.requires_setup, x.name)):
+        # generate the markdown file from the __manifest__.py file.
+
+        manifest = module.manifest
+        for type in manifest['type']:
+            modules_by_type.setdefault(type, []).append(module)
+
+        description = "\n".join(l.lstrip() for l in manifest['description'].split("\n"))
+        types = ", ".join(type_color[t] for t in manifest['type'])
+        readme_str = f"""
+# {manifest['name']}
+```{{admonition}} Module type
+
+{types}
+```
+{description}
+"""     
+        if not manifest['configs']:
+            readme_str += "\n*This module has no configuration options.*\n"
+        else:
+            config_yaml = {}
+            config_table = header_row
+            for key, value in manifest['configs'].items():
+                type = value.get('type', 'string')
+                if type == 'auto_archiver.utils.json_loader':
+                    value['type'] = 'json'
+                elif type == 'str':
+                    type = "string"
+                
+                default = value.get('default', '')
+                config_yaml[key] = default
+                help = "**Required**. " if value.get('required', False) else "Optional. "
+                help += value.get('help', '')
+                config_table += f"| `{module.name}.{key}` | {help} | {value.get('default', '')} | {type} |\n"
+                configs_cheatsheet += f"| `{module.name}.{key}` | {help} | {default} | {type} |\n"
+            readme_str += "\n## Configuration Options\n"
+            readme_str += "\n### YAML\n"
+            yaml_string = io.BytesIO()
+            yaml.dump({module.name: config_yaml}, yaml_string)
+            
+            readme_str += f"```{{code}} yaml\n{yaml_string.getvalue().decode('utf-8')}\n```\n"
+
+            readme_str += "\n### Command Line:\n"
+            readme_str += config_table
+
+        # add a link to the autodoc refs
+        readme_str += f"\n[API Reference](../../../autoapi/{module.name}/index)\n"
+        # create the module.type folder, use the first type just for where to store the file
+        for type in manifest['type']:
+            type_folder = SAVE_FOLDER / type
+            type_folder.mkdir(exist_ok=True)
+            with open(type_folder / f"{module.name}.md", "w") as f:
+                print("writing", SAVE_FOLDER)
+                f.write(readme_str)
+        generate_index(modules_by_type)
+
+    with open(SAVE_FOLDER / "configs_cheatsheet.md", "w") as f:
+        f.write(configs_cheatsheet)
+
+
+def generate_index(modules_by_type):
+    readme_str = ""
+    for type in BaseModule.MODULE_TYPES:
+        modules = modules_by_type.get(type, [])
+        module_str = f"## {type.capitalize()} Modules\n"
+        for module in modules:
+            module_str += f"\n[{module.manifest['name']}](/modules/autogen/{module.type[0]}/{module.name}.md)\n"
+        with open(SAVE_FOLDER / f"{type}.md", "w") as f:
+            print("writing", SAVE_FOLDER / f"{type}.md")
+            f.write(module_str)
+        readme_str += module_str
+
+    with open(SAVE_FOLDER / "module_list.md", "w") as f:
+        print("writing", SAVE_FOLDER / "module_list.md")
+        f.write(readme_str)

docs/source/conf.py

@@ -0,0 +1,79 @@
+# Configuration file for the Sphinx documentation builder.
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+import sys
+import os
+from importlib.metadata import metadata
+
+sys.path.append(os.path.abspath('../scripts'))
+from scripts import generate_module_docs
+
+# -- Project Hooks -----------------------------------------------------------
+# convert the module __manifest__.py files into markdown files
+generate_module_docs()
+
+
+# -- Project information -----------------------------------------------------
+package_metadata = metadata("auto-archiver")
+project = package_metadata["name"]
+authors = "Bellingcat"
+release = package_metadata["version"]
+language = 'en'
+
+# -- General configuration ---------------------------------------------------
+extensions = [
+    "myst_parser",                  # Markdown support
+    "autoapi.extension",            # Generate API documentation from docstrings
+    "sphinxcontrib.mermaid",        # Mermaid diagrams
+    "sphinx.ext.viewcode",          # Source code links
+    "sphinx_copybutton",
+    "sphinx.ext.napoleon",          # Google-style and NumPy-style docstrings
+    "sphinx.ext.autosectionlabel",
+    # 'sphinx.ext.autosummary',       # Summarize module/class/function docs
+]
+
+templates_path = ['_templates']
+exclude_patterns = []
+
+
+# -- AutoAPI Configuration ---------------------------------------------------
+autoapi_type = 'python'
+autoapi_dirs = ["../../src/auto_archiver/core/", "../../src/auto_archiver/utils/"]
+# get all the modules and add them to the autoapi_dirs
+autoapi_dirs.extend([f"../../src/auto_archiver/modules/{m}" for m in os.listdir("../../src/auto_archiver/modules")])
+autodoc_typehints = "signature"     # Include type hints in the signature
+autoapi_ignore = ["*/version.py", ]                 # Ignore specific modules
+autoapi_keep_files = True          # Option to retain intermediate JSON files for debugging
+autoapi_add_toctree_entry = True    # Include API docs in the TOC
+autoapi_python_use_implicit_namespaces = True
+autoapi_template_dir = "../_templates/autoapi"
+autoapi_options = [
+    "members",
+    "undoc-members",
+    "show-inheritance",
+    "imported-members",
+]
+
+
+# -- Markdown Support --------------------------------------------------------
+myst_enable_extensions = [
+    "deflist",              # Definition lists
+    "html_admonition",      # HTML-style admonitions
+    "html_image",           # Inline HTML images
+    "replacements",         # Substitutions like (C)
+    "smartquotes",          # Smart quotes
+    "linkify",              # Auto-detect links
+    "substitution",         # Text substitutions
+]
+myst_heading_anchors = 2
+myst_fence_as_directive = ["mermaid"]
+
+source_suffix = {
+    ".rst": "restructuredtext",
+    ".md": "markdown",
+}
+
+# -- Options for HTML output -------------------------------------------------
+html_theme = 'sphinx_book_theme'
+html_static_path = ["../_static"]
+html_css_files = ["custom.css"]
+

docs/source/contributing.md

@@ -0,0 +1,2 @@
+```{include} ../../CONTRIBUTING.md
+```

docs/source/core_modules.md

@@ -0,0 +1,27 @@
+# Module Documentation
+
+These pages describe the core modules that come with `auto-archiver` and provide the main functionality for archiving websites on the internet. There are five core module types:
+
+1. Feeders - these 'feed' information (the URLs) from various sources to the `auto-archiver` for processing
+2. Extractors - these 'extract' the page data for a given URL that is fed in by a feeder
+3. Enrichers - these 'enrich' the data extracted in the previous step with additional information
+4. Storage - these 'store' the data in a persistent location (on disk, Google Drive etc.)
+5. Databases - these 'store' the status of the entire archiving process in a log file or database.
+
+
+```{include} modules/autogen/module_list.md
+```
+
+
+```{toctree}
+:maxdepth: 1
+:caption: Core Modules
+:hidden:
+
+modules/config_cheatsheet
+modules/feeder
+modules/extractor
+modules/enricher
+modules/storage
+modules/database
+```

docs/source/development/creating_modules.md

@@ -0,0 +1,52 @@
+# Creating Your Own Modules
+
+Modules are what's used to extend `auto-archiver` to process different websites or media, and/or transform the data in a way that suits your needs. In most cases, the [Core Modules](../core_modules.md) should be sufficient for every day use, but the most common use-cases for making your own Modules include:
+
+1. Extracting data from a website which doesn't work with the current core extractors.
+2. Enriching or altering the data before saving with additional information that the core enrichers do not offer.
+3. Storing your data in a different format/location from what the core storage providers offer.
+
+## Setting up the folder structure
+
+1. First, decide what type of module you wish to create. Check the types of modules on the [](../core_modules.md) page to decide what type you need. (Note: a module can be more than one type, more on that below)
+2. Create a new python package (a folder) with the name of your module (in this tutorial, we'll call it `awesome_extractor`).
+3. Create the `__manifest__.py` and an the `awesome_extractor.py` files in this folder.
+
+When done, you should have a module structure as follows:
+
+```
+.
+├── awesome_extractor
+│   ├── __manifest__.py
+│   └── awesome_extractor.py
+``` 
+
+Check out the [core modules](https://github.com/bellingcat/auto-archiver/tree/main/src/auto_archiver/modules) in the `auto-archiver` repository for examples of the folder structure for real-world modules.
+
+## Populating the Manifest File
+
+The manifest file is where you define the core information of your module. It is a python dict containing important information, here's an example file:
+
+```{include} ../../../tests/data/test_modules/example_module/__manifest__.py
+:name: __manifest__.py
+:literal:
+:parser: python
+```
+
+## Creating the Python Code
+
+The next step is to create your module code. First, create a class which should subclass the base module types from `auto_archiver.core`, here's an example class for the `awesome_extractor` module which is an `extractor`:
+
+```{code-block} python
+:filename: awesome_extractor.py
+
+from auto_archiver.core import Extractor, Metadata
+
+def AwesomeExtractor(Extractor):
+
+    def download(self, item: Metadata) -> Metadata | False:
+      url = item.get_url()
+      # download the content and create the metadata object
+      metadata = ...
+      return metadata
+```

docs/source/development/developer_guidelines.md

@@ -0,0 +1,34 @@
+
+# Developer Guidelines
+
+This section of the documentation provides guidelines for developers who want to modify or contribute to the tool.
+
+
+## Developer Install
+
+1. Clone the project using `git clone https://github.com/bellingcat/auto-archiver.git` 
+2. Install poetry using `curl -sSL https://install.python-poetry.org | python3 -` ([other installation methods](https://python-poetry.org/docs/#installation))
+3. Install dependencies with `poetry install`
+
+## Running 
+4. Run the code with `poetry run auto-archiver [my args]`
+
+```{note}
+Add the plugin [poetry-shell-plugin](https://github.com/python-poetry/poetry-plugin-shell) and run `poetry shell` to activate the virtual environment.
+This allows you to run the auto-archiver without the `poetry run` prefix.
+```
+
+### Optional Development Packages
+
+Install development packages (used for unit tests etc.) using:
+`poetry install -with dev`
+
+
+```{toctree}
+:hidden:
+creating_modules
+docker_development
+testing
+docs
+release
+```

docs/source/development/docker_development.md

@@ -0,0 +1,5 @@
+## Docker development
+working with docker locally:
+  * `docker compose up` to build the first time and run a local image with the settings in `secrets/orchestration.yaml`
+  * To modify/pass additional command line args, use `docker compose run auto-archiver --config secrets/orchestration.yaml [OTHER ARGUMENTS]`
+  * To rebuild after code changes, just pass the `--build` flag, e.g. `docker compose up --build`

docs/source/development/docs.md

@@ -0,0 +1,38 @@
+
+### Building the Docs
+
+The documentation is built using [Sphinx](https://www.sphinx-doc.org/en/master/) and [AutoAPI](https://sphinx-autoapi.readthedocs.io/en/latest/) and hosted on ReadTheDocs.
+To build the documentation locally, run the following commands:
+
+**Install required dependencies:**
+- Install the docs group of dependencies: 
+```shell
+# only the docs dependencies
+poetry install --only docs
+
+# or for all dependencies 
+poetry install
+```
+- Either use [poetry-plugin-shell](https://github.com/python-poetry/poetry-plugin-shell) to activate the virtual environment: `poetry shell`
+- Or prepend the following commands with `poetry run`
+
+**Create the documentation:**
+- Build the documentation: 
+```shell
+# Using makefile (Linux/macOS):
+make -C docs html
+
+# or using sphinx directly (Windows/Linux/macOS):
+sphinx-build -b html docs/source docs/_build/html
+```
+- If you make significant changes and want a fresh build run: `make -C docs clean` to remove the old build files.
+
+**Viewing the documentation:**
+```shell
+# to open the documentation in your browser.
+open docs/_build/html/index.html
+
+# or run autobuild to automatically update the documentation when you make changes
+sphinx-autobuild docs/source docs/_build/html
+```
+

docs/source/development/release.md

@@ -0,0 +1,15 @@
+# Release Process
+
+```{note} This is a work in progress.
+```
+
+1. Update the version number in [version.py](src/auto_archiver/version.py)
+2. Go to github releases > new release > use `vx.y.z` for matching version notation
+   1. package is automatically updated in pypi
+   2. docker image is automatically pushed to dockerhup
+
+
+
+manual release to docker hub
+  * `docker image tag auto-archiver bellingcat/auto-archiver:latest`
+  * `docker push bellingcat/auto-archiver`

docs/source/development/testing.md

@@ -0,0 +1,21 @@
+# Testing
+
+`pytest` is used for testing. There are two main types of tests:
+
+1. 'core' tests which should be run on every change
+2. 'download' tests which hit the network. These tests will do things like make API calls (e.g. Twitter, Bluesky etc.) and should be run regularly to make sure that APIs have not changed.
+
+
+## Running Tests 
+
+1. Make sure you've installed the dev dependencies with `pytest install --with dev`
+2. Tests can be run as follows:
+```
+#### Command prefix of 'poetry run' removed here for simplicity
+# run core tests
+pytest -ra -v -m "not download"
+# run download tests
+pytest -ra -v -m "download"
+# run all tests
+pytest -ra -v
+```

docs/source/example.orchestration.yaml

@@ -0,0 +1,79 @@
+# Auto Archiver Configuration
+# Steps are the modules that will be run in the order they are defined
+
+steps:
+  feeders:
+  - cli_feeder
+  extractors:
+  - generic_extractor
+  - telegram_extractor
+  enrichers:
+  - thumbnail_enricher
+  - meta_enricher
+  - pdq_hash_enricher
+  - ssl_enricher
+  - hash_enricher
+  databases:
+  - console_db
+  - csv_db
+  storages:
+  - local_storage
+  formatters:
+  - html_formatter
+
+# Global configuration
+
+# Authentication
+# a dictionary of authentication information that can be used by extractors to login to website. 
+# you can use a comma separated list for multiple domains on the same line (common usecase: x.com,twitter.com)
+# Common login 'types' are username/password, cookie, api key/token.
+# There are two special keys for using cookies, they are: cookies_file and cookies_from_browser. 
+# Some Examples:
+# facebook.com:
+#   username: "my_username"
+#   password: "my_password"
+# or for a site that uses an API key:
+# twitter.com,x.com:
+#   api_key
+#   api_secret
+# youtube.com:
+#   cookie: "login_cookie=value ; other_cookie=123" # multiple 'key=value' pairs should be separated by ;
+
+authentication: {}
+
+# Logging settings for your project. See the logging settings with --help
+
+logging:
+  level: INFO
+
+# These are the global configurations that are used by the modules
+
+  file:
+  rotation:
+local_storage:
+  path_generator: flat
+  filename_generator: static
+  save_to: ./local_archive
+  save_absolute: false
+html_formatter:
+  detect_thumbnails: true
+thumbnail_enricher:
+  thumbnails_per_minute: 60
+  max_thumbnails: 16
+generic_extractor:
+  subtitles: true
+  comments: false
+  livestreams: false
+  live_from_start: false
+  proxy: ''
+  end_means_success: true
+  allow_playlist: false
+  max_downloads: inf
+csv_db:
+  csv_file: db.csv
+ssl_enricher:
+  skip_when_nothing_archived: true
+hash_enricher:
+  algorithm: SHA-256
+  chunksize: 16000000
+

docs/source/flow_overview.md

@@ -0,0 +1,30 @@
+
+# Archiving Overview
+
+The archiver archives web pages using the following workflow
+1. **Feeder** gets the links (from a spreadsheet, from the console, ...)
+2. **Extractor** tries to extract content from the given link (e.g. videos from youtube, images from Twitter...)
+3. **Enricher** adds more info to the content (hashes, thumbnails, ...)
+4. **Formatter** creates a report from all the archived content (HTML, PDF, ...)
+5. **Database** knows what's been archived and also stores the archive result (spreadsheet, CSV, or just the console)
+
+Each step in the workflow is handled by 'modules' that interact with the data in different ways. For example, the Twitter Extractor Module would extract information from the Twitter website. The Screenshot Enricher Module will take screenshots of the given page. See the [core modules page](core_modules.md) for an overview of all the modules that are available.
+
+Auto-archiver must have at least one module defined for each step of the workflow. This is done by setting the [configuration](installation/configurations.md) for your auto-archiver instance.
+
+Here's the complete workflow that the auto-archiver goes through:
+
+```mermaid
+
+graph TD
+    s((start)) --> F(fa:fa-table Feeder)
+    F -->|get and clean URL| D1{fa:fa-database Database}
+    D1 -->|is already archived| e((end))
+    D1 -->|not yet archived| a(fa:fa-download Archivers)
+    a -->|got media| E(fa:fa-chart-line Enrichers)
+    E --> S[fa:fa-box-archive Storages]
+    E --> Fo(fa:fa-code Formatter)
+    Fo --> S
+    Fo -->|update database| D2(fa:fa-database Database)
+    D2 --> e
+```

docs/source/how_to.md

@@ -0,0 +1,47 @@
+# How-To Guides
+
+## How to use Google Sheets to load and store archive information
+The `--gsheet_feeder.sheet` property is the name of the Google Sheet to check for URLs. 
+This sheet must have been shared with the Google Service account used by `gspread`. 
+This sheet must also have specific columns (case-insensitive) in the `header` - see the [Gsheet Feeder Docs](modules/autogen/feeder/gsheet_feeder.md) for more info. The default names of these columns and their purpose is:
+
+Inputs:
+
+* **Link** *(required)*: the URL of the post to archive
+* **Destination folder**: custom folder for archived file (regardless of storage)
+
+Outputs:
+* **Archive status** *(required)*: Status of archive operation
+* **Archive location**: URL of archived post
+* **Archive date**: Date archived
+* **Thumbnail**: Embeds a thumbnail for the post in the spreadsheet
+* **Timestamp**: Timestamp of original post
+* **Title**: Post title
+* **Text**: Post text
+* **Screenshot**: Link to screenshot of post
+* **Hash**: Hash of archived HTML file (which contains hashes of post media) - for checksums/verification
+* **Perceptual Hash**: Perceptual hashes of found images - these can be used for de-duplication of content
+* **WACZ**: Link to a WACZ web archive of post
+* **ReplayWebpage**: Link to a ReplayWebpage viewer of the WACZ archive
+
+For example, this is a spreadsheet configured with all of the columns for the auto archiver and a few URLs to archive. (Note that the column names are not case sensitive.)
+
+![A screenshot of a Google Spreadsheet with column headers defined as above, and several Youtube and Twitter URLs in the "Link" column](../demo-before.png)
+
+Now the auto archiver can be invoked, with this command in this example: `docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver:dockerize --config secrets/orchestration-global.yaml --gsheet_feeder.sheet "Auto archive test 2023-2"`. Note that the sheet name has been overridden/specified in the command line invocation.
+
+When the auto archiver starts running, it updates the "Archive status" column.
+
+![A screenshot of a Google Spreadsheet with column headers defined as above, and several Youtube and Twitter URLs in the "Link" column. The auto archiver has added "archive in progress" to one of the status columns.](../demo-progress.png)
+
+The links are downloaded and archived, and the spreadsheet is updated to the following:
+
+![A screenshot of a Google Spreadsheet with videos archived and metadata added per the description of the columns above.](../demo-after.png)
+
+Note that the first row is skipped, as it is assumed to be a header row (`--gsheet_feeder.header=1` and you can change it if you use more rows above). Rows with an empty URL column, or a non-empty archive column are also skipped. All sheets in the document will be checked.
+
+The "archive location" link contains the path of the archived file, in local storage, S3, or in Google Drive.
+
+![The archive result for a link in the demo sheet.](../demo-archive.png)
+
+---

docs/source/index.md

@@ -0,0 +1,17 @@
+
+```{include} ../../README.md
+```
+
+```{toctree}
+:maxdepth: 2
+:hidden:
+:caption: Contents:
+
+Overview <self>
+contributing
+installation/installation.rst
+core_modules.md
+how_to
+development/developer_guidelines
+autoapi/index.rst
+```

docs/source/installation/config_cheatsheet.md

@@ -0,0 +1,6 @@
+# Configuration Cheat Sheet
+
+Below is a list of all configurations for the core modules in Auto Archiver
+
+```{include} ../modules/autogen/configs_cheatsheet.md
+```

docs/source/installation/configurations.md

@@ -0,0 +1,98 @@
+
+# Configuration
+
+This section of the documentation provides guidelines for configuring the tool.
+
+## Configuring using a file
+
+The recommended way to configure auto-archiver for long-term and deployed projects is a configuration file, typically called `orchestration.yaml`. This is a YAML file containing all the settings for your entire workflow.
+
+The structure of orchestration file is split into 2 parts: `steps` (what [steps](../flow_overview.md) to use) and `configurations` (settings for different modules), here's a simplification:
+
+A default `orchestration.yaml` will be created for you the first time you run auto-archiver (without any arguments). Here's what it looks like:
+
+<details>
+<summary>View exampleorchestration.yaml</summary>
+
+```{literalinclude} ../example.orchestration.yaml
+   :language: yaml
+   :caption: orchestration.yaml
+```
+
+</details>
+
+## Configuring from the Command Line
+
+You can run auto-archiver directy from the command line, without the need for a configuration file, command line arguments are parsed using the format `module_name.config_value`. For example, a config value of `api_key` in the `instagram_extractor` module would be passed on the command line with the flag `--instagram_extractor.api_key=API_KEY`.
+
+The command line arguments are useful for testing or editing config values and enabling/disabling modules on the fly. When you are happy with your settings, you can store them back in your configuration file by passing the `-s/--store` flag on the command line.
+
+```bash
+auto-archiver --instagram_extractor.api_key=123 --other_module.setting --store
+# will store the new settings into the configuration file (default: orchestration.yaml)
+```
+
+```{note} Arguments passed on the command line override those saved in your settings file. Save them to your config file using the -s or --store flag
+```
+
+## Seeing all Configuration Options
+
+View the configurable settings for the core modules on the individual doc pages for each [](../core_modules.md).
+You can also view all settings available for the modules you have on your system using the `--help` flag in auto-archiver.
+
+```{code-block} console
+:caption: Example output when using the --help flag with auto-archiver
+$ auto-archiver --help
+...
+Positional Arguments:
+  urls                  URL(s) to archive, either a single URL or a list of urls, should not come from config.yaml
+
+Options:
+  --help, -h            show a full help message and exit
+  --version             show program's version number and exit
+  --config CONFIG_FILE  the filename of the YAML configuration file (defaults to 'config.yaml')
+  --mode {simple,full}  the mode to run the archiver in
+  -s, --store, --no-store
+                        Store the created config in the config file
+  --module_paths MODULE_PATHS [MODULE_PATHS ...]
+                        additional paths to search for modules
+  --feeders STEPS.FEEDERS [STEPS.FEEDERS ...]
+                        the feeders to use
+  --enrichers STEPS.ENRICHERS [STEPS.ENRICHERS ...]
+                        the enrichers to use
+  --extractors STEPS.EXTRACTORS [STEPS.EXTRACTORS ...]
+                        the extractors to use
+  --databases STEPS.DATABASES [STEPS.DATABASES ...]
+                        the databases to use
+  --storages STEPS.STORAGES [STEPS.STORAGES ...]
+                        the storages to use
+  --formatters STEPS.FORMATTERS [STEPS.FORMATTERS ...]
+                        the formatter to use
+  --authentication AUTHENTICATION
+                        A dictionary of sites and their authentication methods (token, username etc.) that extractors can use to log into a website. If passing this on the command line, use a JSON string. You may
+                        also pass a path to a valid JSON/YAML file which will be parsed.
+  --logging.level {INFO,DEBUG,ERROR,WARNING}
+                        the logging level to use
+  --logging.file LOGGING.FILE
+                        the logging file to write to
+  --logging.rotation LOGGING.ROTATION
+                        the logging rotation to use
+
+Wayback Machine Enricher:
+  Submits the current URL to the Wayback Machine for archiving and returns either a job ID or the...
+
+  --wayback_extractor_enricher.timeout TIMEOUT
+                        seconds to wait for successful archive confirmation from wayback, if more than this passes the result contains the job_id so the status can later be checked manually.
+  --wayback_extractor_enricher.if_not_archived_within IF_NOT_ARCHIVED_WITHIN
+                        only tell wayback to archive if no archive is available before the number of seconds specified, use None to ignore this option. For more information:
+                        https://docs.google.com/document/d/1Nsv52MvSjbLb2PCpHlat0gkzw0EvtSgpKHu4mk0MnrA
+  --wayback_extractor_enricher.key KEY
+                        wayback API key. to get credentials visit https://archive.org/account/s3.php
+  --wayback_extractor_enricher.secret SECRET
+                        wayback API secret. to get credentials visit https://archive.org/account/s3.php
+  --wayback_extractor_enricher.proxy_http PROXY_HTTP
+                        http proxy to use for wayback requests, eg http://proxy-user:password@proxy-ip:port
+  --wayback_extractor_enricher.proxy_https PROXY_HTTPS
+                        https proxy to use for wayback requests, eg https://proxy-user:password@proxy-ip:port
+```
+

docs/source/installation/installation.md

@@ -0,0 +1,92 @@
+# Installing Auto Archiver
+
+```{toctree}
+:depth: 1
+:hidden:
+
+configurations.md
+config_cheatsheet.md
+```
+
+There are 3  main ways to use the auto-archiver:
+1. Easiest: [via docker](#installing-with-docker)
+2. Local Install: [using pip](#installing-locally-with-pip)
+3. Developer Install: [see the developer guidelines](../development/developer_guidelines)
+
+
+But **you always need a configuration/orchestration file**, which is where you'll configure where/what/how to archive. Make sure you read [orchestration](#orchestration).
+
+
+## Installing with Docker
+
+[![dockeri.co](https://dockerico.blankenship.io/image/bellingcat/auto-archiver)](https://hub.docker.com/r/bellingcat/auto-archiver)
+
+Docker works like a virtual machine running inside your computer, it isolates everything and makes installation simple. Since it is an isolated environment when you need to pass it your orchestration file or get downloaded media out of docker you will need to connect folders on your machine with folders inside docker with the `-v` volume flag.
+
+
+1. Install [docker](https://docs.docker.com/get-docker/)
+2. Pull the auto-archiver docker [image](https://hub.docker.com/r/bellingcat/auto-archiver) with `docker pull bellingcat/auto-archiver`
+3. Run the docker image locally in a container: `docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver --config secrets/orchestration.yaml` breaking this command down:
+   1. `docker run` tells docker to start a new container (an instance of the image)
+   2. `--rm` makes sure this container is removed after execution (less garbage locally)
+   3. `-v $PWD/secrets:/app/secrets` - your secrets folder
+      1. `-v` is a volume flag which means a folder that you have on your computer will be connected to a folder inside the docker container
+      2. `$PWD/secrets` points to a `secrets/` folder in your current working directory (where your console points to), we use this folder as a best practice to hold all the secrets/tokens/passwords/... you use
+      3. `/app/secrets` points to the path the docker container where this image can be found
+   4.  `-v $PWD/local_archive:/app/local_archive` - (optional) if you use local_storage
+       1.  `-v` same as above, this is a volume instruction
+       2.  `$PWD/local_archive` is a folder `local_archive/` in case you want to archive locally and have the files accessible outside docker
+       3.  `/app/local_archive` is a folder inside docker that you can reference in your orchestration.yml file 
+
+### Example invocations
+
+The invocations below will run the auto-archiver Docker image using a configuration file that you have specified
+
+```bash
+# all the configurations come from ./secrets/orchestration.yaml
+docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver --config secrets/orchestration.yaml
+# uses the same configurations but for another google docs sheet 
+# with a header on row 2 and with some different column names
+# notice that columns is a dictionary so you need to pass it as JSON and it will override only the values provided
+docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver --config secrets/orchestration.yaml --gsheet_feeder.sheet="use it on another sheets doc" --gsheet_feeder.header=2 --gsheet_feeder.columns='{"url": "link"}'
+# all the configurations come from orchestration.yaml and specifies that s3 files should be private
+docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver --config secrets/orchestration.yaml --s3_storage.private=1
+```
+
+## Installing Locally with Pip
+
+1. Make sure you have python 3.10 or higher installed
+2. Install the package with your preferred package manager: `pip/pipenv/conda install auto-archiver` or `poetry add auto-archiver`
+3. Test it's installed with `auto-archiver --help`
+4. Install other local dependency requirements (for )
+5. Run it with your orchestration file and pass any flags you want in the command line `auto-archiver --config secrets/orchestration.yaml` if your orchestration file is inside a `secrets/`, which we advise
+
+### Example invocations
+
+Once all your [local requirements](#installing-local-requirements) are correctly installed, the
+
+```bash
+# all the configurations come from ./secrets/orchestration.yaml
+auto-archiver --config secrets/orchestration.yaml
+# uses the same configurations but for another google docs sheet 
+# with a header on row 2 and with some different column names
+# notice that columns is a dictionary so you need to pass it as JSON and it will override only the values provided
+auto-archiver --config secrets/orchestration.yaml --gsheet_feeder.sheet="use it on another sheets doc" --gsheet_feeder.header=2 --gsheet_feeder.columns='{"url": "link"}'
+# all the configurations come from orchestration.yaml and specifies that s3 files should be private
+auto-archiver --config secrets/orchestration.yaml --s3_storage.private=1
+```
+
+### Installing Local Requirements
+
+If using the local installation method, you will also need to install the following dependencies locally:
+
+1.[ffmpeg](https://www.ffmpeg.org/) - for handling of downloaded videos
+2. [firefox](https://www.mozilla.org/en-US/firefox/new/) and [geckodriver](https://github.com/mozilla/geckodriver/releases) on a path folder like `/usr/local/bin` - for taking webpage screenshots with the screenshot enricher
+3. (optional) [fonts-noto](https://fonts.google.com/noto) to deal with multiple unicode characters during selenium/geckodriver's screenshots: `sudo apt install fonts-noto -y`.
+4. [Browsertrix Crawler docker image](https://hub.docker.com/r/webrecorder/browsertrix-crawler) for the WACZ enricher/archiver
+
+
+
+## Developer Install
+
+[See the developer guidelines](../development/developer_guidelines)

docs/source/modules/database.md

@@ -0,0 +1,15 @@
+# Database Modules
+
+Database modules are used to store the status and results of the extraction and enrichment processes somewhere. The database modules are responsible for creating and managing entires for each item that has been processed.
+
+The default (enabled) databases are the CSV Database and the Console Database.
+
+```{include} autogen/database.md
+```
+
+```{toctree}
+:depth: 1
+:hidden:
+:glob:
+autogen/database/*
+```

docs/source/modules/enricher.md

@@ -0,0 +1,14 @@
+# Enricher Modules
+
+Enricher modules are used to add additional information to the items  that have been extracted. Common enrichment tasks include adding metadata to items, such as the hash of the item, a screenshot of the webpage when the item was extracted, or general metadata like the date and time the item was extracted.
+
+
+```{include} autogen/enricher.md
+```
+
+```{toctree}
+:depth: 1
+:hidden:
+:glob:
+autogen/enricher/*
+```

docs/source/modules/extractor.md

@@ -0,0 +1,18 @@
+# Extractor Modules
+
+Extractor modules are used to extract the content of a given URL. Typically, one extractor will work for one website or platform (e.g. a Telegram extractor or an Instagram), however, there are several wide-ranging extractors which work for a wide range of websites.
+
+Extractors that are able to extract content from a wide range of websites include:
+1. Generic Extractor: parses videos and images on sites using the powerful yt-dlp library.
+2. Wayback Machine Extractor: sends pages to the Waygback machine for archiving, and stores the link.
+3. WACZ Extractor: runs a web browser to 'browse' the URL and save a copy of the page in WACZ format. 
+
+```{include} autogen/extractor.md
+```
+
+```{toctree}
+:depth: 1
+:hidden:
+:glob:
+autogen/extractor/*
+```

docs/source/modules/feeder.md

@@ -0,0 +1,20 @@
+# Feeder Modules
+
+Feeder modules are used to feed URLs into the `auto-archiver` for processing. Feeders can take these URLs from a variety of sources, such as a file, a database, or the command line.
+
+The default feeder is the command line feeder (`cli_feeder`), which allows you to input URLs directly into the `auto-archiver` from the command line.
+
+Command line feeder usage:
+```{code} bash
+auto-archiver [options] -- URL1 URL2 ...
+```
+
+```{include} autogen/feeder.md
+```
+
+```{toctree}
+:depth: 1
+:glob:
+:hidden:
+autogen/feeder/*
+```

docs/source/modules/formatter.md

@@ -0,0 +1,13 @@
+# Formatter Modules
+
+Formatter modules are used to format the data extracted from a URL into a specific format. Currently the most widely-used formatter is the HTML formatter, which formats the data into an easily viewable HTML page.
+
+```{include} autogen/formatter.md
+```
+
+```{toctree}
+:depth: 1
+:hidden:
+:glob:
+autogen/formatter/*
+```

docs/source/modules/storage.md

@@ -0,0 +1,15 @@
+# Storage Modules
+
+Storage modules are used to store the data extracted from a URL in a persistent location. This can be on your local hard disk, or on a remote server (e.g. S3 or Google Drive).
+
+The default is to store the files downloaded (e.g. images, videos) in a local directory.
+
+```{include} autogen/storage.md
+```
+
+```{toctree}
+:depth: 1
+:hidden:
+:glob:
+autogen/storage/*
+```

docs/source/overview.md

@@ -0,0 +1,16 @@
+
+```{include} ../../README.md
+```
+
+```{toctree}
+:maxdepth: 2
+:hidden:
+:caption: Contents:
+
+Overview <self>
+installation/installation.rst
+core_modules.md
+how_to
+development/developer_guidelines
+autoapi/index.rst
+```

example.orchestration.yaml

@@ -1,125 +0,0 @@
-steps:
-  # only 1 feeder allowed
-  feeder: gsheet_feeder # defaults to cli_feeder
-  archivers: # order matters, uncomment to activate
-    # - vk_archiver
-    # - telethon_archiver
-    # - telegram_archiver
-    # - twitter_archiver
-    # - twitter_api_archiver
-    # - instagram_tbot_archiver
-    # - instagram_archiver
-    # - tiktok_archiver
-    - youtubedl_archiver
-    # - wayback_archiver_enricher
-    # - wacz_archiver_enricher
-  enrichers:
-    - hash_enricher
-    # - metadata_enricher
-    # - screenshot_enricher
-    # - thumbnail_enricher
-    # - wayback_archiver_enricher
-    # - wacz_archiver_enricher
-    # - pdq_hash_enricher # if you want to calculate hashes for thumbnails, include this after thumbnail_enricher
-  formatter: html_formatter # defaults to mute_formatter
-  storages:
-    - local_storage
-    # - s3_storage
-    # - gdrive_storage
-  databases:
-    - console_db
-    # - csv_db
-    # - gsheet_db
-    # - mongo_db
-
-configurations:
-  gsheet_feeder:
-    sheet: "your sheet name"
-    header: 1
-    service_account: "secrets/service_account.json"
-    # allow_worksheets: "only parse this worksheet"
-    # block_worksheets: "blocked sheet 1,blocked sheet 2"
-    use_sheet_names_in_stored_paths: false
-    columns:
-      url: link
-      status: archive status
-      folder: destination folder
-      archive: archive location
-      date: archive date
-      thumbnail: thumbnail
-      timestamp: upload timestamp
-      title: upload title
-      text: textual content
-      screenshot: screenshot
-      hash: hash
-      pdq_hash: perceptual hashes
-      wacz: wacz
-      replaywebpage: replaywebpage
-  instagram_tbot_archiver:
-    api_id: "TELEGRAM_BOT_API_ID"
-    api_hash: "TELEGRAM_BOT_API_HASH"
-    # session_file: "secrets/anon"
-  telethon_archiver:
-    api_id: "TELEGRAM_BOT_API_ID"
-    api_hash: "TELEGRAM_BOT_API_HASH"
-    # session_file: "secrets/anon"
-    join_channels: false
-    channel_invites: # if you want to archive from private channels
-      - invite: https://t.me/+123456789
-        id: 0000000001
-      - invite: https://t.me/+123456788
-        id: 0000000002
-
-  twitter_api_archiver:
-    # either bearer_token only
-    bearer_token: "TWITTER_BEARER_TOKEN"
-    # OR all of the below
-    # consumer_key: ""
-    # consumer_secret: ""
-    # access_token: ""
-    # access_secret: ""
-  instagram_archiver:
-    username: "INSTAGRAM_USERNAME"
-    password: "INSTAGRAM_PASSWORD"
-    # session_file: "secrets/instaloader.session"
-
-  vk_archiver:
-    username: "or phone number"
-    password: "vk pass"
-    session_file: "secrets/vk_config.v2.json"
-
-  screenshot_enricher:
-    width: 1280
-    height: 2300
-  wayback_archiver_enricher:
-    timeout: 10
-    key: "wayback key"
-    secret: "wayback secret"
-  hash_enricher:
-    algorithm: "SHA3-512" # can also be SHA-256
-  wacz_archiver_enricher:
-    profile: secrets/profile.tar.gz
-  local_storage:
-    save_to: "./local_archive"
-    save_absolute: true
-    filename_generator: static
-    path_generator: flat
-  s3_storage:
-    bucket: your-bucket-name
-    region: reg1
-    key: S3_KEY
-    secret: S3_SECRET
-    endpoint_url: "https://{region}.digitaloceanspaces.com"
-    cdn_url: "https://{bucket}.{region}.cdn.digitaloceanspaces.com/{key}"
-    # if private:true S3 urls will not be readable online
-    private: false
-    # with 'random' you can generate a random UUID for the URL instead of a predictable path, useful to still have public but unlisted files, alternative is 'default' or not omitted from config
-    key_path: random
-  gdrive_storage:
-    path_generator: url
-    filename_generator: random
-    root_folder_id: folder_id_from_url
-    oauth_token: secrets/gd-token.json # needs to be generated with scripts/create_update_gdrive_oauth_token.py
-    service_account: "secrets/service_account.json"
-  csv_db:
-    csv_file: "./local_archive/db.csv"

pyproject.toml

@@ -1,4 +1,91 @@
 [build-system]
-requires = ["setuptools", "wheel", "setuptools-pipfile"]
-build-backend = "setuptools.build_meta"
-[tool.setuptools-pipfile]
+requires = ["poetry-core>=2.0.0,<3.0.0"]
+build-backend = "poetry.core.masonry.api"
+
+[project]
+name = "auto-archiver"
+version = "0.13.2"
+description = "Automatically archive links to videos, images, and social media content from Google Sheets (and more)."
+
+requires-python = ">=3.10,<3.13"
+license = "MIT"
+authors = [
+    { name = "Bellingcat", email = "tech@bellingcat.com" },
+]
+readme = "README.md"
+keywords = ["archive", "oosi", "osint", "scraping"]
+classifiers = [
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3"
+]
+
+dependencies = [
+    "gspread (>=0.0.0)",
+    "beautifulsoup4 (>=0.0.0)",
+    "bs4 (>=0.0.0)",
+    "loguru (>=0.0.0)",
+    "ffmpeg-python (>=0.0.0)",
+    "selenium (>=0.0.0)",
+    "telethon (>=0.0.0)",
+    "google-api-python-client (>=0.0.0)",
+    "google-auth-httplib2 (>=0.0.0)",
+    "google-auth-oauthlib (>=0.0.0)",
+    "oauth2client (>=0.0.0)",
+    "pdqhash (>=0.0.0)",
+    "pillow (>=0.0.0)",
+    "python-slugify (>=0.0.0)",
+    "dateparser (>=0.0.0)",
+    "python-twitter-v2 (>=0.0.0)",
+    "instaloader (>=0.0.0)",
+    "tqdm (>=0.0.0)",
+    "jinja2 (>=0.0.0)",
+    "pyOpenSSL (==24.2.1)",
+    "cryptography (>=41.0.0,<42.0.0)",
+    "boto3 (>=1.28.0,<2.0.0)",
+    "dataclasses-json (>=0.0.0)",
+    "yt-dlp (>=2025.1.26,<2026.0.0)",
+    "numpy (==2.1.3)",
+    "vk-url-scraper (>=0.0.0)",
+    "requests[socks] (>=0.0.0)",
+    "warcio (>=0.0.0)",
+    "jsonlines (>=0.0.0)",
+    "pysubs2 (>=0.0.0)",
+    "retrying (>=0.0.0)",
+    "tsp-client (>=0.0.0)",
+    "certvalidator (>=0.0.0)",
+    "rich-argparse (>=1.6.0,<2.0.0)",
+    "ruamel-yaml (>=0.18.10,<0.19.0)",
+]
+
+[tool.poetry.group.dev.dependencies]
+pytest = "^8.3.4"
+autopep8 = "^2.3.1"
+pytest-loguru = "^0.4.0"
+
+[tool.poetry.group.docs.dependencies]
+sphinx = "^8.1.3"
+sphinx-autoapi = "^3.4.0"
+sphinxcontrib-mermaid = "^1.0.0"
+sphinx-autobuild = "^2024.10.3"
+sphinx-copybutton = "^0.5.2"
+myst-parser = "^4.0.0"
+sphinx-book-theme = "^1.1.3"
+linkify-it-py = "^2.0.3"
+
+
+[project.scripts]
+auto-archiver = "auto_archiver.__main__:main"
+
+[project.urls]
+homepage = "https://github.com/bellingcat/auto-archiver"
+repository = "https://github.com/bellingcat/auto-archiver"
+documentation = "https://github.com/bellingcat/auto-archiver"
+
+
+[tool.pytest.ini_options]
+markers = [
+    "download: marks tests that download content from the network",
+    "incremental: marks a class to run tests incrementally. If a test fails in the class, the remaining tests will be skipped",
+]

scripts/create_update_gdrive_oauth_token.py

@@ -12,7 +12,7 @@ from googleapiclient.errors import HttpError
 # Code below from https://developers.google.com/drive/api/quickstart/python
 # Example invocation: py scripts/create_update_gdrive_oauth_token.py -c secrets/credentials.json -t secrets/gd-token.json
 
-SCOPES = ['https://www.googleapis.com/auth/drive']
+SCOPES = ["https://www.googleapis.com/auth/drive.file"]
 
 
 @click.command(
@@ -23,7 +23,7 @@ SCOPES = ['https://www.googleapis.com/auth/drive']
     "-c",
     type=click.Path(exists=True),
     help="path to the credentials.json file downloaded from https://console.cloud.google.com/apis/credentials",
-    required=True
+    required=True,
 )
 @click.option(
     "--token",
@@ -31,59 +31,62 @@ SCOPES = ['https://www.googleapis.com/auth/drive']
     type=click.Path(exists=False),
     default="gd-token.json",
     help="file where to place the OAuth token, defaults to gd-token.json which you must then move to where your orchestration file points to, defaults to gd-token.json",
-    required=True
+    required=True,
 )
 def main(credentials, token):
     # The file token.json stores the user's access and refresh tokens, and is
     # created automatically when the authorization flow completes for the first time.
     creds = None
     if os.path.exists(token):
-        with open(token, 'r') as stream:
+        with open(token, "r") as stream:
             creds_json = json.load(stream)
             # creds = Credentials.from_authorized_user_file(creds_json, SCOPES)
-            creds_json['refresh_token'] = creds_json.get("refresh_token", "")
+            creds_json["refresh_token"] = creds_json.get("refresh_token", "")
             creds = Credentials.from_authorized_user_info(creds_json, SCOPES)
 
     # If there are no (valid) credentials available, let the user log in.
     if not creds or not creds.valid:
         if creds and creds.expired and creds.refresh_token:
-            print('Requesting new token')
+            print("Requesting new token")
             creds.refresh(Request())
         else:
-            print('First run through so putting up login dialog')
+            print("First run through so putting up login dialog")
             # credentials.json downloaded from https://console.cloud.google.com/apis/credentials
             flow = InstalledAppFlow.from_client_secrets_file(credentials, SCOPES)
             creds = flow.run_local_server(port=55192)
         # Save the credentials for the next run
-        with open(token, 'w') as token:
-            print('Saving new token')
+        with open(token, "w") as token:
+            print("Saving new token")
             token.write(creds.to_json())
     else:
-        print('Token valid')
+        print("Token valid")
 
     try:
-        service = build('drive', 'v3', credentials=creds)
+        service = build("drive", "v3", credentials=creds)
 
         # About the user
         results = service.about().get(fields="*").execute()
-        emailAddress = results['user']['emailAddress']
+        emailAddress = results["user"]["emailAddress"]
         print(emailAddress)
 
         # Call the Drive v3 API and return some files
-        results = service.files().list(
-            pageSize=10, fields="nextPageToken, files(id, name)").execute()
-        items = results.get('files', [])
+        results = (
+            service.files()
+            .list(pageSize=10, fields="nextPageToken, files(id, name)")
+            .execute()
+        )
+        items = results.get("files", [])
 
         if not items:
-            print('No files found.')
+            print("No files found.")
             return
-        print('Files:')
+        print("Files:")
         for item in items:
-            print(u'{0} ({1})'.format(item['name'], item['id']))
+            print("{0} ({1})".format(item["name"], item["id"]))
 
     except HttpError as error:
-        print(f'An error occurred: {error}')
+        print(f"An error occurred: {error}")
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()

scripts/release.sh

@@ -1,19 +0,0 @@
-
-#!/bin/bash
-
-set -e
-
-TAG=$(python -c 'from src.auto_archiver.version import __version__; print("v" + __version__)')
-
-read -p "Creating new release for $TAG. Do you want to continue? [Y/n] " prompt
-
-if [[ $prompt == "y" || $prompt == "Y" || $prompt == "yes" || $prompt == "Yes" ]]; then
-    # git add -A
-    # git commit -m "Bump version to $TAG for release" || true && git push
-    echo "Creating new git tag $TAG"
-    git tag "$TAG" -m "$TAG"
-    git push --tags
-else
-    echo "Cancelled"
-    exit 1
-fi

scripts/telegram_setup.py

@@ -0,0 +1,29 @@
+"""
+This script is used to create a new session file for the Telegram client.
+To do this you must first create a Telegram application at https://my.telegram.org/apps
+And store your id and hash in the environment variables TELEGRAM_API_ID and TELEGRAM_API_HASH.
+Create a .env file, or add the following to your environment :
+```
+export TELEGRAM_API_ID=[YOUR_ID_HERE]
+export TELEGRAM_API_HASH=[YOUR_HASH_HERE]
+```
+Then run this script to create a new session file.
+
+You will need to provide your phone number and a 2FA code the first time you run this script.
+"""
+
+
+import os
+from telethon.sync import TelegramClient
+from loguru import logger
+
+
+# Create a
+API_ID = os.getenv("TELEGRAM_API_ID")
+API_HASH = os.getenv("TELEGRAM_API_HASH")
+SESSION_FILE = "secrets/anon-insta"
+
+os.makedirs("secrets", exist_ok=True)
+with TelegramClient(SESSION_FILE, API_ID, API_HASH) as client:
+    logger.success(f"New session file created: {SESSION_FILE}.session")
+

setup.cfg

@@ -1,53 +0,0 @@
-[metadata]
-name = auto_archiver
-version = attr: auto_archiver.version.__version__
-author = Bellingcat
-author_email = tech@bellingcat.com
-description = Easily archive online media content
-long_description = file: README.md
-long_description_content_type = text/markdown
-keywords = archive, oosi, osint, scraping
-license = MIT
-classifiers =
-	Intended Audience :: Developers
-	Intended Audience :: Science/Research
-	License :: OSI Approved :: MIT License
-	Programming Language :: Python :: 3
-project_urls = 
-	Source Code = https://github.com/bellingcat/auto-archiver
-	Bug Tracker = https://github.com/bellingcat/auto-archiver/issues
-	Bellingcat = https://www.bellingcat.com
-platforms = any
-
-[options]
-setup_requires =
-    setuptools-pipfile
-zip_safe = False
-package_dir=
-    =src
-packages=find:
-find_packages=true
-python_requires = >=3.8
-
-[options.package_data]
-* = *.html
-
-[options.entry_points]
-console_scripts =
-    auto-archiver = auto_archiver.__main__:main
-
-# [options.extras_require]
-# pdf = ReportLab>=1.2; RXP
-# rest = docutils>=0.3; pack ==1.1, ==1.3
-
-[options.packages.find]
-where=src
-# include=auto_archiver*
-# exclude =
-#     examples*
-#     .eggs*
-#     build*
-#     secrets*
-#     tmp*
-#     docs*
-#     src.tests*

setup.py

@@ -1,4 +0,0 @@
-from setuptools import setup
-
-if __name__ == "__main__":
-    setup()

src/auto_archiver/__init__.py

@@ -1,7 +0,0 @@
-from . import archivers, databases, enrichers, feeders, formatters, storages, utils, core
-
-# need to manually specify due to cyclical deps
-from .core.orchestrator import ArchivingOrchestrator
-from .core.config import Config
-# making accessible directly
-from .core.metadata import Metadata

src/auto_archiver/__main__.py

@@ -1,12 +1,9 @@
-from . import Config
-from . import ArchivingOrchestrator
+""" Entry point for the auto_archiver package. """
+from auto_archiver.core.orchestrator import ArchivingOrchestrator
+import sys
 
 def main():
-    config = Config()
-    config.parse()
-    orchestrator = ArchivingOrchestrator(config)
-    for r in orchestrator.feed(): pass
-
+    ArchivingOrchestrator().run(sys.argv[1:])
 
 if __name__ == "__main__":
     main()

src/auto_archiver/archivers/__init__.py

@@ -1,10 +0,0 @@
-from .archiver import Archiver
-from .telethon_archiver import TelethonArchiver
-from .twitter_archiver import TwitterArchiver
-from .twitter_api_archiver import TwitterApiArchiver
-from .instagram_archiver import InstagramArchiver
-from .instagram_tbot_archiver import InstagramTbotArchiver
-from .tiktok_archiver import TiktokArchiver
-from .telegram_archiver import TelegramArchiver
-from .vk_archiver import VkArchiver
-from .youtubedl_archiver import YoutubeDLArchiver

src/auto_archiver/archivers/archiver.py

@@ -1,60 +0,0 @@
-from __future__ import annotations
-from abc import abstractmethod
-from dataclasses import dataclass
-import os
-import mimetypes, requests
-
-from ..core import Metadata, Step, ArchivingContext
-
-
-@dataclass
-class Archiver(Step):
-    name = "archiver"
-
-    def __init__(self, config: dict) -> None:
-        # without this STEP.__init__ is not called
-        super().__init__(config)
-
-    def init(name: str, config: dict) -> Archiver:
-        # only for typing...
-        return Step.init(name, config, Archiver)
-
-    def setup(self) -> None:
-        # used when archivers need to login or do other one-time setup
-        pass
-
-    def sanitize_url(self, url: str) -> str:
-        # used to clean unnecessary URL parameters OR unfurl redirect links
-        return url
-
-    def _guess_file_type(self, path: str) -> str:
-        """
-        Receives a URL or filename and returns global mimetype like 'image' or 'video'
-        see https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types
-        """
-        mime = mimetypes.guess_type(path)[0]
-        if mime is not None:
-            return mime.split("/")[0]
-        return ""
-
-    def download_from_url(self, url: str, to_filename: str = None, item: Metadata = None) -> str:
-        """
-        downloads a URL to provided filename, or inferred from URL, returns local filename, if item is present will use its tmp_dir
-        """
-        if not to_filename:
-            to_filename = url.split('/')[-1].split('?')[0]
-            if len(to_filename) > 64:
-                to_filename = to_filename[-64:]
-        if item:
-            to_filename = os.path.join(ArchivingContext.get_tmp_dir(), to_filename)
-        headers = {
-            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36'
-        }
-        d = requests.get(url, headers=headers)
-        assert d.status_code == 200, f"got response code {d.status_code} for {url=}"
-        with open(to_filename, 'wb') as f:
-            f.write(d.content)
-        return to_filename
-
-    @abstractmethod
-    def download(self, item: Metadata) -> Metadata: pass

src/auto_archiver/archivers/instagram_tbot_archiver.py

@@ -1,77 +0,0 @@
-
-from telethon.sync import TelegramClient
-from loguru import logger
-import time, os
-from sqlite3 import OperationalError
-from . import Archiver
-from ..core import Metadata, Media, ArchivingContext
-
-
-class InstagramTbotArchiver(Archiver):
-    """
-    calls a telegram bot to fetch instagram posts/stories... and gets available media from it
-    https://github.com/adw0rd/instagrapi
-    https://t.me/instagram_load_bot
-    """
-    name = "instagram_tbot_archiver"
-
-    def __init__(self, config: dict) -> None:
-        super().__init__(config)
-        self.assert_valid_string("api_id")
-        self.assert_valid_string("api_hash")
-        self.timeout = int(self.timeout)
-        try:
-            self.client = TelegramClient(self.session_file, self.api_id, self.api_hash)
-        except OperationalError as e:
-            logger.error(f"Unable to access the {self.session_file} session, please make sure you don't use the same session file here and in telethon_archiver. if you do then disable at least one of the archivers for the 1st time you setup telethon session: {e}")
-
-    @staticmethod
-    def configs() -> dict:
-        return {
-            "api_id": {"default": None, "help": "telegram API_ID value, go to https://my.telegram.org/apps"},
-            "api_hash": {"default": None, "help": "telegram API_HASH value, go to https://my.telegram.org/apps"},
-            "session_file": {"default": "secrets/anon-insta", "help": "optional, records the telegram login session for future usage, '.session' will be appended to the provided value."},
-            "timeout": {"default": 45, "help": "timeout to fetch the instagram content in seconds."},
-        }
-
-    def setup(self) -> None:
-        logger.info(f"SETUP {self.name} checking login...")
-        with self.client.start():
-            logger.success(f"SETUP {self.name} login works.")
-
-    def download(self, item: Metadata) -> Metadata:
-        url = item.get_url()
-        if not "instagram.com" in url: return False
-
-        result = Metadata()
-        tmp_dir = ArchivingContext.get_tmp_dir()
-        with self.client.start():
-            chat = self.client.get_entity("instagram_load_bot")
-            since_id = self.client.send_message(entity=chat, message=url).id
-
-            attempts = 0
-            seen_media = []
-            message = ""
-            time.sleep(3)
-            # media is added before text by the bot so it can be used as a stop-logic mechanism
-            while attempts < (self.timeout - 3) and (not message or not len(seen_media)):
-                attempts += 1
-                time.sleep(1)
-                for post in self.client.iter_messages(chat, min_id=since_id):
-                    since_id = max(since_id, post.id)
-                    if post.media and post.id not in seen_media:
-                        filename_dest = os.path.join(tmp_dir, f'{chat.id}_{post.id}')
-                        media = self.client.download_media(post.media, filename_dest)
-                        if media: 
-                            result.add_media(Media(media))
-                            seen_media.append(post.id)
-                    if post.message: message += post.message
-
-            if "You must enter a URL to a post" in message: 
-                logger.debug(f"invalid link {url=} for {self.name}: {message}")
-                return False
-                
-            if message:
-                result.set_content(message).set_title(message[:128])
-
-            return result.success("insta-via-bot")

src/auto_archiver/archivers/tiktok_archiver.py

@@ -1,56 +0,0 @@
-import json, os, traceback
-import tiktok_downloader
-from loguru import logger
-
-
-from . import Archiver
-from ..core import Metadata, Media, ArchivingContext
-from ..utils.misc import random_str
-
-
-class TiktokArchiver(Archiver):
-    name = "tiktok_archiver"
-
-    def __init__(self, config: dict) -> None:
-        super().__init__(config)
-
-    @staticmethod
-    def configs() -> dict:
-        return {}
-
-    def download(self, item: Metadata) -> Metadata:
-        url = item.get_url()
-        if 'tiktok.com' not in url:
-            return False
-
-        result = Metadata()
-        try:
-            info = tiktok_downloader.info_post(url)
-            result.set_title(info.desc)
-            result.set_timestamp(info.create_time)
-            result.set_content(json.dumps({
-                "cover": info.cover,
-                "author": info.author,
-                "music_title": info.author,
-                "caption": getattr(info, "caption", info.desc),
-            }, ensure_ascii=False, indent=4))
-        except:
-            error = traceback.format_exc()
-            logger.warning(f'Other Tiktok error {error}')
-
-        try:
-            filename = os.path.join(ArchivingContext.get_tmp_dir(), f'{random_str(8)}.mp4')
-            tiktok_media = tiktok_downloader.snaptik(url).get_media()
-
-            if len(tiktok_media) <= 0:
-                logger.debug(f"TikTok: could not get media from {url=}")
-                return False
-
-            logger.info(f'downloading video {filename=}')
-            tiktok_media[0].download(filename)
-
-            result.add_media(Media(filename))
-            return result.success("tiktok")
-        except:
-            error = traceback.format_exc()
-            logger.warning(f'Other Tiktok error {error}')

src/auto_archiver/archivers/twitter_api_archiver.py

@@ -1,98 +0,0 @@
-
-import json, mimetypes
-from datetime import datetime
-from loguru import logger
-from pytwitter import Api
-from slugify import slugify
-
-from . import Archiver
-from .twitter_archiver import TwitterArchiver
-from ..core import Metadata,Media
-
-
-class TwitterApiArchiver(TwitterArchiver, Archiver):
-    name = "twitter_api_archiver"
-
-    def __init__(self, config: dict) -> None:
-        super().__init__(config)
-
-        if self.bearer_token:
-            self.assert_valid_string("bearer_token")
-            self.api = Api(bearer_token=self.bearer_token)
-        elif self.consumer_key and self.consumer_secret and self.access_token and self.access_secret:
-            self.assert_valid_string("consumer_key")
-            self.assert_valid_string("consumer_secret")
-            self.assert_valid_string("access_token")
-            self.assert_valid_string("access_secret")
-            self.api = Api(
-                consumer_key=self.consumer_key, consumer_secret=self.consumer_secret, access_token=self.access_token, access_secret=self.access_secret)
-        assert hasattr(self, "api") and self.api is not None, "Missing Twitter API configurations, please provide either bearer_token OR (consumer_key, consumer_secret, access_token, access_secret) to use this archiver."
-
-    @staticmethod
-    def configs() -> dict:
-        return {
-            "bearer_token": {"default": None, "help": "twitter API bearer_token which is enough for archiving, if not provided you will need consumer_key, consumer_secret, access_token, access_secret"},
-            "consumer_key": {"default": None, "help": "twitter API consumer_key"},
-            "consumer_secret": {"default": None, "help": "twitter API consumer_secret"},
-            "access_token": {"default": None, "help": "twitter API access_token"},
-            "access_secret": {"default": None, "help": "twitter API access_secret"},
-        }
-
-    def download(self, item: Metadata) -> Metadata:
-        url = item.get_url()
-        # detect URLs that we definitely cannot handle
-        username, tweet_id = self.get_username_tweet_id(url)
-        if not username: return False
-
-        try:
-            tweet = self.api.get_tweet(tweet_id, expansions=["attachments.media_keys"], media_fields=["type", "duration_ms", "url", "variants"], tweet_fields=["attachments", "author_id", "created_at", "entities", "id", "text", "possibly_sensitive"])
-        except Exception as e:
-            logger.error(f"Could not get tweet: {e}")
-            return False
-
-        result = Metadata()
-        result.set_title(tweet.data.text)
-        result.set_timestamp(datetime.strptime(tweet.data.created_at, "%Y-%m-%dT%H:%M:%S.%fZ"))
-
-        urls = []
-        if tweet.includes:
-            for i, m in enumerate(tweet.includes.media):
-                media = Media(filename="")
-                if m.url and len(m.url):
-                    media.set("src", m.url)
-                    media.set("duration", (m.duration_ms or 1) // 1000)
-                    mimetype = "image/jpeg"
-                elif hasattr(m, "variants"):
-                    variant = self.choose_variant(m.variants)
-                    if not variant: continue
-                    media.set("src", variant.url)
-                    mimetype = variant.content_type
-                else:
-                    continue
-                logger.info(f"Found media {media}")
-                ext = mimetypes.guess_extension(mimetype)
-                media.filename = self.download_from_url(media.get("src"), f'{slugify(url)}_{i}{ext}', item)
-                result.add_media(media)
-
-        result.set_content(json.dumps({
-            "id": tweet.data.id,
-            "text": tweet.data.text,
-            "created_at": tweet.data.created_at,
-            "author_id": tweet.data.author_id,
-            "geo": tweet.data.geo,
-            "lang": tweet.data.lang,
-            "media": urls
-        }, ensure_ascii=False, indent=4))
-        return result.success("twitter")
-
-    def choose_variant(self, variants):
-        # choosing the highest quality possible
-        variant, bit_rate = None, -1
-        for var in variants:
-            if var.content_type == "video/mp4":
-                if var.bit_rate > bit_rate:
-                    bit_rate = var.bit_rate
-                    variant = var
-            else:
-                variant = var if not variant else variant
-        return variant

src/auto_archiver/archivers/twitter_archiver.py

@@ -1,152 +0,0 @@
-import re, requests, mimetypes, json
-from datetime import datetime
-from loguru import logger
-from snscrape.modules.twitter import TwitterTweetScraper, Video, Gif, Photo
-from slugify import slugify
-
-from . import Archiver
-from ..core import Metadata, Media
-from ..utils import UrlUtil
-
-
-class TwitterArchiver(Archiver):
-    """
-    This Twitter Archiver uses unofficial scraping methods.
-    """
-
-    name = "twitter_archiver"
-    link_pattern = re.compile(r"(?:twitter|x).com\/(?:\#!\/)?(\w+)\/status(?:es)?\/(\d+)")
-    link_clean_pattern = re.compile(r"(.+(?:twitter|x)\.com\/.+\/\d+)(\?)*.*")
-
-    def __init__(self, config: dict) -> None:
-        super().__init__(config)
-
-    @staticmethod
-    def configs() -> dict:
-        return {}
-
-    def sanitize_url(self, url: str) -> str:
-        # expand URL if t.co and clean tracker GET params
-        if 'https://t.co/' in url:
-            try:
-                r = requests.get(url)
-                logger.debug(f'Expanded url {url} to {r.url}')
-                url = r.url
-            except:
-                logger.error(f'Failed to expand url {url}')
-        # https://twitter.com/MeCookieMonster/status/1617921633456640001?s=20&t=3d0g4ZQis7dCbSDg-mE7-w
-        return self.link_clean_pattern.sub("\\1", url)
-
-    def download(self, item: Metadata) -> Metadata:
-        """
-        if this url is archivable will download post info and look for other posts from the same group with media.
-        can handle private/public channels
-        """
-        url = item.get_url()
-        # detect URLs that we definitely cannot handle
-        username, tweet_id = self.get_username_tweet_id(url)
-        if not username: return False
-
-        result = Metadata()
-
-        scr = TwitterTweetScraper(tweet_id)
-        try:
-            tweet = next(scr.get_items())
-        except Exception as ex:
-            logger.warning(f"can't get tweet: {type(ex).__name__} occurred. args: {ex.args}")
-            return self.download_alternative(item, url, tweet_id)
-
-        result.set_title(tweet.content).set_content(tweet.json()).set_timestamp(tweet.date)
-        if tweet.media is None:
-            logger.debug(f'No media found, archiving tweet text only')
-            return result
-
-        for i, tweet_media in enumerate(tweet.media):
-            media = Media(filename="")
-            mimetype = ""
-            if type(tweet_media) == Video:
-                variant = max(
-                    [v for v in tweet_media.variants if v.bitrate], key=lambda v: v.bitrate)
-                media.set("src", variant.url).set("duration", tweet_media.duration)
-                mimetype = variant.contentType
-            elif type(tweet_media) == Gif:
-                variant = tweet_media.variants[0]
-                media.set("src", variant.url)
-                mimetype = variant.contentType
-            elif type(tweet_media) == Photo:
-                media.set("src", UrlUtil.twitter_best_quality_url(tweet_media.fullUrl))
-                mimetype = "image/jpeg"
-            else:
-                logger.warning(f"Could not get media URL of {tweet_media}")
-                continue
-            ext = mimetypes.guess_extension(mimetype)
-            media.filename = self.download_from_url(media.get("src"), f'{slugify(url)}_{i}{ext}', item)
-            result.add_media(media)
-
-        return result.success("twitter-snscrape")
-
-    def download_alternative(self, item: Metadata, url: str, tweet_id: str) -> Metadata:
-        """
-        Hack alternative working again.
-        https://stackoverflow.com/a/71867055/6196010 (OUTDATED URL)
-        https://github.com/JustAnotherArchivist/snscrape/issues/996#issuecomment-1615937362
-        next to test: https://cdn.embedly.com/widgets/media.html?&schema=twitter&url=https://twitter.com/bellingcat/status/1674700676612386816
-        """
-
-        logger.debug(f"Trying twitter hack for {url=}")
-        result = Metadata()
-
-        hack_url = f"https://cdn.syndication.twimg.com/tweet-result?id={tweet_id}"
-        r = requests.get(hack_url)
-        if r.status_code != 200: return False
-        tweet = r.json()
-
-        urls = []
-        for p in tweet.get("photos", []):
-            urls.append(p["url"])
-
-        # 1 tweet has 1 video max
-        if "video" in tweet:
-            v = tweet["video"]
-            urls.append(self.choose_variant(v.get("variants", [])))
-
-        logger.debug(f"Twitter hack got {urls=}")
-
-        for i, u in enumerate(urls):
-            media = Media(filename="")
-            u = UrlUtil.twitter_best_quality_url(u)
-            media.set("src", u)
-            ext = ""
-            if (mtype := mimetypes.guess_type(UrlUtil.remove_get_parameters(u))[0]):
-                ext = mimetypes.guess_extension(mtype)
-
-            media.filename = self.download_from_url(u, f'{slugify(url)}_{i}{ext}', item)
-            result.add_media(media)
-
-        result.set_title(tweet.get("text")).set_content(json.dumps(tweet, ensure_ascii=False)).set_timestamp(datetime.strptime(tweet["created_at"], "%Y-%m-%dT%H:%M:%S.%fZ"))
-        return result.success("twitter-hack")
-
-    def get_username_tweet_id(self, url):
-        # detect URLs that we definitely cannot handle
-        matches = self.link_pattern.findall(url)
-        if not len(matches): return False, False
-
-        username, tweet_id = matches[0]  # only one URL supported
-        logger.debug(f"Found {username=} and {tweet_id=} in {url=}")
-
-        return username, tweet_id
-
-    def choose_variant(self, variants):
-        # choosing the highest quality possible
-        variant, width, height = None, 0, 0
-        for var in variants:
-            if var.get("type", "") == "video/mp4":
-                width_height = re.search(r"\/(\d+)x(\d+)\/", var["src"])
-                if width_height:
-                    w, h = int(width_height[1]), int(width_height[2])
-                    if w > width or h > height:
-                        width, height = w, h
-                        variant = var.get("src", variant)
-            else:
-                variant = var.get("src") if not variant else variant
-        return variant

src/auto_archiver/archivers/youtubedl_archiver.py

@@ -1,67 +0,0 @@
-import datetime, os, yt_dlp
-from loguru import logger
-
-from . import Archiver
-from ..core import Metadata, Media, ArchivingContext
-
-
-class YoutubeDLArchiver(Archiver):
-    name = "youtubedl_archiver"
-
-    def __init__(self, config: dict) -> None:
-        super().__init__(config)
-
-    @staticmethod
-    def configs() -> dict:
-        return {
-            "facebook_cookie": {"default": None, "help": "optional facebook cookie to have more access to content, from browser, looks like 'cookie: datr= xxxx'"},
-        }
-
-    def download(self, item: Metadata) -> Metadata:
-        #TODO: yt-dlp for transcripts?
-        url = item.get_url()
-
-        if item.netloc in ['facebook.com', 'www.facebook.com'] and self.facebook_cookie:
-            logger.debug('Using Facebook cookie')
-            yt_dlp.utils.std_headers['cookie'] = self.facebook_cookie
-
-        ydl = yt_dlp.YoutubeDL({'outtmpl': os.path.join(ArchivingContext.get_tmp_dir(), f'%(id)s.%(ext)s'), 'quiet': False, 'noplaylist': True})
-
-        try:
-            # don'd download since it can be a live stream
-            info = ydl.extract_info(url, download=False)
-            if info.get('is_live', False):
-                logger.warning("Live streaming media, not archiving now")
-                return False
-        except yt_dlp.utils.DownloadError as e:
-            logger.debug(f'No video - Youtube normal control flow: {e}')
-            return False
-        except Exception as e:
-            logger.debug(f'ytdlp exception which is normal for example a facebook page with images only will cause a IndexError: list index out of range. Exception here is: \n  {e}')
-            return False
-
-        # this time download
-        info = ydl.extract_info(url, download=True)
-        if "entries" in info:
-            entries = info.get("entries", [])
-            if not len(entries):
-                logger.warning('YoutubeDLArchiver could not find any video')
-                return False
-        else: entries = [info]
-
-        result = Metadata()
-        result.set_title(info.get("title"))
-        for entry in entries:
-            filename = ydl.prepare_filename(entry)
-            if not os.path.exists(filename):
-                filename = filename.split('.')[0] + '.mkv'
-            result.add_media(Media(filename).set("duration", info.get("duration")))
-
-        if (timestamp := info.get("timestamp")):
-            timestamp = datetime.datetime.utcfromtimestamp(timestamp).replace(tzinfo=datetime.timezone.utc).isoformat()
-            result.set_timestamp(timestamp)
-        if (upload_date := info.get("upload_date")):
-            upload_date = datetime.datetime.strptime(upload_date, '%Y%m%d').replace(tzinfo=datetime.timezone.utc)
-            result.set("upload_date", upload_date)
-
-        return result.success("yt-dlp")

src/auto_archiver/core/__init__.py

@@ -1,8 +1,17 @@
+""" Core modules to handle things such as orchestration, metadata and configs..
+
+"""
 from .metadata import Metadata
 from .media import Media
-from .step import Step
-from .context import ArchivingContext
+from .module import BaseModule
 
 # cannot import ArchivingOrchestrator/Config to avoid circular dep
 # from .orchestrator import ArchivingOrchestrator
-# from .config import Config
+# from .config import Config
+
+from .database import Database
+from .enricher import Enricher
+from .feeder import Feeder
+from .storage import Storage
+from .extractor import Extractor
+from .formatter import Formatter

src/auto_archiver/core/base_module.py

@@ -0,0 +1,146 @@
+
+from urllib.parse import urlparse
+from typing import  Mapping, Any
+from abc import ABC
+from copy import deepcopy, copy
+from tempfile import TemporaryDirectory
+from auto_archiver.utils import url as UrlUtil
+
+from loguru import logger
+
+class BaseModule(ABC):
+
+    """
+    Base module class. All modules should inherit from this class.
+
+    The exact methods a class implements will depend on the type of module it is,
+    however modules can have a .setup() method to run any setup code
+    (e.g. logging in to a site, spinning up a browser etc.)
+
+    See BaseModule.MODULE_TYPES for the types of modules you can create, noting that
+    a subclass can be of multiple types. For example, a module that extracts data from
+    a website and stores it in a database would be both an 'extractor' and a 'database' module.
+
+    Each module is a python package, and should have a __manifest__.py file in the
+    same directory as the module file. The __manifest__.py specifies the module information
+    like name, author, version, dependencies etc. See BaseModule._DEFAULT_MANIFEST for the
+    default manifest structure.
+
+    """
+
+    MODULE_TYPES = [
+        'feeder',
+        'extractor',
+        'enricher',
+        'database',
+        'storage',
+        'formatter'
+    ]
+
+    _DEFAULT_MANIFEST = {
+    'name': '', # the display name of the module
+    'author': 'Bellingcat', # creator of the module, leave this as Bellingcat or set your own name!
+    'type': [], # the type of the module, can be one or more of BaseModule.MODULE_TYPES
+    'requires_setup': True, # whether or not this module requires additional setup such as setting API Keys or installing additional softare
+    'description': '', # a description of the module
+    'dependencies': {}, # external dependencies, e.g. python packages or binaries, in dictionary format
+    'entry_point': '', # the entry point for the module, in the format 'module_name::ClassName'. This can be left blank to use the default entry point of module_name::ModuleName
+    'version': '1.0', # the version of the module
+    'configs': {} # any configuration options this module has, these will be exposed to the user in the config file or via the command line
+}
+
+    config: Mapping[str, Any]
+    authentication: Mapping[str, Mapping[str, str]]
+    name: str
+
+    # this is set by the orchestrator prior to archiving
+    tmp_dir: TemporaryDirectory = None
+
+    @property
+    def storages(self) -> list:
+        return self.config.get('storages', [])
+
+    def config_setup(self, config: dict):
+
+        authentication = config.get('authentication', {})
+        # extract out concatenated sites
+        for key, val in copy(authentication).items():
+            if "," in key:
+                for site in key.split(","):
+                    authentication[site] = val
+                del authentication[key]
+
+        # this is important. Each instance is given its own deepcopied config, so modules cannot
+        # change values to affect other modules
+        config = deepcopy(config)
+        authentication = deepcopy(config.pop('authentication', {}))
+
+        self.authentication = authentication
+        self.config = config
+        for key, val in config.get(self.name, {}).items():
+            setattr(self, key, val)
+
+    def setup(self):
+        # For any additional setup required by modules, e.g. autehntication
+        pass
+
+    def auth_for_site(self, site: str, extract_cookies=True) -> Mapping[str, Any]:
+        """
+        Returns the authentication information for a given site. This is used to authenticate
+        with a site before extracting data. The site should be the domain of the site, e.g. 'twitter.com'
+        
+        extract_cookies: bool - whether or not to extract cookies from the given browser and return the 
+        cookie jar (disabling can speed up) processing if you don't actually need the cookies jar
+
+        Currently, the dict can have keys of the following types:
+        - username: str - the username to use for login
+        - password: str - the password to use for login
+        - api_key: str - the API key to use for login
+        - api_secret: str - the API secret to use for login
+        - cookie: str - a cookie string to use for login (specific to this site)
+        - cookies_jar: YoutubeDLCookieJar | http.cookiejar.MozillaCookieJar - a cookie jar compatible with requests (e.g. `requests.get(cookies=cookie_jar)`)
+        """
+        # TODO: think about if/how we can deal with sites that have multiple domains (main one is x.com/twitter.com)
+        # for now the user must enter them both, like "x.com,twitter.com" in their config. Maybe we just hard-code?
+
+        site = UrlUtil.domain_for_url(site)
+        # add the 'www' version of the site to the list of sites to check
+        authdict = {}
+
+
+        for to_try in [site, f"www.{site}"]:
+            if to_try in self.authentication:
+                authdict.update(self.authentication[to_try])
+                break
+
+        # do a fuzzy string match just to print a warning - don't use it since it's insecure
+        if not authdict:
+            for key in self.authentication.keys():
+                if key in site or site in key:
+                    logger.debug(f"Could not find exact authentication information for site '{site}'. \
+                                    did find information for '{key}' which is close, is this what you meant? \
+                                    If so, edit your authentication settings to make sure it exactly matches.")
+
+        def get_ytdlp_cookiejar(args):
+            import yt_dlp
+            from yt_dlp import parse_options
+            logger.debug(f"Extracting cookies from settings: {args[1]}")
+            # parse_options returns a named tuple as follows, we only need the ydl_options part
+            # collections.namedtuple('ParsedOptions', ('parser', 'options', 'urls', 'ydl_opts'))
+            ytdlp_opts = getattr(parse_options(args), 'ydl_opts')
+            return yt_dlp.YoutubeDL(ytdlp_opts).cookiejar
+
+        # get the cookies jar, prefer the browser cookies than the file
+        if 'cookies_from_browser' in self.authentication:
+            authdict['cookies_from_browser'] = self.authentication['cookies_from_browser']
+            if extract_cookies:
+                authdict['cookies_jar'] = get_ytdlp_cookiejar(['--cookies-from-browser', self.authentication['cookies_from_browser']])
+        elif 'cookies_file' in self.authentication:
+            authdict['cookies_file'] = self.authentication['cookies_file']
+            if extract_cookies:
+                authdict['cookies_jar'] = get_ytdlp_cookiejar(['--cookies', self.authentication['cookies_file']])
+        
+        return authdict
+    
+    def repr(self):
+        return f"Module<'{self.display_name}' (config: {self.config[self.name]})>"

src/auto_archiver/core/config.py

@@ -1,120 +1,164 @@
+"""
+The Config class initializes and parses configurations for all other steps.
+It supports CLI argument parsing, loading from YAML file, and overrides to allow
+flexible setup in various environments.
 
+"""
+
+import argparse
+from ruamel.yaml import YAML, CommentedMap, add_representer
 
-import argparse, yaml
-from dataclasses import dataclass, field
-from typing import List
-from collections import defaultdict
 from loguru import logger
 
-from ..archivers import Archiver
-from ..feeders import Feeder
-from ..databases import Database
-from ..formatters import Formatter
-from ..storages import Storage
-from ..enrichers import Enricher
-from . import Step
-from ..utils import update_nested_dict
+from copy import deepcopy
+from .module import BaseModule
 
+from typing import Any, List, Type, Tuple
 
-@dataclass
-class Config:
-    configurable_parents = [
-        Feeder,
-        Enricher,
-        Archiver,
-        Database,
-        Storage,
-        Formatter
-        # Util
-    ]
-    feeder: Feeder
-    formatter: Formatter
-    archivers: List[Archiver] = field(default_factory=[])
-    enrichers: List[Enricher] = field(default_factory=[])
-    storages: List[Storage] = field(default_factory=[])
-    databases: List[Database] = field(default_factory=[])
+_yaml: YAML = YAML()
 
-    def __init__(self) -> None:
-        self.defaults = {}
-        self.cli_ops = {}
-        self.config = {}
+EMPTY_CONFIG = _yaml.load("""
+# Auto Archiver Configuration
+# Steps are the modules that will be run in the order they are defined
 
-    def parse(self, use_cli=True, yaml_config_filename: str = None, overwrite_configs: str = {}):
+steps:""" + "".join([f"\n   {module}s: []" for module in BaseModule.MODULE_TYPES]) + \
+"""
+
+# Global configuration
+
+# Authentication
+# a dictionary of authentication information that can be used by extractors to login to website. 
+# you can use a comma separated list for multiple domains on the same line (common usecase: x.com,twitter.com)
+# Common login 'types' are username/password, cookie, api key/token.
+# There are two special keys for using cookies, they are: cookies_file and cookies_from_browser. 
+# Some Examples:
+# facebook.com:
+#   username: "my_username"
+#   password: "my_password"
+# or for a site that uses an API key:
+# twitter.com,x.com:
+#   api_key
+#   api_secret
+# youtube.com:
+#   cookie: "login_cookie=value ; other_cookie=123" # multiple 'key=value' pairs should be separated by ;
+
+authentication: {}
+
+# These are the global configurations that are used by the modules
+
+logging:
+  level: INFO
+
+""")
+# note: 'logging' is explicitly added above in order to better format the config file
+
+class DefaultValidatingParser(argparse.ArgumentParser):
+
+    def error(self, message):
         """
-        if yaml_config_filename is provided, the --config argument is ignored, 
-        useful for library usage when the config values are preloaded
-        overwrite_configs is a dict that overwrites the yaml file contents
+        Override of error to format a nicer looking error message using logger
         """
-        # 1. parse CLI values
-        if use_cli:
-            parser = argparse.ArgumentParser(
-                # prog = "auto-archiver",
-                description="Auto Archiver is a CLI tool to archive media/metadata from online URLs; it can read URLs from many sources (Google Sheets, Command Line, ...); and write results to many destinations too (CSV, Google Sheets, MongoDB, ...)!",
-                epilog="Check the code at https://github.com/bellingcat/auto-archiver"
-            )
+        logger.error("Problem with configuration file (tip: use --help to see the available options):")
+        logger.error(message)
+        self.exit(2)
 
-            parser.add_argument('--config', action='store', dest='config', help='the filename of the YAML configuration file (defaults to \'config.yaml\')', default='orchestration.yaml')
+    def parse_known_args(self, args=None, namespace=None):
+        """
+        Override of parse_known_args to also check the 'defaults' values - which are passed in from the config file
+        """
+        for action in self._actions:
+            if not namespace or action.dest not in namespace:
+                # for actions that are required and already have a default value, remove the 'required' check
+                if action.required and action.default is not None:
+                    action.required = False
 
-        for configurable in self.configurable_parents:
-            child: Step
-            for child in configurable.__subclasses__():
-                assert child.configs() is not None and type(child.configs()) == dict, f"class '{child.name}' should have a configs method returning a dict."
-                for config, details in child.configs().items():
-                    assert "." not in child.name, f"class prop name cannot contain dots('.'): {child.name}"
-                    assert "." not in config, f"config property cannot contain dots('.'): {config}"
-                    config_path = f"{child.name}.{config}"
+                if action.default is not None:
+                    try:
+                        self._check_value(action, action.default)
+                    except argparse.ArgumentError as e:
+                        logger.error(f"You have an invalid setting in your configuration file ({action.dest}):")
+                        logger.error(e)
+                        exit()
 
-                    if use_cli:
-                        try:
-                            parser.add_argument(f'--{config_path}', action='store', dest=config_path, help=f"{details['help']} (defaults to {details['default']})", choices=details.get("choices", None))
-                        except argparse.ArgumentError:
-                            # captures cases when a Step is used in 2 flows, eg: wayback enricher vs wayback archiver
-                            pass
+        return super().parse_known_args(args, namespace)
 
-                    self.defaults[config_path] = details["default"]
-                    if "cli_set" in details:
-                        self.cli_ops[config_path] = details["cli_set"]
 
-        if use_cli:
-            args = parser.parse_args()
-            yaml_config_filename = yaml_config_filename or getattr(args, "config")
-        else: args = {}
+def to_dot_notation(yaml_conf: CommentedMap | dict) -> dict:
+    dotdict = {}
 
-        # 2. read YAML config file (or use provided value)
-        self.yaml_config = self.read_yaml(yaml_config_filename)
-        update_nested_dict(self.yaml_config, overwrite_configs)
+    def process_subdict(subdict, prefix=""):
+        for key, value in subdict.items():
+            if is_dict_type(value):
+                process_subdict(value, f"{prefix}{key}.")
+            else:
+                dotdict[f"{prefix}{key}"] = value
 
-        # 3. CONFIGS: decide value with priority: CLI >> config.yaml >> default
-        self.config = defaultdict(dict)
-        for config_path, default in self.defaults.items():
-            child, config = tuple(config_path.split("."))
-            val = getattr(args, config_path, None)
-            if val is not None and config_path in self.cli_ops:
-                val = self.cli_ops[config_path](val, default)
-            if val is None:
-                val = self.yaml_config.get("configurations", {}).get(child, {}).get(config, default)
-            self.config[child][config] = val
-        self.config = dict(self.config)
+    process_subdict(yaml_conf)
+    return dotdict
 
-        # 4. STEPS: read steps and validate they exist
-        steps = self.yaml_config.get("steps", {})
-        assert "archivers" in steps, "your configuration steps are missing the archivers property"
-        assert "storages" in steps, "your configuration steps are missing the storages property"
+def from_dot_notation(dotdict: dict) -> dict:
+    normal_dict = {}
 
-        self.feeder = Feeder.init(steps.get("feeder", "cli_feeder"), self.config)
-        self.formatter = Formatter.init(steps.get("formatter", "mute_formatter"), self.config)
-        self.enrichers = [Enricher.init(e, self.config) for e in steps.get("enrichers", [])]
-        self.archivers = [Archiver.init(e, self.config) for e in (steps.get("archivers") or [])]
-        self.databases = [Database.init(e, self.config) for e in steps.get("databases", [])]
-        self.storages = [Storage.init(e, self.config) for e in steps.get("storages", [])]
+    def add_part(key, value, current_dict):
+        if "." in key:
+            key_parts = key.split(".")
+            current_dict.setdefault(key_parts[0], {})
+            add_part(".".join(key_parts[1:]), value, current_dict[key_parts[0]])
+        else:
+            current_dict[key] = value
 
-        logger.info(f"FEEDER: {self.feeder.name}")
-        logger.info(f"ENRICHERS: {[x.name for x in self.enrichers]}")
-        logger.info(f"ARCHIVERS: {[x.name for x in self.archivers]}")
-        logger.info(f"DATABASES: {[x.name for x in self.databases]}")
-        logger.info(f"STORAGES: {[x.name for x in self.storages]}")
-        logger.info(f"FORMATTER: {self.formatter.name}")
+    for key, value in dotdict.items():
+        add_part(key, value, normal_dict)
 
-    def read_yaml(self, yaml_filename: str) -> dict:
+    return normal_dict
+
+
+def is_list_type(value):
+    return isinstance(value, list) or isinstance(value, tuple) or isinstance(value, set)
+
+def is_dict_type(value):
+    return isinstance(value, dict) or isinstance(value, CommentedMap)
+
+def merge_dicts(dotdict: dict, yaml_dict: CommentedMap) -> CommentedMap:
+    yaml_dict: CommentedMap = deepcopy(yaml_dict)
+
+    # first deal with lists, since 'update' replaces lists from a in b, but we want to extend
+    def update_dict(subdict, yaml_subdict):
+        for key, value in subdict.items():
+            if not yaml_subdict.get(key):
+                yaml_subdict[key] = value
+                continue
+
+            if is_dict_type(value):
+                update_dict(value, yaml_subdict[key])
+            elif is_list_type(value):
+                yaml_subdict[key].extend(s for s in value if s not in yaml_subdict[key])
+            else:
+                yaml_subdict[key] = value
+
+    update_dict(from_dot_notation(dotdict), yaml_dict)
+
+    return yaml_dict
+
+def read_yaml(yaml_filename: str) -> CommentedMap:
+    config = None
+    try:
         with open(yaml_filename, "r", encoding="utf-8") as inf:
-            return yaml.safe_load(inf)
+            config = _yaml.load(inf)
+    except FileNotFoundError:
+        pass
+
+    if not config:
+        config = EMPTY_CONFIG
+    
+    return config
+
+# TODO: make this tidier/find a way to notify of which keys should not be stored
+
+
+def store_yaml(config: CommentedMap, yaml_filename: str) -> None:
+    config_to_save = deepcopy(config)
+
+    config_to_save.pop('urls', None)
+    with open(yaml_filename, "w", encoding="utf-8") as outf:
+        _yaml.dump(config_to_save, outf)

src/auto_archiver/core/context.py

@@ -1,52 +0,0 @@
-from loguru import logger
-
-
-class ArchivingContext:
-    """
-    Singleton context class.
-    ArchivingContext._get_instance() to retrieve it if needed
-    otherwise just 
-    ArchivingContext.set(key, value)
-    and 
-    ArchivingContext.get(key, default)
-
-    When reset is called, all values are cleared EXCEPT if they were .set(keep_on_reset=True)
-        reset(full_reset=True) will recreate everything including the keep_on_reset status
-    """
-    _instance = None
-
-    def __init__(self):
-        self.configs = {}
-        self.keep_on_reset = set()
-
-    @staticmethod
-    def get_instance():
-        if ArchivingContext._instance is None:
-            ArchivingContext._instance = ArchivingContext()
-        return ArchivingContext._instance
-
-    @staticmethod
-    def set(key, value, keep_on_reset: bool = False):
-        ac = ArchivingContext.get_instance()
-        ac.configs[key] = value
-        if keep_on_reset: ac.keep_on_reset.add(key)
-
-    @staticmethod
-    def get(key: str, default=None):
-        return ArchivingContext.get_instance().configs.get(key, default)
-
-    @staticmethod
-    def reset(full_reset: bool = False):
-        ac = ArchivingContext.get_instance()
-        if full_reset: ac.keep_on_reset = set()
-        ac.configs = {k: v for k, v in ac.configs.items() if k in ac.keep_on_reset}
-
-    # ---- custom getters/setters for widely used context values
-
-    @staticmethod
-    def set_tmp_dir(tmp_dir: str):
-        ArchivingContext.get_instance().configs["tmp_dir"] = tmp_dir
-
-    @staticmethod
-    def get_tmp_dir() -> str:
-        return ArchivingContext.get_instance().configs.get("tmp_dir")

src/auto_archiver/core/database.py

@@ -0,0 +1,39 @@
+"""
+Database module for the auto-archiver that defines the interface for implementing database modules
+in the media archiving framework. 
+"""
+
+from __future__ import annotations
+from abc import abstractmethod
+from typing import Union
+
+from auto_archiver.core import Metadata, BaseModule
+
+class Database(BaseModule):
+    """
+    Base class for implementing database modules in the media archiving framework.
+
+    Subclasses must implement the `fetch` and `done` methods to define platform-specific behavior.
+    """
+
+    def started(self, item: Metadata) -> None:
+        """signals the DB that the given item archival has started"""
+        pass
+
+    def failed(self, item: Metadata, reason:str) -> None:
+        """update DB accordingly for failure"""
+        pass
+
+    def aborted(self, item: Metadata) -> None:
+        """abort notification if user cancelled after start"""
+        pass
+
+    # @abstractmethod
+    def fetch(self, item: Metadata) -> Union[Metadata, bool]:
+        """check and fetch if the given item has been archived already, each database should handle its own caching, and configuration mechanisms"""
+        return False
+
+    @abstractmethod
+    def done(self, item: Metadata, cached: bool=False) -> None:
+        """archival result ready - should be saved to DB"""
+        pass

src/auto_archiver/core/enricher.py

@@ -0,0 +1,28 @@
+"""
+Base module for Enrichers – modular components that enhance archived content by adding
+context, metadata, or additional processing.
+
+These add additional information to the context, such as screenshots, hashes, and metadata.
+They are designed to work within the archiving pipeline, operating on `Metadata` objects after
+the archiving step and before storage or formatting.
+
+Enrichers are optional but highly useful for making the archived data more powerful.
+"""
+from __future__ import annotations
+from abc import abstractmethod
+from auto_archiver.core import Metadata, BaseModule
+
+class Enricher(BaseModule):
+    """Base classes and utilities for enrichers in the Auto-Archiver system.
+    
+    Enricher modules must implement the `enrich` method to define their behavior.
+    """
+
+    @abstractmethod
+    def enrich(self, to_enrich: Metadata) -> None:
+        """
+        Enriches a Metadata object with additional information or context.
+
+        Takes the metadata object to enrich as an argument and modifies it in place, returning None.
+        """
+        pass

src/auto_archiver/core/extractor.py

@@ -0,0 +1,115 @@
+""" The `extractor` module defines the base functionality for implementing extractors in the media archiving framework.
+    This class provides common utility methods and a standard interface for extractors.
+
+    Factory method to initialize an extractor instance based on its name.
+
+
+"""
+from __future__ import annotations
+from pathlib import Path
+from abc import abstractmethod
+from dataclasses import dataclass
+import mimetypes
+import os
+import mimetypes
+import requests
+from loguru import logger
+from retrying import retry
+import re
+
+from auto_archiver.core import Metadata, BaseModule
+
+
+class Extractor(BaseModule):
+    """
+    Base class for implementing extractors in the media archiving framework.
+    Subclasses must implement the `download` method to define platform-specific behavior.
+    """
+
+    valid_url: re.Pattern = None
+
+    def cleanup(self) -> None:
+        """
+        Called when extractors are done, or upon errors, cleanup any resources
+        """
+        pass
+
+    def sanitize_url(self, url: str) -> str:
+        """
+        Used to clean unnecessary URL parameters OR unfurl redirect links
+        """
+        return url
+    
+    def match_link(self, url: str) -> re.Match:
+        """
+        Returns a match object if the given URL matches the valid_url pattern or False/None if not.
+
+        Normally used in the `suitable` method to check if the URL is supported by this extractor.
+
+        """
+        return self.valid_url.match(url)
+
+    def suitable(self, url: str) -> bool:
+        """
+        Returns True if this extractor can handle the given URL
+
+        Should be overridden by subclasses
+
+        """
+        if self.valid_url:
+            return self.match_link(url) is not None
+        
+        return True
+
+    def _guess_file_type(self, path: str) -> str:
+        """
+        Receives a URL or filename and returns global mimetype like 'image' or 'video'
+        see https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types
+        """
+        mime = mimetypes.guess_type(path)[0]
+        if mime is not None:
+            return mime.split("/")[0]
+        return ""
+
+    @retry(wait_random_min=500, wait_random_max=3500, stop_max_attempt_number=5)
+    def download_from_url(self, url: str, to_filename: str = None, verbose=True) -> str:
+        """
+            downloads a URL to provided filename, or inferred from URL, returns local filename
+        """
+        if not to_filename:
+            to_filename = url.split('/')[-1].split('?')[0]
+            if len(to_filename) > 64:
+                to_filename = to_filename[-64:]
+        to_filename = os.path.join(self.tmp_dir, to_filename)
+        if verbose: logger.debug(f"downloading {url[0:50]=} {to_filename=}")
+        headers = {
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36'
+        }
+        try:
+            d = requests.get(url, stream=True, headers=headers, timeout=30)
+            d.raise_for_status()
+
+            # get mimetype from the response headers
+            if not mimetypes.guess_type(to_filename)[0]:
+                content_type = d.headers.get('Content-Type') or self._guess_file_type(url)
+                extension = mimetypes.guess_extension(content_type)
+                if extension:
+                    to_filename += extension
+
+            with open(to_filename, 'wb') as f:
+                for chunk in d.iter_content(chunk_size=8192):
+                    f.write(chunk)
+            return to_filename
+
+        except requests.RequestException as e:
+            logger.warning(f"Failed to fetch the Media URL: {e}")
+
+    @abstractmethod
+    def download(self, item: Metadata) -> Metadata | False:
+        """
+        Downloads the media from the given URL and returns a Metadata object with the downloaded media.
+        
+        If the URL is not supported or the download fails, this method should return False.
+
+        """
+        pass

src/auto_archiver/core/feeder.py

@@ -0,0 +1,25 @@
+"""
+The feeder base module defines the interface for implementing feeders in the media archiving framework. 
+"""
+
+from __future__ import annotations
+from abc import abstractmethod
+from auto_archiver.core import Metadata
+from auto_archiver.core import BaseModule
+
+class Feeder(BaseModule):
+
+    """
+    Base class for implementing feeders in the media archiving framework.
+
+    Subclasses must implement the `__iter__` method to define platform-specific behavior.
+    """
+
+    @abstractmethod
+    def __iter__(self) -> Metadata:
+        """
+        Returns an iterator (use `yield`) over the items to be archived.
+        
+        These should be instances of Metadata, typically created with Metadata().set_url(url).
+        """
+        return None

src/auto_archiver/core/formatter.py

@@ -0,0 +1,24 @@
+"""
+Base module for formatters – modular components that format metadata into media objects for storage.
+
+The most commonly used formatter is the HTML formatter, which takes metadata and formats it into an HTML file for storage.
+"""
+
+from __future__ import annotations
+from abc import abstractmethod
+from auto_archiver.core import Metadata, Media, BaseModule
+
+
+class Formatter(BaseModule):
+    """
+    Base class for implementing formatters in the media archiving framework.
+    
+    Subclasses must implement the `format` method to define their behavior.
+    """
+
+    @abstractmethod
+    def format(self, item: Metadata) -> Media:
+        """
+        Formats a Metadata object into a user-viewable format (e.g. HTML) and stores it if needed.
+        """
+        return None

src/auto_archiver/core/media.py

@@ -1,3 +1,7 @@
+"""
+Manages media files and their associated metadata, supporting storage,
+nested media retrieval, and type validation.
+"""
 
 from __future__ import annotations
 import os
@@ -7,17 +11,22 @@ from dataclasses import dataclass, field
 from dataclasses_json import dataclass_json, config
 import mimetypes
 
-import ffmpeg
-from ffmpeg._run import Error
-
-from .context import ArchivingContext
-
 from loguru import logger
 
 
 @dataclass_json  # annotation order matters
 @dataclass
 class Media:
+    """
+    Represents a media file with associated properties and storage details.
+
+    Attributes:
+    - filename: The file path of the media.
+    - key: An optional identifier for the media.
+    - urls: A list of URLs where the media is stored or accessible.
+    - properties: Additional metadata or transformations for the media.
+    - _mimetype: The media's mimetype (e.g., image/jpeg, video/mp4).
+    """
     filename: str
     key: str = None
     urls: List[str] = field(default_factory=list)
@@ -25,32 +34,38 @@ class Media:
     _mimetype: str = None  # eg: image/jpeg
     _stored: bool = field(default=False, repr=False, metadata=config(exclude=lambda _: True))  # always exclude
 
-    def store(self: Media, override_storages: List = None, url: str = "url-not-available"):
-        # stores the media into the provided/available storages [Storage]
-        # repeats the process for its properties, in case they have inner media themselves
-        # for now it only goes down 1 level but it's easy to make it recursive if needed
-        storages = override_storages or ArchivingContext.get("storages")
+    def store(self: Media, metadata: Any, url: str = "url-not-available", storages: List[Any] = None) -> None:
+        # 'Any' typing for metadata to avoid circular imports. Stores the media
+        # into the provided/available storages [Storage] repeats the process for
+        # its properties, in case they have inner media themselves for now it
+        # only goes down 1 level but it's easy to make it recursive if needed.
         if not len(storages):
             logger.warning(f"No storages found in local context or provided directly for {self.filename}.")
             return
 
         for s in storages:
             for any_media in self.all_inner_media(include_self=True):
-                s.store(any_media, url)
+                s.store(any_media, url, metadata=metadata)
 
     def all_inner_media(self, include_self=False):
-        """ Media can be inside media properties, examples include transformations on original media.
-        This function returns a generator for all the inner media.        
+        """Retrieves all media, including nested media within properties or transformations on original media.
+        This function returns a generator for all the inner media.
+
         """
         if include_self: yield self
         for prop in self.properties.values():
-            if isinstance(prop, Media): yield prop
+            if isinstance(prop, Media): 
+                for inner_media in prop.all_inner_media(include_self=True):
+                    yield inner_media
             if isinstance(prop, list):
                 for prop_media in prop:
-                    if isinstance(prop_media, Media): yield prop_media
+                    if isinstance(prop_media, Media): 
+                        for inner_media in prop_media.all_inner_media(include_self=True):
+                            yield inner_media
 
-    def is_stored(self) -> bool:
-        return len(self.urls) > 0 and len(self.urls) == len(ArchivingContext.get("storages"))
+    def is_stored(self, in_storage) -> bool:
+        # checks if the media is already stored in the given storage
+        return len(self.urls) > 0 and len(self.urls) == len(in_storage.config["steps"]["storages"])
 
     def set(self, key: str, value: Any) -> Media:
         self.properties[key] = value
@@ -65,7 +80,9 @@ class Media:
 
     @property  # getter .mimetype
     def mimetype(self) -> str:
-        assert self.filename is not None and len(self.filename) > 0, "cannot get mimetype from media without filename"
+        if not self.filename or len(self.filename) == 0:
+            logger.warning(f"cannot get mimetype from media without filename: {self}")
+            return ""
         if not self._mimetype:
             self._mimetype = mimetypes.guess_type(self.filename)[0]
         return self._mimetype or ""
@@ -84,6 +101,12 @@ class Media:
         return self.mimetype.startswith("image")
 
     def is_valid_video(self) -> bool:
+        # Note: this is intentional, to only import ffmpeg here - when the method is called
+        # this speeds up loading the module. We check that 'ffmpeg' is available on startup
+        # when we load each manifest file
+        import ffmpeg
+        from ffmpeg._run import Error
+
         # checks for video streams with ffmpeg, or min file size for a video
         # self.is_video() should be used together with this method
         try:

src/auto_archiver/core/metadata.py

@@ -1,3 +1,13 @@
+"""
+Acts as a container for metadata and media objects associated with an archived item.
+
+Key Functionalities:
+- Store and retrieve metadata and associated media.
+- Merge metadata objects with conflict resolution.
+- Validate properties like URLs and timestamps.
+- Manage and deduplicate media objects.
+- Support for flexible metadata querying and appending.
+"""
 
 from __future__ import annotations
 import hashlib
@@ -7,9 +17,9 @@ from dataclasses_json import dataclass_json, config
 import datetime
 from urllib.parse import urlparse
 from dateutil.parser import parse as parse_dt
-from .media import Media
-from .context import ArchivingContext
+from loguru import logger
 
+from .media import Media
 
 @dataclass_json  # annotation order matters
 @dataclass
@@ -19,16 +29,22 @@ class Metadata:
     media: List[Media] = field(default_factory=list)
 
     def __post_init__(self):
-        self.set("_processed_at", datetime.datetime.utcnow())
+        self.set("_processed_at", datetime.datetime.now(datetime.timezone.utc))
+        self._context = {}
 
     def merge(self: Metadata, right: Metadata, overwrite_left=True) -> Metadata:
         """
-        merges two Metadata instances, will overwrite according to overwrite_left flag
+        Merges another `Metadata` instance into this one.
+
+        Conflicts are resolved based on the `overwrite_left` flag:
+        - If `True`, this instance's values are overwritten by `right`.
+        - If `False`, the inverse applies.
         """
         if not right: return self
         if overwrite_left:
             if right.status and len(right.status):
                 self.status = right.status
+            self._context.update(right._context)
             for k, v in right.metadata.items():
                 assert k not in self.metadata or type(v) == type(self.get(k))
                 if type(v) not in [dict, list, set] or k not in self.metadata:
@@ -41,17 +57,22 @@ class Metadata:
             return right.merge(self)
         return self
 
-    def store(self: Metadata, override_storages: List = None):
+    def store(self, storages=[]):
         # calls .store for all contained media. storages [Storage]
         self.remove_duplicate_media_by_hash()
-        storages = override_storages or ArchivingContext.get("storages")
         for media in self.media:
-            media.store(override_storages=storages, url=self.get_url())
+            media.store(url=self.get_url(), metadata=self, storages=storages)
 
     def set(self, key: str, val: Any) -> Metadata:
         self.metadata[key] = val
         return self
 
+    def append(self, key: str, val: Any) -> Metadata:
+        if key not in self.metadata:
+            self.metadata[key] = []    
+        self.metadata[key] = val
+        return self
+
     def get(self, key: str, default: Any = None, create_if_missing=False) -> Union[Metadata, str]:
         # goes through metadata and returns the Metadata available
         if create_if_missing and key not in self.metadata:
@@ -67,7 +88,8 @@ class Metadata:
         return "success" in self.status
 
     def is_empty(self) -> bool:
-        return not self.is_success() and len(self.media) == 0 and len(self.metadata) <= 2  # url, processed_at
+        meaningfull_ids = set(self.metadata.keys()) - set(["_processed_at", "url", "total_bytes", "total_size", "archive_duration_seconds"])
+        return not self.is_success() and len(self.media) == 0 and len(meaningfull_ids) == 0
 
     @property  # getter .netloc
     def netloc(self) -> str:
@@ -106,10 +128,15 @@ class Metadata:
     def get_timestamp(self, utc=True, iso=True) -> datetime.datetime:
         ts = self.get("timestamp")
         if not ts: return 
-        if type(ts) == float: ts = datetime.datetime.fromtimestamp(ts)
-        if utc: ts = ts.replace(tzinfo=datetime.timezone.utc)
-        if iso: return ts.isoformat()
-        return ts
+        try:
+            if type(ts) == str: ts = datetime.datetime.fromisoformat(ts)
+            if type(ts) == float: ts = datetime.datetime.fromtimestamp(ts)
+            if utc: ts = ts.replace(tzinfo=datetime.timezone.utc)
+            if iso: return ts.isoformat()
+            return ts
+        except Exception as e:
+            logger.error(f"Unable to parse timestamp {ts}: {e}")
+            return
 
     def add_media(self, media: Media, id: str = None) -> Metadata:
         # adds a new media, optionally including an id
@@ -165,3 +192,23 @@ class Metadata:
 
     def __str__(self) -> str:
         return self.__repr__()
+
+
+    @staticmethod
+    def choose_most_complete(results: List[Metadata]) -> Metadata:
+        # returns the most complete result from a list of results
+        # prioritizes results with more media, then more metadata
+        if len(results) == 0: return None
+        if len(results) == 1: return results[0]
+        most_complete = results[0]
+        for r in results[1:]:
+            if len(r.media) > len(most_complete.media): most_complete = r
+            elif len(r.media) == len(most_complete.media) and len(r.metadata) > len(most_complete.metadata): most_complete = r
+        return most_complete
+
+    def set_context(self, key: str, val: Any) -> Metadata:
+        self._context[key] = val
+        return self
+    
+    def get_context(self, key: str, default: Any = None) -> Any:
+        return self._context.get(key, default)

src/auto_archiver/core/module.py

@@ -0,0 +1,249 @@
+"""
+Defines the Step abstract base class, which acts as a blueprint for steps in the archiving pipeline
+by handling user configuration, validating the steps properties, and implementing dynamic instantiation.
+
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import List
+import shutil
+import ast
+import copy
+import sys
+from importlib.util import find_spec
+import os
+from os.path import join
+from loguru import logger
+import auto_archiver
+from .base_module import BaseModule
+
+_LAZY_LOADED_MODULES = {}
+
+MANIFEST_FILE = "__manifest__.py"
+
+
+def setup_paths(paths: list[str]) -> None:
+    """
+    Sets up the paths for the modules to be loaded from
+    
+    This is necessary for the modules to be imported correctly
+    
+    """
+    for path in paths:
+        # check path exists, if it doesn't, log a warning
+        if not os.path.exists(path):
+            logger.warning(f"Path '{path}' does not exist. Skipping...")
+            continue
+
+        # see odoo/module/module.py -> initialize_sys_path
+        if path not in auto_archiver.modules.__path__:
+                auto_archiver.modules.__path__.append(path)
+
+    # sort based on the length of the path, so that the longest path is last in the list
+    auto_archiver.modules.__path__ = sorted(auto_archiver.modules.__path__, key=len, reverse=True)
+
+def get_module(module_name: str, config: dict) -> BaseModule:
+    """
+    Gets and sets up a module using the provided config
+    
+    This will actually load and instantiate the module, and load all its dependencies (i.e. not lazy)
+    
+    """
+    return get_module_lazy(module_name).load(config)
+
+def get_module_lazy(module_name: str, suppress_warnings: bool = False) -> LazyBaseModule:
+    """
+    Lazily loads a module, returning a LazyBaseModule
+    
+    This has all the information about the module, but does not load the module itself or its dependencies
+    
+    To load an actual module, call .setup() on a lazy module
+    
+    """
+    if module_name in _LAZY_LOADED_MODULES:
+        return _LAZY_LOADED_MODULES[module_name]
+
+    available = available_modules(limit_to_modules=[module_name], suppress_warnings=suppress_warnings)
+    if not available:
+        raise IndexError(f"Module '{module_name}' not found. Are you sure it's installed/exists?")
+    return available[0]
+
+def available_modules(with_manifest: bool=False, limit_to_modules: List[str]= [], suppress_warnings: bool = False) -> List[LazyBaseModule]:
+    
+    # search through all valid 'modules' paths. Default is 'modules' in the current directory
+
+    # see odoo/modules/module.py -> get_modules
+    def is_really_module(module_path):
+        if os.path.isfile(join(module_path, MANIFEST_FILE)):
+            return True
+
+    all_modules = []
+
+    for module_folder in auto_archiver.modules.__path__:
+        # walk through each module in module_folder and check if it has a valid manifest
+        try:
+            possible_modules = os.listdir(module_folder)
+        except FileNotFoundError:
+            logger.warning(f"Module folder {module_folder} does not exist")
+            continue
+
+        for possible_module in possible_modules:
+            if limit_to_modules and possible_module not in limit_to_modules:
+                continue
+
+            possible_module_path = join(module_folder, possible_module)
+            if not is_really_module(possible_module_path):
+                continue
+            if _LAZY_LOADED_MODULES.get(possible_module):
+                continue
+            lazy_module = LazyBaseModule(possible_module, possible_module_path)
+
+            _LAZY_LOADED_MODULES[possible_module] = lazy_module
+
+            all_modules.append(lazy_module)
+    
+    if not suppress_warnings:
+        for module in limit_to_modules:
+            if not any(module == m.name for m in all_modules):
+                logger.warning(f"Module '{module}' not found. Are you sure it's installed?")
+
+    return all_modules
+
+@dataclass
+class LazyBaseModule:
+
+    """
+    A lazy module class, which only loads the manifest and does not load the module itself.
+
+    This is useful for getting information about a module without actually loading it.
+
+    """
+    name: str
+    type: list
+    description: str
+    path: str
+
+    _manifest: dict = None
+    _instance: BaseModule = None
+    _entry_point: str = None
+
+    def __init__(self, module_name, path):
+        self.name = module_name
+        self.path = path
+
+    @property
+    def entry_point(self):
+        if not self._entry_point and not self.manifest['entry_point']:
+            # try to create the entry point from the module name
+            self._entry_point = f"{self.name}::{self.name.replace('_', ' ').title().replace(' ', '')}"
+        return self._entry_point
+
+    @property
+    def dependencies(self) -> dict:
+        return self.manifest['dependencies']
+    
+    @property
+    def configs(self) -> dict:
+        return self.manifest['configs']
+    
+    @property
+    def requires_setup(self) -> bool:
+        return self.manifest['requires_setup']
+    
+    @property
+    def display_name(self) -> str:
+        return self.manifest['name']
+
+    @property
+    def manifest(self) -> dict:
+        if self._manifest:
+            return self._manifest
+        # print(f"Loading manifest for module {module_path}")
+        # load the manifest file
+        manifest = copy.deepcopy(BaseModule._DEFAULT_MANIFEST)
+
+        with open(join(self.path, MANIFEST_FILE)) as f:
+            try:
+                manifest.update(ast.literal_eval(f.read()))
+            except (ValueError, TypeError, SyntaxError, MemoryError, RecursionError) as e:
+                logger.error(f"Error loading manifest from file {self.path}/{MANIFEST_FILE}: {e}")
+            
+        self._manifest = manifest
+        self.type = manifest['type']
+        self._entry_point = manifest['entry_point']
+        self.description = manifest['description']
+        self.version = manifest['version']
+
+        return manifest
+
+    def load(self, config) -> BaseModule:
+
+        if self._instance:
+            return self._instance
+
+        # check external dependencies are installed
+        def check_deps(deps, check):
+            for dep in deps:
+                if not len(dep):
+                    # clear out any empty strings that a user may have erroneously added
+                    continue
+                if not check(dep):
+                    logger.error(f"Module '{self.name}' requires external dependency '{dep}' which is not available/setup. Have you installed the required dependencies for the '{self.name}' module? See the README for more information.")
+                    exit(1)
+
+        def check_python_dep(dep):
+            # first check if it's a module:
+            try:
+                m = get_module_lazy(dep, suppress_warnings=True)
+                try:
+                # we must now load this module and set it up with the config
+                    m.load(config)
+                    return True
+                except:
+                    logger.error(f"Unable to setup module '{dep}' for use in module '{self.name}'")
+                    return False
+            except IndexError:
+                # not a module, continue
+                pass
+
+            return find_spec(dep)
+
+        check_deps(self.dependencies.get('python', []), check_python_dep)
+        check_deps(self.dependencies.get('bin', []), lambda dep: shutil.which(dep))
+
+
+        logger.debug(f"Loading module '{self.display_name}'...")
+
+        for qualname in [self.name, f'auto_archiver.modules.{self.name}']:
+            try:
+                # first import the whole module, to make sure it's working properly
+                __import__(qualname)
+                break
+            except ImportError:
+                pass
+
+        # then import the file for the entry point
+        file_name, class_name = self.entry_point.split('::')
+        sub_qualname = f'{qualname}.{file_name}'
+
+        __import__(f'{qualname}.{file_name}', fromlist=[self.entry_point])
+        # finally, get the class instance
+        instance: BaseModule = getattr(sys.modules[sub_qualname], class_name)()
+        if not getattr(instance, 'name', None):
+            instance.name = self.name
+
+        if not getattr(instance, 'display_name', None):
+            instance.display_name = self.display_name
+
+        self._instance = instance
+
+        # merge the default config with the user config
+        default_config = dict((k, v['default']) for k, v in self.configs.items() if v.get('default'))
+        config[self.name] = default_config  | config.get(self.name, {})
+        instance.config_setup(config)
+        instance.setup()
+        return instance
+
+    def __repr__(self):
+        return f"Module<'{self.display_name}' ({self.name})>"

src/auto_archiver/core/orchestrator.py

@@ -1,121 +1,497 @@
+""" Orchestrates all archiving steps, including feeding items,
+    archiving them with specific archivers, enrichment, storage,
+    formatting, database operations and clean up.
+
+"""
+
 from __future__ import annotations
-from typing import Generator, Union, List
+from typing import Generator, Union, List, Type
+from urllib.parse import urlparse
+from ipaddress import ip_address
+import argparse
+import os
+import sys
+import json
+from tempfile import TemporaryDirectory
+import traceback
 
-from .context import ArchivingContext
+from rich_argparse import RichHelpFormatter
 
-from ..archivers import Archiver
-from ..feeders import Feeder
-from ..formatters import Formatter
-from ..storages import Storage
-from ..enrichers import Enricher
-from ..databases import Database
-from .metadata import Metadata
 
-import tempfile, traceback
+from .metadata import Metadata, Media
+from auto_archiver.version import __version__
+from .config import _yaml, read_yaml, store_yaml, to_dot_notation, merge_dicts, EMPTY_CONFIG, DefaultValidatingParser
+from .module import available_modules, LazyBaseModule, get_module, setup_paths
+from . import validators, Feeder, Extractor, Database, Storage, Formatter, Enricher
+from .module import BaseModule
+
 from loguru import logger
 
 
-class ArchivingOrchestrator:
-    def __init__(self, config) -> None:
-        self.feeder: Feeder = config.feeder
-        self.formatter: Formatter = config.formatter
-        self.enrichers: List[Enricher] = config.enrichers
-        self.archivers: List[Archiver] = config.archivers
-        self.databases: List[Database] = config.databases
-        self.storages: List[Storage] = config.storages
-        ArchivingContext.set("storages", self.storages, keep_on_reset=True)
+DEFAULT_CONFIG_FILE = "orchestration.yaml"
 
-        for a in self.archivers: a.setup()
+
+class JsonParseAction(argparse.Action):
+    def __call__(self, parser, namespace, values, option_string=None):
+        try:
+            setattr(namespace, self.dest, json.loads(values))
+        except json.JSONDecodeError as e:
+            raise argparse.ArgumentTypeError(f"Invalid JSON input for argument '{self.dest}': {e}")
+
+
+class AuthenticationJsonParseAction(JsonParseAction):
+    def __call__(self, parser, namespace, values, option_string=None):
+        super().__call__(parser, namespace, values, option_string)
+        auth_dict = getattr(namespace, self.dest)
+        if isinstance(auth_dict, str):
+            # if it's a string
+            try:
+                with open(auth_dict, 'r') as f:
+                    try:
+                        auth_dict = json.load(f)
+                    except json.JSONDecodeError:
+                        # maybe it's yaml, try that
+                        auth_dict = _yaml.load(f)
+            except:
+                pass
+
+        if not isinstance(auth_dict, dict):
+            raise argparse.ArgumentTypeError("Authentication must be a dictionary of site names and their authentication methods")
+        for site, auth in auth_dict.items():
+            if not isinstance(site, str) or not isinstance(auth, dict):
+                raise argparse.ArgumentTypeError("Authentication must be a dictionary of site names and their authentication methods")
+        setattr(namespace, self.dest, auth_dict)
+
+
+class UniqueAppendAction(argparse.Action):
+    def __call__(self, parser, namespace, values, option_string=None):
+        if not hasattr(namespace, self.dest):
+            setattr(namespace, self.dest, [])
+        for value in values:
+            if value not in getattr(namespace, self.dest):
+                getattr(namespace, self.dest).append(value)
+
+
+class ArchivingOrchestrator:
+
+    feeders: List[Type[Feeder]]
+    extractors: List[Type[Extractor]]
+    enrichers: List[Type[Enricher]]
+    databases: List[Type[Database]]
+    storages: List[Type[Storage]]
+    formatters: List[Type[Formatter]]
+
+    def setup_basic_parser(self):
+        parser = argparse.ArgumentParser(
+            prog="auto-archiver",
+            add_help=False,
+            description="""
+                Auto Archiver is a CLI tool to archive media/metadata from online URLs;
+                it can read URLs from many sources (Google Sheets, Command Line, ...); and write results to many destinations too (CSV, Google Sheets, MongoDB, ...)!
+                """,
+            epilog="Check the code at https://github.com/bellingcat/auto-archiver",
+            formatter_class=RichHelpFormatter,
+        )
+        parser.add_argument('--help', '-h', action='store_true', dest='help', help='show a full help message and exit')
+        parser.add_argument('--version', action='version', version=__version__)
+        parser.add_argument('--config', action='store', dest="config_file", help='the filename of the YAML configuration file (defaults to \'config.yaml\')', default=DEFAULT_CONFIG_FILE)
+        parser.add_argument('--mode', action='store', dest='mode', type=str, choices=['simple', 'full'], help='the mode to run the archiver in', default='simple')
+        # override the default 'help' so we can inject all the configs and show those
+        parser.add_argument('-s', '--store', dest='store', default=False, help='Store the created config in the config file', action=argparse.BooleanOptionalAction)
+        parser.add_argument('--module_paths', dest='module_paths', nargs='+', default=[], help='additional paths to search for modules', action=UniqueAppendAction)
+
+        self.basic_parser = parser
+        return parser
+
+    def setup_complete_parser(self, basic_config: dict, yaml_config: dict, unused_args: list[str]) -> None:
+        parser = DefaultValidatingParser(
+            add_help=False,
+        )
+        self.add_additional_args(parser)
+
+        # check what mode we're in
+        # if we have a config file, use that to decide which modules to load
+        # if simple, we'll load just the modules that has requires_setup = False
+        # if full, we'll load all modules
+        # TODO: BUG** - basic_config won't have steps in it, since these args aren't added to 'basic_parser'
+        # but should we add them? Or should we just add them to the 'complete' parser?
+        if yaml_config != EMPTY_CONFIG:
+            # only load the modules enabled in config
+            # TODO: if some steps are empty (e.g. 'feeders' is empty), should we default to the 'simple' ones? Or only if they are ALL empty?
+            enabled_modules = []
+            # first loads the modules from the config file, then from the command line
+            for config in [yaml_config['steps'], basic_config.__dict__]:
+                for module_type in BaseModule.MODULE_TYPES:
+                    enabled_modules.extend(config.get(f"{module_type}s", []))
+
+            # clear out duplicates, but keep the order
+            enabled_modules = list(dict.fromkeys(enabled_modules))
+            avail_modules = available_modules(with_manifest=True, limit_to_modules=enabled_modules, suppress_warnings=True)
+            self.add_module_args(avail_modules, parser)
+        elif basic_config.mode == 'simple':
+            simple_modules = [module for module in available_modules(with_manifest=True) if not module.requires_setup]
+            self.add_module_args(simple_modules, parser)
+
+            # for simple mode, we use the cli_feeder and any modules that don't require setup
+            yaml_config['steps']['feeders'] = ['cli_feeder']
+
+            # add them to the config
+            for module in simple_modules:
+                for module_type in module.type:
+                    yaml_config['steps'].setdefault(f"{module_type}s", []).append(module.name)
+        else:
+            # load all modules, they're not using the 'simple' mode
+            self.add_module_args(available_modules(with_manifest=True), parser)
+
+        parser.set_defaults(**to_dot_notation(yaml_config))
+
+        # reload the parser with the new arguments, now that we have them
+        parsed, unknown = parser.parse_known_args(unused_args)
+
+        # merge the new config with the old one
+        self.config = merge_dicts(vars(parsed), yaml_config)
+        # clean out args from the base_parser that we don't want in the config
+        for key in vars(basic_config):
+            self.config.pop(key, None)
+
+        # setup the logging
+        self.setup_logging()
+
+        if unknown:
+            logger.warning(f"Ignoring unknown/unused arguments: {unknown}\nPerhaps you don't have this module enabled?")
+
+        if (self.config != yaml_config and basic_config.store) or not os.path.isfile(basic_config.config_file):
+            logger.info(f"Storing configuration file to {basic_config.config_file}")
+            store_yaml(self.config, basic_config.config_file)
+
+        return self.config
+
+    def add_additional_args(self, parser: argparse.ArgumentParser = None):
+        if not parser:
+            parser = self.parser
+
+        # allow passing URLs directly on the command line
+        parser.add_argument('urls', nargs='*', default=[], help='URL(s) to archive, either a single URL or a list of urls, should not come from config.yaml')
+
+        parser.add_argument('--feeders', dest='steps.feeders', nargs='+', default=['cli_feeder'], help='the feeders to use', action=UniqueAppendAction)
+        parser.add_argument('--enrichers', dest='steps.enrichers', nargs='+', help='the enrichers to use', action=UniqueAppendAction)
+        parser.add_argument('--extractors', dest='steps.extractors', nargs='+', help='the extractors to use', action=UniqueAppendAction)
+        parser.add_argument('--databases', dest='steps.databases', nargs='+', help='the databases to use', action=UniqueAppendAction)
+        parser.add_argument('--storages', dest='steps.storages', nargs='+', help='the storages to use', action=UniqueAppendAction)
+        parser.add_argument('--formatters', dest='steps.formatters', nargs='+', help='the formatter to use', action=UniqueAppendAction)
+
+        parser.add_argument('--authentication', dest='authentication', help='A dictionary of sites and their authentication methods \
+                                                                            (token, username etc.) that extractors can use to log into \
+                                                                            a website. If passing this on the command line, use a JSON string. \
+                                                                            You may also pass a path to a valid JSON/YAML file which will be parsed.',
+                                                                            default={},
+                                                                            action=AuthenticationJsonParseAction)
+        # logging arguments
+        parser.add_argument('--logging.level', action='store', dest='logging.level', choices=['INFO', 'DEBUG', 'ERROR', 'WARNING'], help='the logging level to use', default='INFO', type=str.upper)
+        parser.add_argument('--logging.file', action='store', dest='logging.file', help='the logging file to write to', default=None)
+        parser.add_argument('--logging.rotation', action='store', dest='logging.rotation', help='the logging rotation to use', default=None)
+
+    def add_module_args(self, modules: list[LazyBaseModule] = None, parser: argparse.ArgumentParser = None) -> None:
+
+        if not modules:
+            modules = available_modules(with_manifest=True)
+
+        module: LazyBaseModule
+        for module in modules:
+
+            if not module.configs:
+                # this module has no configs, don't show anything in the help
+                # (TODO: do we want to show something about this module though, like a description?)
+                continue
+
+            group = parser.add_argument_group(module.display_name or module.name, f"{module.description[:100]}...")
+
+            for name, kwargs in module.configs.items():
+                if not kwargs.get('metavar', None):
+                    # make a nicer metavar, metavar is what's used in the help, e.g. --cli_feeder.urls [METAVAR]
+                    kwargs['metavar'] = name.upper()
+
+                if kwargs.get('required', False):
+                    # required args shouldn't have a 'default' value, remove it
+                    kwargs.pop('default', None)
+
+                kwargs.pop('cli_set', None)
+                should_store = kwargs.pop('should_store', False)
+                kwargs['dest'] = f"{module.name}.{kwargs.pop('dest', name)}"
+                try:
+                    kwargs['type'] = getattr(validators, kwargs.get('type', '__invalid__'))
+                except AttributeError:
+                    kwargs['type'] = __builtins__.get(kwargs.get('type'), str)
+                arg = group.add_argument(f"--{module.name}.{name}", **kwargs)
+                arg.should_store = should_store
+
+    def show_help(self, basic_config: dict):
+        # for the help message, we want to load *all* possible modules and show the help
+        # add configs as arg parser arguments
+
+        self.add_additional_args(self.basic_parser)
+        self.add_module_args(parser=self.basic_parser)
+        self.basic_parser.print_help()
+        self.basic_parser.exit()
+
+    def setup_logging(self):
+        # setup loguru logging
+        logger.remove(0)  # remove the default logger
+        logging_config = self.config['logging']
+        logger.add(sys.stderr, level=logging_config['level'])
+        if log_file := logging_config['file']:
+            logger.add(log_file) if not logging_config['rotation'] else logger.add(log_file, rotation=logging_config['rotation'])
+
+    def install_modules(self, modules_by_type):
+        """
+        Traverses all modules in 'steps' and loads them into the orchestrator, storing them in the 
+        orchestrator's attributes (self.feeders, self.extractors etc.). If no modules of a certain type
+        are loaded, the program will exit with an error message.
+        """
+
+        invalid_modules = []
+        for module_type in BaseModule.MODULE_TYPES:
+
+            step_items = []
+            modules_to_load = modules_by_type[f"{module_type}s"]
+            assert modules_to_load, f"No {module_type}s were configured. Make sure to set at least one {module_type} in your configuration file or on the command line (using --{module_type}s)"
+
+            def check_steps_ok():
+                if not len(step_items):
+                    logger.error(f"NO {module_type.upper()}S LOADED. Please check your configuration and try again.")
+                    if len(modules_to_load):
+                        logger.error(f"Tried to load the following modules, but none were available: {modules_to_load}")
+                    exit()
+
+                if (module_type == 'feeder' or module_type == 'formatter') and len(step_items) > 1:
+                    logger.error(f"Only one {module_type} is allowed, found {len(step_items)} {module_type}s. Please remove one of the following from your configuration file: {modules_to_load}")
+                    exit()
+
+            for module in modules_to_load:
+                if module == 'cli_feeder':
+                    # pseudo module, don't load it
+                    urls = self.config['urls']
+                    if not urls:
+                        logger.error("No URLs provided. Please provide at least one URL via the command line, or set up an alternative feeder. Use --help for more information.")
+                        exit()
+                    # cli_feeder is a pseudo module, it just takes the command line args
+
+                    def feed(self) -> Generator[Metadata]:
+                        for url in urls:
+                            logger.debug(f"Processing URL: '{url}'")
+                            yield Metadata().set_url(url)
+
+                    pseudo_module = type('CLIFeeder', (Feeder,), {
+                        'name': 'cli_feeder',
+                        'display_name': 'CLI Feeder',
+                        '__iter__': feed
+
+                    })()
+
+                    pseudo_module.__iter__ = feed
+                    step_items.append(pseudo_module)
+                    continue
+
+                if module in invalid_modules:
+                    continue
+                try:
+                    loaded_module: BaseModule = get_module(module, self.config)
+                except (KeyboardInterrupt, Exception) as e:
+                    logger.error(f"Error during setup of modules: {e}\n{traceback.format_exc()}")
+                    if module_type == 'extractor' and loaded_module.name == module:
+                        loaded_module.cleanup()
+                    exit()
+
+                if not loaded_module:
+                    invalid_modules.append(module)
+                    continue
+                if loaded_module:
+                    step_items.append(loaded_module)
+
+            check_steps_ok()
+            setattr(self, f"{module_type}s", step_items)
+
+    def load_config(self, config_file: str) -> dict:
+        if not os.path.exists(config_file) and config_file != DEFAULT_CONFIG_FILE:
+            logger.error(f"The configuration file {config_file} was  not found. Make sure the file exists and try again, or run without the --config file to use the default settings.")
+            exit()
+
+        return read_yaml(config_file)
+
+    def run(self, args: list) -> Generator[Metadata]:
+
+        self.setup_basic_parser()
+
+        # parse the known arguments for now (basically, we want the config file)
+        basic_config, unused_args = self.basic_parser.parse_known_args(args)
+
+        # setup any custom module paths, so they'll show in the help and for arg parsing
+        setup_paths(basic_config.module_paths)
+
+        # if help flag was called, then show the help
+        if basic_config.help:
+            self.show_help(basic_config)
+
+        yaml_config = self.load_config(basic_config.config_file)
+        self.setup_complete_parser(basic_config, yaml_config, unused_args)
+
+        logger.info(f"======== Welcome to the AUTO ARCHIVER ({__version__}) ==========")
+        self.install_modules(self.config['steps'])
+
+        # log out the modules that were loaded
+        for module_type in BaseModule.MODULE_TYPES:
+            logger.info(f"{module_type.upper()}S: " + ", ".join(m.display_name for m in getattr(self, f"{module_type}s")))
+
+        for result in self.feed():
+            yield result
+
+    def cleanup(self) -> None:
+        logger.info("Cleaning up")
+        for e in self.extractors:
+            e.cleanup()
 
     def feed(self) -> Generator[Metadata]:
-        for item in self.feeder:
-            yield self.feed_item(item)
+
+        url_count = 0
+        for feeder in self.feeders:
+            for item in feeder:
+                yield self.feed_item(item)
+                url_count += 1
+
+        logger.success(f"Processed {url_count} URL(s)")
+        self.cleanup()
 
     def feed_item(self, item: Metadata) -> Metadata:
+        """
+        Takes one item (URL) to archive and calls self.archive, additionally:
+            - catches keyboard interruptions to do a clean exit
+            - catches any unexpected error, logs it, and does a clean exit
+        """
+        tmp_dir: TemporaryDirectory = None
         try:
-            ArchivingContext.reset()
-            with tempfile.TemporaryDirectory(dir="./") as tmp_dir:
-                ArchivingContext.set_tmp_dir(tmp_dir)
-                return self.archive(item)
+            tmp_dir = TemporaryDirectory(dir="./")
+            # set tmp_dir on all modules
+            for m in self.all_modules:
+                m.tmp_dir = tmp_dir.name
+            return self.archive(item)
         except KeyboardInterrupt:
             # catches keyboard interruptions to do a clean exit
             logger.warning(f"caught interrupt on {item=}")
-            for d in self.databases: d.aborted(item)
+            for d in self.databases:
+                d.aborted(item)
+            self.cleanup()
             exit()
         except Exception as e:
             logger.error(f'Got unexpected error on item {item}: {e}\n{traceback.format_exc()}')
-            for d in self.databases: d.failed(item)
-
-        # how does this handle the parameters like folder which can be different for each archiver?
-        # the storage needs to know where to archive!!
-        # solution: feeders have context: extra metadata that they can read or ignore,
-        # all of it should have sensible defaults (eg: folder)
-        # default feeder is a list with 1 element
+            for d in self.databases:
+                if type(e) == AssertionError:
+                    d.failed(item, str(e))
+                else:
+                    d.failed(item, reason="unexpected error")
+        finally:
+            if tmp_dir:
+                # remove the tmp_dir from all modules
+                for m in self.all_modules:
+                    m.tmp_dir = None
+                tmp_dir.cleanup()
 
     def archive(self, result: Metadata) -> Union[Metadata, None]:
-        original_url = result.get_url()
+        """
+            Runs the archiving process for a single URL
+            1. Each archiver can sanitize its own URLs
+            2. Check for cached results in Databases, and signal start to the databases
+            3. Call Archivers until one succeeds
+            4. Call Enrichers
+            5. Store all downloaded/generated media
+            6. Call selected Formatter and store formatted if needed
+        """
 
-        # 1 - cleanup
-        # each archiver is responsible for cleaning/expanding its own URLs
+        original_url = result.get_url().strip()
+        try:
+            self.assert_valid_url(original_url)
+        except AssertionError as e:
+            logger.error(f"Error archiving URL {original_url}: {e}")
+            raise e
+
+        # 1 - sanitize - each archiver is responsible for cleaning/expanding its own URLs
         url = original_url
-        for a in self.archivers: url = a.sanitize_url(url)
+        for a in self.extractors:
+            url = a.sanitize_url(url)
+
         result.set_url(url)
         if original_url != url: result.set("original_url", original_url)
 
-        # 2 - notify start to DB
-        # signal to DB that archiving has started
-        # and propagate already archived if it exists
+        # 2 - notify start to DBs, propagate already archived if feature enabled in DBs
         cached_result = None
         for d in self.databases:
-            # are the databases to decide whether to archive?
-            # they can simply return True by default, otherwise they can avoid duplicates. should this logic be more granular, for example on the archiver level: a tweet will not need be scraped twice, whereas an instagram profile might. the archiver could not decide from the link which parts to archive,
-            # instagram profile example: it would always re-archive everything
-            # maybe the database/storage could use a hash/key to decide if there's a need to re-archive
             d.started(result)
-            if (local_result := d.fetch(result)):
-                cached_result = (cached_result or Metadata()).merge(local_result)
+            if local_result := d.fetch(result):
+                cached_result = (cached_result or Metadata()).merge(local_result).merge(result)
         if cached_result:
             logger.debug("Found previously archived entry")
             for d in self.databases:
-                d.done(cached_result, cached=True)
+                try: d.done(cached_result, cached=True)
+                except Exception as e:
+                    logger.error(f"ERROR database {d.name}: {e}: {traceback.format_exc()}")
             return cached_result
 
-        # 3 - call archivers until one succeeds
-        for a in self.archivers:
-            logger.info(f"Trying archiver {a.name} for {url}")
+        # 3 - call extractors until one succeeds
+        for a in self.extractors:
+            logger.info(f"Trying extractor {a.name} for {url}")
             try:
-                # Q: should this be refactored so it's just a.download(result)?
                 result.merge(a.download(result))
                 if result.is_success(): break
-            except Exception as e: logger.error(f"Unexpected error with archiver {a.name}: {e}: {traceback.format_exc()}")
+            except Exception as e:
+                logger.error(f"ERROR archiver {a.name}: {e}: {traceback.format_exc()}")
 
-        # what if an archiver returns multiple entries and one is to be part of HTMLgenerator?
-        # should it call the HTMLgenerator as if it's not an enrichment?
-        # eg: if it is enable: generates an HTML with all the returned media, should it include enrichers? yes
-        # then how to execute it last? should there also be post-processors? are there other examples?
-        # maybe as a PDF? or a Markdown file
-
-        # 4 - call enrichers: have access to archived content, can generate metadata and Media
-        # eg: screenshot, wacz, webarchive, thumbnails
+        # 4 - call enrichers to work with archived content
         for e in self.enrichers:
             try: e.enrich(result)
-            except Exception as exc: logger.error(f"Unexpected error with enricher {e.name}: {exc}: {traceback.format_exc()}")
-
-        # 5 - store media
-        # looks for Media in result.media and also result.media[x].properties (as list or dict values)
-        result.store()
+            except Exception as exc:
+                logger.error(f"ERROR enricher {e.name}: {exc}: {traceback.format_exc()}")
 
+        # 5 - store all downloaded/generated media
+        result.store(storages=self.storages)
 
         # 6 - format and store formatted if needed
-        # enrichers typically need access to already stored URLs etc
-        if (final_media := self.formatter.format(result)):
-            final_media.store(url=url)
+        final_media: Media
+        if final_media := self.formatters[0].format(result):
+            final_media.store(url=url, metadata=result, storages=self.storages)
             result.set_final_media(final_media)
 
         if result.is_empty():
             result.status = "nothing archived"
 
-        # signal completion to databases (DBs, Google Sheets, CSV, ...)
-        for d in self.databases: d.done(result)
+        # signal completion to databases and archivers
+        for d in self.databases:
+            try: d.done(result)
+            except Exception as e:
+                logger.error(f"ERROR database {d.name}: {e}: {traceback.format_exc()}")
 
         return result
+
+    def assert_valid_url(self, url: str) -> bool:
+        """
+        Blocks localhost, private, reserved, and link-local IPs and all non-http/https schemes.
+        """
+        assert url.startswith("http://") or url.startswith("https://"), f"Invalid URL scheme"
+
+        parsed = urlparse(url)
+        assert parsed.scheme in ["http", "https"], f"Invalid URL scheme"
+        assert parsed.hostname, f"Invalid URL hostname"
+        assert parsed.hostname != "localhost", f"Invalid URL"
+
+        try:  # special rules for IP addresses
+            ip = ip_address(parsed.hostname)
+        except ValueError: pass
+        else:
+            assert ip.is_global, f"Invalid IP used"
+            assert not ip.is_reserved, f"Invalid IP used"
+            assert not ip.is_link_local, f"Invalid IP used"
+            assert not ip.is_private, f"Invalid IP used"
+
+    # Helper Properties
+
+    @property
+    def all_modules(self) -> List[Type[BaseModule]]:
+        return self.feeders + self.extractors + self.enrichers + self.databases + self.storages + self.formatters

src/auto_archiver/core/step.py

@@ -1,37 +0,0 @@
-from __future__ import annotations
-from dataclasses import dataclass
-from inspect import ClassFoundException
-from typing import Type
-from abc import ABC
-
-
-@dataclass
-class Step(ABC):
-    name: str = None
-
-    def __init__(self, config: dict) -> None:
-        # reads the configs into object properties
-        # self.config = config[self.name]
-        for k, v in config.get(self.name, {}).items():
-            self.__setattr__(k, v)
-
-    @staticmethod
-    def configs() -> dict: return {}
-
-    def init(name: str, config: dict, child: Type[Step]) -> Step:
-        """
-        looks into direct subclasses of child for name and returns such an object
-        TODO: cannot find subclasses of child.subclasses
-        """
-        for sub in child.__subclasses__():
-            if sub.name == name:
-                return sub(config)
-        raise ClassFoundException(f"Unable to initialize STEP with {name=}, check your configuration file/step names, and make sure you made the step discoverable by putting it into __init__.py")
-
-    def assert_valid_string(self, prop: str) -> None:
-        """
-        receives a property name an ensures it exists and is a valid non-empty string, raises an exception if not
-        """
-        assert hasattr(self, prop), f"property {prop} not found"
-        s = getattr(self, prop)
-        assert s is not None and type(s) == str and len(s) > 0, f"invalid property {prop} value '{s}', it should be a valid string"

src/auto_archiver/core/storage.py

@@ -0,0 +1,83 @@
+"""
+Base module for Storage modules – modular components that store media objects in various locations.
+"""
+
+from __future__ import annotations
+from abc import abstractmethod
+from typing import IO
+import os
+
+from loguru import logger
+from slugify import slugify
+
+from auto_archiver.utils.misc import random_str
+
+from auto_archiver.core import Media, BaseModule, Metadata
+from auto_archiver.modules.hash_enricher.hash_enricher import HashEnricher
+from auto_archiver.core.module import get_module
+class Storage(BaseModule):
+    
+    """
+    Base class for implementing storage modules in the media archiving framework.
+
+    Subclasses must implement the `get_cdn_url` and `uploadf` methods to define their behavior.
+    """
+
+    def store(self, media: Media, url: str, metadata: Metadata=None) -> None:
+        if media.is_stored(in_storage=self): 
+            logger.debug(f"{media.key} already stored, skipping")
+            return
+        self.set_key(media, url, metadata)
+        self.upload(media, metadata=metadata)
+        media.add_url(self.get_cdn_url(media))
+
+    @abstractmethod
+    def get_cdn_url(self, media: Media) -> str:
+        """
+        Returns the URL of the media object stored in the CDN.
+        """
+        pass
+
+    @abstractmethod
+    def uploadf(self, file: IO[bytes], key: str, **kwargs: dict) -> bool:
+        """
+        Uploads (or saves) a file to the storage service/location.
+        """
+        pass
+
+    def upload(self, media: Media, **kwargs) -> bool:
+        logger.debug(f'[{self.__class__.__name__}] storing file {media.filename} with key {media.key}')
+        with open(media.filename, 'rb') as f:
+            return self.uploadf(f, media, **kwargs)
+
+    def set_key(self, media: Media, url, metadata: Metadata) -> None:
+        """takes the media and optionally item info and generates a key"""
+        if media.key is not None and len(media.key) > 0: return
+        folder = metadata.get_context('folder', '')
+        filename, ext = os.path.splitext(media.filename)
+
+        # Handle path_generator logic
+        path_generator = self.config.get("path_generator", "url")
+        if path_generator == "flat":
+            path = ""
+            filename = slugify(filename)  # Ensure filename is slugified
+        elif path_generator == "url":
+            path = slugify(url)
+        elif path_generator == "random":
+            path = self.config.get("random_path", random_str(24), True)
+        else:
+            raise ValueError(f"Invalid path_generator: {path_generator}")
+
+        # Handle filename_generator logic
+        filename_generator = self.config.get("filename_generator", "random")
+        if filename_generator == "random":
+            filename = random_str(24)
+        elif filename_generator == "static":
+            # load the hash_enricher module
+            he = get_module(HashEnricher, self.config)
+            hd = he.calculate_hash(media.filename)
+            filename = hd[:24]
+        else:
+            raise ValueError(f"Invalid filename_generator: {filename_generator}")
+
+        media.key = os.path.join(folder, path, f"{filename}{ext}")

src/auto_archiver/core/validators.py

@@ -0,0 +1,19 @@
+# used as validators for config values. Should raise an exception if the value is invalid.
+from pathlib import Path
+import argparse
+
+def example_validator(value):
+    if "example" not in value:
+        raise argparse.ArgumentTypeError(f"{value} is not a valid value for this argument")
+    return value
+
+def positive_number(value):
+    if value < 0:
+        raise argparse.ArgumentTypeError(f"{value} is not a positive number")
+    return value
+
+
+def valid_file(value):
+    if not Path(value).is_file():
+        raise argparse.ArgumentTypeError(f"File '{value}' does not exist.")
+    return value

src/auto_archiver/databases/__init__.py

@@ -1,5 +0,0 @@
-from .database import Database
-from .gsheet_db import GsheetsDb
-from .console_db import ConsoleDb
-from .csv_db import CSVDb
-from .api_db import AAApiDb

src/auto_archiver/databases/api_db.py

@@ -1,66 +0,0 @@
-from typing import Union
-import requests, os
-from loguru import logger
-
-from . import Database
-from ..core import Metadata
-
-
-class AAApiDb(Database):
-    """
-        Connects to auto-archiver-api instance
-    """
-    name = "auto_archiver_api_db"
-
-    def __init__(self, config: dict) -> None:
-        # without this STEP.__init__ is not called
-        super().__init__(config)
-        self.allow_rearchive = bool(self.allow_rearchive)
-        self.assert_valid_string("api_endpoint")
-        self.assert_valid_string("api_secret")
-
-    @staticmethod
-    def configs() -> dict:
-        return {
-            "api_endpoint": {"default": None, "help": "API endpoint where calls are made to"},
-            "api_secret": {"default": None, "help": "API Basic authentication secret [deprecating soon]"},
-            "api_token": {"default": None, "help": "API Bearer token, to be preferred over secret (Basic auth) going forward"},
-            "public": {"default": False, "help": "whether the URL should be publicly available via the API"},
-            "author_id": {"default": None, "help": "which email to assign as author"},
-            "group_id": {"default": None, "help": "which group of users have access to the archive in case public=false as author"},
-            "allow_rearchive": {"default": True, "help": "if False then the API database will be queried prior to any archiving operations and stop if the link has already been archived"},
-            "tags": {"default": [], "help": "what tags to add to the archived URL", "cli_set": lambda cli_val, cur_val: set(cli_val.split(","))},
-        }
-    def fetch(self, item: Metadata) -> Union[Metadata, bool]:
-        """ query the database for the existence of this item"""
-        if not self.allow_rearchive: return
-        
-        params = {"url": item.get_url(), "limit": 1}
-        headers = {"Authorization": f"Bearer {self.api_token}", "accept": "application/json"}
-        response = requests.get(os.path.join(self.api_endpoint, "tasks/search-url"), params=params, headers=headers)
-
-        if response.status_code == 200:
-            logger.success(f"API returned a previously archived instance: {response.json()}")
-            # TODO: can we do better than just returning the first result?
-            return Metadata.from_dict(response.json()[0]["result"])
-        
-        logger.error(f"AA API FAIL ({response.status_code}): {response.json()}")
-        return False
-
-
-    def done(self, item: Metadata, cached: bool=False) -> None:
-        """archival result ready - should be saved to DB"""
-        if cached: 
-            logger.debug(f"skipping saving archive of {item.get_url()} to the AA API because it was cached")
-            return
-        logger.debug(f"saving archive of {item.get_url()} to the AA API.")
-
-        payload = {'result': item.to_json(), 'public': self.public, 'author_id': self.author_id, 'group_id': self.group_id, 'tags': list(self.tags)}
-        response = requests.post(os.path.join(self.api_endpoint, "submit-archive"), json=payload, auth=("abc", self.api_secret))
-
-        if response.status_code == 200:
-            logger.success(f"AA API: {response.json()}")
-        else:
-            logger.error(f"AA API FAIL ({response.status_code}): {response.json()}")
-
-    

src/auto_archiver/databases/database.py

@@ -1,41 +0,0 @@
-from __future__ import annotations
-from dataclasses import dataclass
-from abc import abstractmethod, ABC
-from typing import Union
-
-from ..core import Metadata, Step
-
-
-@dataclass
-class Database(Step, ABC):
-    name = "database"
-
-    def __init__(self, config: dict) -> None:
-        # without this STEP.__init__ is not called
-        super().__init__(config)
-
-    def init(name: str, config: dict) -> Database:
-        # only for typing...
-        return Step.init(name, config, Database)
-
-    def started(self, item: Metadata) -> None:
-        """signals the DB that the given item archival has started"""
-        pass
-
-    def failed(self, item: Metadata) -> None:
-        """update DB accordingly for failure"""
-        pass
-
-    def aborted(self, item: Metadata) -> None:
-        """abort notification if user cancelled after start"""
-        pass
-
-    # @abstractmethod
-    def fetch(self, item: Metadata) -> Union[Metadata, bool]:
-        """check if the given item has been archived already"""
-        return False
-
-    @abstractmethod
-    def done(self, item: Metadata, cached: bool=False) -> None:
-        """archival result ready - should be saved to DB"""
-        pass

src/auto_archiver/databases/gsheet_db.py

@@ -1,107 +0,0 @@
-from typing import Union, Tuple
-import datetime
-from urllib.parse import quote
-
-from loguru import logger
-
-from . import Database
-from ..core import Metadata, Media, ArchivingContext
-from ..utils import GWorksheet
-
-
-class GsheetsDb(Database):
-    """
-        NB: only works if GsheetFeeder is used. 
-        could be updated in the future to support non-GsheetFeeder metadata 
-    """
-    name = "gsheet_db"
-
-    def __init__(self, config: dict) -> None:
-        # without this STEP.__init__ is not called
-        super().__init__(config)
-
-    @staticmethod
-    def configs() -> dict:
-        return {}
-
-    def started(self, item: Metadata) -> None:
-        logger.warning(f"STARTED {item}")
-        gw, row = self._retrieve_gsheet(item)
-        gw.set_cell(row, 'status', 'Archive in progress')
-
-    def failed(self, item: Metadata) -> None:
-        logger.error(f"FAILED {item}")
-        self._safe_status_update(item, 'Archive failed')
-
-    def aborted(self, item: Metadata) -> None:
-        logger.warning(f"ABORTED {item}")
-        self._safe_status_update(item, '')
-
-    def fetch(self, item: Metadata) -> Union[Metadata, bool]:
-        """check if the given item has been archived already"""
-        return False
-
-    def done(self, item: Metadata, cached: bool=False) -> None:
-        """archival result ready - should be saved to DB"""
-        logger.success(f"DONE {item.get_url()}")
-        gw, row = self._retrieve_gsheet(item)
-        # self._safe_status_update(item, 'done')
-
-        cell_updates = []
-        row_values = gw.get_row(row)
-
-        def batch_if_valid(col, val, final_value=None):
-            final_value = final_value or val
-            try:
-                if val and gw.col_exists(col) and gw.get_cell(row_values, col) == '':
-                    cell_updates.append((row, col, final_value))
-            except Exception as e:
-                logger.error(f"Unable to batch {col}={final_value} due to {e}")
-        status_message = item.status
-        if cached:
-            status_message = f"[cached] {status_message}"
-        cell_updates.append((row, 'status', status_message))
-
-        media: Media = item.get_final_media()
-        if hasattr(media, "urls"):
-            batch_if_valid('archive', "\n".join(media.urls))
-        batch_if_valid('date', True, datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc).isoformat())
-        batch_if_valid('title', item.get_title())
-        batch_if_valid('text', item.get("content", ""))
-        batch_if_valid('timestamp', item.get_timestamp())
-        if media: batch_if_valid('hash', media.get("hash", "not-calculated"))
-
-        # merge all pdq hashes into a single string, if present
-        pdq_hashes = []
-        all_media = item.get_all_media()
-        for m in all_media:
-            if pdq := m.get("pdq_hash"):
-                pdq_hashes.append(pdq)
-        if len(pdq_hashes):
-            batch_if_valid('pdq_hash', ",".join(pdq_hashes))
-
-        if (screenshot := item.get_media_by_id("screenshot")) and hasattr(screenshot, "urls"):
-            batch_if_valid('screenshot', "\n".join(screenshot.urls))
-
-        if (thumbnail := item.get_first_image("thumbnail")):
-            if hasattr(thumbnail, "urls"):
-                batch_if_valid('thumbnail', f'=IMAGE("{thumbnail.urls[0]}")')
-
-        if (browsertrix := item.get_media_by_id("browsertrix")):
-            batch_if_valid('wacz', "\n".join(browsertrix.urls))
-            batch_if_valid('replaywebpage', "\n".join([f'https://replayweb.page/?source={quote(wacz)}#view=pages&url={quote(item.get_url())}' for wacz in browsertrix.urls]))
-
-        gw.batch_set_cell(cell_updates)
-
-    def _safe_status_update(self, item: Metadata, new_status: str) -> None:
-        try:
-            gw, row = self._retrieve_gsheet(item)
-            gw.set_cell(row, 'status', new_status)
-        except Exception as e:
-            logger.debug(f"Unable to update sheet: {e}")
-
-    def _retrieve_gsheet(self, item: Metadata) -> Tuple[GWorksheet, int]:
-        # TODO: to make gsheet_db less coupled with gsheet_feeder's "gsheet" parameter, this method could 1st try to fetch "gsheet" from ArchivingContext and, if missing, manage its own singleton - not needed for now
-        gw: GWorksheet = ArchivingContext.get("gsheet").get("worksheet")
-        row: int = ArchivingContext.get("gsheet").get("row")
-        return gw, row

src/auto_archiver/enrichers/__init__.py

@@ -1,9 +0,0 @@
-from .enricher import Enricher
-from .screenshot_enricher import ScreenshotEnricher 
-from .wayback_enricher import WaybackArchiverEnricher
-from .hash_enricher import HashEnricher
-from .thumbnail_enricher import ThumbnailEnricher
-from .wacz_enricher import WaczArchiverEnricher
-from .whisper_enricher import WhisperEnricher
-from .pdq_hash_enricher import PdqHashEnricher
-from .metadata_enricher import MetadataEnricher

src/auto_archiver/enrichers/enricher.py

@@ -1,20 +0,0 @@
-from __future__ import annotations
-from dataclasses import dataclass
-from abc import abstractmethod, ABC
-from ..core import Metadata, Step
-
-@dataclass
-class Enricher(Step, ABC):
-    name = "enricher"
-
-    def __init__(self, config: dict) -> None:
-        # without this STEP.__init__ is not called
-        super().__init__(config)
-        
-
-    # only for typing...
-    def init(name: str, config: dict) -> Enricher:
-        return Step.init(name, config, Enricher)
-
-    @abstractmethod
-    def enrich(self, to_enrich: Metadata) -> None: pass

src/auto_archiver/enrichers/hash_enricher.py

@@ -1,49 +0,0 @@
-import hashlib
-from loguru import logger
-
-from . import Enricher
-from ..core import Metadata, ArchivingContext
-
-
-class HashEnricher(Enricher):
-    """
-    Calculates hashes for Media instances
-    """
-    name = "hash_enricher"
-
-    def __init__(self, config: dict) -> None:
-        # without this STEP.__init__ is not called
-        super().__init__(config)
-        algo_choices = self.configs()["algorithm"]["choices"]
-        assert self.algorithm in algo_choices, f"Invalid hash algorithm selected, must be one of {algo_choices} (you selected {self.algorithm})."
-        self.chunksize = int(self.chunksize)
-        ArchivingContext.set("hash_enricher.algorithm", self.algorithm, keep_on_reset=True)
-
-    @staticmethod
-    def configs() -> dict:
-        return {
-            "algorithm": {"default": "SHA-256", "help": "hash algorithm to use", "choices": ["SHA-256", "SHA3-512"]},
-            "chunksize": {"default": int(1.6e7), "help": "number of bytes to use when reading files in chunks (if this value is too large you will run out of RAM), default is 16MB"},
-        }
-
-    def enrich(self, to_enrich: Metadata) -> None:
-        url = to_enrich.get_url()
-        logger.debug(f"calculating media hashes for {url=} (using {self.algorithm})")
-
-        for i, m in enumerate(to_enrich.media):
-            if len(hd := self.calculate_hash(m.filename)):
-                to_enrich.media[i].set("hash", f"{self.algorithm}:{hd}")
-
-    def calculate_hash(self, filename) -> str:
-        hash = None
-        if self.algorithm == "SHA-256":
-            hash = hashlib.sha256()
-        elif self.algorithm == "SHA3-512":
-            hash = hashlib.sha3_512()
-        else: return ""
-        with open(filename, "rb") as f:
-            while True:
-                buf = f.read(self.chunksize)
-                if not buf: break
-                hash.update(buf)
-        return hash.hexdigest()

src/auto_archiver/enrichers/thumbnail_enricher.py

@@ -1,46 +0,0 @@
-import ffmpeg, os
-from loguru import logger
-
-from . import Enricher
-from ..core import Media, Metadata, ArchivingContext
-from ..utils.misc import random_str
-
-
-class ThumbnailEnricher(Enricher):
-    """
-    Generates thumbnails for all the media
-    """
-    name = "thumbnail_enricher"
-
-    def __init__(self, config: dict) -> None:
-        # without this STEP.__init__ is not called
-        super().__init__(config)
-
-    @staticmethod
-    def configs() -> dict:
-        return {}
-
-    def enrich(self, to_enrich: Metadata) -> None:
-        logger.debug(f"generating thumbnails")
-        for i, m in enumerate(to_enrich.media[::]):
-            if m.is_video():
-                folder = os.path.join(ArchivingContext.get_tmp_dir(), random_str(24))
-                os.makedirs(folder, exist_ok=True)
-                logger.debug(f"generating thumbnails for {m.filename}")
-                fps, duration = 0.5, m.get("duration")
-                if duration is not None:
-                    duration = float(duration)
-                    if duration < 60: fps = 10.0 / duration
-                    elif duration < 120: fps = 20.0 / duration
-                    else: fps = 40.0 / duration
-
-                stream = ffmpeg.input(m.filename)
-                stream = ffmpeg.filter(stream, 'fps', fps=fps).filter('scale', 512, -1)
-                stream.output(os.path.join(folder, 'out%d.jpg')).run()
-
-                thumbnails = os.listdir(folder)
-                thumbnails_media = []
-                for t, fname in enumerate(thumbnails):
-                    if fname[-3:] == 'jpg':
-                        thumbnails_media.append(Media(filename=os.path.join(folder, fname)).set("id", f"thumbnail_{t}"))
-                to_enrich.media[i].set("thumbnails", thumbnails_media)

src/auto_archiver/feeders/__init__.py

@@ -1,3 +0,0 @@
-from.feeder import Feeder
-from .gsheet_feeder import GsheetsFeeder
-from .cli_feeder import CLIFeeder

src/auto_archiver/feeders/cli_feeder.py

@@ -1,32 +0,0 @@
-from loguru import logger
-
-from . import Feeder
-from ..core import Metadata, ArchivingContext
-
-
-class CLIFeeder(Feeder):
-    name = "cli_feeder"
-
-    def __init__(self, config: dict) -> None:
-        # without this STEP.__init__ is not called
-        super().__init__(config)
-        if type(self.urls) != list or len(self.urls) == 0:
-            raise Exception("CLI Feeder did not receive any URL to process")
-
-    @staticmethod
-    def configs() -> dict:
-        return {
-            "urls": {
-                "default": None,
-                "help": "URL(s) to archive, either a single URL or a list of urls, should not come from config.yaml",
-                "cli_set": lambda cli_val, cur_val: list(set(cli_val.split(",")))
-            },
-        }
-
-    def __iter__(self) -> Metadata:
-        for url in self.urls:
-            logger.debug(f"Processing {url}")
-            yield Metadata().set_url(url)
-            ArchivingContext.set("folder", "cli")
-
-        logger.success(f"Processed {len(self.urls)} URL(s)")

src/auto_archiver/feeders/feeder.py

@@ -1,21 +0,0 @@
-from __future__ import annotations
-from dataclasses import dataclass
-from abc import abstractmethod
-from ..core import Metadata
-from ..core import Step
-
-
-@dataclass
-class Feeder(Step):
-    name = "feeder"
-
-    def __init__(self, config: dict) -> None:
-        # without this STEP.__init__ is not called
-        super().__init__(config)
-
-    def init(name: str, config: dict) -> Feeder:
-        # only for code typing
-        return Step.init(name, config, Feeder)
-
-    @abstractmethod
-    def __iter__(self) -> Metadata: return None

src/auto_archiver/feeders/gsheet_feeder.py

@@ -1,95 +0,0 @@
-import gspread, os
-
-from loguru import logger
-from slugify import slugify
-
-# from . import Enricher
-from . import Feeder
-from ..core import Metadata, ArchivingContext
-from ..utils import Gsheets, GWorksheet
-
-
-class GsheetsFeeder(Gsheets, Feeder):
-    name = "gsheet_feeder"
-
-    def __init__(self, config: dict) -> None:
-        # without this STEP.__init__ is not called
-        super().__init__(config)
-        self.gsheets_client = gspread.service_account(filename=self.service_account)
-
-    @staticmethod
-    def configs() -> dict:
-        return dict(
-            Gsheets.configs(),
-            ** {
-                "allow_worksheets": {
-                    "default": set(),
-                    "help": "(CSV) only worksheets whose name is included in allow are included (overrides worksheet_block), leave empty so all are allowed",
-                    "cli_set": lambda cli_val, cur_val: set(cli_val.split(","))
-                },
-                "block_worksheets": {
-                    "default": set(),
-                    "help": "(CSV) explicitly block some worksheets from being processed",
-                    "cli_set": lambda cli_val, cur_val: set(cli_val.split(","))
-                },
-                "use_sheet_names_in_stored_paths": {
-                    "default": True,
-                    "help": "if True the stored files path will include 'workbook_name/worksheet_name/...'",
-                }
-            })
-
-    def __iter__(self) -> Metadata:
-        sh = self.open_sheet()
-        for ii, wks in enumerate(sh.worksheets()):
-            if not self.should_process_sheet(wks.title):
-                logger.debug(f"SKIPPED worksheet '{wks.title}' due to allow/block rules")
-                continue
-
-            logger.info(f'Opening worksheet {ii=}: {wks.title=} header={self.header}')
-            gw = GWorksheet(wks, header_row=self.header, columns=self.columns)
-
-            if len(missing_cols := self.missing_required_columns(gw)):
-                logger.warning(f"SKIPPED worksheet '{wks.title}' due to missing required column(s) for {missing_cols}")
-                continue
-
-            for row in range(1 + self.header, gw.count_rows() + 1):
-                url = gw.get_cell(row, 'url').strip()
-                if not len(url): continue
-
-                original_status = gw.get_cell(row, 'status')
-                status = gw.get_cell(row, 'status', fresh=original_status in ['', None])
-                # TODO: custom status parser(?) aka should_retry_from_status
-                if status not in ['', None]: continue
-
-                # All checks done - archival process starts here
-                m = Metadata().set_url(url)
-                ArchivingContext.set("gsheet", {"row": row, "worksheet": gw}, keep_on_reset=True)
-                if gw.get_cell_or_default(row, 'folder', "") is None:
-                    folder = ''
-                else:
-                    folder = slugify(gw.get_cell_or_default(row, 'folder', "").strip())
-                if len(folder):
-                    if self.use_sheet_names_in_stored_paths:
-                        ArchivingContext.set("folder", os.path.join(folder, slugify(self.sheet), slugify(wks.title)), True)
-                    else:
-                        ArchivingContext.set("folder", folder, True)
-
-                yield m
-
-            logger.success(f'Finished worksheet {wks.title}')
-
-    def should_process_sheet(self, sheet_name: str) -> bool:
-        if len(self.allow_worksheets) and sheet_name not in self.allow_worksheets:
-            # ALLOW rules exist AND sheet name not explicitly allowed
-            return False
-        if len(self.block_worksheets) and sheet_name in self.block_worksheets:
-            # BLOCK rules exist AND sheet name is blocked
-            return False
-        return True
-
-    def missing_required_columns(self, gw: GWorksheet) -> list:
-        missing = []
-        for required_col in ['url', 'status']:
-            if not gw.col_exists(required_col):
-                missing.append(required_col)
-        return missing

src/auto_archiver/formatters/__init__.py

@@ -1,3 +0,0 @@
-from .formatter import Formatter
-from .html_formatter import HtmlFormatter
-from .mute_formatter import MuteFormatter

src/auto_archiver/formatters/formatter.py

@@ -1,20 +0,0 @@
-from __future__ import annotations
-from dataclasses import dataclass
-from abc import abstractmethod
-from ..core import Metadata, Media, Step
-
-
-@dataclass
-class Formatter(Step):
-    name = "formatter"
-
-    def __init__(self, config: dict) -> None:
-        # without this STEP.__init__ is not called
-        super().__init__(config)
-
-    def init(name: str, config: dict) -> Formatter:
-        # only for code typing
-        return Step.init(name, config, Formatter)
-
-    @abstractmethod
-    def format(self, item: Metadata) -> Media: return None

src/auto_archiver/formatters/mute_formatter.py

@@ -1,16 +0,0 @@
-from __future__ import annotations
-from dataclasses import dataclass
-
-from ..core import Metadata, Media
-from . import Formatter
-
-
-@dataclass
-class MuteFormatter(Formatter):
-    name = "mute_formatter"
-
-    def __init__(self, config: dict) -> None:
-        # without this STEP.__init__ is not called
-        super().__init__(config)
-
-    def format(self, item: Metadata) -> Media: return None

src/auto_archiver/formatters/templates/html_template.html

@@ -1,280 +0,0 @@
-{# templates/results.html #}
-{% import 'macros.html' as macros %}
-<!DOCTYPE html>
-<html lang="en">
-
-<head>
-    <meta charset="utf-8">
-    <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300italic,700,700italic">
-    <title>{{ url }}</title>
-    <style>
-        html {
-            font-family: 'Roboto', sans-serif;
-        }
-
-        table {
-            table-layout: fixed;
-            width: 90%;
-        }
-
-        table td {
-            word-wrap: break-word;
-            overflow-wrap: break-word;
-            padding: 5px;
-        }
-
-        table,
-        th,
-        td {
-            margin: auto;
-            border: 1px solid;
-            border-collapse: collapse;
-            vertical-align: top;
-        }
-
-        table.metadata td:first-child {
-            text-align: center;
-        }
-
-        table.content td:nth-child(2),
-        .center {
-            text-align: center;
-        }
-
-        .copy:hover {
-            background: aliceblue;
-            cursor: copy;
-        }
-
-        #notification {
-            position: fixed;
-            right: 20px;
-            top: 20px;
-            background: aquamarine;
-            box-shadow: 6px 8px 5px 0px #000000;
-            padding: 10px;
-            font-size: large;
-            display: none;
-        }
-
-        img,
-        video {
-            filter: gray;
-            -webkit-filter: grayscale(1);
-            filter: grayscale(1);
-        }
-
-        /* Disable grayscale on hover */
-        /* img:hover,
-        video:hover {
-            -webkit-filter: grayscale(0);
-            filter: none;
-        } */
-
-
-        .collapsible {
-            background-color: #777;
-            color: white;
-            cursor: pointer;
-            padding: 5px;
-            margin: 10px;
-            width: 100%;
-            border: none;
-            text-align: left;
-            outline: none;
-            font-size: 15px;
-        }
-
-        .active,
-        .collapsible:hover {
-            background-color: #555;
-        }
-
-        .collapsible-content {
-            padding: 0 18px;
-            display: none;
-            overflow: hidden;
-            background-color: #f1f1f1;
-        }
-    </style>
-</head>
-
-<body>
-    <div id="notification"></div>
-    <h2>Archived media for <a href="{{ url }}">{{ url }}</a></h2>
-    {% if title | string | length > 0 %}
-    <p><b>title:</b> '<span class="copy">{{ title }}</span>'</p>
-    {% endif %}
-    <h2 class="center">content {{ media | length }} item(s)</h2>
-    <form class="center">
-        <label>
-            <input type="checkbox" id="safe-media-view" checked>
-            Safe Media View
-        </label>
-    </form>
-    <table class="content">
-        <tr>
-            <th>about</th>
-            <th>preview(s)</th>
-        </tr>
-        <tbody>
-            {% for m in media %}
-            <tr>
-                <td>
-                    <ul>
-                        <li><b>key:</b> <span class="copy">{{ m.key }}</span></li>
-                        <li><b>type:</b> <span class="copy">{{ m.mimetype }}</span></li>
-
-                        {% for prop in m.properties %}
-
-                        {% if m.properties[prop] | is_list %}
-                        <p></p>
-                        <div>
-                            <b class="collapsible" title="expand">{{ prop }}:</b>
-                            <p></p>
-                            <div class="collapsible-content">
-                                {% for subprop in m.properties[prop] %}
-                                {% if subprop | is_media %}
-                                {{ macros.display_media(subprop, true, url) }}
-
-                                <ul>
-                                    {% for subprop_prop in subprop.properties %}
-                                    <li><b>{{ subprop_prop }}:</b>
-                                        {{ macros.copy_urlize(subprop.properties[subprop_prop]) }}</li>
-                                    {% endfor %}
-                                </ul>
-
-                                {% else %}
-                                {{ subprop }}
-                                {% endif %}
-                                {% endfor %}
-                            </div>
-                        </div>
-                        <p></p>
-                        {% elif m.properties[prop] | string | length > 1 %}
-                        <li><b>{{ prop }}:</b> {{ macros.copy_urlize(m.properties[prop]) }}</li>
-                        {% endif %}
-
-                        {% endfor %}
-                    </ul>
-                </td>
-                <td>
-                    {{ macros.display_media(m, true, url) }}
-                </td>
-            </tr>
-            {% endfor %}
-        </tbody>
-    </table>
-    <h2 class="center">metadata</h2>
-    <table class="metadata">
-        <tr>
-            <th>key</th>
-            <th>value</th>
-        </tr>
-        {% for key in metadata %}
-        <tr>
-            <td>{{ key }}</td>
-            <td>
-                {{ macros.copy_urlize(metadata[key]) }}
-            </td>
-        </tr>
-        {% endfor %}
-    </table>
-
-    <p style="text-align:center;">Made with <a
-            href="https://github.com/bellingcat/auto-archiver">bellingcat/auto-archiver</a> v{{ version }}</p>
-</body>
-<script defer>
-    // notification logic
-    const notification = document.getElementById("notification");
-
-    function showNotification(message, miliseconds) {
-        notification.style.display = "block";
-        notification.innerText = message;
-        setTimeout(() => {
-            notification.style.display = "none";
-            notification.innerText = "";
-        }, miliseconds || 1000)
-    }
-
-    // copy logic
-    Array.from(document.querySelectorAll(".copy")).forEach(el => {
-        el.onclick = () => {
-            document.execCommand("copy");
-        }
-        el.addEventListener("copy", (e) => {
-            e.preventDefault();
-            if (e.clipboardData) {
-                if (el.hasAttribute("copy-value")) {
-                    e.clipboardData.setData("text/plain", el.getAttribute("copy-value"));
-                } else {
-                    e.clipboardData.setData("text/plain", el.textContent);
-                }
-                console.log(e.clipboardData.getData("text"))
-                showNotification("copied!")
-            }
-        })
-    })
-
-    // collapsibles
-    let coll = document.getElementsByClassName("collapsible");
-    let i;
-
-    for (i = 0; i < coll.length; i++) {
-        coll[i].addEventListener("click", function () {
-            this.classList.toggle("active");
-            // let content = this.nextElementSibling;
-            let content = this.parentElement.querySelector(".collapsible-content");
-            if (content.style.display === "block") {
-                content.style.display = "none";
-            } else {
-                content.style.display = "block";
-            }
-        });
-    }
-
-    // logic for enabled/disabled greyscale
-    // Get references to the checkboxes and images/videos
-    const safeImageViewCheckbox = document.getElementById('safe-media-view');
-    const imagesVideos = document.querySelectorAll('img, video');
-
-    // Function to toggle grayscale effect
-    function toggleGrayscale() {
-        imagesVideos.forEach(element => {
-            if (safeImageViewCheckbox.checked) {
-                // Enable grayscale effect
-                element.style.filter = 'grayscale(1)';
-                element.style.webkitFilter = 'grayscale(1)';
-            } else {
-                // Disable grayscale effect
-                element.style.filter = 'none';
-                element.style.webkitFilter = 'none';
-            }
-        });
-    }
-
-    // Add event listener to the checkbox to trigger the toggleGrayscale function
-    safeImageViewCheckbox.addEventListener('change', toggleGrayscale);
-
-    // Handle the hover effect using JavaScript
-    imagesVideos.forEach(element => {
-        element.addEventListener('mouseenter', () => {
-            // Disable grayscale effect on hover
-            element.style.filter = 'none';
-            element.style.webkitFilter = 'none';
-        });
-
-        element.addEventListener('mouseleave', () => {
-            // Re-enable grayscale effect if checkbox is checked
-            if (safeImageViewCheckbox.checked) {
-                element.style.filter = 'grayscale(1)';
-                element.style.webkitFilter = 'grayscale(1)';
-            }
-        });
-    });
-
-    // Call the function on page load to apply the initial state
-    toggleGrayscale();
-</script>
-
-</html>

src/auto_archiver/formatters/templates/macros.html

@@ -1,74 +0,0 @@
-{% macro display_media(m, links, main_url) -%}
-
-{% for url in m.urls %}
-{% if url | length == 0 %}
-No URL available for {{ m.key }}.
-{% elif 'http' in url %}
-{% if 'image' in m.mimetype %}
-<div>
-    <a href="{{ url }}">
-        <img src="{{ url }}" style="max-height:400px;max-width:400px;"></img>
-    </a>
-
-    <div>
-        Reverse Image Search:&nbsp;
-        <a href="https://www.google.com/searchbyimage?sbisrc=4chanx&image_url={{ url | quote }}&safe=off">Google</a>,&nbsp;
-        <a href="https://lens.google.com/uploadbyurl?url={{ url | quote }}">Google Lens</a>,&nbsp;
-        <a href="https://yandex.ru/images/touch/search?rpt=imageview&url={{ url | quote }}">Yandex</a>,&nbsp;
-        <a href="https://www.bing.com/images/search?view=detailv2&iss=sbi&form=SBIVSP&sbisrc=UrlPaste&q=imgurl:{{ url | quote }}">Bing</a>,&nbsp;
-        <a href="https://www.tineye.com/search/?url={{ url | quote }}">Tineye</a>
-    </div>
-    <p></p>
-</div>
-{% elif 'video' in m.mimetype %}
-<div>
-    <video src="{{ url }}" controls style="max-height:400px;max-width:600px;">
-        Your browser does not support the video element.
-    </video>
-</div>
-{% elif 'audio' in m.mimetype %}
-<div>
-    <audio controls>
-        <source src="{{ url }}" type="{{ m.mimetype }}">
-        Your browser does not support the audio element.
-    </audio>
-</div>
-{% elif m.filename | get_extension == ".wacz" %}
-<a href="https://replayweb.page/?source={{ url | quote }}#view=pages&url={{ main_url }}">replayweb</a>
-{% else %}
-No preview available for {{ m.key }}.
-{% endif %}
-{% else %}
-{{ m.url | urlize }}
-{% endif %}
-{% if links %}
-<a href="{{ url }}">open</a> or
-<a href="{{ url }}" download="">download</a> or
-{{ copy_urlize(url, "copy") }}
-
-<br>
-{% endif %}
-{% endfor %}
-
-{%- endmacro -%}
-
-{% macro copy_urlize(val, href_text) -%}
-
-{% if val is mapping %}
-<ul>
-    {% for key in val %}
-    <li>
-        <b>{{ key }}:</b> {{ copy_urlize(val[key]) }}
-    </li>
-    {% endfor %}
-</ul>
-
-{% else %}
-{% if href_text  | length == 0 %}
-<span class="copy">{{ val | string | urlize }}</span>
-{% else %}
-<span class="copy" copy-value="{{val}}">{{ href_text | string | urlize }}</span>
-{% endif %}
-{% endif %}
-
-{%- endmacro -%}