makes orchestrator.run return the results to allow for code integration (#196)

* disables tests on ubuntu-latest

* drops fixed oscrypto version for git commit

* version bump

.github/workflows/docker-publish.yaml

@@ -0,0 +1,54 @@
+name: Docker
+
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+on:
+  release:
+    types: [published]
+
+env:
+  # Use docker.io for Docker Hub if empty
+  REGISTRY: ghcr.io
+  # github.repository as <account>/<repo>
+  IMAGE_NAME: ${{ github.repository }}
+
+
+jobs:
+  push_to_registry:
+    name: Push Docker image to Docker Hub
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v3
+        
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+      # https://github.com/docker/setup-buildx-action
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96
+        with:
+          images: bellingcat/auto-archiver
+      
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          platforms: linux/amd64,linux/arm64
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/python-publish.yaml

@@ -1,4 +1,4 @@
-# This workflow will upload a Python Package using Twine when a release is created
+# This workflow uploads a Python Package to PyPI using Poetry when a release is created
 # For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
 
 # This workflow uses actions that are not certified by GitHub.
@@ -6,41 +6,42 @@
 # separate terms of service, privacy policy, and support
 # documentation.
 
-name: Upload Python Package
+name: Pypi
 
 on:
   release:
     types: [published]
-  
-  push:
-    branches:
-      - dockerize
-    tags:
-      - 'v*.*.*'
 
 permissions:
   contents: read
 
 jobs:
   deploy:
-
+    name: Publish python package
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
-    - name: Set up Python 3.10
-      uses: actions/setup-python@v4
+    - name: Checkout Repository
+      uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v5
       with:
-        python-version: "3.10"
+        python-version-file: pyproject.toml
+
+    - name: Install Poetry
+      run: |
+        pipx install "poetry>=2.0.0,<3.0.0"
 
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade pip
-        pip install build
-    - name: Build package
-      run: python -m build
-    - name: Publish package
-      uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
-      with:
-        user: __token__
-        password: ${{ secrets.PYPI_API_TOKEN }}
+        poetry install --no-interaction --no-root
+
+    - name: Build the package
+      run: |
+        poetry build
+
+    # Step 6: Publish to PyPI
+    - name: Publish to PyPI
+      run: |
+        poetry publish --username __token__ --password ${{ secrets.PYPI_API_TOKEN }}

.github/workflows/tests-core.yaml

@@ -0,0 +1,43 @@
+name: Core Tests
+
+on:
+  push:
+    branches: [ main ]
+    paths:
+      - src/**
+  pull_request:
+    paths:
+      - src/**
+
+jobs:
+  tests:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+        os: [ubuntu-22.04]
+        #TODO: re-enable ubuntu-latest, this is disabled as oscrypto cannot be pinned to github commit and pushed to pypi
+    defaults:
+      run:
+        working-directory: ./
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Poetry
+        run: pipx install poetry
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: 'poetry'
+
+      - name: Install dependencies
+        run: poetry install --no-interaction --with dev
+
+      - name: Run Core Tests
+        run: |
+          poetry run auto-archiver --version || true
+          poetry run pytest -ra -v -m "not download"

.github/workflows/tests-download.yaml

@@ -0,0 +1,40 @@
+name: Download Tests
+
+on:
+  schedule:
+    - cron: '35 14 * * 1'
+  pull_request:
+    branches: [ main ]
+    paths:
+      - src/**
+
+jobs:
+  tests:
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10"] # only run expensive downloads on one (lowest) python version
+    defaults:
+      run:
+        working-directory: ./
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install poetry
+        run: pipx install poetry
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: 'poetry'
+
+      - name: Install dependencies
+        run: poetry install --no-interaction --with dev
+
+      - name: Run Download Tests
+        run: poetry run pytest -ra -v -x -m "download"
+        env:
+          TWITTER_BEARER_TOKEN: ${{ secrets.TWITTER_BEARER_TOKEN }}

.gitignore

@@ -27,4 +27,9 @@ instaloader.session
 orchestration.yaml
 auto_archiver.egg-info*
 logs*
-*.csv
+*.csv
+archived/
+dist*
+docs/_build/
+docs/source/autoapi/
+docs/source/modules/autogen/

.pylintrc

@@ -0,0 +1,3 @@
+[MAIN]
+
+ignore-patterns=(.*tests.*.py, __manifest__.py)

.readthedocs.yaml

@@ -0,0 +1,22 @@
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+# Required
+version: 2
+
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.10"
+  jobs:
+    post_install:
+      - pip install poetry
+      # https://python-poetry.org/docs/managing-dependencies/#dependency-groups
+      # VIRTUAL_ENV needs to be set manually for now.
+      # See https://github.com/readthedocs/readthedocs.org/pull/11152/
+      - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH poetry install --with docs
+
+
+sphinx:
+  configuration: docs/source/conf.py

CONTRIBUTING.md

@@ -0,0 +1,49 @@
+# Contributing to Auto Archiver
+
+Thank you for your interest in contributing to Auto Archiver! Your contributions help improve the project and make it more useful for everyone. Please follow the guidelines below to ensure a smooth collaboration.
+
+### 1. Reporting a Bug
+
+If you encounter a bug, please create an issue on GitHub with the following details:
+
+* Describe the bug: Provide a clear and concise description of the issue.
+* Steps to reproduce: Include the steps needed to reproduce the bug.
+* Expected behavior: Describe what you expected to happen.
+* Actual behavior: Explain what actually happened.
+* Screenshots/logs: If applicable, attach screenshots or logs to help diagnose the problem.
+* Environment: Mention the OS, Ruby version, and any other relevant details.
+
+### 2. Writing a Patch/Fix and Submitting Pull Requests
+
+If you’d like to fix a bug or improve existing code:
+
+1. Open a pull request on GitHub and link it to the relevant issue.
+2. Make sure to document your pull request with a clear description of what changes were made and why.
+3. Wait for review and make any requested changes.
+
+### 3. Creating New Modules
+
+If you want to add a new module to Auto Archiver:
+
+1. Ensure your module follows the existing [coding style and project structure](https://auto-archiver.readthedocs.io/en/latest/development/creating_modules.html).
+2. Write clear documentation explaining what your module does and how to use it.
+3. Ideally, include unit tests for your module!
+4. Follow the steps in Section 2 to submit a pull request.
+
+### 4. Do You Have Questions About the Source Code?
+
+If you have any questions about how the source code works or need help using Auto Archiver
+
+📝 Check the [Auto Archiver](https://auto-archiver.readthedocs.io/en/latest/) documentation.
+
+👉 Ask your questions in the [Bellingcat Discord](https://www.bellingcat.com/follow-bellingcat-on-social-media/).
+
+### 5. Do You Want to Contribute to the Documentation?
+
+We welcome contributions to the documentation!
+
+📖 Please read [Contributing to the Auto Archiver Documentation](https://auto-archiver.readthedocs.io/en/latest/development/docs.html) to learn how you can help improve the project's documentation.
+
+------------------
+
+Thank you for contributing to Auto Archiver! 🚀

Dockerfile

@@ -1,35 +1,60 @@
-# stage 1 - all dependencies
-From python:3.10
+FROM webrecorder/browsertrix-crawler:1.4.2 AS base
+
+ENV RUNNING_IN_DOCKER=1 \
+    LANG=C.UTF-8 \
+    LC_ALL=C.UTF-8 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONFAULTHANDLER=1 \
+    PATH="/root/.local/bin:$PATH"
+
+# Installing system dependencies
+RUN add-apt-repository ppa:mozillateam/ppa && \
+	apt-get update && \
+    apt-get install -y --no-install-recommends gcc ffmpeg fonts-noto exiftool && \
+	apt-get install -y --no-install-recommends firefox-esr && \
+    ln -s /usr/bin/firefox-esr /usr/bin/firefox && \
+    wget https://github.com/mozilla/geckodriver/releases/download/v0.35.0/geckodriver-v0.35.0-linux64.tar.gz && \
+    tar -xvzf geckodriver* -C /usr/local/bin && \
+    chmod +x /usr/local/bin/geckodriver && \
+    rm geckodriver-v* && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+
+# Poetry and runtime
+FROM base AS runtime
+
+ENV POETRY_NO_INTERACTION=1 \
+    POETRY_VIRTUALENVS_IN_PROJECT=1 \
+    POETRY_VIRTUALENVS_CREATE=1
+
+
+# Create a virtual environment for poetry and install it
+RUN python3 -m venv /poetry-venv && \
+    /poetry-venv/bin/python -m pip install --upgrade pip && \
+    /poetry-venv/bin/python -m pip install "poetry>=2.0.0,<3.0.0"
 
 WORKDIR /app
 
-# TODO: use custom ffmpeg builds instead of apt-get install
-RUN pip install --upgrade pip && \
-	pip install pipenv && \
-	apt-get update && \
-	apt-get install -y gcc ffmpeg fonts-noto firefox-esr && \
-	wget https://github.com/mozilla/geckodriver/releases/download/v0.32.0/geckodriver-v0.32.0-linux64.tar.gz && \
-	tar -xvzf geckodriver* -C /usr/local/bin && \
-	chmod +x /usr/local/bin/geckodriver && \
-	rm geckodriver-v* 
+
+COPY pyproject.toml poetry.lock README.md ./
+# Copy dependency files and install dependencies (excluding the package itself)
+RUN /poetry-venv/bin/poetry install --only main --no-root --no-cache
 
 
-# install docker for WACZ
-# TODO: currently disabled see https://github.com/bellingcat/auto-archiver/issues/66
-# RUN curl -fsSL https://get.docker.com | sh
+# Copy code: This is needed for poetry to install the package itself,
+# but the environment should be cached from the previous step if toml and lock files haven't changed
+COPY ./src/ .
+RUN /poetry-venv/bin/poetry install --only main --no-cache
 
-# TODO: avoid copying unnecessary files, including .git
-COPY Pipfile Pipfile.lock ./
-RUN pipenv install --python=3.10 --system --deploy
-ENV IS_DOCKER=1
-# doing this at the end helps during development, builds are quick
-COPY ./src/ . 
 
-# TODO: figure out how to make volumes not be root, does it depend on host or dockerfile?
-# RUN useradd --system --groups sudo --shell /bin/bash archiver && chown -R archiver:sudo .
-# USER archiver
-ENTRYPOINT ["python"]
-# ENTRYPOINT ["docker-entrypoint.sh"]
+# Update PATH to include virtual environment binaries
+# Allowing entry point to run the application directly with Python
+ENV VIRTUAL_ENV=/app/.venv \
+    PATH="/app/.venv/bin:$PATH"
+
+ENTRYPOINT ["python3", "-m", "auto_archiver"]
+
+# should be executed with 2 volumes (3 if local_storage is used)
+# docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive aa pipenv run python3 -m auto_archiver --config secrets/orchestration.yaml
 
-# should be executed with 2 volumes (3 if local_storage)
-# docker run -v /var/run/docker.sock:/var/run/docker.sock -v $PWD/secrets:/app/secrets  -v $PWD/local_archive:/app/local_archive aa --help

Pipfile

@@ -1,39 +0,0 @@
-[[source]]
-url = "https://pypi.org/simple"
-verify_ssl = true
-name = "pypi"
-
-[packages]
-gspread = "*"
-boto3 = "*"
-argparse = "*"
-beautifulsoup4 = "*"
-tiktok-downloader = "*"
-bs4 = "*"
-loguru = "*"
-ffmpeg-python = "*"
-selenium = "*"
-snscrape = "*"
-yt-dlp = "*"
-telethon = "*"
-google-api-python-client = "*"
-google-auth-httplib2 = "*"
-google-auth-oauthlib = "*"
-oauth2client = "*"
-python-slugify = "*"
-pyyaml = "*"
-dateparser = "*"
-vk-url-scraper = "*"
-python-twitter-v2 = "*"
-instaloader = "*"
-tqdm = "*"
-jinja2 = "*"
-cryptography = "==38.0.4"
-dataclasses-json = "*"
-
-[requires]
-python_version = "3.9"
-
-[dev-packages]
-autopep8 = "*"
-setuptools-pipfile = "*"

README.md

@@ -1,238 +1,39 @@
-# Auto Archiver
+<h1 align="center">Auto Archiver</h1>
+
+[![PyPI version](https://badge.fury.io/py/auto-archiver.svg)](https://badge.fury.io/py/auto-archiver)
+[![Docker Image Version (latest by date)](https://img.shields.io/docker/v/bellingcat/auto-archiver?label=version&logo=docker)](https://hub.docker.com/r/bellingcat/auto-archiver)
+[![Core Test Status](https://github.com/bellingcat/auto-archiver/workflows/Core%20Tests/badge.svg)](https://github.com/bellingcat/auto-archiver/actions/workflows/tests-core.yaml)
+[![Download Test Status](https://github.com/bellingcat/auto-archiver/workflows/Download%20Tests/badge.svg)](https://github.com/bellingcat/auto-archiver/actions/workflows/tests-download.yaml)
+<!-- ![Docker Pulls](https://img.shields.io/docker/pulls/bellingcat/auto-archiver) -->
+<!-- [![PyPI download month](https://img.shields.io/pypi/dm/auto-archiver.svg)](https://pypi.python.org/pypi/auto-archiver/) -->
+<!-- [![Documentation Status](https://readthedocs.org/projects/vk-url-scraper/badge/?version=latest)](https://vk-url-scraper.readthedocs.io/en/latest/?badge=latest) -->
+
+
+
+Auto Archiver is a Python tool to automatically archive content on the web in a secure and verifiable way. It takes URLs from different sources (e.g. a CSV file, Google Sheets, command line etc.) and archives the content of each one. It can archive social media posts, videos, images and webpages. Content can enriched, then saved either locally or remotely (S3 bucket, Google Drive). The status of the archiving process can be appended to a CSV report, or if using Google Sheets – back to the original sheet.
+
+<div class="hidden_rtd">
+  
+**[See the Auto Archiver documentation for more information.](https://auto-archiver.readthedocs.io/en/latest/)**
+
+</div>
+
 Read the [article about Auto Archiver on bellingcat.com](https://www.bellingcat.com/resources/2022/09/22/preserve-vital-online-content-with-bellingcats-auto-archiver-tool/).
 
 
-Python tool to automatically archive social media posts, videos, and images from a Google Sheets, the console, and more. Uses different archivers depending on the platform, and can save content to local storage, S3 bucket (Digital Ocean Spaces, AWS, ...), and Google Drive. If using Google Sheets as the source for links, it will be updated with information about the archived content. It can be run manually or on an automated basis.
+## Installation
 
-There are 3 ways to use the auto-archiver
-1. (simplest) via docker `docker ... TODO`
-2. (pypi) `pip install auto-archiver`
-3. (legacy) clone and manually install from repo (see legacy [tutorial video](https://youtu.be/VfAhcuV2tLQ))
+View the [Installation Guide](installation/installation.md) for full instructions
 
+To get started quickly using Docker:
 
+`docker pull bellingcat/auto-archiver && docker run`
 
-### Examples
+Or pip:
 
+`pip install auto-archiver && auto-archiver --help`
 
+## Contributing
 
-# Requirement configurations
-# Running with docker
-# Running without docker
-
-
-
-### Setup checklist
-Use this to make sure you help making sure you did all the required steps:
-* [ ] you have a `/secrets` folder with all your configuration files including
-  * [ ] a configuration file eg: `config.yaml` pointing to the correct location of other files
-  * [ ] you have a `service_account.json` 
-  * [ ] (optional for telegram) a `anon.session` which appears after the 1st run to avoid logging into the 
-  * [ ] (optional for VK) a `vk_config.v2.json`
-  * [ ] (optional for using GoogleDrive storage) `gd-token.json`
-  * [ ] (optional for instagram) `instaloader.session` file which appears after the 1st run and login in telegram
-  * [ ] (optional for browsertrix) `profile.tar.gz` file
-
-### Private telegram channels
-* Cannot use bot token
-* Should have one with bot token, one without
-* Setup join all private invite links at the start
-* 
-
-## Setup
-### Always required
-1. [A Google Service account is necessary for use with `gspread`.](https://gspread.readthedocs.io/en/latest/oauth2.html#for-bots-using-service-account) Credentials for this account should be stored in `service_account.json`, in the same directory as the script.
-2. A configuration file, see [Configuration file](#configuration-file).
-
-### With docker image
-[Docker](https://www.docker.com/) is like a virtual machine program that isolates all the installation dependencies needed for the auto-archiver and it should be the only thing you need to install.
-
-<!-- TODO add further instructions for docker -->
-
-### Without docker
-Check this [tutorial video](https://youtu.be/VfAhcuV2tLQ) for setup without the docker image.
-
-If you are using `pipenv` (recommended), `pipenv install` is sufficient to install Python prerequisites.
-
-You need to install the following requirements on your machine:
-1. [A Google Service account is necessary for use with `gspread`.](https://gspread.readthedocs.io/en/latest/oauth2.html#for-bots-using-service-account) Credentials for this account should be stored in `service_account.json`, in the same directory as the script.
-2. [ffmpeg](https://www.ffmpeg.org/) must also be installed locally for this tool to work. 
-3. [firefox](https://www.mozilla.org/en-US/firefox/new/) and [geckodriver](https://github.com/mozilla/geckodriver/releases) on a path folder like `/usr/local/bin`. 
-4. [fonts-noto](https://fonts.google.com/noto) to deal with multiple unicode characters during selenium/geckodriver's screenshots: `sudo apt install fonts-noto -y`. 
-5. Internet Archive credentials can be retrieved from https://archive.org/account/s3.php.
-6. If you would like to take archival [WACZ](https://specs.webrecorder.net/wacz/1.1.1/) snapshots using [browsertrix-crawler](https://github.com/webrecorder/browsertrix-crawler) in addition to screenshots you will need to install [Docker](https://www.docker.com/). 
-   1. To improve the websites browsertrix can archive you can also create a custom profile by running `docker run -p 9222:9222 -p 9223:9223 -v $PWD/browsertrix/crawls/profiles:/crawls/profiles/ -it webrecorder/browsertrix-crawler create-login-profile --interactive --url "https://youtube.com"`, going to [http://localhost:9223/](http://localhost:9223/) and accepting the cookies prompt on youtube, and then navigating to other websites and logging in as per your needs, so as to access more publicly blocked content, and then specifying the created `profile.tar.gz` in your config file under `execution.browsertrix.profile`. 
-
-### Configuration file
-Configuration is done via a config.yaml file (see [example.config.yaml](example.config.yaml)) and some properties of that file can be overwritten via command line arguments. Make a copy of that file and rename it to your liking eg. `config-test.yaml` . Here is the current result from running the `python auto_archive.py --help`:
-
-<details><summary><code>python auto_archive.py --help</code></summary>
-
-
-
-```js
-usage: auto_archive.py [-h] [--config CONFIG] [--storage {s3,local,gd}] [--sheet SHEET] [--header HEADER] [--check-if-exists] [--save-logs] [--s3-private] [--col-url URL] [--col-status STATUS] [--col-folder FOLDER]
-                       [--col-archive ARCHIVE] [--col-date DATE] [--col-thumbnail THUMBNAIL] [--col-thumbnail_index THUMBNAIL_INDEX] [--col-timestamp TIMESTAMP] [--col-title TITLE] [--col-duration DURATION]
-                       [--col-screenshot SCREENSHOT] [--col-hash HASH]
-
-Automatically archive social media posts, videos, and images from a Google Sheets document. 
-The command line arguments will always override the configurations in the provided YAML config file (--config), only some high-level options
-are allowed via the command line and the YAML configuration file is the preferred method. The sheet must have the "url" and "status" for the archiver to work.
-
-optional arguments:
-  -h, --help            show this help message and exit
-  --config CONFIG       the filename of the YAML configuration file (defaults to 'config.yaml')
-  --storage {s3,local,gd}
-                        which storage to use [execution.storage in config.yaml]
-  --sheet SHEET         the name of the google sheets document [execution.sheet in config.yaml]
-  --header HEADER       1-based index for the header row [execution.header in config.yaml]
-  --check-if-exists     when possible checks if the URL has been archived before and does not archive the same URL twice [exceution.check_if_exists]
-  --save-logs           creates or appends execution logs to files logs/LEVEL.log [exceution.save_logs]
-  --s3-private          Store content without public access permission (only for storage=s3) [secrets.s3.private in config.yaml]
-  --col-url URL         the name of the column to READ url FROM (default='link')
-  --col-status STATUS   the name of the column to FILL WITH status (default='archive status')
-  --col-folder FOLDER   the name of the column to READ folder FROM (default='destination folder')
-  --col-archive ARCHIVE
-                        the name of the column to FILL WITH archive (default='archive location')
-  --col-date DATE       the name of the column to FILL WITH date (default='archive date')
-  --col-thumbnail THUMBNAIL
-                        the name of the column to FILL WITH thumbnail (default='thumbnail')
-  --col-thumbnail_index THUMBNAIL_INDEX
-                        the name of the column to FILL WITH thumbnail_index (default='thumbnail index')
-  --col-timestamp TIMESTAMP
-                        the name of the column to FILL WITH timestamp (default='upload timestamp')
-  --col-title TITLE     the name of the column to FILL WITH title (default='upload title')
-  --col-duration DURATION
-                        the name of the column to FILL WITH duration (default='duration')
-  --col-screenshot SCREENSHOT
-                        the name of the column to FILL WITH screenshot (default='screenshot')
-  --col-hash HASH       the name of the column to FILL WITH hash (default='hash')
-```
-
-</details><br/>
-
-#### Example invocations
-All the configurations can be specified in the YAML config file, but sometimes it is useful to override only some of those like the sheet that we are running the archival on, here are some examples (possibly prepended by `pipenv run`):
-
-```bash
-# all the configurations come from config.yaml
-python auto_archive.py
-
-# all the configurations come from config.yaml,
-# checks if URL is not archived twice and saves logs to logs/ folder
-python auto_archive.py --check-if-exists --save_logs
-
-# all the configurations come from my_config.yaml
-python auto_archive.py --config my_config.yaml
-
-# reads the configurations but saves archived content to google drive instead
-python auto_archive.py --config my_config.yaml --storage gd
-
-# uses the configurations but for another google docs sheet 
-# with a header on row 2 and with some different column names
-python auto_archive.py --config my_config.yaml --sheet="use it on another sheets doc" --header=2 --col-link="put urls here"
-
-# all the configurations come from config.yaml and specifies that s3 files should be private
-python auto_archive.py --s3-private
-```
-
-### Extra notes on configuration
-#### Google Drive
-To use Google Drive storage you need the id of the shared folder in the `config.yaml` file which must be shared with the service account eg `autoarchiverservice@auto-archiver-111111.iam.gserviceaccount.com` and then you can use `--storage=gd`
-
-#### Telethon (Telegrams API Library)
-The first time you run, you will be prompted to do a authentication with the phone number associated, alternatively you can put your `anon.session` in the root.
-
-
-## Running
-The `--sheet name` property (or `execution.sheet` in the YAML file) is the name of the Google Sheet to check for URLs. 
-This sheet must have been shared with the Google Service account used by `gspread`. 
-This sheet must also have specific columns (case-insensitive) in the `header` row (see `COLUMN_NAMES` in [gworksheet.py](utils/gworksheet.py)), only the `link` and `status` columns are mandatory:
-* `Link` (required): the location of the media to be archived. This is the only column that should be supplied with data initially
-* `Archive status` (required): the status of the auto archiver script. Any row with text in this column will be skipped automatically.
-* `Destination folder`: (optional) by default files are saved to a folder called `name-of-sheets-document/name-of-sheets-tab/` using this option you can organize documents into folder from the sheet. 
-* `Archive location`: the location of the archived version. For files that were not able to be auto archived, this can be manually updated.
-* `Archive date`: the date that the auto archiver script ran for this file
-* `Upload timestamp`: the timestamp extracted from the video. (For YouTube, this unfortunately does not currently include the time)
-* `Upload title`: the "title" of the video from the original source
-* `Hash`: a hash of the first video or image found
-* `Screenshot`: a screenshot taken with from a browser view of opening the page
-* in case of videos
-  * `Duration`: duration in seconds
-  * `Thumbnail`: an image thumbnail of the video (resize row height to make this more visible)
-  * `Thumbnail index`: a link to a page that shows many thumbnails for the video, useful for quickly seeing video content
-
-
-For example, for use with this spreadsheet:
-
-![A screenshot of a Google Spreadsheet with column headers defined as above, and several Youtube and Twitter URLs in the "Media URL" column](docs/demo-before.png)
-
-```pipenv run python auto_archive.py --sheet archiver-test```
-
-When the auto archiver starts running, it updates the "Archive status" column.
-
-![A screenshot of a Google Spreadsheet with column headers defined as above, and several Youtube and Twitter URLs in the "Media URL" column. The auto archiver has added "archive in progress" to one of the status columns.](docs/demo-progress.png)
-
-The links are downloaded and archived, and the spreadsheet is updated to the following:
-
-![A screenshot of a Google Spreadsheet with videos archived and metadata added per the description of the columns above.](docs/demo-after.png)
-
-Note that the first row is skipped, as it is assumed to be a header row (`--header=1` and you can change it if you use more rows above). Rows with an empty URL column, or a non-empty archive column are also skipped. All sheets in the document will be checked.
-
-## Automating
-
-The auto-archiver can be run automatically via cron. An example crontab entry that runs the archiver every minute is as follows.
-
-```* * * * * python auto_archive.py --sheet archiver-test```
-
-With this configuration, the archiver should archive and store all media added to the Google Sheet every 60 seconds. Of course, additional logging information, etc. might be required.
-
-# auto_auto_archiver
-
-To make it easier to set up new auto-archiver sheets, the auto-auto-archiver will look at a particular sheet and run the auto-archiver on every sheet name in column A, starting from row 11. (It starts here to support instructional text in the first rows of the sheet, as shown below.) You can simply use your default config as for `auto_archiver.py` but use `--sheet` to specify the name of the sheet that lists the names of sheets to archive.It must be shared with the same service account.
-
-![A screenshot of a Google Spreadsheet configured to show instructional text and a list of sheet names to check with auto-archiver.](docs/auto-auto.png)
-
-# Docker development
-* working with docker locally:
-  * `docker build . -t auto-archiver` to build a local image
-  * `docker run --rm -v $PWD/secrets:/app/secrets aa --config secrets/config.yaml`
-    * to use local archive, also create a volume `-v` for it by adding `-v $PWD/local_archive:/app/local_archive`
-* release to docker hub
-  * `docker image tag auto-archiver bellingcat/auto-archiver:latest`
-  * `docker push bellingcat/auto-archiver` (validate [here]())
-
-# Code structure
-Code is split into functional concepts:
-1. [Archivers](archivers/) - receive a URL that they try to archive
-2. [Storages](storages/) - they deal with where the archived files go
-3. [Utilities](utils/)
-   1. [GWorksheet](utils/gworksheet.py) - facilitates some of the reading/writing tasks for a Google Worksheet
-
-### Current Archivers
-Archivers are tested in a meaningful order with Wayback Machine being the failsafe, that can easily be changed in the code. 
-
-> Note: We have 2 Twitter Archivers (`TwitterArchiver`, `TwitterApiArchiver`) because one requires Twitter API V2 credentials and has better results and the other does not rely on official APIs and misses out on some content. 
-
-https://mermaid.js.org/syntax/flowchart.html
-```mermaid
-graph TD
-    A(Archiver) -->|parent of| B(TelethonArchiver)
-    A -->|parent of| C(TiktokArchiver)
-    A -->|parent of| D(YoutubeDLArchiver)
-    A -->|parent of| D(InstagramArchiver)
-    A -->|parent of| E(TelegramArchiver)
-    A -->|parent of| F(TwitterArchiver)
-    A -->|parent of| G(VkArchiver)
-    A -->|parent of| H(WaybackArchiver)
-    F -->|parent of| I(TwitterApiArchiver)
-```
-### Current Storages
-```mermaid
-graph TD
-    A(BaseStorage) -->|parent of| B(S3Storage)
-    A(BaseStorage) -->|parent of| C(LocalStorage)
-    A(BaseStorage) -->|parent of| D(GoogleDriveStorage)
-```
-
-
+We welcome contributions to the Auto Archiver project! See the [Contributing Guide](https://auto-archiver.readthedocs.io/en/latest/contributing.html) for how to get involved!
 

docker-compose.yaml

@@ -0,0 +1,16 @@
+version: '3.8'
+
+services:
+  auto-archiver:
+  # point to the local dockerfile
+    build:
+      context: .
+      dockerfile: Dockerfile
+    container_name: auto-archiver
+    volumes:
+      - ./secrets:/app/secrets
+      - ./local_archive:/app/local_archive
+    environment:
+      - WACZ_ENABLE_DOCKER=true
+      - RUNNING_IN_DOCKER=true
+    command: --config secrets/orchestration.yaml

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = source
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/_static/custom.css

@@ -0,0 +1,4 @@
+.hidden_rtd {
+    display:none;
+}
+

docs/_templates/autoapi/index.rst

@@ -0,0 +1,46 @@
+API Reference
+=============
+
+These pages are intended for developers of the `auto-archiver` package, 
+and include documentation on the core classes and functions used by 
+the auto-archiver
+
+
+Core Classes
+------------
+
+
+.. toctree::
+   :titlesonly:
+
+   {% for page in pages|selectattr("is_top_level_object") %}
+   {% if page.name == 'core' %}
+   {{ page.include_path }}
+   {% endif %}
+   {% endfor %}
+
+Util Functions
+--------------
+
+.. toctree::
+   :titlesonly:
+
+   {% for page in pages|selectattr("is_top_level_object") %}
+   {% if page.name == 'utils' %}
+   {{ page.include_path }}
+   {% endif %}
+   {% endfor %}
+
+
+Core Modules
+------------
+
+.. toctree::
+   :titlesonly:
+
+   {% for page in pages|selectattr("is_top_level_object") %}
+   {% if page.name != 'core' and page.name != 'utils' %}
+   {{ page.include_path }}
+   {% endif %}
+   {% endfor %}
+

docs/_templates/autoapi/python/attribute.rst

@@ -0,0 +1 @@
+{% extends "python/data.rst" %}

docs/_templates/autoapi/python/class.rst

@@ -0,0 +1,104 @@
+{% if obj.display %}
+   {% if is_own_page %}
+{{ obj.id }}
+{{ "=" * obj.id | length }}
+
+   {% endif %}
+   {% set visible_children = obj.children|selectattr("display")|list %}
+   {% set own_page_children = visible_children|selectattr("type", "in", own_page_types)|list %}
+   {% if is_own_page and own_page_children %}
+.. toctree::
+   :hidden:
+
+      {% for child in own_page_children %}
+   {{ child.include_path }}
+      {% endfor %}
+
+   {% endif %}
+.. py:{{ obj.type }}:: {% if is_own_page %}{{ obj.id }}{% else %}{{ obj.short_name }}{% endif %}{% if obj.args %}({{ obj.args }}){% endif %}
+
+   {% for (args, return_annotation) in obj.overloads %}
+      {{ " " * (obj.type | length) }}   {{ obj.short_name }}{% if args %}({{ args }}){% endif %}
+
+   {% endfor %}
+   {% if obj.bases %}
+      {% if "show-inheritance" in autoapi_options %}
+
+   Bases: {% for base in obj.bases %}{{ base|link_objs }}{% if not loop.last %}, {% endif %}{% endfor %}
+      {% endif %}
+
+
+      {% if "show-inheritance-diagram" in autoapi_options and obj.bases != ["object"] %}
+   .. autoapi-inheritance-diagram:: {{ obj.obj["full_name"] }}
+      :parts: 1
+         {% if "private-members" in autoapi_options %}
+      :private-bases:
+         {% endif %}
+
+      {% endif %}
+   {% endif %}
+   {% if obj.docstring %}
+
+   {{ obj.docstring|indent(3) }}
+   {% endif %}
+   {% for obj_item in visible_children %}
+      {% if obj_item.type not in own_page_types %}
+
+   {{ obj_item.render()|indent(3) }}
+      {% endif %}
+   {% endfor %}
+   {% if is_own_page and own_page_children %}
+      {% set visible_attributes = own_page_children|selectattr("type", "equalto", "attribute")|list %}
+      {% if visible_attributes %}
+Attributes
+----------
+
+.. autoapisummary::
+
+         {% for attribute in visible_attributes %}
+   {{ attribute.id }}
+         {% endfor %}
+
+
+      {% endif %}
+      {% set visible_exceptions = own_page_children|selectattr("type", "equalto", "exception")|list %}
+      {% if visible_exceptions %}
+Exceptions
+----------
+
+.. autoapisummary::
+
+         {% for exception in visible_exceptions %}
+   {{ exception.id }}
+         {% endfor %}
+
+
+      {% endif %}
+      {% set visible_classes = own_page_children|selectattr("type", "equalto", "class")|list %}
+      {% if visible_classes %}
+Classes
+-------
+
+.. autoapisummary::
+
+         {% for klass in visible_classes %}
+   {{ klass.id }}
+         {% endfor %}
+
+
+      {% endif %}
+      {% set visible_methods = own_page_children|selectattr("type", "equalto", "method")|list %}
+      {% if visible_methods %}
+Methods
+-------
+
+.. autoapisummary::
+
+            {% for method in visible_methods %}
+   {{ method.id }}
+            {% endfor %}
+
+
+      {% endif %}
+   {% endif %}
+{% endif %}

docs/_templates/autoapi/python/data.rst

@@ -0,0 +1,38 @@
+{% if obj.display %}
+   {% if is_own_page %}
+{{ obj.id }}
+{{ "=" * obj.id | length }}
+
+   {% endif %}
+.. py:{{ obj.type }}:: {% if is_own_page %}{{ obj.id }}{% else %}{{ obj.name }}{% endif %}
+   {% if obj.annotation is not none %}
+
+   :type: {% if obj.annotation %} {{ obj.annotation }}{% endif %}
+   {% endif %}
+   {% if obj.value is not none %}
+
+      {% if obj.value.splitlines()|count > 1 %}
+   :value: Multiline-String
+
+   .. raw:: html
+
+      <details><summary>Show Value</summary>
+
+   .. code-block:: python
+
+      {{ obj.value|indent(width=6,blank=true) }}
+
+   .. raw:: html
+
+      </details>
+
+      {% else %}
+   :value: {{ obj.value|truncate(100) }}
+      {% endif %}
+   {% endif %}
+
+   {% if obj.docstring %}
+
+   {{ obj.docstring|indent(3) }}
+   {% endif %}
+{% endif %}

docs/_templates/autoapi/python/exception.rst

@@ -0,0 +1 @@
+{% extends "python/class.rst" %}

docs/_templates/autoapi/python/function.rst

@@ -0,0 +1,21 @@
+{% if obj.display %}
+   {% if is_own_page %}
+{{ obj.id }}
+{{ "=" * obj.id | length }}
+
+   {% endif %}
+.. py:function:: {% if is_own_page %}{{ obj.id }}{% else %}{{ obj.short_name }}{% endif %}({{ obj.args }}){% if obj.return_annotation is not none %} -> {{ obj.return_annotation }}{% endif %}
+   {% for (args, return_annotation) in obj.overloads %}
+
+                 {%+ if is_own_page %}{{ obj.id }}{% else %}{{ obj.short_name }}{% endif %}({{ args }}){% if return_annotation is not none %} -> {{ return_annotation }}{% endif %}
+   {% endfor %}
+   {% for property in obj.properties %}
+
+   :{{ property }}:
+   {% endfor %}
+
+   {% if obj.docstring %}
+
+   {{ obj.docstring|indent(3) }}
+   {% endif %}
+{% endif %}

docs/_templates/autoapi/python/method.rst

@@ -0,0 +1,21 @@
+{% if obj.display %}
+   {% if is_own_page %}
+{{ obj.id }}
+{{ "=" * obj.id | length }}
+
+   {% endif %}
+.. py:method:: {% if is_own_page %}{{ obj.id }}{% else %}{{ obj.short_name }}{% endif %}({{ obj.args }}){% if obj.return_annotation is not none %} -> {{ obj.return_annotation }}{% endif %}
+   {% for (args, return_annotation) in obj.overloads %}
+
+               {%+ if is_own_page %}{{ obj.id }}{% else %}{{ obj.short_name }}{% endif %}({{ args }}){% if return_annotation is not none %} -> {{ return_annotation }}{% endif %}
+   {% endfor %}
+   {% for property in obj.properties %}
+
+   :{{ property }}:
+   {% endfor %}
+
+   {% if obj.docstring %}
+
+   {{ obj.docstring|indent(3) }}
+   {% endif %}
+{% endif %}

docs/_templates/autoapi/python/module.rst

@@ -0,0 +1,156 @@
+{% if obj.display %}
+   {% if is_own_page %}
+{{ obj.id }}
+{{ "=" * obj.id|length }}
+
+.. py:module:: {{ obj.name }}
+
+      {% if obj.docstring %}
+.. autoapi-nested-parse::
+
+   {{ obj.docstring|indent(3) }}
+
+      {% endif %}
+
+      {% block submodules %}
+         {% set visible_subpackages = obj.subpackages|selectattr("display")|list %}
+         {% set visible_submodules = obj.submodules|selectattr("display")|list %}
+         {% set visible_submodules = (visible_subpackages + visible_submodules)|sort %}
+         {% if visible_submodules %}
+Submodules
+----------
+
+.. toctree::
+   :maxdepth: 1
+
+            {% for submodule in visible_submodules %}
+   {{ submodule.include_path }}
+            {% endfor %}
+
+
+         {% endif %}
+      {% endblock %}
+      {% block content %}
+         {% set visible_children = obj.children|selectattr("display")|list %}
+         {% if visible_children %}
+            {% set visible_attributes = visible_children|selectattr("type", "equalto", "data")|list %}
+            {% if visible_attributes %}
+               {% if "attribute" in own_page_types or "show-module-summary" in autoapi_options %}
+Attributes
+----------
+
+                  {% if "attribute" in own_page_types %}
+.. toctree::
+   :hidden:
+
+                     {% for attribute in visible_attributes %}
+   {{ attribute.include_path }}
+                     {% endfor %}
+
+                  {% endif %}
+.. autoapisummary::
+
+                  {% for attribute in visible_attributes %}
+   {{ attribute.id }}
+                  {% endfor %}
+               {% endif %}
+
+
+            {% endif %}
+            {% set visible_exceptions = visible_children|selectattr("type", "equalto", "exception")|list %}
+            {% if visible_exceptions %}
+               {% if "exception" in own_page_types or "show-module-summary" in autoapi_options %}
+Exceptions
+----------
+
+                  {% if "exception" in own_page_types %}
+.. toctree::
+   :hidden:
+
+                     {% for exception in visible_exceptions %}
+   {{ exception.include_path }}
+                     {% endfor %}
+
+                  {% endif %}
+.. autoapisummary::
+
+                  {% for exception in visible_exceptions %}
+   {{ exception.id }}
+                  {% endfor %}
+               {% endif %}
+
+
+            {% endif %}
+            {% set visible_classes = visible_children|selectattr("type", "equalto", "class")|list %}
+            {% if visible_classes %}
+               {% if "class" in own_page_types or "show-module-summary" in autoapi_options %}
+Classes
+-------
+
+                  {% if "class" in own_page_types %}
+.. toctree::
+   :hidden:
+
+                     {% for klass in visible_classes %}
+   {{ klass.include_path }}
+                     {% endfor %}
+
+                  {% endif %}
+.. autoapisummary::
+
+                  {% for klass in visible_classes %}
+   {{ klass.id }}
+                  {% endfor %}
+               {% endif %}
+
+
+            {% endif %}
+            {% set visible_functions = visible_children|selectattr("type", "equalto", "function")|list %}
+            {% if visible_functions %}
+               {% if "function" in own_page_types or "show-module-summary" in autoapi_options %}
+Functions
+---------
+
+                  {% if "function" in own_page_types %}
+.. toctree::
+   :hidden:
+
+                     {% for function in visible_functions %}
+   {{ function.include_path }}
+                     {% endfor %}
+
+                  {% endif %}
+.. autoapisummary::
+
+                  {% for function in visible_functions %}
+   {{ function.id }}
+                  {% endfor %}
+               {% endif %}
+
+
+            {% endif %}
+            {% set this_page_children = visible_children|rejectattr("type", "in", own_page_types)|list %}
+            {% if this_page_children %}
+{{ obj.type|title }} Contents
+{{ "-" * obj.type|length }}---------
+
+               {% for obj_item in this_page_children %}
+{{ obj_item.render()|indent(0) }}
+               {% endfor %}
+            {% endif %}
+         {% endif %}
+      {% endblock %}
+   {% else %}
+.. py:module:: {{ obj.name }}
+
+      {% if obj.docstring %}
+   .. autoapi-nested-parse::
+
+      {{ obj.docstring|indent(6) }}
+
+      {% endif %}
+      {% for obj_item in visible_children %}
+   {{ obj_item.render()|indent(3) }}
+      {% endfor %}
+   {% endif %}
+{% endif %}

docs/_templates/autoapi/python/package.rst

@@ -0,0 +1 @@
+{% extends "python/module.rst" %}

docs/_templates/autoapi/python/property.rst

@@ -0,0 +1,21 @@
+{% if obj.display %}
+   {% if is_own_page %}
+{{ obj.id }}
+{{ "=" * obj.id | length }}
+
+   {% endif %}
+.. py:property:: {% if is_own_page %}{{ obj.id}}{% else %}{{ obj.short_name }}{% endif %}
+   {% if obj.annotation %}
+
+   :type: {{ obj.annotation }}
+   {% endif %}
+   {% for property in obj.properties %}
+
+   :{{ property }}:
+   {% endfor %}
+
+   {% if obj.docstring %}
+
+   {{ obj.docstring|indent(3) }}
+   {% endif %}
+{% endif %}

docs/scripts/__init__.py

@@ -0,0 +1 @@
+from scripts import generate_module_docs

docs/scripts/scripts.py

@@ -0,0 +1,105 @@
+# iterate through all the modules in auto_archiver.modules and turn the __manifest__.py file into a markdown table
+from pathlib import Path
+from auto_archiver.core.module import available_modules
+from auto_archiver.core.base_module import BaseModule
+from ruamel.yaml import YAML
+import io
+
+MODULES_FOLDER = Path(__file__).parent.parent.parent.parent / "src" / "auto_archiver" / "modules"
+SAVE_FOLDER = Path(__file__).parent.parent / "source" / "modules" / "autogen"
+
+type_color = {
+    'feeder': "<span style='color: #FFA500'>[feeder](/core_modules.md#feeder-modules)</a></span>",
+    'extractor': "<span style='color: #00FF00'>[extractor](/core_modules.md#extractor-modules)</a></span>",
+    'enricher': "<span style='color: #0000FF'>[enricher](/core_modules.md#enricher-modules)</a></span>",
+    'database': "<span style='color: #FF00FF'>[database](/core_modules.md#database-modules)</a></span>",
+    'storage': "<span style='color: #FFFF00'>[storage](/core_modules.md#storage-modules)</a></span>",
+    'formatter': "<span style='color: #00FFFF'>[formatter](/core_modules.md#formatter-modules)</a></span>",
+}
+
+TABLE_HEADER = ("Option", "Description", "Default", "Type")
+
+def generate_module_docs():
+    yaml = YAML()
+    SAVE_FOLDER.mkdir(exist_ok=True)
+    modules_by_type = {}
+
+    header_row = "| " + " | ".join(TABLE_HEADER) + "|\n" + "| --- " * len(TABLE_HEADER) + "|\n"
+    configs_cheatsheet = "\n## Configuration Options\n"
+    configs_cheatsheet += header_row
+
+    for module in sorted(available_modules(with_manifest=True), key=lambda x: (x.requires_setup, x.name)):
+        # generate the markdown file from the __manifest__.py file.
+
+        manifest = module.manifest
+        for type in manifest['type']:
+            modules_by_type.setdefault(type, []).append(module)
+
+        description = "\n".join(l.lstrip() for l in manifest['description'].split("\n"))
+        types = ", ".join(type_color[t] for t in manifest['type'])
+        readme_str = f"""
+# {manifest['name']}
+```{{admonition}} Module type
+
+{types}
+```
+{description}
+"""     
+        if not manifest['configs']:
+            readme_str += "\n*This module has no configuration options.*\n"
+        else:
+            config_yaml = {}
+            config_table = header_row
+            for key, value in manifest['configs'].items():
+                type = value.get('type', 'string')
+                if type == 'auto_archiver.utils.json_loader':
+                    value['type'] = 'json'
+                elif type == 'str':
+                    type = "string"
+                
+                default = value.get('default', '')
+                config_yaml[key] = default
+                help = "**Required**. " if value.get('required', False) else "Optional. "
+                help += value.get('help', '')
+                config_table += f"| `{module.name}.{key}` | {help} | {value.get('default', '')} | {type} |\n"
+                configs_cheatsheet += f"| `{module.name}.{key}` | {help} | {default} | {type} |\n"
+            readme_str += "\n## Configuration Options\n"
+            readme_str += "\n### YAML\n"
+            yaml_string = io.BytesIO()
+            yaml.dump({module.name: config_yaml}, yaml_string)
+            
+            readme_str += f"```{{code}} yaml\n{yaml_string.getvalue().decode('utf-8')}\n```\n"
+
+            readme_str += "\n### Command Line:\n"
+            readme_str += config_table
+
+        # add a link to the autodoc refs
+        readme_str += f"\n[API Reference](../../../autoapi/{module.name}/index)\n"
+        # create the module.type folder, use the first type just for where to store the file
+        for type in manifest['type']:
+            type_folder = SAVE_FOLDER / type
+            type_folder.mkdir(exist_ok=True)
+            with open(type_folder / f"{module.name}.md", "w") as f:
+                print("writing", SAVE_FOLDER)
+                f.write(readme_str)
+        generate_index(modules_by_type)
+
+    with open(SAVE_FOLDER / "configs_cheatsheet.md", "w") as f:
+        f.write(configs_cheatsheet)
+
+
+def generate_index(modules_by_type):
+    readme_str = ""
+    for type in BaseModule.MODULE_TYPES:
+        modules = modules_by_type.get(type, [])
+        module_str = f"## {type.capitalize()} Modules\n"
+        for module in modules:
+            module_str += f"\n[{module.manifest['name']}](/modules/autogen/{module.type[0]}/{module.name}.md)\n"
+        with open(SAVE_FOLDER / f"{type}.md", "w") as f:
+            print("writing", SAVE_FOLDER / f"{type}.md")
+            f.write(module_str)
+        readme_str += module_str
+
+    with open(SAVE_FOLDER / "module_list.md", "w") as f:
+        print("writing", SAVE_FOLDER / "module_list.md")
+        f.write(readme_str)

docs/source/conf.py

@@ -0,0 +1,79 @@
+# Configuration file for the Sphinx documentation builder.
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+import sys
+import os
+from importlib.metadata import metadata
+
+sys.path.append(os.path.abspath('../scripts'))
+from scripts import generate_module_docs
+
+# -- Project Hooks -----------------------------------------------------------
+# convert the module __manifest__.py files into markdown files
+generate_module_docs()
+
+
+# -- Project information -----------------------------------------------------
+package_metadata = metadata("auto-archiver")
+project = package_metadata["name"]
+authors = "Bellingcat"
+release = package_metadata["version"]
+language = 'en'
+
+# -- General configuration ---------------------------------------------------
+extensions = [
+    "myst_parser",                  # Markdown support
+    "autoapi.extension",            # Generate API documentation from docstrings
+    "sphinxcontrib.mermaid",        # Mermaid diagrams
+    "sphinx.ext.viewcode",          # Source code links
+    "sphinx_copybutton",
+    "sphinx.ext.napoleon",          # Google-style and NumPy-style docstrings
+    "sphinx.ext.autosectionlabel",
+    # 'sphinx.ext.autosummary',       # Summarize module/class/function docs
+]
+
+templates_path = ['_templates']
+exclude_patterns = []
+
+
+# -- AutoAPI Configuration ---------------------------------------------------
+autoapi_type = 'python'
+autoapi_dirs = ["../../src/auto_archiver/core/", "../../src/auto_archiver/utils/"]
+# get all the modules and add them to the autoapi_dirs
+autoapi_dirs.extend([f"../../src/auto_archiver/modules/{m}" for m in os.listdir("../../src/auto_archiver/modules")])
+autodoc_typehints = "signature"     # Include type hints in the signature
+autoapi_ignore = ["*/version.py", ]                 # Ignore specific modules
+autoapi_keep_files = True          # Option to retain intermediate JSON files for debugging
+autoapi_add_toctree_entry = True    # Include API docs in the TOC
+autoapi_python_use_implicit_namespaces = True
+autoapi_template_dir = "../_templates/autoapi"
+autoapi_options = [
+    "members",
+    "undoc-members",
+    "show-inheritance",
+    "imported-members",
+]
+
+
+# -- Markdown Support --------------------------------------------------------
+myst_enable_extensions = [
+    "deflist",              # Definition lists
+    "html_admonition",      # HTML-style admonitions
+    "html_image",           # Inline HTML images
+    "replacements",         # Substitutions like (C)
+    "smartquotes",          # Smart quotes
+    "linkify",              # Auto-detect links
+    "substitution",         # Text substitutions
+]
+myst_heading_anchors = 2
+myst_fence_as_directive = ["mermaid"]
+
+source_suffix = {
+    ".rst": "restructuredtext",
+    ".md": "markdown",
+}
+
+# -- Options for HTML output -------------------------------------------------
+html_theme = 'sphinx_book_theme'
+html_static_path = ["../_static"]
+html_css_files = ["custom.css"]
+

docs/source/contributing.md

@@ -0,0 +1,2 @@
+```{include} ../../CONTRIBUTING.md
+```

docs/source/core_modules.md

@@ -0,0 +1,27 @@
+# Module Documentation
+
+These pages describe the core modules that come with `auto-archiver` and provide the main functionality for archiving websites on the internet. There are five core module types:
+
+1. Feeders - these 'feed' information (the URLs) from various sources to the `auto-archiver` for processing
+2. Extractors - these 'extract' the page data for a given URL that is fed in by a feeder
+3. Enrichers - these 'enrich' the data extracted in the previous step with additional information
+4. Storage - these 'store' the data in a persistent location (on disk, Google Drive etc.)
+5. Databases - these 'store' the status of the entire archiving process in a log file or database.
+
+
+```{include} modules/autogen/module_list.md
+```
+
+
+```{toctree}
+:maxdepth: 1
+:caption: Core Modules
+:hidden:
+
+modules/config_cheatsheet
+modules/feeder
+modules/extractor
+modules/enricher
+modules/storage
+modules/database
+```

docs/source/development/creating_modules.md

@@ -0,0 +1,52 @@
+# Creating Your Own Modules
+
+Modules are what's used to extend `auto-archiver` to process different websites or media, and/or transform the data in a way that suits your needs. In most cases, the [Core Modules](../core_modules.md) should be sufficient for every day use, but the most common use-cases for making your own Modules include:
+
+1. Extracting data from a website which doesn't work with the current core extractors.
+2. Enriching or altering the data before saving with additional information that the core enrichers do not offer.
+3. Storing your data in a different format/location from what the core storage providers offer.
+
+## Setting up the folder structure
+
+1. First, decide what type of module you wish to create. Check the types of modules on the [](../core_modules.md) page to decide what type you need. (Note: a module can be more than one type, more on that below)
+2. Create a new python package (a folder) with the name of your module (in this tutorial, we'll call it `awesome_extractor`).
+3. Create the `__manifest__.py` and an the `awesome_extractor.py` files in this folder.
+
+When done, you should have a module structure as follows:
+
+```
+.
+├── awesome_extractor
+│   ├── __manifest__.py
+│   └── awesome_extractor.py
+``` 
+
+Check out the [core modules](https://github.com/bellingcat/auto-archiver/tree/main/src/auto_archiver/modules) in the `auto-archiver` repository for examples of the folder structure for real-world modules.
+
+## Populating the Manifest File
+
+The manifest file is where you define the core information of your module. It is a python dict containing important information, here's an example file:
+
+```{include} ../../../tests/data/test_modules/example_module/__manifest__.py
+:name: __manifest__.py
+:literal:
+:parser: python
+```
+
+## Creating the Python Code
+
+The next step is to create your module code. First, create a class which should subclass the base module types from `auto_archiver.core`, here's an example class for the `awesome_extractor` module which is an `extractor`:
+
+```{code-block} python
+:filename: awesome_extractor.py
+
+from auto_archiver.core import Extractor, Metadata
+
+def AwesomeExtractor(Extractor):
+
+    def download(self, item: Metadata) -> Metadata | False:
+      url = item.get_url()
+      # download the content and create the metadata object
+      metadata = ...
+      return metadata
+```

docs/source/development/developer_guidelines.md

@@ -0,0 +1,34 @@
+
+# Developer Guidelines
+
+This section of the documentation provides guidelines for developers who want to modify or contribute to the tool.
+
+
+## Developer Install
+
+1. Clone the project using `git clone https://github.com/bellingcat/auto-archiver.git` 
+2. Install poetry using `curl -sSL https://install.python-poetry.org | python3 -` ([other installation methods](https://python-poetry.org/docs/#installation))
+3. Install dependencies with `poetry install`
+
+## Running 
+4. Run the code with `poetry run auto-archiver [my args]`
+
+```{note}
+Add the plugin [poetry-shell-plugin](https://github.com/python-poetry/poetry-plugin-shell) and run `poetry shell` to activate the virtual environment.
+This allows you to run the auto-archiver without the `poetry run` prefix.
+```
+
+### Optional Development Packages
+
+Install development packages (used for unit tests etc.) using:
+`poetry install -with dev`
+
+
+```{toctree}
+:hidden:
+creating_modules
+docker_development
+testing
+docs
+release
+```

docs/source/development/docker_development.md

@@ -0,0 +1,5 @@
+## Docker development
+working with docker locally:
+  * `docker compose up` to build the first time and run a local image with the settings in `secrets/orchestration.yaml`
+  * To modify/pass additional command line args, use `docker compose run auto-archiver --config secrets/orchestration.yaml [OTHER ARGUMENTS]`
+  * To rebuild after code changes, just pass the `--build` flag, e.g. `docker compose up --build`

docs/source/development/docs.md

@@ -0,0 +1,38 @@
+
+### Building the Docs
+
+The documentation is built using [Sphinx](https://www.sphinx-doc.org/en/master/) and [AutoAPI](https://sphinx-autoapi.readthedocs.io/en/latest/) and hosted on ReadTheDocs.
+To build the documentation locally, run the following commands:
+
+**Install required dependencies:**
+- Install the docs group of dependencies: 
+```shell
+# only the docs dependencies
+poetry install --only docs
+
+# or for all dependencies 
+poetry install
+```
+- Either use [poetry-plugin-shell](https://github.com/python-poetry/poetry-plugin-shell) to activate the virtual environment: `poetry shell`
+- Or prepend the following commands with `poetry run`
+
+**Create the documentation:**
+- Build the documentation: 
+```shell
+# Using makefile (Linux/macOS):
+make -C docs html
+
+# or using sphinx directly (Windows/Linux/macOS):
+sphinx-build -b html docs/source docs/_build/html
+```
+- If you make significant changes and want a fresh build run: `make -C docs clean` to remove the old build files.
+
+**Viewing the documentation:**
+```shell
+# to open the documentation in your browser.
+open docs/_build/html/index.html
+
+# or run autobuild to automatically update the documentation when you make changes
+sphinx-autobuild docs/source docs/_build/html
+```
+

docs/source/development/release.md

@@ -0,0 +1,15 @@
+# Release Process
+
+```{note} This is a work in progress.
+```
+
+1. Update the version number in [version.py](src/auto_archiver/version.py)
+2. Go to github releases > new release > use `vx.y.z` for matching version notation
+   1. package is automatically updated in pypi
+   2. docker image is automatically pushed to dockerhup
+
+
+
+manual release to docker hub
+  * `docker image tag auto-archiver bellingcat/auto-archiver:latest`
+  * `docker push bellingcat/auto-archiver`

docs/source/development/testing.md

@@ -0,0 +1,21 @@
+# Testing
+
+`pytest` is used for testing. There are two main types of tests:
+
+1. 'core' tests which should be run on every change
+2. 'download' tests which hit the network. These tests will do things like make API calls (e.g. Twitter, Bluesky etc.) and should be run regularly to make sure that APIs have not changed.
+
+
+## Running Tests 
+
+1. Make sure you've installed the dev dependencies with `pytest install --with dev`
+2. Tests can be run as follows:
+```
+#### Command prefix of 'poetry run' removed here for simplicity
+# run core tests
+pytest -ra -v -m "not download"
+# run download tests
+pytest -ra -v -m "download"
+# run all tests
+pytest -ra -v
+```

docs/source/example.orchestration.yaml

@@ -0,0 +1,79 @@
+# Auto Archiver Configuration
+# Steps are the modules that will be run in the order they are defined
+
+steps:
+  feeders:
+  - cli_feeder
+  extractors:
+  - generic_extractor
+  - telegram_extractor
+  enrichers:
+  - thumbnail_enricher
+  - meta_enricher
+  - pdq_hash_enricher
+  - ssl_enricher
+  - hash_enricher
+  databases:
+  - console_db
+  - csv_db
+  storages:
+  - local_storage
+  formatters:
+  - html_formatter
+
+# Global configuration
+
+# Authentication
+# a dictionary of authentication information that can be used by extractors to login to website. 
+# you can use a comma separated list for multiple domains on the same line (common usecase: x.com,twitter.com)
+# Common login 'types' are username/password, cookie, api key/token.
+# There are two special keys for using cookies, they are: cookies_file and cookies_from_browser. 
+# Some Examples:
+# facebook.com:
+#   username: "my_username"
+#   password: "my_password"
+# or for a site that uses an API key:
+# twitter.com,x.com:
+#   api_key
+#   api_secret
+# youtube.com:
+#   cookie: "login_cookie=value ; other_cookie=123" # multiple 'key=value' pairs should be separated by ;
+
+authentication: {}
+
+# Logging settings for your project. See the logging settings with --help
+
+logging:
+  level: INFO
+
+# These are the global configurations that are used by the modules
+
+  file:
+  rotation:
+local_storage:
+  path_generator: flat
+  filename_generator: static
+  save_to: ./local_archive
+  save_absolute: false
+html_formatter:
+  detect_thumbnails: true
+thumbnail_enricher:
+  thumbnails_per_minute: 60
+  max_thumbnails: 16
+generic_extractor:
+  subtitles: true
+  comments: false
+  livestreams: false
+  live_from_start: false
+  proxy: ''
+  end_means_success: true
+  allow_playlist: false
+  max_downloads: inf
+csv_db:
+  csv_file: db.csv
+ssl_enricher:
+  skip_when_nothing_archived: true
+hash_enricher:
+  algorithm: SHA-256
+  chunksize: 16000000
+

docs/source/flow_overview.md

@@ -0,0 +1,30 @@
+
+# Archiving Overview
+
+The archiver archives web pages using the following workflow
+1. **Feeder** gets the links (from a spreadsheet, from the console, ...)
+2. **Extractor** tries to extract content from the given link (e.g. videos from youtube, images from Twitter...)
+3. **Enricher** adds more info to the content (hashes, thumbnails, ...)
+4. **Formatter** creates a report from all the archived content (HTML, PDF, ...)
+5. **Database** knows what's been archived and also stores the archive result (spreadsheet, CSV, or just the console)
+
+Each step in the workflow is handled by 'modules' that interact with the data in different ways. For example, the Twitter Extractor Module would extract information from the Twitter website. The Screenshot Enricher Module will take screenshots of the given page. See the [core modules page](core_modules.md) for an overview of all the modules that are available.
+
+Auto-archiver must have at least one module defined for each step of the workflow. This is done by setting the [configuration](installation/configurations.md) for your auto-archiver instance.
+
+Here's the complete workflow that the auto-archiver goes through:
+
+```mermaid
+
+graph TD
+    s((start)) --> F(fa:fa-table Feeder)
+    F -->|get and clean URL| D1{fa:fa-database Database}
+    D1 -->|is already archived| e((end))
+    D1 -->|not yet archived| a(fa:fa-download Archivers)
+    a -->|got media| E(fa:fa-chart-line Enrichers)
+    E --> S[fa:fa-box-archive Storages]
+    E --> Fo(fa:fa-code Formatter)
+    Fo --> S
+    Fo -->|update database| D2(fa:fa-database Database)
+    D2 --> e
+```

docs/source/how_to.md

@@ -0,0 +1,47 @@
+# How-To Guides
+
+## How to use Google Sheets to load and store archive information
+The `--gsheet_feeder.sheet` property is the name of the Google Sheet to check for URLs. 
+This sheet must have been shared with the Google Service account used by `gspread`. 
+This sheet must also have specific columns (case-insensitive) in the `header` - see the [Gsheet Feeder Docs](modules/autogen/feeder/gsheet_feeder.md) for more info. The default names of these columns and their purpose is:
+
+Inputs:
+
+* **Link** *(required)*: the URL of the post to archive
+* **Destination folder**: custom folder for archived file (regardless of storage)
+
+Outputs:
+* **Archive status** *(required)*: Status of archive operation
+* **Archive location**: URL of archived post
+* **Archive date**: Date archived
+* **Thumbnail**: Embeds a thumbnail for the post in the spreadsheet
+* **Timestamp**: Timestamp of original post
+* **Title**: Post title
+* **Text**: Post text
+* **Screenshot**: Link to screenshot of post
+* **Hash**: Hash of archived HTML file (which contains hashes of post media) - for checksums/verification
+* **Perceptual Hash**: Perceptual hashes of found images - these can be used for de-duplication of content
+* **WACZ**: Link to a WACZ web archive of post
+* **ReplayWebpage**: Link to a ReplayWebpage viewer of the WACZ archive
+
+For example, this is a spreadsheet configured with all of the columns for the auto archiver and a few URLs to archive. (Note that the column names are not case sensitive.)
+
+![A screenshot of a Google Spreadsheet with column headers defined as above, and several Youtube and Twitter URLs in the "Link" column](../demo-before.png)
+
+Now the auto archiver can be invoked, with this command in this example: `docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver:dockerize --config secrets/orchestration-global.yaml --gsheet_feeder.sheet "Auto archive test 2023-2"`. Note that the sheet name has been overridden/specified in the command line invocation.
+
+When the auto archiver starts running, it updates the "Archive status" column.
+
+![A screenshot of a Google Spreadsheet with column headers defined as above, and several Youtube and Twitter URLs in the "Link" column. The auto archiver has added "archive in progress" to one of the status columns.](../demo-progress.png)
+
+The links are downloaded and archived, and the spreadsheet is updated to the following:
+
+![A screenshot of a Google Spreadsheet with videos archived and metadata added per the description of the columns above.](../demo-after.png)
+
+Note that the first row is skipped, as it is assumed to be a header row (`--gsheet_feeder.header=1` and you can change it if you use more rows above). Rows with an empty URL column, or a non-empty archive column are also skipped. All sheets in the document will be checked.
+
+The "archive location" link contains the path of the archived file, in local storage, S3, or in Google Drive.
+
+![The archive result for a link in the demo sheet.](../demo-archive.png)
+
+---

docs/source/index.md

@@ -0,0 +1,17 @@
+
+```{include} ../../README.md
+```
+
+```{toctree}
+:maxdepth: 2
+:hidden:
+:caption: Contents:
+
+Overview <self>
+contributing
+installation/installation.rst
+core_modules.md
+how_to
+development/developer_guidelines
+autoapi/index.rst
+```

docs/source/installation/config_cheatsheet.md

@@ -0,0 +1,6 @@
+# Configuration Cheat Sheet
+
+Below is a list of all configurations for the core modules in Auto Archiver
+
+```{include} ../modules/autogen/configs_cheatsheet.md
+```

docs/source/installation/configurations.md

@@ -0,0 +1,98 @@
+
+# Configuration
+
+This section of the documentation provides guidelines for configuring the tool.
+
+## Configuring using a file
+
+The recommended way to configure auto-archiver for long-term and deployed projects is a configuration file, typically called `orchestration.yaml`. This is a YAML file containing all the settings for your entire workflow.
+
+The structure of orchestration file is split into 2 parts: `steps` (what [steps](../flow_overview.md) to use) and `configurations` (settings for different modules), here's a simplification:
+
+A default `orchestration.yaml` will be created for you the first time you run auto-archiver (without any arguments). Here's what it looks like:
+
+<details>
+<summary>View exampleorchestration.yaml</summary>
+
+```{literalinclude} ../example.orchestration.yaml
+   :language: yaml
+   :caption: orchestration.yaml
+```
+
+</details>
+
+## Configuring from the Command Line
+
+You can run auto-archiver directy from the command line, without the need for a configuration file, command line arguments are parsed using the format `module_name.config_value`. For example, a config value of `api_key` in the `instagram_extractor` module would be passed on the command line with the flag `--instagram_extractor.api_key=API_KEY`.
+
+The command line arguments are useful for testing or editing config values and enabling/disabling modules on the fly. When you are happy with your settings, you can store them back in your configuration file by passing the `-s/--store` flag on the command line.
+
+```bash
+auto-archiver --instagram_extractor.api_key=123 --other_module.setting --store
+# will store the new settings into the configuration file (default: orchestration.yaml)
+```
+
+```{note} Arguments passed on the command line override those saved in your settings file. Save them to your config file using the -s or --store flag
+```
+
+## Seeing all Configuration Options
+
+View the configurable settings for the core modules on the individual doc pages for each [](../core_modules.md).
+You can also view all settings available for the modules you have on your system using the `--help` flag in auto-archiver.
+
+```{code-block} console
+:caption: Example output when using the --help flag with auto-archiver
+$ auto-archiver --help
+...
+Positional Arguments:
+  urls                  URL(s) to archive, either a single URL or a list of urls, should not come from config.yaml
+
+Options:
+  --help, -h            show a full help message and exit
+  --version             show program's version number and exit
+  --config CONFIG_FILE  the filename of the YAML configuration file (defaults to 'config.yaml')
+  --mode {simple,full}  the mode to run the archiver in
+  -s, --store, --no-store
+                        Store the created config in the config file
+  --module_paths MODULE_PATHS [MODULE_PATHS ...]
+                        additional paths to search for modules
+  --feeders STEPS.FEEDERS [STEPS.FEEDERS ...]
+                        the feeders to use
+  --enrichers STEPS.ENRICHERS [STEPS.ENRICHERS ...]
+                        the enrichers to use
+  --extractors STEPS.EXTRACTORS [STEPS.EXTRACTORS ...]
+                        the extractors to use
+  --databases STEPS.DATABASES [STEPS.DATABASES ...]
+                        the databases to use
+  --storages STEPS.STORAGES [STEPS.STORAGES ...]
+                        the storages to use
+  --formatters STEPS.FORMATTERS [STEPS.FORMATTERS ...]
+                        the formatter to use
+  --authentication AUTHENTICATION
+                        A dictionary of sites and their authentication methods (token, username etc.) that extractors can use to log into a website. If passing this on the command line, use a JSON string. You may
+                        also pass a path to a valid JSON/YAML file which will be parsed.
+  --logging.level {INFO,DEBUG,ERROR,WARNING}
+                        the logging level to use
+  --logging.file LOGGING.FILE
+                        the logging file to write to
+  --logging.rotation LOGGING.ROTATION
+                        the logging rotation to use
+
+Wayback Machine Enricher:
+  Submits the current URL to the Wayback Machine for archiving and returns either a job ID or the...
+
+  --wayback_extractor_enricher.timeout TIMEOUT
+                        seconds to wait for successful archive confirmation from wayback, if more than this passes the result contains the job_id so the status can later be checked manually.
+  --wayback_extractor_enricher.if_not_archived_within IF_NOT_ARCHIVED_WITHIN
+                        only tell wayback to archive if no archive is available before the number of seconds specified, use None to ignore this option. For more information:
+                        https://docs.google.com/document/d/1Nsv52MvSjbLb2PCpHlat0gkzw0EvtSgpKHu4mk0MnrA
+  --wayback_extractor_enricher.key KEY
+                        wayback API key. to get credentials visit https://archive.org/account/s3.php
+  --wayback_extractor_enricher.secret SECRET
+                        wayback API secret. to get credentials visit https://archive.org/account/s3.php
+  --wayback_extractor_enricher.proxy_http PROXY_HTTP
+                        http proxy to use for wayback requests, eg http://proxy-user:password@proxy-ip:port
+  --wayback_extractor_enricher.proxy_https PROXY_HTTPS
+                        https proxy to use for wayback requests, eg https://proxy-user:password@proxy-ip:port
+```
+

docs/source/installation/installation.md

@@ -0,0 +1,92 @@
+# Installing Auto Archiver
+
+```{toctree}
+:depth: 1
+:hidden:
+
+configurations.md
+config_cheatsheet.md
+```
+
+There are 3  main ways to use the auto-archiver:
+1. Easiest: [via docker](#installing-with-docker)
+2. Local Install: [using pip](#installing-locally-with-pip)
+3. Developer Install: [see the developer guidelines](../development/developer_guidelines)
+
+
+But **you always need a configuration/orchestration file**, which is where you'll configure where/what/how to archive. Make sure you read [orchestration](#orchestration).
+
+
+## Installing with Docker
+
+[![dockeri.co](https://dockerico.blankenship.io/image/bellingcat/auto-archiver)](https://hub.docker.com/r/bellingcat/auto-archiver)
+
+Docker works like a virtual machine running inside your computer, it isolates everything and makes installation simple. Since it is an isolated environment when you need to pass it your orchestration file or get downloaded media out of docker you will need to connect folders on your machine with folders inside docker with the `-v` volume flag.
+
+
+1. Install [docker](https://docs.docker.com/get-docker/)
+2. Pull the auto-archiver docker [image](https://hub.docker.com/r/bellingcat/auto-archiver) with `docker pull bellingcat/auto-archiver`
+3. Run the docker image locally in a container: `docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver --config secrets/orchestration.yaml` breaking this command down:
+   1. `docker run` tells docker to start a new container (an instance of the image)
+   2. `--rm` makes sure this container is removed after execution (less garbage locally)
+   3. `-v $PWD/secrets:/app/secrets` - your secrets folder
+      1. `-v` is a volume flag which means a folder that you have on your computer will be connected to a folder inside the docker container
+      2. `$PWD/secrets` points to a `secrets/` folder in your current working directory (where your console points to), we use this folder as a best practice to hold all the secrets/tokens/passwords/... you use
+      3. `/app/secrets` points to the path the docker container where this image can be found
+   4.  `-v $PWD/local_archive:/app/local_archive` - (optional) if you use local_storage
+       1.  `-v` same as above, this is a volume instruction
+       2.  `$PWD/local_archive` is a folder `local_archive/` in case you want to archive locally and have the files accessible outside docker
+       3.  `/app/local_archive` is a folder inside docker that you can reference in your orchestration.yml file 
+
+### Example invocations
+
+The invocations below will run the auto-archiver Docker image using a configuration file that you have specified
+
+```bash
+# all the configurations come from ./secrets/orchestration.yaml
+docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver --config secrets/orchestration.yaml
+# uses the same configurations but for another google docs sheet 
+# with a header on row 2 and with some different column names
+# notice that columns is a dictionary so you need to pass it as JSON and it will override only the values provided
+docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver --config secrets/orchestration.yaml --gsheet_feeder.sheet="use it on another sheets doc" --gsheet_feeder.header=2 --gsheet_feeder.columns='{"url": "link"}'
+# all the configurations come from orchestration.yaml and specifies that s3 files should be private
+docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver --config secrets/orchestration.yaml --s3_storage.private=1
+```
+
+## Installing Locally with Pip
+
+1. Make sure you have python 3.10 or higher installed
+2. Install the package with your preferred package manager: `pip/pipenv/conda install auto-archiver` or `poetry add auto-archiver`
+3. Test it's installed with `auto-archiver --help`
+4. Install other local dependency requirements (for )
+5. Run it with your orchestration file and pass any flags you want in the command line `auto-archiver --config secrets/orchestration.yaml` if your orchestration file is inside a `secrets/`, which we advise
+
+### Example invocations
+
+Once all your [local requirements](#installing-local-requirements) are correctly installed, the
+
+```bash
+# all the configurations come from ./secrets/orchestration.yaml
+auto-archiver --config secrets/orchestration.yaml
+# uses the same configurations but for another google docs sheet 
+# with a header on row 2 and with some different column names
+# notice that columns is a dictionary so you need to pass it as JSON and it will override only the values provided
+auto-archiver --config secrets/orchestration.yaml --gsheet_feeder.sheet="use it on another sheets doc" --gsheet_feeder.header=2 --gsheet_feeder.columns='{"url": "link"}'
+# all the configurations come from orchestration.yaml and specifies that s3 files should be private
+auto-archiver --config secrets/orchestration.yaml --s3_storage.private=1
+```
+
+### Installing Local Requirements
+
+If using the local installation method, you will also need to install the following dependencies locally:
+
+1.[ffmpeg](https://www.ffmpeg.org/) - for handling of downloaded videos
+2. [firefox](https://www.mozilla.org/en-US/firefox/new/) and [geckodriver](https://github.com/mozilla/geckodriver/releases) on a path folder like `/usr/local/bin` - for taking webpage screenshots with the screenshot enricher
+3. (optional) [fonts-noto](https://fonts.google.com/noto) to deal with multiple unicode characters during selenium/geckodriver's screenshots: `sudo apt install fonts-noto -y`.
+4. [Browsertrix Crawler docker image](https://hub.docker.com/r/webrecorder/browsertrix-crawler) for the WACZ enricher/archiver
+
+
+
+## Developer Install
+
+[See the developer guidelines](../development/developer_guidelines)

docs/source/modules/database.md

@@ -0,0 +1,15 @@
+# Database Modules
+
+Database modules are used to store the status and results of the extraction and enrichment processes somewhere. The database modules are responsible for creating and managing entires for each item that has been processed.
+
+The default (enabled) databases are the CSV Database and the Console Database.
+
+```{include} autogen/database.md
+```
+
+```{toctree}
+:depth: 1
+:hidden:
+:glob:
+autogen/database/*
+```

docs/source/modules/enricher.md

@@ -0,0 +1,14 @@
+# Enricher Modules
+
+Enricher modules are used to add additional information to the items  that have been extracted. Common enrichment tasks include adding metadata to items, such as the hash of the item, a screenshot of the webpage when the item was extracted, or general metadata like the date and time the item was extracted.
+
+
+```{include} autogen/enricher.md
+```
+
+```{toctree}
+:depth: 1
+:hidden:
+:glob:
+autogen/enricher/*
+```

docs/source/modules/extractor.md

@@ -0,0 +1,18 @@
+# Extractor Modules
+
+Extractor modules are used to extract the content of a given URL. Typically, one extractor will work for one website or platform (e.g. a Telegram extractor or an Instagram), however, there are several wide-ranging extractors which work for a wide range of websites.
+
+Extractors that are able to extract content from a wide range of websites include:
+1. Generic Extractor: parses videos and images on sites using the powerful yt-dlp library.
+2. Wayback Machine Extractor: sends pages to the Waygback machine for archiving, and stores the link.
+3. WACZ Extractor: runs a web browser to 'browse' the URL and save a copy of the page in WACZ format. 
+
+```{include} autogen/extractor.md
+```
+
+```{toctree}
+:depth: 1
+:hidden:
+:glob:
+autogen/extractor/*
+```

docs/source/modules/feeder.md

@@ -0,0 +1,20 @@
+# Feeder Modules
+
+Feeder modules are used to feed URLs into the `auto-archiver` for processing. Feeders can take these URLs from a variety of sources, such as a file, a database, or the command line.
+
+The default feeder is the command line feeder (`cli_feeder`), which allows you to input URLs directly into the `auto-archiver` from the command line.
+
+Command line feeder usage:
+```{code} bash
+auto-archiver [options] -- URL1 URL2 ...
+```
+
+```{include} autogen/feeder.md
+```
+
+```{toctree}
+:depth: 1
+:glob:
+:hidden:
+autogen/feeder/*
+```

docs/source/modules/formatter.md

@@ -0,0 +1,13 @@
+# Formatter Modules
+
+Formatter modules are used to format the data extracted from a URL into a specific format. Currently the most widely-used formatter is the HTML formatter, which formats the data into an easily viewable HTML page.
+
+```{include} autogen/formatter.md
+```
+
+```{toctree}
+:depth: 1
+:hidden:
+:glob:
+autogen/formatter/*
+```

docs/source/modules/storage.md

@@ -0,0 +1,15 @@
+# Storage Modules
+
+Storage modules are used to store the data extracted from a URL in a persistent location. This can be on your local hard disk, or on a remote server (e.g. S3 or Google Drive).
+
+The default is to store the files downloaded (e.g. images, videos) in a local directory.
+
+```{include} autogen/storage.md
+```
+
+```{toctree}
+:depth: 1
+:hidden:
+:glob:
+autogen/storage/*
+```

docs/source/overview.md

@@ -0,0 +1,16 @@
+
+```{include} ../../README.md
+```
+
+```{toctree}
+:maxdepth: 2
+:hidden:
+:caption: Contents:
+
+Overview <self>
+installation/installation.rst
+core_modules.md
+how_to
+development/developer_guidelines
+autoapi/index.rst
+```

example.config.yaml

@@ -1,143 +0,0 @@
----
-secrets:
-  # needed if you use storage=s3
-  s3:
-    # contains S3 info on region, bucket, key and secret
-    region: reg1
-    bucket: my-bucket
-    key: "s3 API key"
-    secret: "s3 API secret"
-    # use region format like such
-    endpoint_url: "https://{region}.digitaloceanspaces.com"
-    # endpoint_url: "https://s3.{region}.amazonaws.com"
-    #use bucket, region, and key (key is the archived file path generated when executing) format like such as:
-    cdn_url: "https://{bucket}.{region}.cdn.digitaloceanspaces.com/{key}"
-    # if private:true S3 urls will not be readable online
-    private: false
-    # with 'random' you can generate a random UUID for the URL instead of a predictable path, useful to still have public but unlisted files, alternative is 'default' or not omitted from config
-    key_path: random
-
-  # needed if you use storage=gd
-  google_drive:
-    # To authenticate with google you have two options (1. service account OR 2. OAuth token)
-
-    # 1. service account - storage space will count towards the developer account
-    # filename can be the same or different file from google_sheets.service_account, defaults to "service_account.json"
-    # service_account: "service_account.json"
-
-    # 2. OAuth token  - storage space will count towards the owner of the GDrive folder
-    # (only 1. or 2. - if both specified then this 2. takes precedence)
-    # needs write access on the server so refresh flow works
-    # To get the token, run the file `create_update_test_oauth_token.py`
-    # you can edit that file if you want a different token filename, default is "gd-token.json"
-    oauth_token_filename: "gd-token.json"
-
-    root_folder_id: copy XXXX from https://drive.google.com/drive/folders/XXXX
-
-  # needed if you use storage=local
-  local:
-    # local path to save files in
-    save_to: "./local_archive"
-
-  wayback:
-    # to get credentials visit https://archive.org/account/s3.php
-    key: your API key
-    secret: your API secret
-
-  telegram:
-    # to get credentials see: https://telegra.ph/How-to-get-Telegram-APP-ID--API-HASH-05-27
-    api_id: your API key, see
-    api_hash: your API hash
-    # optional, but allows access to more content such as large videos, talk to @botfather
-    bot_token: your bot-token
-    # optional, defaults to ./anon, records the telegram login session for future usage
-    session_file: "secrets/anon"
-
-  # twitter configuration - API V2 only
-  # if you don't provide credentials the less-effective unofficial TwitterArchiver will be used instead
-  twitter:
-    # either bearer_token only
-    bearer_token: ""
-    # OR all of the below
-    consumer_key: ""
-    consumer_secret: ""
-    access_token: ""
-    access_secret: ""
-
-  # vkontakte (vk.com) credentials
-  vk:
-    username: "phone number or email"
-    password: "password"
-    # optional, defaults to ./vk_config.v2.json, records VK login session for future usage
-    session_file: "secrets/vk_config.v2.json"
-
-  # instagram  credentials
-  instagram:
-    username: "username"
-    password: "password"
-    session_file: "instaloader.session" # <- default value
-
-  google_sheets:
-    # local filename: defaults to service_account.json, see https://gspread.readthedocs.io/en/latest/oauth2.html#for-bots-using-service-account
-    service_account: "service_account.json"
-
-  facebook:
-    # optional facebook cookie to have more access to content, from browser, looks like 'cookie: datr= xxxx'
-    cookie: ""
-execution:
-  # can be overwritten with CMD --sheet=
-  sheet: your-sheet-name
-
-  # block or allow worksheets by name, instead of defaulting to checking all worksheets in a Spreadsheet
-  # worksheet_allow and worksheet_block can be single values or lists
-  # if worksheet_allow is specified, worksheet_block is ignored
-  # worksheet_allow:
-  #   - Sheet1
-  #   - "Sheet 2"
-  # worksheet_block: BlockedSheet
-
-  # which row of your tabs contains the header, can be overwritten with CMD --header=
-  header: 1
-  # which storage to use, can be overwritten with CMD --storage=
-  storage: s3
-  # defaults to false, when true will try to avoid duplicate URL archives
-  check_if_exists: true
-
-  # choose a hash algorithm (either SHA-256 or SHA3-512, defaults to SHA-256)
-  # hash_algorithm: SHA-256
-
-  # optional configurations for the selenium browser that takes screenshots, these are the defaults
-  selenium:
-    # values under 10s might mean screenshots fail to grab screenshot
-    timeout_seconds: 120
-    window_width: 1400
-    window_height: 2000
-
-  # optional browsertrix configuration (for profile generation see https://github.com/webrecorder/browsertrix-crawler#creating-and-using-browser-profiles)
-  # browsertrix will capture a WACZ archive of the page which can then be seen as the original on replaywebpage
-  browsertrix:
-    enabled: true # defaults to false
-    profile: "./browsertrix/crawls/profile.tar.gz"
-    timeout_seconds: 120 # defaults to 90s
-  # puts execution logs into /logs folder, defaults to false
-  save_logs: true
-  # custom column names, only needed if different from default, can be overwritten with CMD --col-NAME="VALUE"
-  # url and status are the only columns required to be present in the google sheet
-  column_names:
-    url: link
-    status: archive status
-    archive: archive location
-    # use this column to override default location data
-    folder: folder
-    date: archive date
-    thumbnail: thumbnail
-    thumbnail_index: thumbnail index
-    timestamp: upload timestamp
-    title: upload title
-    duration: duration
-    screenshot: screenshot
-    hash: hash
-    wacz: wacz
-    # if you want the replaypage to work, make sure to allow CORS on your bucket, see https://replayweb.page/docs/embedding#cors-restrictions
-    replaywebpage: replaywebpage
-

orchestration.example.yaml

@@ -1,82 +0,0 @@
-steps:
-  # only 1 feeder allowed
-  # a feeder could be in an "infinite loop" for example: gsheets_infinite feeder which holds-> this could be an easy logic addiction by modifying for each to while not feeder.done() if it becomes necessary
-  feeder: gsheet_feeder # default -> only expects URL from CLI
-  archivers: # order matters
-    - telethon
-    # - tiktok
-    # - twitter
-    # - instagram
-    # - webarchive # this way it runs as a failsafe only
-  # enrichers:
-  #   - screenshot
-    # - wacz
-    # - webarchive # this way it runs for every case, webarchive extends archiver and enrichment
-    # - thumbnails
-  formatters:
-    - HTMLFormater
-    - PdfFormater
-  storages:
-    - local_storage
-    - s3
-  databases:
-    - gsheets_db
-    - mongo_db
-
-
-
-configurations:
-  global:
-    - save_logs: False
-  gsheet_feeder:
-    sheet: my-auto-archiver
-    header: 2 # defaults to 1 in GSheetsFeeder
-    service_account: "secrets/service_account.json"
-    # allow_worksheets: "allowed"
-    # block_worksheets: "blocked1,blocked2"
-    columns:
-        'url': 'link'
-        'status': 'archive status'
-        'folder': 'destination folder'
-        'archive': 'archive location'
-        'date': 'archive date'
-        'thumbnail': 'thumbnail'
-        'thumbnail_index': 'thumbnail index'
-        'timestamp': 'upload timestamp'
-        'title': 'upload title'
-        'duration': 'duration'
-        'screenshot': 'screenshot'
-        'hash': 'hash'
-        'wacz': 'wacz'
-        'replaywebpage': 'replaywebpage'
-  telethon:
-    api_id: "1234567"
-    api_hash: "examplehash"
-    session_file: "secrets/anon"
-    channel_invites:
-      - invite: https://t.me/+XXXXXXXXXXXXXX
-        id: 1000000000
-      - invite: https://t.me/joinchat/XXXXXXXXXXXXXX
-        id: 1000000001
-
-  tiktok:
-    api_keys:
-      - username: 1
-        password: 2
-      - username: 3
-        password: 4
-    username: "abc"
-    password: "123"
-    token: "here"
-  screenshot:
-    width: 1280
-    height: 4600
-  wacz:
-    profile: secrets/profile.tar.gz
-  webarchive:
-    api_key: "12345"
-  s3: 
-    - bucket: 123
-    - region: "nyc3"
-    - cdn: "{region}{bucket}"
-

pyproject.toml

@@ -1,5 +1,91 @@
 [build-system]
-requires = ["setuptools", "wheel", "setuptools-pipfile"]
-build-backend = "setuptools.build_meta"
-[tool]
-setuptools-pipfile = "../Pipfile"
+requires = ["poetry-core>=2.0.0,<3.0.0"]
+build-backend = "poetry.core.masonry.api"
+
+[project]
+name = "auto-archiver"
+version = "0.13.2"
+description = "Automatically archive links to videos, images, and social media content from Google Sheets (and more)."
+
+requires-python = ">=3.10,<3.13"
+license = "MIT"
+authors = [
+    { name = "Bellingcat", email = "tech@bellingcat.com" },
+]
+readme = "README.md"
+keywords = ["archive", "oosi", "osint", "scraping"]
+classifiers = [
+    "Intended Audience :: Developers",
+    "Intended Audience :: Science/Research",
+    "License :: OSI Approved :: MIT License",
+    "Programming Language :: Python :: 3"
+]
+
+dependencies = [
+    "gspread (>=0.0.0)",
+    "beautifulsoup4 (>=0.0.0)",
+    "bs4 (>=0.0.0)",
+    "loguru (>=0.0.0)",
+    "ffmpeg-python (>=0.0.0)",
+    "selenium (>=0.0.0)",
+    "telethon (>=0.0.0)",
+    "google-api-python-client (>=0.0.0)",
+    "google-auth-httplib2 (>=0.0.0)",
+    "google-auth-oauthlib (>=0.0.0)",
+    "oauth2client (>=0.0.0)",
+    "pdqhash (>=0.0.0)",
+    "pillow (>=0.0.0)",
+    "python-slugify (>=0.0.0)",
+    "dateparser (>=0.0.0)",
+    "python-twitter-v2 (>=0.0.0)",
+    "instaloader (>=0.0.0)",
+    "tqdm (>=0.0.0)",
+    "jinja2 (>=0.0.0)",
+    "pyOpenSSL (==24.2.1)",
+    "cryptography (>=41.0.0,<42.0.0)",
+    "boto3 (>=1.28.0,<2.0.0)",
+    "dataclasses-json (>=0.0.0)",
+    "yt-dlp (>=2025.1.26,<2026.0.0)",
+    "numpy (==2.1.3)",
+    "vk-url-scraper (>=0.0.0)",
+    "requests[socks] (>=0.0.0)",
+    "warcio (>=0.0.0)",
+    "jsonlines (>=0.0.0)",
+    "pysubs2 (>=0.0.0)",
+    "retrying (>=0.0.0)",
+    "tsp-client (>=0.0.0)",
+    "certvalidator (>=0.0.0)",
+    "rich-argparse (>=1.6.0,<2.0.0)",
+    "ruamel-yaml (>=0.18.10,<0.19.0)",
+]
+
+[tool.poetry.group.dev.dependencies]
+pytest = "^8.3.4"
+autopep8 = "^2.3.1"
+pytest-loguru = "^0.4.0"
+
+[tool.poetry.group.docs.dependencies]
+sphinx = "^8.1.3"
+sphinx-autoapi = "^3.4.0"
+sphinxcontrib-mermaid = "^1.0.0"
+sphinx-autobuild = "^2024.10.3"
+sphinx-copybutton = "^0.5.2"
+myst-parser = "^4.0.0"
+sphinx-book-theme = "^1.1.3"
+linkify-it-py = "^2.0.3"
+
+
+[project.scripts]
+auto-archiver = "auto_archiver.__main__:main"
+
+[project.urls]
+homepage = "https://github.com/bellingcat/auto-archiver"
+repository = "https://github.com/bellingcat/auto-archiver"
+documentation = "https://github.com/bellingcat/auto-archiver"
+
+
+[tool.pytest.ini_options]
+markers = [
+    "download: marks tests that download content from the network",
+    "incremental: marks a class to run tests incrementally. If a test fails in the class, the remaining tests will be skipped",
+]

scripts/create_update_gdrive_oauth_token.py

@@ -10,8 +10,9 @@ from googleapiclient.errors import HttpError
 # You can run this code to get a new token and verify it belongs to the correct user
 # This token will be refresh automatically by the auto-archiver
 # Code below from https://developers.google.com/drive/api/quickstart/python
+# Example invocation: py scripts/create_update_gdrive_oauth_token.py -c secrets/credentials.json -t secrets/gd-token.json
 
-SCOPES = ['https://www.googleapis.com/auth/drive']
+SCOPES = ["https://www.googleapis.com/auth/drive.file"]
 
 
 @click.command(
@@ -22,7 +23,7 @@ SCOPES = ['https://www.googleapis.com/auth/drive']
     "-c",
     type=click.Path(exists=True),
     help="path to the credentials.json file downloaded from https://console.cloud.google.com/apis/credentials",
-    required=True
+    required=True,
 )
 @click.option(
     "--token",
@@ -30,59 +31,62 @@ SCOPES = ['https://www.googleapis.com/auth/drive']
     type=click.Path(exists=False),
     default="gd-token.json",
     help="file where to place the OAuth token, defaults to gd-token.json which you must then move to where your orchestration file points to, defaults to gd-token.json",
-    required=True
+    required=True,
 )
 def main(credentials, token):
     # The file token.json stores the user's access and refresh tokens, and is
     # created automatically when the authorization flow completes for the first time.
     creds = None
     if os.path.exists(token):
-        with open(token, 'r') as stream:
+        with open(token, "r") as stream:
             creds_json = json.load(stream)
             # creds = Credentials.from_authorized_user_file(creds_json, SCOPES)
-            creds_json['refresh_token'] = creds_json.get("refresh_token", "")
+            creds_json["refresh_token"] = creds_json.get("refresh_token", "")
             creds = Credentials.from_authorized_user_info(creds_json, SCOPES)
 
     # If there are no (valid) credentials available, let the user log in.
     if not creds or not creds.valid:
         if creds and creds.expired and creds.refresh_token:
-            print('Requesting new token')
+            print("Requesting new token")
             creds.refresh(Request())
         else:
-            print('First run through so putting up login dialog')
+            print("First run through so putting up login dialog")
             # credentials.json downloaded from https://console.cloud.google.com/apis/credentials
             flow = InstalledAppFlow.from_client_secrets_file(credentials, SCOPES)
             creds = flow.run_local_server(port=55192)
         # Save the credentials for the next run
-        with open(token, 'w') as token:
-            print('Saving new token')
+        with open(token, "w") as token:
+            print("Saving new token")
             token.write(creds.to_json())
     else:
-        print('Token valid')
+        print("Token valid")
 
     try:
-        service = build('drive', 'v3', credentials=creds)
+        service = build("drive", "v3", credentials=creds)
 
         # About the user
         results = service.about().get(fields="*").execute()
-        emailAddress = results['user']['emailAddress']
+        emailAddress = results["user"]["emailAddress"]
         print(emailAddress)
 
         # Call the Drive v3 API and return some files
-        results = service.files().list(
-            pageSize=10, fields="nextPageToken, files(id, name)").execute()
-        items = results.get('files', [])
+        results = (
+            service.files()
+            .list(pageSize=10, fields="nextPageToken, files(id, name)")
+            .execute()
+        )
+        items = results.get("files", [])
 
         if not items:
-            print('No files found.')
+            print("No files found.")
             return
-        print('Files:')
+        print("Files:")
         for item in items:
-            print(u'{0} ({1})'.format(item['name'], item['id']))
+            print("{0} ({1})".format(item["name"], item["id"]))
 
     except HttpError as error:
-        print(f'An error occurred: {error}')
+        print(f"An error occurred: {error}")
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     main()

scripts/release.sh

@@ -1,19 +0,0 @@
-
-#!/bin/bash
-
-set -e
-
-TAG=$(python -c 'from src.auto_archiver.version import VERSION; print("v" + VERSION)')
-
-read -p "Creating new release for $TAG. Do you want to continue? [Y/n] " prompt
-
-if [[ $prompt == "y" || $prompt == "Y" || $prompt == "yes" || $prompt == "Yes" ]]; then
-    git add -A
-    git commit -m "Bump version to $TAG for release" || true && git push
-    echo "Creating new git tag $TAG"
-    git tag "$TAG" -m "$TAG"
-    git push --tags
-else
-    echo "Cancelled"
-    exit 1
-fi

scripts/telegram_setup.py

@@ -0,0 +1,29 @@
+"""
+This script is used to create a new session file for the Telegram client.
+To do this you must first create a Telegram application at https://my.telegram.org/apps
+And store your id and hash in the environment variables TELEGRAM_API_ID and TELEGRAM_API_HASH.
+Create a .env file, or add the following to your environment :
+```
+export TELEGRAM_API_ID=[YOUR_ID_HERE]
+export TELEGRAM_API_HASH=[YOUR_HASH_HERE]
+```
+Then run this script to create a new session file.
+
+You will need to provide your phone number and a 2FA code the first time you run this script.
+"""
+
+
+import os
+from telethon.sync import TelegramClient
+from loguru import logger
+
+
+# Create a
+API_ID = os.getenv("TELEGRAM_API_ID")
+API_HASH = os.getenv("TELEGRAM_API_HASH")
+SESSION_FILE = "secrets/anon-insta"
+
+os.makedirs("secrets", exist_ok=True)
+with TelegramClient(SESSION_FILE, API_ID, API_HASH) as client:
+    logger.success(f"New session file created: {SESSION_FILE}.session")
+

setup.cfg

@@ -1,49 +0,0 @@
-[metadata]
-name = auto_archiver
-version = 2.0.0
-author = Bellingcat
-author_email = tech@bellingcat.com
-description = Easily archive online media content
-long_description = file: README.md, LICENSE
-keywords = archive, oosi, osint, scraping
-license = MIT
-classifiers =
-	Intended Audience :: Developers,
-	Intended Audience :: Science/Research,
-	License :: OSI Approved :: MIT License,
-	Programming Language :: Python :: 3,
-
-[options]
-setup_requires =
-    setuptools-pipfile
-zip_safe = False
-include_package_data = True
-package_dir=
-    =src
-packages=find:
-find_packages=true
-python_requires = >=3.8
-
-# [options.package_data]
-# * = *.txt, *.rst
-# hello = *.msg
-
-[options.entry_points]
-console_scripts =
-    auto-archiver = auto_archiver.__main__:main
-
-# [options.extras_require]
-# pdf = ReportLab>=1.2; RXP
-# rest = docutils>=0.3; pack ==1.1, ==1.3
-
-[options.packages.find]
-where=src
-# include=auto_archiver*
-# exclude =
-#     examples*
-#     .eggs*
-#     build*
-#     secrets*
-#     tmp*
-#     docs*
-#     src.tests*

src/auto_archiver/__init__.py

@@ -1,7 +0,0 @@
-from . import archivers, databases, enrichers, feeders, formatters, storages, utils, core
-
-# need to manually specify due to cyclical deps
-from .core.orchestrator import ArchivingOrchestrator
-from .core.config import Config
-# making accessible directly
-from .core.metadata import Metadata

src/auto_archiver/__main__.py

@@ -1,12 +1,9 @@
-from . import Config
-from . import ArchivingOrchestrator
+""" Entry point for the auto_archiver package. """
+from auto_archiver.core.orchestrator import ArchivingOrchestrator
+import sys
 
 def main():
-    config = Config()
-    config.parse()
-    orchestrator = ArchivingOrchestrator(config)
-    orchestrator.feed()
-
+    ArchivingOrchestrator().run(sys.argv[1:])
 
 if __name__ == "__main__":
     main()

src/auto_archiver/archivers/__init__.py

@@ -1,9 +0,0 @@
-from .archiver import Archiver
-from .telethon_archiver import TelethonArchiver
-from .twitter_archiver import TwitterArchiver
-from .twitter_api_archiver import TwitterApiArchiver
-from .instagram_archiver import InstagramArchiver
-from .tiktok_archiver import TiktokArchiver
-from .telegram_archiver import TelegramArchiver
-from .vk_archiver import VkArchiver
-from .youtubedl_archiver import YoutubeDLArchiver

src/auto_archiver/archivers/archiver.py

@@ -1,64 +0,0 @@
-from __future__ import annotations
-from abc import abstractmethod
-from dataclasses import dataclass
-import os
-import mimetypes, requests
-from ..core import Metadata
-from ..core import Step
-
-
-@dataclass
-class Archiver(Step):
-    name = "archiver"
-
-    def __init__(self, config: dict) -> None:
-        # without this STEP.__init__ is not called
-        super().__init__(config)
-
-    def init(name: str, config: dict) -> Archiver:
-        # only for typing...
-        return Step.init(name, config, Archiver)
-
-    def setup(self) -> None:
-        # used when archivers need to login or do other one-time setup
-        pass
-
-    def sanitize_url(self, url: str) -> str:
-        # used to clean unnecessary URL parameters OR unfurl redirect links
-        return url
-
-    def is_rearchivable(self, url: str) -> bool:
-        # archivers can signal if it does not make sense to rearchive a piece of content
-        # default is rearchiving
-        return True
-
-    def _guess_file_type(self, path: str) -> str:
-        """
-        Receives a URL or filename and returns global mimetype like 'image' or 'video'
-        see https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types
-        """
-        mime = mimetypes.guess_type(path)[0]
-        if mime is not None:
-            return mime.split("/")[0]
-        return ""
-
-    def download_from_url(self, url: str, to_filename: str = None, item: Metadata = None) -> str:
-        """
-        downloads a URL to provided filename, or inferred from URL, returns local filename, if item is present will use its tmp_dir
-        """
-        if not to_filename:
-            to_filename = url.split('/')[-1].split('?')[0]
-            if len(to_filename) > 64:
-                to_filename = to_filename[-64:]
-        if item:
-            to_filename = os.path.join(item.get_tmp_dir(), to_filename)
-        headers = {
-            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36'
-        }
-        d = requests.get(url, headers=headers)
-        with open(to_filename, 'wb') as f:
-            f.write(d.content)
-        return to_filename
-
-    @abstractmethod
-    def download(self, item: Metadata) -> Metadata: pass

src/auto_archiver/archivers/tiktok_archiver.py

@@ -1,58 +0,0 @@
-import json, os, traceback, uuid
-import tiktok_downloader
-from loguru import logger
-
-from . import Archiver
-from ..core import Metadata, Media
-
-
-class TiktokArchiver(Archiver):
-    name = "tiktok_archiver"
-
-    def __init__(self, config: dict) -> None:
-        super().__init__(config)
-
-    @staticmethod
-    def configs() -> dict:
-        return {}
-
-    def is_rearchivable(self, url: str) -> bool:
-        # TikTok posts are static
-        return False
-
-    def download(self, item: Metadata) -> Metadata:
-        url = item.get_url()
-        if 'tiktok.com' not in url:
-            return False
-
-        result = Metadata()
-        try:
-            info = tiktok_downloader.info_post(url)
-            result.set_title(info.desc)
-            result.set_timestamp(info.create_time)
-            result.set_content(json.dumps({
-                "cover": info.cover,
-                "author": info.author,
-                "music_title": info.author,
-                "caption": getattr(info, "caption", info.desc),
-            }, ensure_ascii=False, indent=4))
-        except:
-            error = traceback.format_exc()
-            logger.warning(f'Other Tiktok error {error}')
-
-        try:
-            filename = os.path.join(item.get_tmp_dir(), f'{str(uuid.uuid4())[0:8]}.mp4')
-            tiktok_media = tiktok_downloader.snaptik(url).get_media()
-
-            if len(tiktok_media) <= 0:
-                logger.debug(f"TikTok: could not get media from {url=}")
-                return False
-
-            logger.info(f'downloading video {filename=}')
-            tiktok_media[0].download(filename)
-
-            result.add_media(Media(filename))
-            return result.success("tiktok")
-        except:
-            error = traceback.format_exc()
-            logger.warning(f'Other Tiktok error {error}')

src/auto_archiver/archivers/twitter_api_archiver.py

@@ -1,98 +0,0 @@
-
-import json, mimetypes
-from datetime import datetime
-from loguru import logger
-from pytwitter import Api
-from slugify import slugify
-
-from . import Archiver
-from .twitter_archiver import TwitterArchiver
-from ..core import Metadata,Media
-
-
-class TwitterApiArchiver(TwitterArchiver, Archiver):
-    name = "twitter_api_archiver"
-
-    def __init__(self, config: dict) -> None:
-        super().__init__(config)
-
-        if self.bearer_token:
-            self.assert_valid_string("bearer_token")
-            self.api = Api(bearer_token=self.bearer_token)
-        elif self.consumer_key and self.consumer_secret and self.access_token and self.access_secret:
-            self.assert_valid_string("consumer_key")
-            self.assert_valid_string("consumer_secret")
-            self.assert_valid_string("access_token")
-            self.assert_valid_string("access_secret")
-            self.api = Api(
-                consumer_key=self.consumer_key, consumer_secret=self.consumer_secret, access_token=self.access_token, access_secret=self.access_secret)
-        assert hasattr(self, "api") and self.api is not None, "Missing Twitter API configurations, please provide either bearer_token OR (consumer_key, consumer_secret, access_token, access_secret) to use this archiver."
-
-    @staticmethod
-    def configs() -> dict:
-        return {
-            "bearer_token": {"default": None, "help": "twitter API bearer_token which is enough for archiving, if not provided you will need consumer_key, consumer_secret, access_token, access_secret"},
-            "consumer_key": {"default": None, "help": "twitter API consumer_key"},
-            "consumer_secret": {"default": None, "help": "twitter API consumer_secret"},
-            "access_token": {"default": None, "help": "twitter API access_token"},
-            "access_secret": {"default": None, "help": "twitter API access_secret"},
-        }
-
-    def download(self, item: Metadata) -> Metadata:
-        url = item.get_url()
-        # detect URLs that we definitely cannot handle
-        username, tweet_id = self.get_username_tweet_id(url)
-        if not username: return False
-
-        try:
-            tweet = self.api.get_tweet(tweet_id, expansions=["attachments.media_keys"], media_fields=["type", "duration_ms", "url", "variants"], tweet_fields=["attachments", "author_id", "created_at", "entities", "id", "text", "possibly_sensitive"])
-        except Exception as e:
-            logger.error(f"Could not get tweet: {e}")
-            return False
-
-        result = Metadata()
-        result.set_title(tweet.data.text)
-        result.set_timestamp(datetime.strptime(tweet.data.created_at, "%Y-%m-%dT%H:%M:%S.%fZ"))
-
-        urls = []
-        if tweet.includes:
-            for i, m in enumerate(tweet.includes.media):
-                media = Media(filename="")
-                if m.url and len(m.url):
-                    media.set("src", m.url)
-                    media.set("duration", (m.duration_ms or 1) // 1000)
-                    mimetype = "image/jpeg"
-                elif hasattr(m, "variants"):
-                    variant = self.choose_variant(m.variants)
-                    if not variant: continue
-                    media.set("src", variant.url)
-                    mimetype = variant.content_type
-                else:
-                    continue
-                logger.info(f"Found media {media}")
-                ext = mimetypes.guess_extension(mimetype)
-                media.filename = self.download_from_url(media.get("src"), f'{slugify(url)}_{i}{ext}', item)
-                result.add_media(media)
-
-        result.set_content(json.dumps({
-            "id": tweet.data.id,
-            "text": tweet.data.text,
-            "created_at": tweet.data.created_at,
-            "author_id": tweet.data.author_id,
-            "geo": tweet.data.geo,
-            "lang": tweet.data.lang,
-            "media": urls
-        }, ensure_ascii=False, indent=4))
-        return result.success("twitter")
-
-    def choose_variant(self, variants):
-        # choosing the highest quality possible
-        variant, bit_rate = None, -1
-        for var in variants:
-            if var.content_type == "video/mp4":
-                if var.bit_rate > bit_rate:
-                    bit_rate = var.bit_rate
-                    variant = var
-            else:
-                variant = var if not variant else variant
-        return variant

src/auto_archiver/archivers/twitter_archiver.py

@@ -1,148 +0,0 @@
-import re, requests, mimetypes, json
-from datetime import datetime
-from loguru import logger
-from snscrape.modules.twitter import TwitterTweetScraper, Video, Gif, Photo
-from slugify import slugify
-
-from . import Archiver
-from ..core import Metadata, Media
-
-
-class TwitterArchiver(Archiver):
-    """
-    This Twitter Archiver uses unofficial scraping methods.
-    """
-
-    name = "twitter_archiver"
-    link_pattern = re.compile(r"twitter.com\/(?:\#!\/)?(\w+)\/status(?:es)?\/(\d+)")
-    link_clean_pattern = re.compile(r"(.+twitter\.com\/.+\/\d+)(\?)*.*")
-
-    def __init__(self, config: dict) -> None:
-        super().__init__(config)
-
-    @staticmethod
-    def configs() -> dict:
-        return {}
-
-    def sanitize_url(self, url: str) -> str:
-        # expand URL if t.co and clean tracker GET params
-        if 'https://t.co/' in url:
-            try:
-                r = requests.get(url)
-                logger.debug(f'Expanded url {url} to {r.url}')
-                url = r.url
-            except:
-                logger.error(f'Failed to expand url {url}')
-        # https://twitter.com/MeCookieMonster/status/1617921633456640001?s=20&t=3d0g4ZQis7dCbSDg-mE7-w
-        return self.link_clean_pattern.sub("\\1", url)
-
-    def is_rearchivable(self, url: str) -> bool:
-        # Twitter posts are static
-        return False
-
-    def download(self, item: Metadata) -> Metadata:
-        """
-        if this url is archivable will download post info and look for other posts from the same group with media.
-        can handle private/public channels
-        """
-        url = item.get_url()
-        # detect URLs that we definitely cannot handle
-        username, tweet_id = self.get_username_tweet_id(url)
-        if not username: return False
-
-        result = Metadata()
-
-        scr = TwitterTweetScraper(tweet_id)
-        try:
-            tweet = next(scr.get_items())
-        except Exception as ex:
-            logger.warning(f"can't get tweet: {type(ex).__name__} occurred. args: {ex.args}")
-            return self.download_alternative(item, url, tweet_id)
-
-        result.set_title(tweet.content).set_content(tweet.json()).set_timestamp(tweet.date)
-        if tweet.media is None:
-            logger.debug(f'No media found, archiving tweet text only')
-            return result
-
-        for i, tweet_media in enumerate(tweet.media):
-            media = Media(filename="")
-            mimetype = ""
-            if type(tweet_media) == Video:
-                variant = max(
-                    [v for v in tweet_media.variants if v.bitrate], key=lambda v: v.bitrate)
-                media.set("src", variant.url).set("duration", tweet_media.duration)
-                mimetype = variant.contentType
-            elif type(tweet_media) == Gif:
-                variant = tweet_media.variants[0]
-                media.set("src", variant.url)
-                mimetype = variant.contentType
-            elif type(tweet_media) == Photo:
-                media.set("src", tweet_media.fullUrl.replace('name=large', 'name=orig'))
-                mimetype = "image/jpeg"
-            else:
-                logger.warning(f"Could not get media URL of {tweet_media}")
-                continue
-            ext = mimetypes.guess_extension(mimetype)
-            media.filename = self.download_from_url(media.get("src"), f'{slugify(url)}_{i}{ext}', item)
-            result.add_media(media)
-
-        return result.success("twitter")
-
-    def download_alternative(self, item: Metadata, url: str, tweet_id: str) -> Metadata:
-        """
-        CURRENTLY STOPPED WORKING
-        """
-        return False
-        # https://stackoverflow.com/a/71867055/6196010
-        logger.debug(f"Trying twitter hack for {url=}")
-        result = Metadata()
-
-        hack_url = f"https://cdn.syndication.twimg.com/tweet?id={tweet_id}"
-        r = requests.get(hack_url)
-        if r.status_code != 200: return False
-        tweet = r.json()
-
-        urls = []
-        for p in tweet["photos"]:
-            urls.append(p["url"])
-
-        # 1 tweet has 1 video max
-        if "video" in tweet:
-            v = tweet["video"]
-            urls.append(self.choose_variant(v.get("variants", [])))
-
-        logger.debug(f"Twitter hack got {urls=}")
-
-        for u in urls:
-            media = Media()
-            media.set("src", u)
-            media.filename = self.download_from_url(u, f'{slugify(url)}_{i}', item)
-            result.add_media(media)
-
-        result.set_content(json.dumps(tweet, ensure_ascii=False)).set_timestamp(datetime.strptime(tweet["created_at"], "%Y-%m-%dT%H:%M:%S.%fZ"))
-        return result
-
-    def get_username_tweet_id(self, url):
-        # detect URLs that we definitely cannot handle
-        matches = self.link_pattern.findall(url)
-        if not len(matches): return False, False
-
-        username, tweet_id = matches[0]  # only one URL supported
-        logger.debug(f"Found {username=} and {tweet_id=} in {url=}")
-
-        return username, tweet_id
-
-    def choose_variant(self, variants):
-        # choosing the highest quality possible
-        variant, width, height = None, 0, 0
-        for var in variants:
-            if var.get("type", "") == "video/mp4":
-                width_height = re.search(r"\/(\d+)x(\d+)\/", var["src"])
-                if width_height:
-                    w, h = int(width_height[1]), int(width_height[2])
-                    if w > width or h > height:
-                        width, height = w, h
-                        variant = var.get("src", variant)
-            else:
-                variant = var.get("src") if not variant else variant
-        return variant

src/auto_archiver/archivers/youtubedl_archiver.py

@@ -1,67 +0,0 @@
-import datetime, os, yt_dlp
-from loguru import logger
-
-from . import Archiver
-from ..core import Metadata, Media
-
-
-class YoutubeDLArchiver(Archiver):
-    name = "youtubedl_enricher"
-
-    def __init__(self, config: dict) -> None:
-        super().__init__(config)
-
-    @staticmethod
-    def configs() -> dict:
-        return {
-            "facebook_cookie": {"default": None, "help": "optional facebook cookie to have more access to content, from browser, looks like 'cookie: datr= xxxx'"},
-        }
-
-    def download(self, item: Metadata) -> Metadata:
-        #TODO: yt-dlp for transcripts?
-        url = item.get_url()
-
-        if item.netloc in ['facebook.com', 'www.facebook.com'] and self.facebook_cookie:
-            logger.debug('Using Facebook cookie')
-            yt_dlp.utils.std_headers['cookie'] = self.facebook_cookie
-
-        ydl = yt_dlp.YoutubeDL({'outtmpl': os.path.join(item.get_tmp_dir(), f'%(id)s.%(ext)s'), 'quiet': False})
-
-        try:
-            # don'd download since it can be a live stream
-            info = ydl.extract_info(url, download=False)
-            if info.get('is_live', False):
-                logger.warning("Live streaming media, not archiving now")
-                return False
-        except yt_dlp.utils.DownloadError as e:
-            logger.debug(f'No video - Youtube normal control flow: {e}')
-            return False
-        except Exception as e:
-            logger.debug(f'ytdlp exception which is normal for example a facebook page with images only will cause a IndexError: list index out of range. Exception here is: \n  {e}')
-            return False
-
-        # this time download
-        info = ydl.extract_info(url, download=True)
-        if "entries" in info:
-            entries = info.get("entries", [])
-            if not len(entries):
-                logger.warning('YoutubeDLArchiver could not find any video')
-                return False
-        else: entries = [info]
-
-        result = Metadata()
-        result.set_title(info.get("title"))
-        for entry in entries:
-            filename = ydl.prepare_filename(entry)
-            if not os.path.exists(filename):
-                filename = filename.split('.')[0] + '.mkv'
-            result.add_media(Media(filename).set("duration", info.get("duration")))
-
-        if (timestamp := info.get("timestamp")):
-            timestamp = datetime.datetime.utcfromtimestamp(timestamp).replace(tzinfo=datetime.timezone.utc).isoformat()
-            result.set_timestamp(timestamp)
-        if (upload_date := info.get("upload_date")):
-            upload_date = datetime.datetime.strptime(upload_date, '%Y%m%d').replace(tzinfo=datetime.timezone.utc)
-            result.set("upload_date", upload_date)
-
-        return result.success("yt-dlp")

src/auto_archiver/auto_auto_archive.py

@@ -1,34 +0,0 @@
-
-#TODO: refactor GDriveStorage before merging to main
-# is it possible to have something like this with the new pipeline?
-
-
-# # import tempfile
-# import auto_archive
-# from loguru import logger
-# from configs import Config
-# from storages import Storage
-
-
-# def main():
-#     c = Config()
-#     c.parse()
-#     logger.info(f'Opening document {c.sheet} to look for sheet names to archive')
-
-#     gc = c.gsheets_client
-#     sh = gc.open(c.sheet)
-
-#     wks = sh.get_worksheet(0)
-#     values = wks.get_all_values()
-
-#     with tempfile.TemporaryDirectory(dir="./") as tmpdir:
-#         Storage.TMP_FOLDER = tmpdir
-#         for i in range(11, len(values)):
-#             c.sheet = values[i][0]
-#             logger.info(f"Processing {c.sheet}")
-#             auto_archive.process_sheet(c)
-#         c.destroy_webdriver()
-
-
-# if __name__ == "__main__":
-#     main()

src/auto_archiver/core/__init__.py

@@ -1,7 +1,17 @@
-from .media import Media
+""" Core modules to handle things such as orchestration, metadata and configs..
+
+"""
 from .metadata import Metadata
-from .step import Step
+from .media import Media
+from .module import BaseModule
 
 # cannot import ArchivingOrchestrator/Config to avoid circular dep
 # from .orchestrator import ArchivingOrchestrator
-# from .config import Config
+# from .config import Config
+
+from .database import Database
+from .enricher import Enricher
+from .feeder import Feeder
+from .storage import Storage
+from .extractor import Extractor
+from .formatter import Formatter

src/auto_archiver/core/base_module.py

@@ -0,0 +1,146 @@
+
+from urllib.parse import urlparse
+from typing import  Mapping, Any
+from abc import ABC
+from copy import deepcopy, copy
+from tempfile import TemporaryDirectory
+from auto_archiver.utils import url as UrlUtil
+
+from loguru import logger
+
+class BaseModule(ABC):
+
+    """
+    Base module class. All modules should inherit from this class.
+
+    The exact methods a class implements will depend on the type of module it is,
+    however modules can have a .setup() method to run any setup code
+    (e.g. logging in to a site, spinning up a browser etc.)
+
+    See BaseModule.MODULE_TYPES for the types of modules you can create, noting that
+    a subclass can be of multiple types. For example, a module that extracts data from
+    a website and stores it in a database would be both an 'extractor' and a 'database' module.
+
+    Each module is a python package, and should have a __manifest__.py file in the
+    same directory as the module file. The __manifest__.py specifies the module information
+    like name, author, version, dependencies etc. See BaseModule._DEFAULT_MANIFEST for the
+    default manifest structure.
+
+    """
+
+    MODULE_TYPES = [
+        'feeder',
+        'extractor',
+        'enricher',
+        'database',
+        'storage',
+        'formatter'
+    ]
+
+    _DEFAULT_MANIFEST = {
+    'name': '', # the display name of the module
+    'author': 'Bellingcat', # creator of the module, leave this as Bellingcat or set your own name!
+    'type': [], # the type of the module, can be one or more of BaseModule.MODULE_TYPES
+    'requires_setup': True, # whether or not this module requires additional setup such as setting API Keys or installing additional softare
+    'description': '', # a description of the module
+    'dependencies': {}, # external dependencies, e.g. python packages or binaries, in dictionary format
+    'entry_point': '', # the entry point for the module, in the format 'module_name::ClassName'. This can be left blank to use the default entry point of module_name::ModuleName
+    'version': '1.0', # the version of the module
+    'configs': {} # any configuration options this module has, these will be exposed to the user in the config file or via the command line
+}
+
+    config: Mapping[str, Any]
+    authentication: Mapping[str, Mapping[str, str]]
+    name: str
+
+    # this is set by the orchestrator prior to archiving
+    tmp_dir: TemporaryDirectory = None
+
+    @property
+    def storages(self) -> list:
+        return self.config.get('storages', [])
+
+    def config_setup(self, config: dict):
+
+        authentication = config.get('authentication', {})
+        # extract out concatenated sites
+        for key, val in copy(authentication).items():
+            if "," in key:
+                for site in key.split(","):
+                    authentication[site] = val
+                del authentication[key]
+
+        # this is important. Each instance is given its own deepcopied config, so modules cannot
+        # change values to affect other modules
+        config = deepcopy(config)
+        authentication = deepcopy(config.pop('authentication', {}))
+
+        self.authentication = authentication
+        self.config = config
+        for key, val in config.get(self.name, {}).items():
+            setattr(self, key, val)
+
+    def setup(self):
+        # For any additional setup required by modules, e.g. autehntication
+        pass
+
+    def auth_for_site(self, site: str, extract_cookies=True) -> Mapping[str, Any]:
+        """
+        Returns the authentication information for a given site. This is used to authenticate
+        with a site before extracting data. The site should be the domain of the site, e.g. 'twitter.com'
+        
+        extract_cookies: bool - whether or not to extract cookies from the given browser and return the 
+        cookie jar (disabling can speed up) processing if you don't actually need the cookies jar
+
+        Currently, the dict can have keys of the following types:
+        - username: str - the username to use for login
+        - password: str - the password to use for login
+        - api_key: str - the API key to use for login
+        - api_secret: str - the API secret to use for login
+        - cookie: str - a cookie string to use for login (specific to this site)
+        - cookies_jar: YoutubeDLCookieJar | http.cookiejar.MozillaCookieJar - a cookie jar compatible with requests (e.g. `requests.get(cookies=cookie_jar)`)
+        """
+        # TODO: think about if/how we can deal with sites that have multiple domains (main one is x.com/twitter.com)
+        # for now the user must enter them both, like "x.com,twitter.com" in their config. Maybe we just hard-code?
+
+        site = UrlUtil.domain_for_url(site)
+        # add the 'www' version of the site to the list of sites to check
+        authdict = {}
+
+
+        for to_try in [site, f"www.{site}"]:
+            if to_try in self.authentication:
+                authdict.update(self.authentication[to_try])
+                break
+
+        # do a fuzzy string match just to print a warning - don't use it since it's insecure
+        if not authdict:
+            for key in self.authentication.keys():
+                if key in site or site in key:
+                    logger.debug(f"Could not find exact authentication information for site '{site}'. \
+                                    did find information for '{key}' which is close, is this what you meant? \
+                                    If so, edit your authentication settings to make sure it exactly matches.")
+
+        def get_ytdlp_cookiejar(args):
+            import yt_dlp
+            from yt_dlp import parse_options
+            logger.debug(f"Extracting cookies from settings: {args[1]}")
+            # parse_options returns a named tuple as follows, we only need the ydl_options part
+            # collections.namedtuple('ParsedOptions', ('parser', 'options', 'urls', 'ydl_opts'))
+            ytdlp_opts = getattr(parse_options(args), 'ydl_opts')
+            return yt_dlp.YoutubeDL(ytdlp_opts).cookiejar
+
+        # get the cookies jar, prefer the browser cookies than the file
+        if 'cookies_from_browser' in self.authentication:
+            authdict['cookies_from_browser'] = self.authentication['cookies_from_browser']
+            if extract_cookies:
+                authdict['cookies_jar'] = get_ytdlp_cookiejar(['--cookies-from-browser', self.authentication['cookies_from_browser']])
+        elif 'cookies_file' in self.authentication:
+            authdict['cookies_file'] = self.authentication['cookies_file']
+            if extract_cookies:
+                authdict['cookies_jar'] = get_ytdlp_cookiejar(['--cookies', self.authentication['cookies_file']])
+        
+        return authdict
+    
+    def repr(self):
+        return f"Module<'{self.display_name}' (config: {self.config[self.name]})>"

src/auto_archiver/core/config.py

@@ -1,117 +1,164 @@
+"""
+The Config class initializes and parses configurations for all other steps.
+It supports CLI argument parsing, loading from YAML file, and overrides to allow
+flexible setup in various environments.
 
+"""
+
+import argparse
+from ruamel.yaml import YAML, CommentedMap, add_representer
 
-import argparse, yaml
-from dataclasses import dataclass, field
-from typing import List
-from collections import defaultdict
 from loguru import logger
 
-from ..archivers import Archiver
-from ..feeders import Feeder
-from ..databases import Database
-from ..formatters import Formatter
-from ..storages import Storage
-from ..enrichers import Enricher
-from . import Step
+from copy import deepcopy
+from .module import BaseModule
 
+from typing import Any, List, Type, Tuple
 
-@dataclass
-class Config:
-    configurable_parents = [
-        Feeder,
-        Enricher,
-        Archiver,
-        Database,
-        Storage,
-        Formatter
-        # Util
-    ]
-    feeder: Feeder
-    formatter: Formatter
-    archivers: List[Archiver] = field(default_factory=[])
-    enrichers: List[Enricher] = field(default_factory=[])
-    storages: List[Storage] = field(default_factory=[])
-    databases: List[Database] = field(default_factory=[])
+_yaml: YAML = YAML()
 
-    def __init__(self) -> None:
-        self.defaults = {}
-        self.cli_ops = {}
-        self.config = {}
+EMPTY_CONFIG = _yaml.load("""
+# Auto Archiver Configuration
+# Steps are the modules that will be run in the order they are defined
 
-    def parse(self, use_cli=True, yaml_config_filename: str = None):
+steps:""" + "".join([f"\n   {module}s: []" for module in BaseModule.MODULE_TYPES]) + \
+"""
+
+# Global configuration
+
+# Authentication
+# a dictionary of authentication information that can be used by extractors to login to website. 
+# you can use a comma separated list for multiple domains on the same line (common usecase: x.com,twitter.com)
+# Common login 'types' are username/password, cookie, api key/token.
+# There are two special keys for using cookies, they are: cookies_file and cookies_from_browser. 
+# Some Examples:
+# facebook.com:
+#   username: "my_username"
+#   password: "my_password"
+# or for a site that uses an API key:
+# twitter.com,x.com:
+#   api_key
+#   api_secret
+# youtube.com:
+#   cookie: "login_cookie=value ; other_cookie=123" # multiple 'key=value' pairs should be separated by ;
+
+authentication: {}
+
+# These are the global configurations that are used by the modules
+
+logging:
+  level: INFO
+
+""")
+# note: 'logging' is explicitly added above in order to better format the config file
+
+class DefaultValidatingParser(argparse.ArgumentParser):
+
+    def error(self, message):
         """
-        if yaml_config_filename is provided, the --config argument is ignored, 
-        useful for library usage when the config values are preloaded
+        Override of error to format a nicer looking error message using logger
         """
-        # 1. parse CLI values
-        if use_cli:
-            parser = argparse.ArgumentParser(
-                # prog = "auto-archiver",
-                description="Auto Archiver is a CLI tool to archive media/metadata from online URLs; it can read URLs from many sources (Google Sheets, Command Line, ...); and write results to many destinations too (CSV, Google Sheets, MongoDB, ...)!",
-                epilog="Check the code at https://github.com/bellingcat/auto-archiver"
-            )
+        logger.error("Problem with configuration file (tip: use --help to see the available options):")
+        logger.error(message)
+        self.exit(2)
 
-            parser.add_argument('--config', action='store', dest='config', help='the filename of the YAML configuration file (defaults to \'config.yaml\')', default='config.yaml')
+    def parse_known_args(self, args=None, namespace=None):
+        """
+        Override of parse_known_args to also check the 'defaults' values - which are passed in from the config file
+        """
+        for action in self._actions:
+            if not namespace or action.dest not in namespace:
+                # for actions that are required and already have a default value, remove the 'required' check
+                if action.required and action.default is not None:
+                    action.required = False
 
-        for configurable in self.configurable_parents:
-            child: Step
-            for child in configurable.__subclasses__():
-                assert child.configs() is not None and type(child.configs()) == dict, f"class '{child.name}' should have a configs method returning a dict."
-                for config, details in child.configs().items():
-                    assert "." not in child.name, f"class prop name cannot contain dots('.'): {child.name}"
-                    assert "." not in config, f"config property cannot contain dots('.'): {config}"
-                    config_path = f"{child.name}.{config}"
+                if action.default is not None:
+                    try:
+                        self._check_value(action, action.default)
+                    except argparse.ArgumentError as e:
+                        logger.error(f"You have an invalid setting in your configuration file ({action.dest}):")
+                        logger.error(e)
+                        exit()
 
-                    if use_cli:
-                        try:
-                            parser.add_argument(f'--{config_path}', action='store', dest=config_path, help=f"{details['help']} (defaults to {details['default']})", choices=details.get("choices", None))
-                        except argparse.ArgumentError:
-                            # captures cases when a Step is used in 2 flows, eg: wayback enricher vs wayback archiver
-                            pass
+        return super().parse_known_args(args, namespace)
 
-                    self.defaults[config_path] = details["default"]
-                    if "cli_set" in details:
-                        self.cli_ops[config_path] = details["cli_set"]
 
-        if use_cli:
-            args = parser.parse_args()
-            yaml_config_filename = yaml_config_filename or getattr(args, "config")
-        else: args = {}
+def to_dot_notation(yaml_conf: CommentedMap | dict) -> dict:
+    dotdict = {}
 
-        # 2. read YAML config file (or use provided value)
-        self.yaml_config = self.read_yaml(yaml_config_filename)
+    def process_subdict(subdict, prefix=""):
+        for key, value in subdict.items():
+            if is_dict_type(value):
+                process_subdict(value, f"{prefix}{key}.")
+            else:
+                dotdict[f"{prefix}{key}"] = value
 
-        # 3. CONFIGS: decide value with priority: CLI >> config.yaml >> default
-        self.config = defaultdict(dict)
-        for config_path, default in self.defaults.items():
-            child, config = tuple(config_path.split("."))
-            val = getattr(args, config_path, None)
-            if val is not None and config_path in self.cli_ops:
-                val = self.cli_ops[config_path](val, default)
-            if val is None:
-                val = self.yaml_config.get("configurations", {}).get(child, {}).get(config, default)
-            self.config[child][config] = val
-        self.config = dict(self.config)
+    process_subdict(yaml_conf)
+    return dotdict
 
-        # 4. STEPS: read steps and validate they exist
-        steps = self.yaml_config.get("steps", {})
-        assert "archivers" in steps, "your configuration steps are missing the archivers property"
-        assert "storages" in steps, "your configuration steps are missing the storages property"
+def from_dot_notation(dotdict: dict) -> dict:
+    normal_dict = {}
 
-        self.feeder = Feeder.init(steps.get("feeder", "cli_feeder"), self.config)
-        self.formatter = Formatter.init(steps.get("formatter", "mute_formatter"), self.config)
-        self.enrichers = [Enricher.init(e, self.config) for e in steps.get("enrichers", [])]
-        self.archivers = [Archiver.init(e, self.config) for e in (steps.get("archivers") or [])]
-        self.databases = [Database.init(e, self.config) for e in steps.get("databases", [])]
-        self.storages = [Storage.init(e, self.config) for e in steps.get("storages", [])]
+    def add_part(key, value, current_dict):
+        if "." in key:
+            key_parts = key.split(".")
+            current_dict.setdefault(key_parts[0], {})
+            add_part(".".join(key_parts[1:]), value, current_dict[key_parts[0]])
+        else:
+            current_dict[key] = value
 
-        logger.info(f"FEEDER: {self.feeder.name}")
-        logger.info(f"ENRICHERS: {[x.name for x in self.enrichers]}")
-        logger.info(f"ARCHIVERS: {[x.name for x in self.archivers]}")
-        logger.info(f"DATABASES: {[x.name for x in self.databases]}")
-        logger.info(f"STORAGES: {[x.name for x in self.storages]}")
-        logger.info(f"FORMATTER: {self.formatter.name}")
+    for key, value in dotdict.items():
+        add_part(key, value, normal_dict)
 
-    def read_yaml(self, yaml_filename: str) -> dict:
+    return normal_dict
+
+
+def is_list_type(value):
+    return isinstance(value, list) or isinstance(value, tuple) or isinstance(value, set)
+
+def is_dict_type(value):
+    return isinstance(value, dict) or isinstance(value, CommentedMap)
+
+def merge_dicts(dotdict: dict, yaml_dict: CommentedMap) -> CommentedMap:
+    yaml_dict: CommentedMap = deepcopy(yaml_dict)
+
+    # first deal with lists, since 'update' replaces lists from a in b, but we want to extend
+    def update_dict(subdict, yaml_subdict):
+        for key, value in subdict.items():
+            if not yaml_subdict.get(key):
+                yaml_subdict[key] = value
+                continue
+
+            if is_dict_type(value):
+                update_dict(value, yaml_subdict[key])
+            elif is_list_type(value):
+                yaml_subdict[key].extend(s for s in value if s not in yaml_subdict[key])
+            else:
+                yaml_subdict[key] = value
+
+    update_dict(from_dot_notation(dotdict), yaml_dict)
+
+    return yaml_dict
+
+def read_yaml(yaml_filename: str) -> CommentedMap:
+    config = None
+    try:
         with open(yaml_filename, "r", encoding="utf-8") as inf:
-            return yaml.safe_load(inf)
+            config = _yaml.load(inf)
+    except FileNotFoundError:
+        pass
+
+    if not config:
+        config = EMPTY_CONFIG
+    
+    return config
+
+# TODO: make this tidier/find a way to notify of which keys should not be stored
+
+
+def store_yaml(config: CommentedMap, yaml_filename: str) -> None:
+    config_to_save = deepcopy(config)
+
+    config_to_save.pop('urls', None)
+    with open(yaml_filename, "w", encoding="utf-8") as outf:
+        _yaml.dump(config_to_save, outf)

src/auto_archiver/core/database.py

@@ -0,0 +1,39 @@
+"""
+Database module for the auto-archiver that defines the interface for implementing database modules
+in the media archiving framework. 
+"""
+
+from __future__ import annotations
+from abc import abstractmethod
+from typing import Union
+
+from auto_archiver.core import Metadata, BaseModule
+
+class Database(BaseModule):
+    """
+    Base class for implementing database modules in the media archiving framework.
+
+    Subclasses must implement the `fetch` and `done` methods to define platform-specific behavior.
+    """
+
+    def started(self, item: Metadata) -> None:
+        """signals the DB that the given item archival has started"""
+        pass
+
+    def failed(self, item: Metadata, reason:str) -> None:
+        """update DB accordingly for failure"""
+        pass
+
+    def aborted(self, item: Metadata) -> None:
+        """abort notification if user cancelled after start"""
+        pass
+
+    # @abstractmethod
+    def fetch(self, item: Metadata) -> Union[Metadata, bool]:
+        """check and fetch if the given item has been archived already, each database should handle its own caching, and configuration mechanisms"""
+        return False
+
+    @abstractmethod
+    def done(self, item: Metadata, cached: bool=False) -> None:
+        """archival result ready - should be saved to DB"""
+        pass

src/auto_archiver/core/enricher.py

@@ -0,0 +1,28 @@
+"""
+Base module for Enrichers – modular components that enhance archived content by adding
+context, metadata, or additional processing.
+
+These add additional information to the context, such as screenshots, hashes, and metadata.
+They are designed to work within the archiving pipeline, operating on `Metadata` objects after
+the archiving step and before storage or formatting.
+
+Enrichers are optional but highly useful for making the archived data more powerful.
+"""
+from __future__ import annotations
+from abc import abstractmethod
+from auto_archiver.core import Metadata, BaseModule
+
+class Enricher(BaseModule):
+    """Base classes and utilities for enrichers in the Auto-Archiver system.
+    
+    Enricher modules must implement the `enrich` method to define their behavior.
+    """
+
+    @abstractmethod
+    def enrich(self, to_enrich: Metadata) -> None:
+        """
+        Enriches a Metadata object with additional information or context.
+
+        Takes the metadata object to enrich as an argument and modifies it in place, returning None.
+        """
+        pass

src/auto_archiver/core/extractor.py

@@ -0,0 +1,115 @@
+""" The `extractor` module defines the base functionality for implementing extractors in the media archiving framework.
+    This class provides common utility methods and a standard interface for extractors.
+
+    Factory method to initialize an extractor instance based on its name.
+
+
+"""
+from __future__ import annotations
+from pathlib import Path
+from abc import abstractmethod
+from dataclasses import dataclass
+import mimetypes
+import os
+import mimetypes
+import requests
+from loguru import logger
+from retrying import retry
+import re
+
+from auto_archiver.core import Metadata, BaseModule
+
+
+class Extractor(BaseModule):
+    """
+    Base class for implementing extractors in the media archiving framework.
+    Subclasses must implement the `download` method to define platform-specific behavior.
+    """
+
+    valid_url: re.Pattern = None
+
+    def cleanup(self) -> None:
+        """
+        Called when extractors are done, or upon errors, cleanup any resources
+        """
+        pass
+
+    def sanitize_url(self, url: str) -> str:
+        """
+        Used to clean unnecessary URL parameters OR unfurl redirect links
+        """
+        return url
+    
+    def match_link(self, url: str) -> re.Match:
+        """
+        Returns a match object if the given URL matches the valid_url pattern or False/None if not.
+
+        Normally used in the `suitable` method to check if the URL is supported by this extractor.
+
+        """
+        return self.valid_url.match(url)
+
+    def suitable(self, url: str) -> bool:
+        """
+        Returns True if this extractor can handle the given URL
+
+        Should be overridden by subclasses
+
+        """
+        if self.valid_url:
+            return self.match_link(url) is not None
+        
+        return True
+
+    def _guess_file_type(self, path: str) -> str:
+        """
+        Receives a URL or filename and returns global mimetype like 'image' or 'video'
+        see https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types
+        """
+        mime = mimetypes.guess_type(path)[0]
+        if mime is not None:
+            return mime.split("/")[0]
+        return ""
+
+    @retry(wait_random_min=500, wait_random_max=3500, stop_max_attempt_number=5)
+    def download_from_url(self, url: str, to_filename: str = None, verbose=True) -> str:
+        """
+            downloads a URL to provided filename, or inferred from URL, returns local filename
+        """
+        if not to_filename:
+            to_filename = url.split('/')[-1].split('?')[0]
+            if len(to_filename) > 64:
+                to_filename = to_filename[-64:]
+        to_filename = os.path.join(self.tmp_dir, to_filename)
+        if verbose: logger.debug(f"downloading {url[0:50]=} {to_filename=}")
+        headers = {
+            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36'
+        }
+        try:
+            d = requests.get(url, stream=True, headers=headers, timeout=30)
+            d.raise_for_status()
+
+            # get mimetype from the response headers
+            if not mimetypes.guess_type(to_filename)[0]:
+                content_type = d.headers.get('Content-Type') or self._guess_file_type(url)
+                extension = mimetypes.guess_extension(content_type)
+                if extension:
+                    to_filename += extension
+
+            with open(to_filename, 'wb') as f:
+                for chunk in d.iter_content(chunk_size=8192):
+                    f.write(chunk)
+            return to_filename
+
+        except requests.RequestException as e:
+            logger.warning(f"Failed to fetch the Media URL: {e}")
+
+    @abstractmethod
+    def download(self, item: Metadata) -> Metadata | False:
+        """
+        Downloads the media from the given URL and returns a Metadata object with the downloaded media.
+        
+        If the URL is not supported or the download fails, this method should return False.
+
+        """
+        pass

src/auto_archiver/core/feeder.py

@@ -0,0 +1,25 @@
+"""
+The feeder base module defines the interface for implementing feeders in the media archiving framework. 
+"""
+
+from __future__ import annotations
+from abc import abstractmethod
+from auto_archiver.core import Metadata
+from auto_archiver.core import BaseModule
+
+class Feeder(BaseModule):
+
+    """
+    Base class for implementing feeders in the media archiving framework.
+
+    Subclasses must implement the `__iter__` method to define platform-specific behavior.
+    """
+
+    @abstractmethod
+    def __iter__(self) -> Metadata:
+        """
+        Returns an iterator (use `yield`) over the items to be archived.
+        
+        These should be instances of Metadata, typically created with Metadata().set_url(url).
+        """
+        return None

src/auto_archiver/core/formatter.py

@@ -0,0 +1,24 @@
+"""
+Base module for formatters – modular components that format metadata into media objects for storage.
+
+The most commonly used formatter is the HTML formatter, which takes metadata and formats it into an HTML file for storage.
+"""
+
+from __future__ import annotations
+from abc import abstractmethod
+from auto_archiver.core import Metadata, Media, BaseModule
+
+
+class Formatter(BaseModule):
+    """
+    Base class for implementing formatters in the media archiving framework.
+    
+    Subclasses must implement the `format` method to define their behavior.
+    """
+
+    @abstractmethod
+    def format(self, item: Metadata) -> Media:
+        """
+        Formats a Metadata object into a user-viewable format (e.g. HTML) and stores it if needed.
+        """
+        return None

src/auto_archiver/core/media.py

@@ -1,20 +1,71 @@
+"""
+Manages media files and their associated metadata, supporting storage,
+nested media retrieval, and type validation.
+"""
 
 from __future__ import annotations
-from ast import List
-from typing import Any
+import os
+import traceback
+from typing import Any, List
 from dataclasses import dataclass, field
-from dataclasses_json import dataclass_json
+from dataclasses_json import dataclass_json, config
 import mimetypes
 
-# annotation order matters
-@dataclass_json
+from loguru import logger
+
+
+@dataclass_json  # annotation order matters
 @dataclass
 class Media:
+    """
+    Represents a media file with associated properties and storage details.
+
+    Attributes:
+    - filename: The file path of the media.
+    - key: An optional identifier for the media.
+    - urls: A list of URLs where the media is stored or accessible.
+    - properties: Additional metadata or transformations for the media.
+    - _mimetype: The media's mimetype (e.g., image/jpeg, video/mp4).
+    """
     filename: str
     key: str = None
     urls: List[str] = field(default_factory=list)
-    _mimetype: str = None  # eg: image/jpeg
     properties: dict = field(default_factory=dict)
+    _mimetype: str = None  # eg: image/jpeg
+    _stored: bool = field(default=False, repr=False, metadata=config(exclude=lambda _: True))  # always exclude
+
+    def store(self: Media, metadata: Any, url: str = "url-not-available", storages: List[Any] = None) -> None:
+        # 'Any' typing for metadata to avoid circular imports. Stores the media
+        # into the provided/available storages [Storage] repeats the process for
+        # its properties, in case they have inner media themselves for now it
+        # only goes down 1 level but it's easy to make it recursive if needed.
+        if not len(storages):
+            logger.warning(f"No storages found in local context or provided directly for {self.filename}.")
+            return
+
+        for s in storages:
+            for any_media in self.all_inner_media(include_self=True):
+                s.store(any_media, url, metadata=metadata)
+
+    def all_inner_media(self, include_self=False):
+        """Retrieves all media, including nested media within properties or transformations on original media.
+        This function returns a generator for all the inner media.
+
+        """
+        if include_self: yield self
+        for prop in self.properties.values():
+            if isinstance(prop, Media): 
+                for inner_media in prop.all_inner_media(include_self=True):
+                    yield inner_media
+            if isinstance(prop, list):
+                for prop_media in prop:
+                    if isinstance(prop_media, Media): 
+                        for inner_media in prop_media.all_inner_media(include_self=True):
+                            yield inner_media
+
+    def is_stored(self, in_storage) -> bool:
+        # checks if the media is already stored in the given storage
+        return len(self.urls) > 0 and len(self.urls) == len(in_storage.config["steps"]["storages"])
 
     def set(self, key: str, value: Any) -> Media:
         self.properties[key] = value
@@ -29,7 +80,9 @@ class Media:
 
     @property  # getter .mimetype
     def mimetype(self) -> str:
-        assert self.filename is not None and len(self.filename) > 0, "cannot get mimetype from media without filename"
+        if not self.filename or len(self.filename) == 0:
+            logger.warning(f"cannot get mimetype from media without filename: {self}")
+            return ""
         if not self._mimetype:
             self._mimetype = mimetypes.guess_type(self.filename)[0]
         return self._mimetype or ""
@@ -40,3 +93,32 @@ class Media:
 
     def is_video(self) -> bool:
         return self.mimetype.startswith("video")
+
+    def is_audio(self) -> bool:
+        return self.mimetype.startswith("audio")
+
+    def is_image(self) -> bool:
+        return self.mimetype.startswith("image")
+
+    def is_valid_video(self) -> bool:
+        # Note: this is intentional, to only import ffmpeg here - when the method is called
+        # this speeds up loading the module. We check that 'ffmpeg' is available on startup
+        # when we load each manifest file
+        import ffmpeg
+        from ffmpeg._run import Error
+
+        # checks for video streams with ffmpeg, or min file size for a video
+        # self.is_video() should be used together with this method
+        try:
+            streams = ffmpeg.probe(self.filename, select_streams='v')['streams']
+            logger.warning(f"STREAMS FOR {self.filename} {streams}")
+            return any(s.get("duration_ts", 0) > 0 for s in streams)
+        except Error: return False # ffmpeg errors when reading bad files
+        except Exception as e:
+            logger.error(e)
+            logger.error(traceback.format_exc())
+            try:
+                fsize = os.path.getsize(self.filename)
+                return fsize > 20_000
+            except: pass
+        return True

src/auto_archiver/core/metadata.py

@@ -1,36 +1,50 @@
+"""
+Acts as a container for metadata and media objects associated with an archived item.
+
+Key Functionalities:
+- Store and retrieve metadata and associated media.
+- Merge metadata objects with conflict resolution.
+- Validate properties like URLs and timestamps.
+- Manage and deduplicate media objects.
+- Support for flexible metadata querying and appending.
+"""
 
 from __future__ import annotations
-from ast import List, Set
-from typing import Any, Union, Dict
+import hashlib
+from typing import Any, List, Union, Dict
 from dataclasses import dataclass, field
-from dataclasses_json import dataclass_json
+from dataclasses_json import dataclass_json, config
 import datetime
 from urllib.parse import urlparse
 from dateutil.parser import parse as parse_dt
+from loguru import logger
+
 from .media import Media
 
-
-# annotation order matters
-@dataclass_json
+@dataclass_json  # annotation order matters
 @dataclass
 class Metadata:
     status: str = "no archiver"
-    _processed_at: datetime = field(default_factory=datetime.datetime.utcnow)
     metadata: Dict[str, Any] = field(default_factory=dict)
-    tmp_keys: Set[str] = field(default_factory=set, repr=False, metadata={"exclude": True})  # keys that are not to be saved in DBs
     media: List[Media] = field(default_factory=list)
-    rearchivable: bool = True  # defaults to true, archivers can overwrite
+
+    def __post_init__(self):
+        self.set("_processed_at", datetime.datetime.now(datetime.timezone.utc))
+        self._context = {}
 
     def merge(self: Metadata, right: Metadata, overwrite_left=True) -> Metadata:
         """
-        merges two Metadata instances, will overwrite according to overwrite_left flag
+        Merges another `Metadata` instance into this one.
+
+        Conflicts are resolved based on the `overwrite_left` flag:
+        - If `True`, this instance's values are overwritten by `right`.
+        - If `False`, the inverse applies.
         """
         if not right: return self
         if overwrite_left:
             if right.status and len(right.status):
                 self.status = right.status
-            self.rearchivable |= right.rearchivable
-            self.tmp_keys |= right.tmp_keys
+            self._context.update(right._context)
             for k, v in right.metadata.items():
                 assert k not in self.metadata or type(v) == type(self.get(k))
                 if type(v) not in [dict, list, set] or k not in self.metadata:
@@ -43,10 +57,20 @@ class Metadata:
             return right.merge(self)
         return self
 
-    def set(self, key: str, val: Any, is_tmp=False) -> Metadata:
-        # if not self.metadata: self.metadata = {}
+    def store(self, storages=[]):
+        # calls .store for all contained media. storages [Storage]
+        self.remove_duplicate_media_by_hash()
+        for media in self.media:
+            media.store(url=self.get_url(), metadata=self, storages=storages)
+
+    def set(self, key: str, val: Any) -> Metadata:
+        self.metadata[key] = val
+        return self
+
+    def append(self, key: str, val: Any) -> Metadata:
+        if key not in self.metadata:
+            self.metadata[key] = []    
         self.metadata[key] = val
-        if is_tmp: self.tmp_keys.add(key)
         return self
 
     def get(self, key: str, default: Any = None, create_if_missing=False) -> Union[Metadata, str]:
@@ -63,6 +87,10 @@ class Metadata:
     def is_success(self) -> bool:
         return "success" in self.status
 
+    def is_empty(self) -> bool:
+        meaningfull_ids = set(self.metadata.keys()) - set(["_processed_at", "url", "total_bytes", "total_size", "archive_duration_seconds"])
+        return not self.is_success() and len(self.media) == 0 and len(meaningfull_ids) == 0
+
     @property  # getter .netloc
     def netloc(self) -> str:
         return urlparse(self.get_url()).netloc
@@ -82,7 +110,8 @@ class Metadata:
 
     def set_content(self, content: str) -> Metadata:
         # a dump with all the relevant content
-        return self.set("content", content)
+        append_content = (self.get("content", "") + content + "\n").strip()
+        return self.set("content", append_content)
 
     def set_title(self, title: str) -> Metadata:
         return self.set("title", title)
@@ -90,12 +119,6 @@ class Metadata:
     def get_title(self) -> str:
         return self.get("title")
 
-    def set_tmp_dir(self, tmp_dir: str) -> Metadata:
-        return self.set("tmp_dir", tmp_dir, True)
-
-    def get_tmp_dir(self) -> str:
-        return self.get("tmp_dir")
-
     def set_timestamp(self, timestamp: datetime.datetime) -> Metadata:
         if type(timestamp) == str:
             timestamp = parse_dt(timestamp)
@@ -104,10 +127,16 @@ class Metadata:
 
     def get_timestamp(self, utc=True, iso=True) -> datetime.datetime:
         ts = self.get("timestamp")
-        if not ts: return ts
-        if utc: ts = ts.replace(tzinfo=datetime.timezone.utc)
-        if iso: return ts.isoformat()
-        return ts
+        if not ts: return 
+        try:
+            if type(ts) == str: ts = datetime.datetime.fromisoformat(ts)
+            if type(ts) == float: ts = datetime.datetime.fromtimestamp(ts)
+            if utc: ts = ts.replace(tzinfo=datetime.timezone.utc)
+            if iso: return ts.isoformat()
+            return ts
+        except Exception as e:
+            logger.error(f"Unable to parse timestamp {ts}: {e}")
+            return
 
     def add_media(self, media: Media, id: str = None) -> Metadata:
         # adds a new media, optionally including an id
@@ -122,7 +151,28 @@ class Metadata:
         for m in self.media:
             if m.get("id") == id: return m
         return default
-    
+
+    def remove_duplicate_media_by_hash(self) -> None:
+        # iterates all media, calculates a hash if it's missing and deletes duplicates
+        def calculate_hash_in_chunks(hash_algo, chunksize, filename) -> str:
+            # taken from hash_enricher, cannot be isolated to misc due to circular imports
+            with open(filename, "rb") as f:
+                while True:
+                    buf = f.read(chunksize)
+                    if not buf: break
+                    hash_algo.update(buf)
+            return hash_algo.hexdigest()
+
+        media_hashes = set()
+        new_media = []
+        for m in self.media:
+            h = m.get("hash")
+            if not h: h = calculate_hash_in_chunks(hashlib.sha256(), int(1.6e7), m.filename)
+            if len(h) and h in media_hashes: continue
+            media_hashes.add(h)
+            new_media.append(m)
+        self.media = new_media
+
     def get_first_image(self, default=None) -> Media:
         for m in self.media:
             if "image" in m.mimetype: return m
@@ -136,8 +186,29 @@ class Metadata:
         _default = self.media[0] if len(self.media) else None
         return self.get_media_by_id("_final_media", _default)
 
-    def get_clean_metadata(self) -> Metadata:
-        return dict(
-            {k: v for k, v in self.metadata.items() if k not in self.tmp_keys},
-            **{"processed_at": self._processed_at}
-        )
+    def get_all_media(self) -> List[Media]:
+        # returns a list with all the media and inner media
+        return [inner for m in self.media for inner in m.all_inner_media(True)]
+
+    def __str__(self) -> str:
+        return self.__repr__()
+
+
+    @staticmethod
+    def choose_most_complete(results: List[Metadata]) -> Metadata:
+        # returns the most complete result from a list of results
+        # prioritizes results with more media, then more metadata
+        if len(results) == 0: return None
+        if len(results) == 1: return results[0]
+        most_complete = results[0]
+        for r in results[1:]:
+            if len(r.media) > len(most_complete.media): most_complete = r
+            elif len(r.media) == len(most_complete.media) and len(r.metadata) > len(most_complete.metadata): most_complete = r
+        return most_complete
+
+    def set_context(self, key: str, val: Any) -> Metadata:
+        self._context[key] = val
+        return self
+    
+    def get_context(self, key: str, default: Any = None) -> Any:
+        return self._context.get(key, default)

src/auto_archiver/core/module.py

@@ -0,0 +1,249 @@
+"""
+Defines the Step abstract base class, which acts as a blueprint for steps in the archiving pipeline
+by handling user configuration, validating the steps properties, and implementing dynamic instantiation.
+
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import List
+import shutil
+import ast
+import copy
+import sys
+from importlib.util import find_spec
+import os
+from os.path import join
+from loguru import logger
+import auto_archiver
+from .base_module import BaseModule
+
+_LAZY_LOADED_MODULES = {}
+
+MANIFEST_FILE = "__manifest__.py"
+
+
+def setup_paths(paths: list[str]) -> None:
+    """
+    Sets up the paths for the modules to be loaded from
+    
+    This is necessary for the modules to be imported correctly
+    
+    """
+    for path in paths:
+        # check path exists, if it doesn't, log a warning
+        if not os.path.exists(path):
+            logger.warning(f"Path '{path}' does not exist. Skipping...")
+            continue
+
+        # see odoo/module/module.py -> initialize_sys_path
+        if path not in auto_archiver.modules.__path__:
+                auto_archiver.modules.__path__.append(path)
+
+    # sort based on the length of the path, so that the longest path is last in the list
+    auto_archiver.modules.__path__ = sorted(auto_archiver.modules.__path__, key=len, reverse=True)
+
+def get_module(module_name: str, config: dict) -> BaseModule:
+    """
+    Gets and sets up a module using the provided config
+    
+    This will actually load and instantiate the module, and load all its dependencies (i.e. not lazy)
+    
+    """
+    return get_module_lazy(module_name).load(config)
+
+def get_module_lazy(module_name: str, suppress_warnings: bool = False) -> LazyBaseModule:
+    """
+    Lazily loads a module, returning a LazyBaseModule
+    
+    This has all the information about the module, but does not load the module itself or its dependencies
+    
+    To load an actual module, call .setup() on a lazy module
+    
+    """
+    if module_name in _LAZY_LOADED_MODULES:
+        return _LAZY_LOADED_MODULES[module_name]
+
+    available = available_modules(limit_to_modules=[module_name], suppress_warnings=suppress_warnings)
+    if not available:
+        raise IndexError(f"Module '{module_name}' not found. Are you sure it's installed/exists?")
+    return available[0]
+
+def available_modules(with_manifest: bool=False, limit_to_modules: List[str]= [], suppress_warnings: bool = False) -> List[LazyBaseModule]:
+    
+    # search through all valid 'modules' paths. Default is 'modules' in the current directory
+
+    # see odoo/modules/module.py -> get_modules
+    def is_really_module(module_path):
+        if os.path.isfile(join(module_path, MANIFEST_FILE)):
+            return True
+
+    all_modules = []
+
+    for module_folder in auto_archiver.modules.__path__:
+        # walk through each module in module_folder and check if it has a valid manifest
+        try:
+            possible_modules = os.listdir(module_folder)
+        except FileNotFoundError:
+            logger.warning(f"Module folder {module_folder} does not exist")
+            continue
+
+        for possible_module in possible_modules:
+            if limit_to_modules and possible_module not in limit_to_modules:
+                continue
+
+            possible_module_path = join(module_folder, possible_module)
+            if not is_really_module(possible_module_path):
+                continue
+            if _LAZY_LOADED_MODULES.get(possible_module):
+                continue
+            lazy_module = LazyBaseModule(possible_module, possible_module_path)
+
+            _LAZY_LOADED_MODULES[possible_module] = lazy_module
+
+            all_modules.append(lazy_module)
+    
+    if not suppress_warnings:
+        for module in limit_to_modules:
+            if not any(module == m.name for m in all_modules):
+                logger.warning(f"Module '{module}' not found. Are you sure it's installed?")
+
+    return all_modules
+
+@dataclass
+class LazyBaseModule:
+
+    """
+    A lazy module class, which only loads the manifest and does not load the module itself.
+
+    This is useful for getting information about a module without actually loading it.
+
+    """
+    name: str
+    type: list
+    description: str
+    path: str
+
+    _manifest: dict = None
+    _instance: BaseModule = None
+    _entry_point: str = None
+
+    def __init__(self, module_name, path):
+        self.name = module_name
+        self.path = path
+
+    @property
+    def entry_point(self):
+        if not self._entry_point and not self.manifest['entry_point']:
+            # try to create the entry point from the module name
+            self._entry_point = f"{self.name}::{self.name.replace('_', ' ').title().replace(' ', '')}"
+        return self._entry_point
+
+    @property
+    def dependencies(self) -> dict:
+        return self.manifest['dependencies']
+    
+    @property
+    def configs(self) -> dict:
+        return self.manifest['configs']
+    
+    @property
+    def requires_setup(self) -> bool:
+        return self.manifest['requires_setup']
+    
+    @property
+    def display_name(self) -> str:
+        return self.manifest['name']
+
+    @property
+    def manifest(self) -> dict:
+        if self._manifest:
+            return self._manifest
+        # print(f"Loading manifest for module {module_path}")
+        # load the manifest file
+        manifest = copy.deepcopy(BaseModule._DEFAULT_MANIFEST)
+
+        with open(join(self.path, MANIFEST_FILE)) as f:
+            try:
+                manifest.update(ast.literal_eval(f.read()))
+            except (ValueError, TypeError, SyntaxError, MemoryError, RecursionError) as e:
+                logger.error(f"Error loading manifest from file {self.path}/{MANIFEST_FILE}: {e}")
+            
+        self._manifest = manifest
+        self.type = manifest['type']
+        self._entry_point = manifest['entry_point']
+        self.description = manifest['description']
+        self.version = manifest['version']
+
+        return manifest
+
+    def load(self, config) -> BaseModule:
+
+        if self._instance:
+            return self._instance
+
+        # check external dependencies are installed
+        def check_deps(deps, check):
+            for dep in deps:
+                if not len(dep):
+                    # clear out any empty strings that a user may have erroneously added
+                    continue
+                if not check(dep):
+                    logger.error(f"Module '{self.name}' requires external dependency '{dep}' which is not available/setup. Have you installed the required dependencies for the '{self.name}' module? See the README for more information.")
+                    exit(1)
+
+        def check_python_dep(dep):
+            # first check if it's a module:
+            try:
+                m = get_module_lazy(dep, suppress_warnings=True)
+                try:
+                # we must now load this module and set it up with the config
+                    m.load(config)
+                    return True
+                except:
+                    logger.error(f"Unable to setup module '{dep}' for use in module '{self.name}'")
+                    return False
+            except IndexError:
+                # not a module, continue
+                pass
+
+            return find_spec(dep)
+
+        check_deps(self.dependencies.get('python', []), check_python_dep)
+        check_deps(self.dependencies.get('bin', []), lambda dep: shutil.which(dep))
+
+
+        logger.debug(f"Loading module '{self.display_name}'...")
+
+        for qualname in [self.name, f'auto_archiver.modules.{self.name}']:
+            try:
+                # first import the whole module, to make sure it's working properly
+                __import__(qualname)
+                break
+            except ImportError:
+                pass
+
+        # then import the file for the entry point
+        file_name, class_name = self.entry_point.split('::')
+        sub_qualname = f'{qualname}.{file_name}'
+
+        __import__(f'{qualname}.{file_name}', fromlist=[self.entry_point])
+        # finally, get the class instance
+        instance: BaseModule = getattr(sys.modules[sub_qualname], class_name)()
+        if not getattr(instance, 'name', None):
+            instance.name = self.name
+
+        if not getattr(instance, 'display_name', None):
+            instance.display_name = self.display_name
+
+        self._instance = instance
+
+        # merge the default config with the user config
+        default_config = dict((k, v['default']) for k, v in self.configs.items() if v.get('default'))
+        config[self.name] = default_config  | config.get(self.name, {})
+        instance.config_setup(config)
+        instance.setup()
+        return instance
+
+    def __repr__(self):
+        return f"Module<'{self.display_name}' ({self.name})>"

src/auto_archiver/core/orchestrator.py

@@ -1,130 +1,497 @@
+""" Orchestrates all archiving steps, including feeding items,
+    archiving them with specific archivers, enrichment, storage,
+    formatting, database operations and clean up.
+
+"""
+
 from __future__ import annotations
-from ast import List
-from typing import Union
+from typing import Generator, Union, List, Type
+from urllib.parse import urlparse
+from ipaddress import ip_address
+import argparse
+import os
+import sys
+import json
+from tempfile import TemporaryDirectory
+import traceback
 
-from ..archivers import Archiver
-from ..feeders import Feeder
-from ..formatters import Formatter
-from ..storages import Storage
-from ..enrichers import Enricher
-from ..databases import Database
-from .media import Media
-from .metadata import Metadata
+from rich_argparse import RichHelpFormatter
+
+
+from .metadata import Metadata, Media
+from auto_archiver.version import __version__
+from .config import _yaml, read_yaml, store_yaml, to_dot_notation, merge_dicts, EMPTY_CONFIG, DefaultValidatingParser
+from .module import available_modules, LazyBaseModule, get_module, setup_paths
+from . import validators, Feeder, Extractor, Database, Storage, Formatter, Enricher
+from .module import BaseModule
 
-import tempfile, traceback
 from loguru import logger
 
 
+DEFAULT_CONFIG_FILE = "orchestration.yaml"
+
+
+class JsonParseAction(argparse.Action):
+    def __call__(self, parser, namespace, values, option_string=None):
+        try:
+            setattr(namespace, self.dest, json.loads(values))
+        except json.JSONDecodeError as e:
+            raise argparse.ArgumentTypeError(f"Invalid JSON input for argument '{self.dest}': {e}")
+
+
+class AuthenticationJsonParseAction(JsonParseAction):
+    def __call__(self, parser, namespace, values, option_string=None):
+        super().__call__(parser, namespace, values, option_string)
+        auth_dict = getattr(namespace, self.dest)
+        if isinstance(auth_dict, str):
+            # if it's a string
+            try:
+                with open(auth_dict, 'r') as f:
+                    try:
+                        auth_dict = json.load(f)
+                    except json.JSONDecodeError:
+                        # maybe it's yaml, try that
+                        auth_dict = _yaml.load(f)
+            except:
+                pass
+
+        if not isinstance(auth_dict, dict):
+            raise argparse.ArgumentTypeError("Authentication must be a dictionary of site names and their authentication methods")
+        for site, auth in auth_dict.items():
+            if not isinstance(site, str) or not isinstance(auth, dict):
+                raise argparse.ArgumentTypeError("Authentication must be a dictionary of site names and their authentication methods")
+        setattr(namespace, self.dest, auth_dict)
+
+
+class UniqueAppendAction(argparse.Action):
+    def __call__(self, parser, namespace, values, option_string=None):
+        if not hasattr(namespace, self.dest):
+            setattr(namespace, self.dest, [])
+        for value in values:
+            if value not in getattr(namespace, self.dest):
+                getattr(namespace, self.dest).append(value)
+
+
 class ArchivingOrchestrator:
-    def __init__(self, config) -> None:
-        self.feeder: Feeder = config.feeder
-        self.formatter: Formatter = config.formatter
-        self.enrichers: List[Enricher] = config.enrichers
-        self.archivers: List[Archiver] = config.archivers
-        self.databases: List[Database] = config.databases
-        self.storages: List[Storage] = config.storages
 
-        for a in self.archivers: a.setup()
+    feeders: List[Type[Feeder]]
+    extractors: List[Type[Extractor]]
+    enrichers: List[Type[Enricher]]
+    databases: List[Type[Database]]
+    storages: List[Type[Storage]]
+    formatters: List[Type[Formatter]]
 
-    def feed(self) -> None:
-        for item in self.feeder:
-            self.feed_item(item)
+    def setup_basic_parser(self):
+        parser = argparse.ArgumentParser(
+            prog="auto-archiver",
+            add_help=False,
+            description="""
+                Auto Archiver is a CLI tool to archive media/metadata from online URLs;
+                it can read URLs from many sources (Google Sheets, Command Line, ...); and write results to many destinations too (CSV, Google Sheets, MongoDB, ...)!
+                """,
+            epilog="Check the code at https://github.com/bellingcat/auto-archiver",
+            formatter_class=RichHelpFormatter,
+        )
+        parser.add_argument('--help', '-h', action='store_true', dest='help', help='show a full help message and exit')
+        parser.add_argument('--version', action='version', version=__version__)
+        parser.add_argument('--config', action='store', dest="config_file", help='the filename of the YAML configuration file (defaults to \'config.yaml\')', default=DEFAULT_CONFIG_FILE)
+        parser.add_argument('--mode', action='store', dest='mode', type=str, choices=['simple', 'full'], help='the mode to run the archiver in', default='simple')
+        # override the default 'help' so we can inject all the configs and show those
+        parser.add_argument('-s', '--store', dest='store', default=False, help='Store the created config in the config file', action=argparse.BooleanOptionalAction)
+        parser.add_argument('--module_paths', dest='module_paths', nargs='+', default=[], help='additional paths to search for modules', action=UniqueAppendAction)
+
+        self.basic_parser = parser
+        return parser
+
+    def setup_complete_parser(self, basic_config: dict, yaml_config: dict, unused_args: list[str]) -> None:
+        parser = DefaultValidatingParser(
+            add_help=False,
+        )
+        self.add_additional_args(parser)
+
+        # check what mode we're in
+        # if we have a config file, use that to decide which modules to load
+        # if simple, we'll load just the modules that has requires_setup = False
+        # if full, we'll load all modules
+        # TODO: BUG** - basic_config won't have steps in it, since these args aren't added to 'basic_parser'
+        # but should we add them? Or should we just add them to the 'complete' parser?
+        if yaml_config != EMPTY_CONFIG:
+            # only load the modules enabled in config
+            # TODO: if some steps are empty (e.g. 'feeders' is empty), should we default to the 'simple' ones? Or only if they are ALL empty?
+            enabled_modules = []
+            # first loads the modules from the config file, then from the command line
+            for config in [yaml_config['steps'], basic_config.__dict__]:
+                for module_type in BaseModule.MODULE_TYPES:
+                    enabled_modules.extend(config.get(f"{module_type}s", []))
+
+            # clear out duplicates, but keep the order
+            enabled_modules = list(dict.fromkeys(enabled_modules))
+            avail_modules = available_modules(with_manifest=True, limit_to_modules=enabled_modules, suppress_warnings=True)
+            self.add_module_args(avail_modules, parser)
+        elif basic_config.mode == 'simple':
+            simple_modules = [module for module in available_modules(with_manifest=True) if not module.requires_setup]
+            self.add_module_args(simple_modules, parser)
+
+            # for simple mode, we use the cli_feeder and any modules that don't require setup
+            yaml_config['steps']['feeders'] = ['cli_feeder']
+
+            # add them to the config
+            for module in simple_modules:
+                for module_type in module.type:
+                    yaml_config['steps'].setdefault(f"{module_type}s", []).append(module.name)
+        else:
+            # load all modules, they're not using the 'simple' mode
+            self.add_module_args(available_modules(with_manifest=True), parser)
+
+        parser.set_defaults(**to_dot_notation(yaml_config))
+
+        # reload the parser with the new arguments, now that we have them
+        parsed, unknown = parser.parse_known_args(unused_args)
+
+        # merge the new config with the old one
+        self.config = merge_dicts(vars(parsed), yaml_config)
+        # clean out args from the base_parser that we don't want in the config
+        for key in vars(basic_config):
+            self.config.pop(key, None)
+
+        # setup the logging
+        self.setup_logging()
+
+        if unknown:
+            logger.warning(f"Ignoring unknown/unused arguments: {unknown}\nPerhaps you don't have this module enabled?")
+
+        if (self.config != yaml_config and basic_config.store) or not os.path.isfile(basic_config.config_file):
+            logger.info(f"Storing configuration file to {basic_config.config_file}")
+            store_yaml(self.config, basic_config.config_file)
+
+        return self.config
+
+    def add_additional_args(self, parser: argparse.ArgumentParser = None):
+        if not parser:
+            parser = self.parser
+
+        # allow passing URLs directly on the command line
+        parser.add_argument('urls', nargs='*', default=[], help='URL(s) to archive, either a single URL or a list of urls, should not come from config.yaml')
+
+        parser.add_argument('--feeders', dest='steps.feeders', nargs='+', default=['cli_feeder'], help='the feeders to use', action=UniqueAppendAction)
+        parser.add_argument('--enrichers', dest='steps.enrichers', nargs='+', help='the enrichers to use', action=UniqueAppendAction)
+        parser.add_argument('--extractors', dest='steps.extractors', nargs='+', help='the extractors to use', action=UniqueAppendAction)
+        parser.add_argument('--databases', dest='steps.databases', nargs='+', help='the databases to use', action=UniqueAppendAction)
+        parser.add_argument('--storages', dest='steps.storages', nargs='+', help='the storages to use', action=UniqueAppendAction)
+        parser.add_argument('--formatters', dest='steps.formatters', nargs='+', help='the formatter to use', action=UniqueAppendAction)
+
+        parser.add_argument('--authentication', dest='authentication', help='A dictionary of sites and their authentication methods \
+                                                                            (token, username etc.) that extractors can use to log into \
+                                                                            a website. If passing this on the command line, use a JSON string. \
+                                                                            You may also pass a path to a valid JSON/YAML file which will be parsed.',
+                                                                            default={},
+                                                                            action=AuthenticationJsonParseAction)
+        # logging arguments
+        parser.add_argument('--logging.level', action='store', dest='logging.level', choices=['INFO', 'DEBUG', 'ERROR', 'WARNING'], help='the logging level to use', default='INFO', type=str.upper)
+        parser.add_argument('--logging.file', action='store', dest='logging.file', help='the logging file to write to', default=None)
+        parser.add_argument('--logging.rotation', action='store', dest='logging.rotation', help='the logging rotation to use', default=None)
+
+    def add_module_args(self, modules: list[LazyBaseModule] = None, parser: argparse.ArgumentParser = None) -> None:
+
+        if not modules:
+            modules = available_modules(with_manifest=True)
+
+        module: LazyBaseModule
+        for module in modules:
+
+            if not module.configs:
+                # this module has no configs, don't show anything in the help
+                # (TODO: do we want to show something about this module though, like a description?)
+                continue
+
+            group = parser.add_argument_group(module.display_name or module.name, f"{module.description[:100]}...")
+
+            for name, kwargs in module.configs.items():
+                if not kwargs.get('metavar', None):
+                    # make a nicer metavar, metavar is what's used in the help, e.g. --cli_feeder.urls [METAVAR]
+                    kwargs['metavar'] = name.upper()
+
+                if kwargs.get('required', False):
+                    # required args shouldn't have a 'default' value, remove it
+                    kwargs.pop('default', None)
+
+                kwargs.pop('cli_set', None)
+                should_store = kwargs.pop('should_store', False)
+                kwargs['dest'] = f"{module.name}.{kwargs.pop('dest', name)}"
+                try:
+                    kwargs['type'] = getattr(validators, kwargs.get('type', '__invalid__'))
+                except AttributeError:
+                    kwargs['type'] = __builtins__.get(kwargs.get('type'), str)
+                arg = group.add_argument(f"--{module.name}.{name}", **kwargs)
+                arg.should_store = should_store
+
+    def show_help(self, basic_config: dict):
+        # for the help message, we want to load *all* possible modules and show the help
+        # add configs as arg parser arguments
+
+        self.add_additional_args(self.basic_parser)
+        self.add_module_args(parser=self.basic_parser)
+        self.basic_parser.print_help()
+        self.basic_parser.exit()
+
+    def setup_logging(self):
+        # setup loguru logging
+        logger.remove(0)  # remove the default logger
+        logging_config = self.config['logging']
+        logger.add(sys.stderr, level=logging_config['level'])
+        if log_file := logging_config['file']:
+            logger.add(log_file) if not logging_config['rotation'] else logger.add(log_file, rotation=logging_config['rotation'])
+
+    def install_modules(self, modules_by_type):
+        """
+        Traverses all modules in 'steps' and loads them into the orchestrator, storing them in the 
+        orchestrator's attributes (self.feeders, self.extractors etc.). If no modules of a certain type
+        are loaded, the program will exit with an error message.
+        """
+
+        invalid_modules = []
+        for module_type in BaseModule.MODULE_TYPES:
+
+            step_items = []
+            modules_to_load = modules_by_type[f"{module_type}s"]
+            assert modules_to_load, f"No {module_type}s were configured. Make sure to set at least one {module_type} in your configuration file or on the command line (using --{module_type}s)"
+
+            def check_steps_ok():
+                if not len(step_items):
+                    logger.error(f"NO {module_type.upper()}S LOADED. Please check your configuration and try again.")
+                    if len(modules_to_load):
+                        logger.error(f"Tried to load the following modules, but none were available: {modules_to_load}")
+                    exit()
+
+                if (module_type == 'feeder' or module_type == 'formatter') and len(step_items) > 1:
+                    logger.error(f"Only one {module_type} is allowed, found {len(step_items)} {module_type}s. Please remove one of the following from your configuration file: {modules_to_load}")
+                    exit()
+
+            for module in modules_to_load:
+                if module == 'cli_feeder':
+                    # pseudo module, don't load it
+                    urls = self.config['urls']
+                    if not urls:
+                        logger.error("No URLs provided. Please provide at least one URL via the command line, or set up an alternative feeder. Use --help for more information.")
+                        exit()
+                    # cli_feeder is a pseudo module, it just takes the command line args
+
+                    def feed(self) -> Generator[Metadata]:
+                        for url in urls:
+                            logger.debug(f"Processing URL: '{url}'")
+                            yield Metadata().set_url(url)
+
+                    pseudo_module = type('CLIFeeder', (Feeder,), {
+                        'name': 'cli_feeder',
+                        'display_name': 'CLI Feeder',
+                        '__iter__': feed
+
+                    })()
+
+                    pseudo_module.__iter__ = feed
+                    step_items.append(pseudo_module)
+                    continue
+
+                if module in invalid_modules:
+                    continue
+                try:
+                    loaded_module: BaseModule = get_module(module, self.config)
+                except (KeyboardInterrupt, Exception) as e:
+                    logger.error(f"Error during setup of modules: {e}\n{traceback.format_exc()}")
+                    if module_type == 'extractor' and loaded_module.name == module:
+                        loaded_module.cleanup()
+                    exit()
+
+                if not loaded_module:
+                    invalid_modules.append(module)
+                    continue
+                if loaded_module:
+                    step_items.append(loaded_module)
+
+            check_steps_ok()
+            setattr(self, f"{module_type}s", step_items)
+
+    def load_config(self, config_file: str) -> dict:
+        if not os.path.exists(config_file) and config_file != DEFAULT_CONFIG_FILE:
+            logger.error(f"The configuration file {config_file} was  not found. Make sure the file exists and try again, or run without the --config file to use the default settings.")
+            exit()
+
+        return read_yaml(config_file)
+
+    def run(self, args: list) -> Generator[Metadata]:
+
+        self.setup_basic_parser()
+
+        # parse the known arguments for now (basically, we want the config file)
+        basic_config, unused_args = self.basic_parser.parse_known_args(args)
+
+        # setup any custom module paths, so they'll show in the help and for arg parsing
+        setup_paths(basic_config.module_paths)
+
+        # if help flag was called, then show the help
+        if basic_config.help:
+            self.show_help(basic_config)
+
+        yaml_config = self.load_config(basic_config.config_file)
+        self.setup_complete_parser(basic_config, yaml_config, unused_args)
+
+        logger.info(f"======== Welcome to the AUTO ARCHIVER ({__version__}) ==========")
+        self.install_modules(self.config['steps'])
+
+        # log out the modules that were loaded
+        for module_type in BaseModule.MODULE_TYPES:
+            logger.info(f"{module_type.upper()}S: " + ", ".join(m.display_name for m in getattr(self, f"{module_type}s")))
+
+        for result in self.feed():
+            yield result
+
+    def cleanup(self) -> None:
+        logger.info("Cleaning up")
+        for e in self.extractors:
+            e.cleanup()
+
+    def feed(self) -> Generator[Metadata]:
+
+        url_count = 0
+        for feeder in self.feeders:
+            for item in feeder:
+                yield self.feed_item(item)
+                url_count += 1
+
+        logger.success(f"Processed {url_count} URL(s)")
+        self.cleanup()
 
     def feed_item(self, item: Metadata) -> Metadata:
-        print("ARCHIVING", item)
+        """
+        Takes one item (URL) to archive and calls self.archive, additionally:
+            - catches keyboard interruptions to do a clean exit
+            - catches any unexpected error, logs it, and does a clean exit
+        """
+        tmp_dir: TemporaryDirectory = None
         try:
-            with tempfile.TemporaryDirectory(dir="./") as tmp_dir:
-                item.set_tmp_dir(tmp_dir)
-                return self.archive(item)
+            tmp_dir = TemporaryDirectory(dir="./")
+            # set tmp_dir on all modules
+            for m in self.all_modules:
+                m.tmp_dir = tmp_dir.name
+            return self.archive(item)
         except KeyboardInterrupt:
             # catches keyboard interruptions to do a clean exit
             logger.warning(f"caught interrupt on {item=}")
-            for d in self.databases: d.aborted(item)
+            for d in self.databases:
+                d.aborted(item)
+            self.cleanup()
             exit()
         except Exception as e:
             logger.error(f'Got unexpected error on item {item}: {e}\n{traceback.format_exc()}')
-            for d in self.databases: d.failed(item)
-
-        # how does this handle the parameters like folder which can be different for each archiver?
-        # the storage needs to know where to archive!!
-        # solution: feeders have context: extra metadata that they can read or ignore,
-        # all of it should have sensible defaults (eg: folder)
-        # default feeder is a list with 1 element
+            for d in self.databases:
+                if type(e) == AssertionError:
+                    d.failed(item, str(e))
+                else:
+                    d.failed(item, reason="unexpected error")
+        finally:
+            if tmp_dir:
+                # remove the tmp_dir from all modules
+                for m in self.all_modules:
+                    m.tmp_dir = None
+                tmp_dir.cleanup()
 
     def archive(self, result: Metadata) -> Union[Metadata, None]:
-        original_url = result.get_url()
+        """
+            Runs the archiving process for a single URL
+            1. Each archiver can sanitize its own URLs
+            2. Check for cached results in Databases, and signal start to the databases
+            3. Call Archivers until one succeeds
+            4. Call Enrichers
+            5. Store all downloaded/generated media
+            6. Call selected Formatter and store formatted if needed
+        """
 
-        # 1 - cleanup
-        # each archiver is responsible for cleaning/expanding its own URLs
+        original_url = result.get_url().strip()
+        try:
+            self.assert_valid_url(original_url)
+        except AssertionError as e:
+            logger.error(f"Error archiving URL {original_url}: {e}")
+            raise e
+
+        # 1 - sanitize - each archiver is responsible for cleaning/expanding its own URLs
         url = original_url
-        for a in self.archivers: url = a.sanitize_url(url)
+        for a in self.extractors:
+            url = a.sanitize_url(url)
+
         result.set_url(url)
         if original_url != url: result.set("original_url", original_url)
 
-        # 2 - rearchiving logic + notify start to DB
-        # archivers can signal whether the content is rearchivable: eg: tweet vs webpage
-        for a in self.archivers: result.rearchivable |= a.is_rearchivable(url)
-        logger.debug(f"{result.rearchivable=} for {url=}")
-
-        # signal to DB that archiving has started
-        # and propagate already archived if it exists
+        # 2 - notify start to DBs, propagate already archived if feature enabled in DBs
         cached_result = None
         for d in self.databases:
-            # are the databases to decide whether to archive?
-            # they can simply return True by default, otherwise they can avoid duplicates. should this logic be more granular, for example on the archiver level: a tweet will not need be scraped twice, whereas an instagram profile might. the archiver could not decide from the link which parts to archive,
-            # instagram profile example: it would always re-archive everything
-            # maybe the database/storage could use a hash/key to decide if there's a need to re-archive
             d.started(result)
-            if (local_result := d.fetch(result)):
-                cached_result = (cached_result or Metadata()).merge(local_result)
-        if cached_result and not cached_result.rearchivable:
+            if local_result := d.fetch(result):
+                cached_result = (cached_result or Metadata()).merge(local_result).merge(result)
+        if cached_result:
             logger.debug("Found previously archived entry")
             for d in self.databases:
-                d.done(cached_result)
+                try: d.done(cached_result, cached=True)
+                except Exception as e:
+                    logger.error(f"ERROR database {d.name}: {e}: {traceback.format_exc()}")
             return cached_result
 
-        # 3 - call archivers until one succeeds
-        for a in self.archivers:
-            logger.info(f"Trying archiver {a.name} for {url}")
+        # 3 - call extractors until one succeeds
+        for a in self.extractors:
+            logger.info(f"Trying extractor {a.name} for {url}")
             try:
-                # Q: should this be refactored so it's just a.download(result)?
                 result.merge(a.download(result))
                 if result.is_success(): break
-            except Exception as e: logger.error(f"Unexpected error with archiver {a.name}: {e}")
+            except Exception as e:
+                logger.error(f"ERROR archiver {a.name}: {e}: {traceback.format_exc()}")
 
-        # what if an archiver returns multiple entries and one is to be part of HTMLgenerator?
-        # should it call the HTMLgenerator as if it's not an enrichment?
-        # eg: if it is enable: generates an HTML with all the returned media, should it include enrichers? yes
-        # then how to execute it last? should there also be post-processors? are there other examples?
-        # maybe as a PDF? or a Markdown file
-
-        # 4 - call enrichers: have access to archived content, can generate metadata and Media
-        # eg: screenshot, wacz, webarchive, thumbnails
+        # 4 - call enrichers to work with archived content
         for e in self.enrichers:
             try: e.enrich(result)
-            except Exception as exc: logger.error(f"Unexpected error with enricher {e.name}: {exc}")
+            except Exception as exc:
+                logger.error(f"ERROR enricher {e.name}: {exc}: {traceback.format_exc()}")
 
-        # 5 - store media
-        # looks for Media in result.media and also result.media[x].properties (as list or dict values)
-        for s in self.storages:
-            for m in result.media:
-                s.store(m, result)  # modifies media
-                # Media can be inside media properties, examples include transformations on original media
-                for prop in m.properties.values():
-                    if isinstance(prop, Media):
-                        s.store(prop, result)
-                    if isinstance(prop, list) and len(prop) > 0 and isinstance(prop[0], Media):
-                        for prop_media in prop:
-                            s.store(prop_media, result)
+        # 5 - store all downloaded/generated media
+        result.store(storages=self.storages)
 
         # 6 - format and store formatted if needed
-        # enrichers typically need access to already stored URLs etc
-        if (final_media := self.formatter.format(result)):
-            for s in self.storages:
-                s.store(final_media, result)
+        final_media: Media
+        if final_media := self.formatters[0].format(result):
+            final_media.store(url=url, metadata=result, storages=self.storages)
             result.set_final_media(final_media)
 
-        # signal completion to databases (DBs, Google Sheets, CSV, ...)
-        for d in self.databases: d.done(result)
+        if result.is_empty():
+            result.status = "nothing archived"
+
+        # signal completion to databases and archivers
+        for d in self.databases:
+            try: d.done(result)
+            except Exception as e:
+                logger.error(f"ERROR database {d.name}: {e}: {traceback.format_exc()}")
 
         return result
+
+    def assert_valid_url(self, url: str) -> bool:
+        """
+        Blocks localhost, private, reserved, and link-local IPs and all non-http/https schemes.
+        """
+        assert url.startswith("http://") or url.startswith("https://"), f"Invalid URL scheme"
+
+        parsed = urlparse(url)
+        assert parsed.scheme in ["http", "https"], f"Invalid URL scheme"
+        assert parsed.hostname, f"Invalid URL hostname"
+        assert parsed.hostname != "localhost", f"Invalid URL"
+
+        try:  # special rules for IP addresses
+            ip = ip_address(parsed.hostname)
+        except ValueError: pass
+        else:
+            assert ip.is_global, f"Invalid IP used"
+            assert not ip.is_reserved, f"Invalid IP used"
+            assert not ip.is_link_local, f"Invalid IP used"
+            assert not ip.is_private, f"Invalid IP used"
+
+    # Helper Properties
+
+    @property
+    def all_modules(self) -> List[Type[BaseModule]]:
+        return self.feeders + self.extractors + self.enrichers + self.databases + self.storages + self.formatters

src/auto_archiver/core/step.py

@@ -1,38 +0,0 @@
-from __future__ import annotations
-from dataclasses import dataclass, field
-from inspect import ClassFoundException
-from typing import Type
-from abc import ABC
-# from collections.abc import Iterable
-
-
-@dataclass
-class Step(ABC):
-    name: str = None
-
-    def __init__(self, config: dict) -> None:
-        # reads the configs into object properties
-        # self.config = config[self.name]
-        for k, v in config.get(self.name, {}).items():
-            self.__setattr__(k, v)
-
-    @staticmethod
-    def configs() -> dict: return {}
-
-    def init(name: str, config: dict, child: Type[Step]) -> Step:
-        """
-        looks into direct subclasses of child for name and returns such ab object
-        TODO: cannot find subclasses of child.subclasses
-        """
-        for sub in child.__subclasses__():
-            if sub.name == name:
-                return sub(config)
-        raise ClassFoundException(f"Unable to initialize STEP with {name=}, check your configuration file/step names, and make sure you made the step discoverable by putting it into __init__.py")
-
-    def assert_valid_string(self, prop: str) -> None:
-        """
-        receives a property name an ensures it exists and is a valid non-empty string, raises an exception if not
-        """
-        assert hasattr(self, prop), f"property {prop} not found"
-        s = getattr(self, prop)
-        assert s is not None and type(s) == str and len(s) > 0, f"invalid property {prop} value '{s}', it should be a valid string"

src/auto_archiver/core/storage.py

@@ -0,0 +1,83 @@
+"""
+Base module for Storage modules – modular components that store media objects in various locations.
+"""
+
+from __future__ import annotations
+from abc import abstractmethod
+from typing import IO
+import os
+
+from loguru import logger
+from slugify import slugify
+
+from auto_archiver.utils.misc import random_str
+
+from auto_archiver.core import Media, BaseModule, Metadata
+from auto_archiver.modules.hash_enricher.hash_enricher import HashEnricher
+from auto_archiver.core.module import get_module
+class Storage(BaseModule):
+    
+    """
+    Base class for implementing storage modules in the media archiving framework.
+
+    Subclasses must implement the `get_cdn_url` and `uploadf` methods to define their behavior.
+    """
+
+    def store(self, media: Media, url: str, metadata: Metadata=None) -> None:
+        if media.is_stored(in_storage=self): 
+            logger.debug(f"{media.key} already stored, skipping")
+            return
+        self.set_key(media, url, metadata)
+        self.upload(media, metadata=metadata)
+        media.add_url(self.get_cdn_url(media))
+
+    @abstractmethod
+    def get_cdn_url(self, media: Media) -> str:
+        """
+        Returns the URL of the media object stored in the CDN.
+        """
+        pass
+
+    @abstractmethod
+    def uploadf(self, file: IO[bytes], key: str, **kwargs: dict) -> bool:
+        """
+        Uploads (or saves) a file to the storage service/location.
+        """
+        pass
+
+    def upload(self, media: Media, **kwargs) -> bool:
+        logger.debug(f'[{self.__class__.__name__}] storing file {media.filename} with key {media.key}')
+        with open(media.filename, 'rb') as f:
+            return self.uploadf(f, media, **kwargs)
+
+    def set_key(self, media: Media, url, metadata: Metadata) -> None:
+        """takes the media and optionally item info and generates a key"""
+        if media.key is not None and len(media.key) > 0: return
+        folder = metadata.get_context('folder', '')
+        filename, ext = os.path.splitext(media.filename)
+
+        # Handle path_generator logic
+        path_generator = self.config.get("path_generator", "url")
+        if path_generator == "flat":
+            path = ""
+            filename = slugify(filename)  # Ensure filename is slugified
+        elif path_generator == "url":
+            path = slugify(url)
+        elif path_generator == "random":
+            path = self.config.get("random_path", random_str(24), True)
+        else:
+            raise ValueError(f"Invalid path_generator: {path_generator}")
+
+        # Handle filename_generator logic
+        filename_generator = self.config.get("filename_generator", "random")
+        if filename_generator == "random":
+            filename = random_str(24)
+        elif filename_generator == "static":
+            # load the hash_enricher module
+            he = get_module(HashEnricher, self.config)
+            hd = he.calculate_hash(media.filename)
+            filename = hd[:24]
+        else:
+            raise ValueError(f"Invalid filename_generator: {filename_generator}")
+
+        media.key = os.path.join(folder, path, f"{filename}{ext}")

src/auto_archiver/core/validators.py

@@ -0,0 +1,19 @@
+# used as validators for config values. Should raise an exception if the value is invalid.
+from pathlib import Path
+import argparse
+
+def example_validator(value):
+    if "example" not in value:
+        raise argparse.ArgumentTypeError(f"{value} is not a valid value for this argument")
+    return value
+
+def positive_number(value):
+    if value < 0:
+        raise argparse.ArgumentTypeError(f"{value} is not a positive number")
+    return value
+
+
+def valid_file(value):
+    if not Path(value).is_file():
+        raise argparse.ArgumentTypeError(f"File '{value}' does not exist.")
+    return value

src/auto_archiver/databases/__init__.py

@@ -1,4 +0,0 @@
-from .database import Database
-from .gsheet_db import GsheetsDb
-from .console_db import ConsoleDb
-from .csv_db import CSVDb

src/auto_archiver/databases/console_db.py

@@ -1,32 +0,0 @@
-from loguru import logger
-
-from . import Database
-from ..core import Metadata
-
-
-class ConsoleDb(Database):
-    """
-        Outputs results to the console
-    """
-    name = "console_db"
-
-    def __init__(self, config: dict) -> None:
-        # without this STEP.__init__ is not called
-        super().__init__(config)
-
-    @staticmethod
-    def configs() -> dict:
-        return {}
-
-    def started(self, item: Metadata) -> None:
-        logger.warning(f"STARTED {item}")
-
-    def failed(self, item: Metadata) -> None:
-        logger.error(f"FAILED {item}")
-
-    def aborted(self, item: Metadata) -> None:
-        logger.warning(f"ABORTED {item}")
-
-    def done(self, item: Metadata) -> None:
-        """archival result ready - should be saved to DB"""
-        logger.success(f"DONE {item}")

src/auto_archiver/databases/database.py

@@ -1,41 +0,0 @@
-from __future__ import annotations
-from dataclasses import dataclass
-from abc import abstractmethod, ABC
-from typing import Union
-
-from ..core import Metadata, Step
-
-
-@dataclass
-class Database(Step, ABC):
-    name = "database"
-
-    def __init__(self, config: dict) -> None:
-        # without this STEP.__init__ is not called
-        super().__init__(config)
-
-    def init(name: str, config: dict) -> Database:
-        # only for typing...
-        return Step.init(name, config, Database)
-
-    def started(self, item: Metadata) -> None:
-        """signals the DB that the given item archival has started"""
-        pass
-
-    def failed(self, item: Metadata) -> None:
-        """update DB accordingly for failure"""
-        pass
-
-    def aborted(self, item: Metadata) -> None:
-        """abort notification if user cancelled after start"""
-        pass
-
-    # @abstractmethod
-    def fetch(self, item: Metadata) -> Union[Metadata, bool]:
-        """check if the given item has been archived already"""
-        return False
-
-    @abstractmethod
-    def done(self, item: Metadata) -> None:
-        """archival result ready - should be saved to DB"""
-        pass

src/auto_archiver/databases/gsheet_db.py

@@ -1,94 +0,0 @@
-from typing import Union, Tuple
-import datetime
-from urllib.parse import quote
-
-# from metadata import Metadata
-from loguru import logger
-
-# from . import Enricher
-from . import Database
-from ..core import Metadata
-from ..core import Media
-from ..utils import GWorksheet
-
-
-class GsheetsDb(Database):
-    """
-        NB: only works if GsheetFeeder is used. 
-        could be updated in the future to support non-GsheetFeeder metadata 
-    """
-    name = "gsheet_db"
-
-    def __init__(self, config: dict) -> None:
-        # without this STEP.__init__ is not called
-        super().__init__(config)
-
-    @staticmethod
-    def configs() -> dict:
-        return {}
-
-    def started(self, item: Metadata) -> None:
-        logger.warning(f"STARTED {item}")
-        gw, row = self._retrieve_gsheet(item)
-        gw.set_cell(row, 'status', 'Archive in progress')
-
-    def failed(self, item: Metadata) -> None:
-        logger.error(f"FAILED {item}")
-        self._safe_status_update(item, 'Archive failed')
-
-    def aborted(self, item: Metadata) -> None:
-        logger.warning(f"ABORTED {item}")
-        self._safe_status_update(item, '')
-
-    def fetch(self, item: Metadata) -> Union[Metadata, bool]:
-        """check if the given item has been archived already"""
-        return False
-
-    def done(self, item: Metadata) -> None:
-        """archival result ready - should be saved to DB"""
-        logger.success(f"DONE {item.get_url()}")
-        gw, row = self._retrieve_gsheet(item)
-        # self._safe_status_update(item, 'done')
-
-        cell_updates = []
-        row_values = gw.get_row(row)
-
-        def batch_if_valid(col, val, final_value=None):
-            final_value = final_value or val
-            if val and gw.col_exists(col) and gw.get_cell(row_values, col) == '':
-                cell_updates.append((row, col, final_value))
-
-        cell_updates.append((row, 'status', item.status))
-
-        media: Media = item.get_final_media()
-
-        batch_if_valid('archive', "\n".join(media.urls))
-        batch_if_valid('date', True, datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc).isoformat())
-        batch_if_valid('title', item.get_title())
-        batch_if_valid('text', item.get("content", "")[:500])
-        batch_if_valid('timestamp', item.get_timestamp())
-        if (screenshot := item.get_media_by_id("screenshot")):
-            batch_if_valid('screenshot', "\n".join(screenshot.urls))
-
-        if (thumbnail := item.get_first_image("thumbnail")):
-            if hasattr(thumbnail, "urls"):
-                batch_if_valid('thumbnail', f'=IMAGE("{thumbnail.urls[0]}")')
-
-        if (browsertrix := item.get_media_by_id("browsertrix")):
-            batch_if_valid('wacz', "\n".join(browsertrix.urls))
-            batch_if_valid('replaywebpage', "\n".join([f'https://replayweb.page/?source={quote(wacz)}#view=pages&url={quote(item.get_url())}' for wacz in browsertrix.urls]))
-
-        gw.batch_set_cell(cell_updates)
-
-    def _safe_status_update(self, item: Metadata, new_status: str) -> None:
-        try:
-            gw, row = self._retrieve_gsheet(item)
-            gw.set_cell(row, 'status', new_status)
-        except Exception as e:
-            logger.debug(f"Unable to update sheet: {e}")
-
-    def _retrieve_gsheet(self, item: Metadata) -> Tuple[GWorksheet, int]:
-        # TODO: to make gsheet_db less coupled with gsheet_feeder's "gsheet" parameter, this method could 1st try to fetch "gsheet" from item and, if missing, manage its own singleton - not needed for now
-        gw: GWorksheet = item.get("gsheet").get("worksheet")
-        row: int = item.get("gsheet").get("row")
-        return gw, row

src/auto_archiver/enrichers/__init__.py

@@ -1,6 +0,0 @@
-from .enricher import Enricher
-from .screenshot_enricher import ScreenshotEnricher 
-from .wayback_enricher import WaybackArchiverEnricher
-from .hash_enricher import HashEnricher
-from .thumbnail_enricher import ThumbnailEnricher
-from .wacz_enricher import WaczEnricher

src/auto_archiver/enrichers/enricher.py

@@ -1,20 +0,0 @@
-from __future__ import annotations
-from dataclasses import dataclass
-from abc import abstractmethod, ABC
-from ..core import Metadata, Step
-
-@dataclass
-class Enricher(Step, ABC):
-    name = "enricher"
-
-    def __init__(self, config: dict) -> None:
-        # without this STEP.__init__ is not called
-        super().__init__(config)
-        
-
-    # only for typing...
-    def init(name: str, config: dict) -> Enricher:
-        return Step.init(name, config, Enricher)
-
-    @abstractmethod
-    def enrich(self, to_enrich: Metadata) -> None: pass

src/auto_archiver/enrichers/hash_enricher.py

@@ -1,39 +0,0 @@
-import hashlib
-from loguru import logger
-
-from . import Enricher
-from ..core import Metadata
-
-
-class HashEnricher(Enricher):
-    """
-    Calculates hashes for Media instances
-    """
-    name = "hash_enricher"
-
-    def __init__(self, config: dict) -> None:
-        # without this STEP.__init__ is not called
-        super().__init__(config)
-        algo_choices = self.configs()["algorithm"]["choices"]
-        assert self.algorithm in algo_choices, f"Invalid hash algorithm selected, must be one of {algo_choices} (you selected {self.algorithm})."
-
-    @staticmethod
-    def configs() -> dict:
-        return {
-            "algorithm": {"default": "SHA-256", "help": "hash algorithm to use", "choices": ["SHA-256", "SHA3-512"]}
-        }
-
-    def enrich(self, to_enrich: Metadata) -> None:
-        url = to_enrich.get_url()
-        logger.debug(f"calculating media hashes for {url=} (using {self.algorithm})")
-
-        for i, m in enumerate(to_enrich.media):
-            with open(m.filename, "rb") as f:
-                bytes = f.read()  # read entire file as bytes
-                hash = None
-                if self.algorithm == "SHA-256":
-                    hash = hashlib.sha256(bytes)
-                elif self.algorithm == "SHA3-512":
-                    hash = hashlib.sha3_512(bytes)
-                else: continue
-                to_enrich.media[i].set("hash", f"{self.algorithm}:{hash.hexdigest()}")

src/auto_archiver/enrichers/screenshot_enricher.py

@@ -1,34 +0,0 @@
-from loguru import logger
-import time, uuid, os
-from selenium.common.exceptions import TimeoutException
-
-from . import Enricher
-from ..utils import Webdriver
-from ..core import Media, Metadata
-
-class ScreenshotEnricher(Enricher):
-    name = "screenshot_enricher"
-
-    @staticmethod
-    def configs() -> dict:
-        return {
-            "width": {"default": 1280, "help": "width of the screenshots"},
-            "height": {"default": 720, "help": "height of the screenshots"},
-            "timeout": {"default": 60, "help": "timeout for taking the screenshot"}
-        }
-
-    def enrich(self, to_enrich: Metadata) -> None:
-        url = to_enrich.get_url()
-        logger.debug(f"Enriching screenshot for {url=}")
-        with Webdriver(self.width, self.height, self.timeout, 'facebook.com' in url) as driver:
-            try:
-                driver.get(url)
-                time.sleep(2)
-                screenshot_file = os.path.join(to_enrich.get_tmp_dir(), f"screenshot_{str(uuid.uuid4())[0:8]}.png")
-                driver.save_screenshot(screenshot_file)
-                to_enrich.add_media(Media(filename=screenshot_file), id="screenshot")
-            except TimeoutException:
-                logger.info("TimeoutException loading page for screenshot")
-            except Exception as e:
-                logger.error(f"Got error while loading webdriver for screenshot enricher: {e}")
-        # return None

src/auto_archiver/enrichers/thumbnail_enricher.py

@@ -1,45 +0,0 @@
-import ffmpeg, os, uuid
-from loguru import logger
-
-from . import Enricher
-from ..core import Media, Metadata
-
-
-class ThumbnailEnricher(Enricher):
-    """
-    Generates thumbnails for all the media
-    """
-    name = "thumbnail_enricher"
-
-    def __init__(self, config: dict) -> None:
-        # without this STEP.__init__ is not called
-        super().__init__(config)
-
-    @staticmethod
-    def configs() -> dict:
-        return {}
-
-    def enrich(self, to_enrich: Metadata) -> None:
-        logger.debug(f"generating thumbnails")
-        for i, m in enumerate(to_enrich.media[::]):
-            if m.is_video():
-                folder = os.path.join(to_enrich.get_tmp_dir(), str(uuid.uuid4()))
-                os.makedirs(folder, exist_ok=True)
-                logger.debug(f"generating thumbnails for {m.filename}")
-                fps, duration = 0.5, m.get("duration")
-                if duration is not None:
-                    duration = float(duration)
-                    if duration < 60: fps = 10.0 / duration
-                    elif duration < 120: fps = 20.0 / duration
-                    else: fps = 40.0 / duration
-
-                stream = ffmpeg.input(m.filename)
-                stream = ffmpeg.filter(stream, 'fps', fps=fps).filter('scale', 512, -1)
-                stream.output(os.path.join(folder, 'out%d.jpg')).run()
-
-                thumbnails = os.listdir(folder)
-                thumbnails_media = []
-                for t, fname in enumerate(thumbnails):
-                    if fname[-3:] == 'jpg':
-                        thumbnails_media.append(Media(filename=os.path.join(folder, fname)).set("id", f"thumbnail_{t}"))
-                to_enrich.media[i].set("thumbnails", thumbnails_media)

src/auto_archiver/enrichers/wacz_enricher.py

@@ -1,65 +0,0 @@
-import os, shutil, subprocess, uuid
-from loguru import logger
-
-from ..core import Media, Metadata
-from . import Enricher
-
-
-class WaczEnricher(Enricher):
-    """
-    Submits the current URL to the webarchive and returns a job_id or completed archive
-    """
-    name = "wacz_enricher"
-
-    def __init__(self, config: dict) -> None:
-        # without this STEP.__init__ is not called
-        super().__init__(config)
-
-    @staticmethod
-    def configs() -> dict:
-        return {
-            "profile": {"default": None, "help": "browsertrix-profile (for profile generation see https://github.com/webrecorder/browsertrix-crawler#creating-and-using-browser-profiles)."},
-            "timeout": {"default": 90, "help": "timeout for WACZ generation in seconds"},
-        }
-
-    def enrich(self, to_enrich: Metadata) -> bool:
-        # TODO: figure out support for browsertrix in docker
-        url = to_enrich.get_url()
-        logger.debug(f"generating WACZ for {url=}")
-        collection = str(uuid.uuid4())[0:8]
-        browsertrix_home = os.path.abspath(to_enrich.get_tmp_dir())
-        cmd = [
-            "docker", "run",
-            "--rm",  # delete container once it has completed running
-            "-v", f"{browsertrix_home}:/crawls/",
-            # "-it", # this leads to "the input device is not a TTY"
-            "webrecorder/browsertrix-crawler", "crawl",
-            "--url", url,
-            "--scopeType", "page",
-            "--generateWACZ",
-            "--text",
-            "--collection", collection,
-            "--behaviors", "autoscroll,autoplay,autofetch,siteSpecific",
-            "--behaviorTimeout", str(self.timeout),
-            "--timeout", str(self.timeout)
-        ]
-        if self.profile:
-            profile_fn = os.path.join(browsertrix_home, "profile.tar.gz")
-            shutil.copyfile(self.profile, profile_fn)
-            # TODO: test which is right
-            cmd.extend(["--profile", profile_fn])
-            # cmd.extend(["--profile", "/crawls/profile.tar.gz"])
-
-        try:
-            logger.info(f"Running browsertrix-crawler: {' '.join(cmd)}")
-            subprocess.run(cmd, check=True)
-        except Exception as e:
-            logger.error(f"WACZ generation failed: {e}")
-            return False
-
-        filename = os.path.join(browsertrix_home, "collections", collection, f"{collection}.wacz")
-        if not os.path.exists(filename):
-            logger.warning(f"Unable to locate and upload WACZ  {filename=}")
-            return False
-
-        to_enrich.add_media(Media(filename), "browsertrix")

src/auto_archiver/feeders/__init__.py

@@ -1,3 +0,0 @@
-from.feeder import Feeder
-from .gsheet_feeder import GsheetsFeeder
-from .cli_feeder import CLIFeeder

src/auto_archiver/feeders/cli_feeder.py

@@ -1,30 +0,0 @@
-from loguru import logger
-
-from . import Feeder
-from ..core import Metadata
-
-
-class CLIFeeder(Feeder):
-    name = "cli_feeder"
-
-    def __init__(self, config: dict) -> None:
-        # without this STEP.__init__ is not called
-        super().__init__(config)
-        if type(self.urls) != list or len(self.urls) == 0:
-            raise Exception("CLI Feeder did not receive any URL to process")
-
-    @staticmethod
-    def configs() -> dict:
-        return {
-            "urls": {
-                "default": None,
-                "help": "URL(s) to archive, either a single URL or a list of urls, should not come from config.yaml",
-                "cli_set": lambda cli_val, cur_val: list(set(cli_val.split(",")))
-            },
-        }
-
-    def __iter__(self) -> Metadata:
-        for url in self.urls:
-            logger.debug(f"Processing {url}")
-            yield Metadata().set_url(url).set("folder", "cli", True)
-        logger.success(f"Processed {len(self.urls)} URL(s)")

src/auto_archiver/feeders/feeder.py

@@ -1,21 +0,0 @@
-from __future__ import annotations
-from dataclasses import dataclass
-from abc import abstractmethod
-from ..core import Metadata
-from ..core import Step
-
-
-@dataclass
-class Feeder(Step):
-    name = "feeder"
-
-    def __init__(self, config: dict) -> None:
-        # without this STEP.__init__ is not called
-        super().__init__(config)
-
-    def init(name: str, config: dict) -> Feeder:
-        # only for code typing
-        return Step.init(name, config, Feeder)
-
-    @abstractmethod
-    def __iter__(self) -> Metadata: return None