Merge pull request #312 from bellingcat/dev v1.1.0

v1.1.0 WIP

.github/dependabot.yml

@@ -0,0 +1,40 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "/"
+    groups:
+      python:
+        patterns:
+          - "*"
+    schedule:
+      interval: "monthly"
+  
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    groups:
+      actions:
+        patterns:
+          - "*"
+    schedule:
+      interval: "monthly"
+
+  - package-ecosystem: "npm"
+    directory: "/scripts/settings/"
+    groups:
+      actions:
+        patterns:
+          - "*"
+    schedule:
+      interval: "monthly"
+
+  - package-ecosystem: "docker"
+    # Look for a `Dockerfile` in the `root` directory
+    directory: "/"
+    # Check for updates once a week
+    schedule:
+      interval: "monthly"

.github/workflows/docker-publish.yaml

@@ -22,7 +22,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
@@ -33,14 +33,14 @@ jobs:
         uses: docker/setup-buildx-action@v3
 
       - name: Log in to Docker Hub
-        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96
+        uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804
         with:
           images: bellingcat/auto-archiver
       

.github/workflows/ruff.yaml

@@ -3,8 +3,18 @@ name: Ruff Formatting & Linting
 on:
   push:
     branches: [ main ]
+    paths-ignore:
+      - "README.md"
+      - ".github"
+      - "poetry.lock"
+      - "scripts/settings"
   pull_request:
     branches: [ main ]
+    paths-ignore:
+      - "README.md"
+      - ".github"
+      - "poetry.lock"
+      - "scripts/settings"
 
 jobs:
   build:

.github/workflows/tests-core.yaml

@@ -20,8 +20,7 @@ jobs:
       fail-fast: false
       matrix:
         python-version: ["3.10", "3.11", "3.12"]
-        os: [ubuntu-22.04]
-        #TODO: re-enable ubuntu-latest, this is disabled as oscrypto cannot be pinned to github commit and pushed to pypi
+        os: [ubuntu-22.04, ubuntu-latest]
     defaults:
       run:
         working-directory: ./
@@ -29,16 +28,26 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Install Poetry
-        run: pipx install poetry
+      - name: Install ffmpeg
+        run: sudo apt-get update && sudo apt-get install -y ffmpeg
 
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-          cache: 'poetry'
 
-      - name: Install dependencies
+      - name: Install latest Poetry
+        run: pipx install poetry
+      
+      - name: Cache Poetry and pip artifacts
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/pypoetry
+            ~/.cache/pip
+          key: poetry-${{ matrix.os }}-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }}
+
+      - name: Install dependencies from source only
         run: poetry install --no-interaction --with dev
 
       - name: Run Core Tests

.github/workflows/tests-download.yaml

@@ -22,16 +22,26 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Install poetry
-        run: pipx install poetry
+      - name: Install ffmpeg
+        run: sudo apt-get update && sudo apt-get install -y ffmpeg
 
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-          cache: 'poetry'
 
-      - name: Install dependencies
+      - name: Install latest Poetry
+        run: pipx install poetry
+
+      - name: Cache Poetry and pip artifacts
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/pypoetry
+            ~/.cache/pip
+          key: poetry-${{ matrix.os }}-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }}
+
+      - name: Install dependencies from source only
         run: poetry install --no-interaction --with dev
 
       - name: Run Download Tests

.gitignore

@@ -1,9 +1,11 @@
 tmp*/
 temp/
 .env*
+!.env*.example
 .DS_Store
 expmt/
 service_account.json
+service_account-*.json
 __pycache__/
 ._*
 anu.html
@@ -36,3 +38,7 @@ docs/source/modules/autogen/
 scripts/settings_page.html
 scripts/settings/src/schema.json 
 .vite
+downloaded_files
+latest_logs
+# for launch.json
+.vscode

Dockerfile

@@ -1,4 +1,4 @@
-FROM webrecorder/browsertrix-crawler:1.4.2 AS base
+FROM webrecorder/browsertrix-crawler:1.6.3 AS base
 
 ENV RUNNING_IN_DOCKER=1 \
     LANG=C.UTF-8 \
@@ -11,26 +11,8 @@ ENV RUNNING_IN_DOCKER=1 \
 ARG TARGETARCH
 
 # Installing system dependencies
-RUN add-apt-repository ppa:mozillateam/ppa && \
-	apt-get update && \
-    apt-get install -y --no-install-recommends gcc ffmpeg fonts-noto exiftool && \
-	apt-get install -y --no-install-recommends firefox-esr && \
-    ln -s /usr/bin/firefox-esr /usr/bin/firefox
-
-ARG GECKODRIVER_VERSION=0.36.0
-
-RUN if [ $(uname -m) = "aarch64" ]; then \
-        GECKODRIVER_ARCH=linux-aarch64; \
-    else \
-        GECKODRIVER_ARCH=linux64; \
-    fi && \
-    wget https://github.com/mozilla/geckodriver/releases/download/v${GECKODRIVER_VERSION}/geckodriver-v${GECKODRIVER_VERSION}-${GECKODRIVER_ARCH}.tar.gz && \
-    tar -xvzf geckodriver* -C /usr/local/bin && \
-    chmod +x /usr/local/bin/geckodriver && \
-    rm geckodriver-v* && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
-
+RUN	apt-get update && \
+    apt-get install -y --no-install-recommends gcc ffmpeg fonts-noto exiftool python3-tk 
 
 # Poetry and runtime
 FROM base AS runtime

README.md

@@ -1,12 +1,13 @@
 <h1 align="center">Auto Archiver</h1>
 
+[![Documentation Status](https://readthedocs.org/projects/auto-archiver/badge/?version=latest)](https://auto-archiver.readthedocs.io/en/latest/?badge=latest)
 [![PyPI version](https://badge.fury.io/py/auto-archiver.svg)](https://badge.fury.io/py/auto-archiver)
-[![Docker Image Version (latest by date)](https://img.shields.io/docker/v/bellingcat/auto-archiver?label=version&logo=docker)](https://hub.docker.com/r/bellingcat/auto-archiver)
+[![Docker Image Version (latest by date)](https://img.shields.io/docker/v/bellingcat/auto-archiver?sort=semver&logo=docker&color=#69F0AE)](https://hub.docker.com/r/bellingcat/auto-archiver)
 [![Core Test Status](https://github.com/bellingcat/auto-archiver/workflows/Core%20Tests/badge.svg)](https://github.com/bellingcat/auto-archiver/actions/workflows/tests-core.yaml)
-[![Download Test Status](https://github.com/bellingcat/auto-archiver/workflows/Download%20Tests/badge.svg)](https://github.com/bellingcat/auto-archiver/actions/workflows/tests-download.yaml)
+<!-- [![Download Test Status](https://github.com/bellingcat/auto-archiver/workflows/Download%20Tests/badge.svg)](https://github.com/bellingcat/auto-archiver/actions/workflows/tests-download.yaml) -->
+
 <!-- ![Docker Pulls](https://img.shields.io/docker/pulls/bellingcat/auto-archiver) -->
 <!-- [![PyPI download month](https://img.shields.io/pypi/dm/auto-archiver.svg)](https://pypi.python.org/pypi/auto-archiver/) -->
-<!-- [![Documentation Status](https://readthedocs.org/projects/vk-url-scraper/badge/?version=latest)](https://vk-url-scraper.readthedocs.io/en/latest/?badge=latest) -->
 
 
 

docker-compose.yaml

@@ -1,4 +1,3 @@
-version: '3.8'
 
 services:
   auto-archiver:
@@ -10,7 +9,4 @@ services:
     volumes:
       - ./secrets:/app/secrets
       - ./local_archive:/app/local_archive
-    environment:
-      - WACZ_ENABLE_DOCKER=true
-      - RUNNING_IN_DOCKER=true
     command: --config secrets/orchestration.yaml

docs/scripts/scripts.py

@@ -47,7 +47,6 @@ def generate_module_docs():
 
     for module in sorted(ModuleFactory().available_modules(), key=lambda x: (x.requires_setup, x.name)):
         # generate the markdown file from the __manifest__.py file.
-
         manifest = module.manifest
         for type in manifest["type"]:
             modules_by_type.setdefault(type, []).append(module)
@@ -64,6 +63,27 @@ def generate_module_docs():
 """
         steps_str = "\n".join(f"  {t}s:\n  - {module.name}" for t in manifest["type"])
 
+        if manifest.get("autodoc_dropins"):
+            loaded_module = module.load({})
+            dropins = loaded_module.load_dropins()
+            dropin_str = "\n##### Available Dropins\n"
+            for dropin in dropins:
+                if not (ddoc := dropin.documentation()):
+                    continue
+                dropin_str += f"\n###### {ddoc.get('name', dropin.__name__)}\n\n"
+                dropin_str += f"{ddoc.get('description')}\n\n"
+                if ddoc.get("site"):
+                    dropin_str += f"**Site**: {ddoc['site']}\n\n"
+                if dauth := ddoc.get("authentication"):
+                    dropin_str += "**YAML configuration**:\n"
+                    dropin_auth_yaml = "authentication:\n...\n"
+                    for site, creds in dauth.items():
+                        dropin_auth_yaml += f"  {site}:\n"
+                        for k, v in creds.items():
+                            dropin_auth_yaml += f'    {k}: "{v}"\n'
+                    dropin_str += f"```{{code}} yaml\n{dropin_auth_yaml}...\n```\n"
+            readme_str += dropin_str
+
         if not manifest["configs"]:
             config_string = f"# No configuration options for {module.name}.*\n"
         else:

docs/source/development/testing.md

@@ -3,14 +3,14 @@
 `pytest` is used for testing. There are two main types of tests:
 
 1. 'core' tests which should be run on every change
-2. 'download' tests which hit the network. These tests will do things like make API calls (e.g. Twitter, Bluesky etc.) and should be run regularly to make sure that APIs have not changed.
+2. 'download' tests which hit the network. These tests will do things like make API calls (e.g. Twitter, Bluesky etc.) and should be run regularly to make sure that APIs have not changed, they take longer.
 
 
 ## Running Tests 
 
 1. Make sure you've installed the dev dependencies with `pytest install --with dev`
 2. Tests can be run as follows:
-```
+```{code} bash
 #### Command prefix of 'poetry run' removed here for simplicity
 # run core tests
 pytest -ra -v -m "not download"
@@ -18,4 +18,15 @@ pytest -ra -v -m "not download"
 pytest -ra -v -m "download"
 # run all tests
 pytest -ra -v
+
+
+# run a specific test file
+pytest -ra -v tests/test_file.py
+# run a specific test function
+pytest -ra -v tests/test_file.py::test_function_name
+```
+
+3. Some tests require environment variables to be set. You can use the example `.env.test.example` file as a template. Copy it to `.env.test` and fill in the required values. This file will be loaded automatically by `pytest`.
+```{code} bash
+cp .env.test.example .env.test
 ```

docs/source/flow_overview.md

@@ -8,7 +8,7 @@ The archiver archives web pages using the following workflow
 4. **Formatter** creates a report from all the archived content (HTML, PDF, ...)
 5. **Database** knows what's been archived and also stores the archive result (spreadsheet, CSV, or just the console)
 
-Each step in the workflow is handled by 'modules' that interact with the data in different ways. For example, the Twitter Extractor Module would extract information from the Twitter website. The Screenshot Enricher Module will take screenshots of the given page. See the [core modules page](core_modules.md) for an overview of all the modules that are available.
+Each step in the workflow is handled by 'modules' that interact with the data in different ways. For example, the Twitter Extractor Module would extract information from the Twitter website. The AntiBot Module will download HTML and take screenshots of the given page. See the [core modules page](core_modules.md) for an overview of all the modules that are available.
 
 Auto-archiver must have at least one module defined for each step of the workflow. This is done by setting the [configuration](installation/configurations.md) for your auto-archiver instance.
 

docs/source/how_to/01_authentication_how_to.md

@@ -106,5 +106,117 @@ Finally,Some important things to remember:
 
 ## Authenticating on XXXX site with username/password
 
-```{note} This section is still under construction 🚧
+```{note} 
+This section is still under construction 🚧
 ```
+
+
+# Proof of Origin Tokens
+
+YouTube uses **Proof of Origin Tokens (POT)** as part of its bot detection system to verify that requests originate from valid clients. If a token is missing or invalid, some videos may return errors like "Sign in to confirm you're not a bot."
+
+yt-dlp provides [a detailed guide to POTs](https://github.com/yt-dlp/yt-dlp/wiki/PO-Token-Guide).
+
+### How Auto Archiver Uses POT
+This feature is enabled for the Generic Archiver via two yt-dlp plugins:
+
+- **Client-side plugin**: [yt-dlp-get-pot](https://github.com/coletdjnz/yt-dlp-get-pot)  
+  Detects when a token is required and requests one from a provider.
+
+- **Provider plugin**: [bgutil-ytdlp-pot-provider](https://github.com/Brainicism/bgutil-ytdlp-pot-provider)  
+  Includes both a Python plugin and a **Node.js server or script** to generate the token.
+
+These are installed in our Poetry environment.
+
+### Integration Methods
+
+**Docker (Recommended)**:
+
+When running the Auto Archiver using the Docker image, we use the [Node.js token generation script](https://github.com/Brainicism/bgutil-ytdlp-pot-provider/tree/master/server).
+This is to avoid managing a separate server process, and is handled automatically inside the Docker container when needed.
+
+This is already included in the Docker image, however if you need to disable this you can set the config option `bguils_po_token_method` under the `generic_extractor` section of your `orchestration.yaml` config file to "disabled".
+```yaml
+generic_extractor:
+  bguils_po_token_method: "disabled"
+```
+
+**PyPi/ Local**:
+
+When using the Auto Archiver PyPI package, or running locally, you will need additional system requirements to run the token generation script, namely either Docker, or Node.js and Yarn.
+
+See the [bgutil-ytdlp-pot-provider](https://github.com/Brainicism/bgutil-ytdlp-pot-provider?tab=readme-ov-file#a-http-server-option) documentation for more details.
+
+⚠️WARNING⚠️: This will add the server scripts to the home directory of wherever this is running.
+
+- You can set the config option `bguils_po_token_method` under the `generic_extractor` section of your `orchestration.yaml` config file to "script" to enable the token generation script process locally.
+- Alternatively you can run the bgutil-ytdlp-pot-provider server separately using their Docker image or Node.js server.
+
+### Notes
+
+- The token generation script is only triggered when needed by yt-dlp, so it should have no effect unless YouTube requests a POT.
+- If you're running the Auto Archiver in Docker, this is set up automatically.
+- If you're running locally, you'll need to run the setup script manually or enable the feature in your config.
+- You can set up both the server and the script, and the plugin will fallback on each other if needed. This is recommended for robustness!
+
+### Configurations: 
+
+## Configurations Summary
+
+| Option     | Behavior                                                                                                                                   | Docker Default? |
+|------------| ------------------------------------------------------------------------------------------------------------------------------------------ | --------------- |
+| `auto`     | Docker: Automatically downloads and uses the token generation script. Local: Does nothing; assumes a separate server is running externally. | ✅ Yes           |
+| `script`   | Explicitly downloads and uses the token generation script, even locally.                                                                   | ❌ No            |
+| `disabled` | Disables token generation completely.                                                                                                      | ❌ No            |
+
+Example configuration:
+
+ 
+```yaml
+generic_extractor:
+  # ...  
+  bguils_po_token_method: "script"
+  # For debugging add the verbose flag here:
+  ytdlp_args: "--no-abort-on-error --abort-on-error --verbose"
+
+```
+
+**Advanced Configuration:**
+
+If you change the default port of the bgutil-ytdlp-pot-provider server, you can pass the updated values using our `extractor_args` option for the gereric extractor.
+
+```yaml
+generic_extractor:
+  ytdlp_args: "--no-abort-on-error --abort-on-error --verbose"
+  ytdlp_update_interval: 5
+  bguils_po_token_method: "script"
+  extractor_args:
+    youtube:
+      getpot_bgutil_baseurl: "http://127.0.0.1:8080"
+      player_client: web,tv
+```
+For more details on this for bgutils see [here](https://github.com/Brainicism/bgutil-ytdlp-pot-provider?tab=readme-ov-file#usage)
+
+### Checking the logs
+
+To verify that the POT process working, look for the following lines in your log after adding the config option:
+
+```shell
+[GetPOT] BgUtilScript: Generating POT via script: /Users/you/bgutil-ytdlp-pot-provider/server/build/generate_once.js
+[debug] [GetPOT] BgUtilScript: Executing command to get POT via script: /Users/you/.nvm/versions/node/v20.18.0/bin/node /Users/you/bgutil-ytdlp-pot-provider/server/build/generate_once.js -v ymCMy8OflKM
+[debug] [GetPOT] BgUtilScript: stdout:
+{"poToken":"MlMxojNFhEJvUzGeHEkVRSK_luXtwcDnwSNIOgaUutqB7t99nmlNvtWgYayboopG6ZopZgmQ-6PJCWEMHv89MIiFGGlJRY25Fkwzxmia_8uYgf5AWf==","generatedAt":"2025-03-26T10:45:26.156Z","visitIdentifier":"ymCMy8OflKM"}
+[debug] [GetPOT] Fetching gvs PO Token for tv client
+```
+
+If it can't find the script or something, you'll see something like this:
+```shell
+[debug] [GetPOT] Fetching player PO Token for tv client
+WARNING: [GetPOT] BgUtilScript: Script path doesn't exist: /Users/you/bgutil-ytdlp-pot-provider/server/build/generate_once.js. Please make sure the script has been transpiled correctly.
+WARNING: [GetPOT] BgUtilHTTP: Error reaching GET http://127.0.0.1:4416/ping (caused by TransportError). Please make sure that the server is reachable at http://127.0.0.1:4416.
+[debug] [GetPOT] No player PO Token provider available for tv client
+```
+
+In this case check that the script has been transpiled correctly and is available at the path specified in the log, 
+or that the server is running and reachable.
+

docs/source/how_to/02_gsheets_setup.md

@@ -6,12 +6,43 @@ This guide explains how to set up Google Sheets to process URLs automatically an
 2. Setting up a service account so Auto Archiver can access the sheet
 3. Setting the Auto Archiver settings
 
-### 1. Setting up your Google Sheet
 
-Any Google sheet must have at least *one* column, with the name 'link' (you can change this name afterwards). This is the column with the URLs that you want the Auto Archiver to archive. 
-Your sheet can have many other columns that the Auto Archiver can use, and you can also include any additional columns for your own personal use. The order of the columns does not matter, the naming just needs to be correctly assigned to its corresponding value in the configuration file.
+## 1. Setting up a Google Service Account
 
-We recommend copying [this template Google Sheet](https://docs.google.com/spreadsheets/d/1NJZo_XZUBKTI1Ghlgi4nTPVvCfb0HXAs6j5tNGas72k/edit?usp=sharing) as a starting point for your project, as this matches the default column names.
+Once your Google Sheet is set up, you need to create what's called a 'service account' that will allow the Auto Archiver to access it.
+
+To do this, you can either:
+* a) follow the steps in [this guide](https://gspread.readthedocs.io/en/latest/oauth2.html) all the way up until step 8. You should have downloaded a file called `service_account.json` and should save it in the `secrets/` folder
+* b) run the following script to automatically generate the file:
+```{code} bash
+https://raw.githubusercontent.com/bellingcat/auto-archiver/refs/heads/main/scripts/generate_google_services.sh | bash -s --
+```
+This uses gcloud to create a new project, a new user and downloads the service account automatically for you. The service account file will have the name `service_account-XXXXXXX.json` where XXXXXXX is a random 16 letter/digit string for the project created.
+
+```{note}
+To save the generated file to a different folder, pass an argument as follows:
+```{code} bash
+https://raw.githubusercontent.com/bellingcat/auto-archiver/refs/heads/main/scripts/generate_google_services.sh | bash -s -- /path/to/secrets
+```
+
+----------
+
+Once you've downloaded the file, you can save it to `secrets/service_account.json` (the default name), or to another file and then change the location in the settings (see step 4).
+
+Also make sure to **note down** the email address for this service account. You'll need that for step 3.
+
+```{note}
+The email address created in this step can be found either by opening the `service_account.json` file, or if you used b) the `generate_google_services.sh` script, then the script will have printed it out for you.
+
+The email address will look something like `user@project-name.iam.gserviceaccount.com`
+```
+
+
+## 2. Setting up your Google Sheet
+
+We recommend copying [this template Google Sheet](https://docs.google.com/spreadsheets/d/1NJZo_XZUBKTI1Ghlgi4nTPVvCfb0HXAs6j5tNGas72k/edit?usp=sharing) as a starting point for your project, as this matches all the columns required.
+
+But if you like, you can also create your own custom sheet. The only columns required are 'link', 'archive status', and 'archive location'. 'link' is the column with the URLs that you want the Auto Archiver to archive, the other two record the archival status and result. 
 
 Here's an overview of all the columns, and what a complete sheet would look like.
 
@@ -46,21 +77,18 @@ In this example the Ghseet Feeder and Gsheet DB are being used, and the archive
 
 ![A screenshot of a Google Spreadsheet with column headers defined as above, and several Youtube and Twitter URLs in the "Link" column](../../demo-before.png)
 
-We'll change the name of the 'Destination Folder' column in step 3.
+We'll change the name of the 'Destination Folder' column in the Step 4a.
 
-## 2. Setting up your Service Account
+## 3. Share your Google Sheet with your Service Account email address
 
-Once your Google Sheet is set up, you need to create what's called a 'service account' that will allow the Auto Archiver to access it.
+Remember that email address you copied in Step 1? Now that you've set up your Google sheet, click 'Share' in the top
+right hand corner and enter the email address. Make sure to give the account **Editor** access. Here's how that looks:
 
-To do this, follow the steps in [this guide](https://gspread.readthedocs.io/en/latest/oauth2.html) all the way up until step 8. You should have downloaded a file called `service_account.json` and shared the Google Sheet with the log 'client_email' email address in this file.
+![Share sheet](share_sheet.png)
 
-Once you've downloaded the file, save it to `secrets/service_account.json`
+## 4. Setting up the configuration file
 
-## 3. Setting up the configuration file
-
-Now that you've set up your Google sheet, and you've set up the service account so Auto Archiver can access the sheet, the final step is to set your configuration.
-
-First, make sure you have `gsheet_feeder_db` set in the `steps.feeders` section of your config. If you wish to store the results of the archiving process back in your Google sheet, make sure to also set the `ghseet_db` settig in the `steps.databases` section. Here's how this might look:
+The final step is to set your configuration. First, make sure you have `gsheet_feeder_db` set in the `steps.feeders` section of your config. If you wish to store the results of the archiving process back in your Google sheet, make sure to also put `gsheet_feeder_db` setting in the `steps.databases` section. Here's how this might look:
 
 ```{code} yaml
 steps:
@@ -75,12 +103,15 @@ steps:
 Next, set up the `gsheet_feeder_db` configuration settings in the 'Configurations' part of the config `orchestration.yaml` file. Open up the file, and set the `gsheet_feeder_db.sheet` setting or the `gsheet_feeder_db.sheet_id` setting. The `sheet` should be the name of your sheet, as it shows in the top left of the sheet. 
 For example, the sheet [here](https://docs.google.com/spreadsheets/d/1NJZo_XZUBKTI1Ghlgi4nTPVvCfb0HXAs6j5tNGas72k/edit?gid=0#gid=0) is called 'Public Auto Archiver template'.
 
+If you saved your `service_account.json` file to anywhere other than the default location (`secrets/service_account.json`), then also make sure to change that now:
+
 Here's how this might look:
 
 ```{code} yaml
 ...
 gsheet_feeder_db:
     sheet: 'My Awesome Sheet'
+    service_account: secrets/service_account-XXXXX.json # or leave as secrets/service_account.json
     ...
 ```
 
@@ -90,7 +121,7 @@ You can also pass these settings directly on the command line without having to
 
 Here, the sheet name has been overridden/specified in the command line invocation.
 
-### 3a. (Optional) Changing the column names
+### 4a. (Optional) Changing the column names
 
 In step 1, we said we would change the name of the 'Destination Folder'. Perhaps you don't like this name, or already have a sheet with a different name. In our example here, we want to name this column 'Save Folder'. To do this, we need to edit the `ghseet_feeder_db.column` setting in the configuration file. 
 For more information on this setting, see the [Gsheet Feeder Database docs](../modules/autogen/feeder/gsheet_feeder_db.md#configuration-options). We will first copy the default settings from the Gsheet Feeder docs for the 'column' settings, and then edit the 'Destination Folder' section to rename it 'Save Folder'. Our final configuration section looks like:

docs/source/how_to/03_logging.md

@@ -1,6 +1,6 @@
 # Keeping Logs
 
-Auto Archiver's logs can be helpful for debugging problematic archiving processes. This guide shows you how to use the logs to 
+Auto Archiver's logs can be helpful for debugging problematic archiving processes. This guide shows you how to use the logs configuration.
 
 ## Setting up logging
 
@@ -8,10 +8,10 @@ Logging settings can be set on the command line or using the orchestration confi
 
 #### Enabling or Disabling Logging
 
-Logging to the console is enabled by default. If you want to globally disable Auto Archiver's logging, then you can set `enabled: false` in your `logging` config:
+Logging to the console is enabled by default. If you want to globally disable Auto Archiver's logging, then you can set `enabled: false` in your `logging` config file:
 
 ```{code} yaml
-
+:caption: orchestration.yaml
 ...
 logging:
    enabled: false
@@ -24,7 +24,7 @@ This will disable all logs from Auto Archiver, but it does not disable logs for
 
 #### Logging Level
 
-There are 7 logging levels in total, with 4 commonly used levels. They are: `DEBUG`, `INFO`, `WARNING` and `ERROR`.
+There are 7 logging levels in total, with 5 of them used in this tool. They are: `DEBUG`, `INFO`, `SUCCESS`, `WARNING` and `ERROR`.
 
 Change the warning level by setting the value in your orchestration config file:
 
@@ -44,7 +44,7 @@ For normal usage, it is recommended to use the `INFO` level, or if you prefer qu
 
 ### Logging to a file
 
-As default, auto-archiver will log to the console. But if you wish to store your logs for future reference, or you are running the auto-archiver from within code a implementation, then you may with to enable file logging. This can be done by setting the `file:` config value in the logging settings.
+As default, auto-archiver will log to the console. But if you wish to store your logs for future reference, or you are running the auto-archiver from within code a implementation, then you may wish to enable file logging. This can be done by setting the `file:` config value in the logging settings.
 
 **Rotation:** For file logging, you can choose to 'rotate' your log files (creating new log files) so they do not get too large. Change this by setting the 'rotation' option in your logging settings. For a full list of rotation options, see the [loguru docs](https://loguru.readthedocs.io/en/stable/overview.html#easier-file-logging-with-rotation-retention-compression).
 
@@ -57,15 +57,33 @@ logging:
     rotation: 1 day
 ```
 
-### Full logging example
+### Logging each level to a different file
+If you want to log each level to a different file, you can do this by setting the `each_level_in_separate_file:` option to `true` and also setting your `file:` name, a new file will be created for each of the 5 levels used, by appending the `0_level` name to the file like so `your_file.log.1_error`. In this case the `level:` option is ignored, and all levels will be logged. 
 
-The below example logs only `WARNING` logs to the console and to the file `/my/file.log`, rotating that file once per week:
 
 ```{code} yaml
 :caption: orchestration.yaml
 
 logging:
-    level: WARNING
+    each_level_in_separate_file: true
+    file: /my/logs/file.log 
+```
+This will create the following files:
+- `/my/logs/file.log.1_debug`
+- `/my/logs/file.log.2_info`
+- `/my/logs/file.log.3_success`
+- `/my/logs/file.log.4_warning`
+- `/my/logs/file.log.5_error`
+
+### Full logging example
+
+The below example logs only `DEBUG` logs to the console and to the file `/my/file.log`, rotating that file once per week:
+
+```{code} yaml
+:caption: orchestration.yaml
+
+logging:
+    level: DEBUG
     file: /my/file.log
     rotation: 1 week
 ```

docs/source/how_to/04_run_instagrapi_server.md

@@ -0,0 +1,169 @@
+# InstagrAPI Server
+
+The instagram API Extractor requires access to a running instance of the InstagrAPI server. 
+We have a lightweight script with the endpoints required for our Instagram API Extractor module which you can run locally, or via Docker.
+
+
+
+⚠️ Warning: Remember that it's best not to use your own personal account for archiving. [Here's why](../installation/authentication.md#recommendations-for-authentication).
+## Quick Start: Using Docker
+
+We've provided a convenient shell script (`run_instagrapi_server.sh`) that simplifies the process of setting up and running the Instagrapi server in Docker. This script handles building the Docker image, setting up credentials, and starting the container.
+
+### 🔧 Running the script:
+
+Run this script either from the repository root or from within the `scripts/instagrapi_server` directory:
+
+```bash
+./scripts/instagrapi_server/run_instagrapi_server.sh
+```
+
+This script will:
+- Prompt for your Instagram username and password.
+- Create the necessary `.env` file.
+- Build the Docker image.
+- Start the Docker container and authenticate with Instagram, creating a session automatically.
+
+### ⏱ To run the server again later:
+```bash
+docker start ig-instasrv
+```
+
+### 🐛 Debugging:
+View logs:
+```bash
+docker logs ig-instasrv
+```
+
+
+### Overview: How the Setup Works
+
+1. You enter your Instagram credentials in a local `.env` file
+2. You run the server **once locally** to generate a session file
+3. After that, you can choose to run the server again locally or inside Docker without needing to log in again
+
+---
+
+## Optional: Manual / Local Setup
+
+If you'd prefer to run the server manually (without Docker), you can follow these steps:
+
+
+1. **Navigate to the server folder (and stay there for the rest of this guide)**:
+   ```bash
+   cd scripts/instagrapi_server
+   ```
+
+2. **Create a `secrets/` folder** (if it doesn't already exist in `scripts/instagrapi_server`):
+   ```bash
+   mkdir -p secrets
+   ```
+
+3. **Create a `.env` file** inside `secrets/` with your Instagram credentials:
+   ```dotenv
+   INSTAGRAM_USERNAME="your_username"
+   INSTAGRAM_PASSWORD="your_password"
+   ```
+
+4. **Install dependencies** using the pyproject.toml file:
+  
+   ```bash
+   poetry install --no-root
+   ```
+
+5. **Run the server locally**:
+   ```bash
+   poetry run uvicorn src.instaserver:app --port 8000
+   ```
+
+6. **Watch for the message**:
+   ```
+   Login successful, session saved.
+   ```
+
+✅ Your session is now saved to `secrets/instagrapi_session.json`.
+
+### To run it again locally:
+```bash
+poetry run uvicorn src.instaserver:app --port 8000
+```
+
+---
+
+## Adding the API Endpoint to Auto Archiver
+
+The server should now be running within that session, and accessible at  http://127.0.0.1:8000 
+
+You can set this in the Auto Archiver orchestration.yaml file like this:
+```yaml
+instagram_api_extractor:
+  api_endpoint: http://127.0.0.1:8000
+```
+
+
+---
+
+## 2. Running the Server Again
+
+Once the session file is created, you should be able to run the server without logging in again.
+
+### To run it locally (from scripts/instagrapi_server):
+```bash
+poetry run uvicorn src.instgrapinstance.instaserver:app --port 8000
+```
+
+---
+
+## 3. Running via Docker (After Setup is Complete, either locally or via the script)
+
+Once the `instagrapi_session.json` and `.env` files are set up, you can pass them Docker and it should authenticate successfully.
+
+### 🔨 Build the Docker image manually:
+```bash
+docker build -t instagrapi-server .
+```
+
+### ▶️ Run the container:
+```bash
+docker run -d \
+  --env-file secrets/.env \
+  -v "$(pwd)/secrets:/app/secrets" \
+  -p 8000:8000 \
+  --name ig-instasrv \
+  instagrapi-server
+```
+
+This passes the /secrets/ directory to docker as well as the environment variables from the `.env` file.
+
+
+
+---
+
+## 4. Optional Cleanup
+
+- **Stop the Docker container**:
+  ```bash
+  docker stop ig-instasrv
+  ```
+
+- **Remove the container**:
+  ```bash
+  docker rm ig-instasrv
+  ```
+
+- **Remove the Docker image**:
+  ```bash
+  docker rmi instagrapi-server
+  ```
+
+### ⏱ To run again later:
+```bash
+docker start ig-instasrv
+```
+
+---
+
+##  Notes
+
+- Never share your `.env` or `instagrapi_session.json` — these contain sensitive login data. 
+- If you want to reset your session, simply delete the `secrets/instagrapi_session.json` file and re-run the local server.

docs/source/how_to/05_upgrading_to_1_1_0.md

@@ -0,0 +1,54 @@
+# Upgrading from v1.0.1
+
+```{note} This how-to is only relevant for people who used Auto Archiver before June 2025 (versions prior to 1.1.0).
+
+If you are new to Auto Archiver, then you are already using the latest configuration format and this how-to is not relevant for you.
+```
+
+Versions 1.1.0+ of Auto Archiver has breaking changes in the configuration format, which means earlier configuration formats will not work without slight modifications.
+
+
+## Dropping `vk_extractor` module
+We have dropped the `vk_extractor` because of problems in a project we relied on. You will need to remove it from your configuration file, otherwise you will see an error like:
+
+```{code} console
+Module 'vk_extractor' not found. Are you sure it's installed/exists?
+```
+
+## Dropping `screenshot_enricher` module
+We have dropped the `screenshot_enricher` module because a new `antibot_extractor_enricher` (see below) module replaces its functionality more robustly and with less dependency hassle on geckodriver/firefox. You will need to remove it from your configuration file, otherwise you will see an error like:
+
+```{code} console
+Module 'screenshot_enricher' not found. Are you sure it's installed/exists?
+```
+
+
+## New `antibot_extractor_enricher` module and VkDropin
+We have added a new [`antibot_extractor_enricher`](../modules/autogen/extractor/antibot_extractor_enricher.md) module that uses a computer-controlled browser to extract content from websites that use anti-bot measures. You can add it to your configuration file like this:
+
+```{code} yaml
+steps:
+  extractors:
+    - antibot_extractor_enricher
+
+  # or alternatively, if you want to use it as an enricher:
+  enrichers:
+    - antibot_extractor_enricher
+```
+
+It will take a full page screenshot, a PDF capture, extract HTML source code, and any other relevant media. 
+
+It comes with Dropins that we will be adding and maintaining. 
+
+> Dropin: A module with site-specific behaviours that is loaded automatically. You don't need to add them to your configuration steps for them to run. Sometimes they need `authentication` configurations though.
+
+One such Dropin is the VkDropin which uses this automated browser to access VKontakte (VK) pages. You should add a username/password to the configuration file if you get authentication blocks from VK, to do so use the [authentication settings](authentication_how_to.md):
+
+```{code} yaml
+authentication:
+  vk.com:
+    username: your_username
+    password: your_password
+```
+
+See all available Dropins in [the source code](https://github.com/bellingcat/auto-archiver/tree/main/src/auto_archiver/modules/antibot_extractor_enricher/dropins). Usually each Dropin needs its own authentication settings, similarly to the VkDropin.

docs/source/how_to/06_new_config_format.md

@@ -71,7 +71,6 @@ The names of the actual modules have also changed, so for any extractor modules
 - `telethon_archiver` → `telethon_extractor`
 - `wacz_archiver_enricher` → `wacz_extractor_enricher`
 - `wayback_archiver_enricher` → `wayback_extractor_enricher`
-- `vk_archiver` → `vk_extractor`
 
 
 #### c) Module Renaming

docs/source/installation/authentication.md

@@ -6,6 +6,15 @@ There are two main use cases for authentication:
 * Some websites require some kind of authentication in order to view the content. Examples include Facebook, Telegram etc.
 * Some websites use anti-bot systems to block bot-like tools from accessing the website. Adding real login information to auto-archiver can sometimes bypass this.
 
+```{note}
+
+The Authentication framework currently only works with the following modules:
+* [Generic Extractor](../modules/autogen/extractor/generic_extractor.md) - the main module for extracting content from websites
+* [Antibot Extractor/Enricher](../modules/autogen/extractor/antibot_extractor_enricher.md)
+
+To authenticate for WACZ archiving, see the instructions on the [](../modules/autogen/enricher/wacz_extractor_enricher.md) page.
+```
+
 ## The Authentication Config
 
 You can save your authentication information directly inside your orchestration config file, or as a separate file (for security/multi-deploy purposes). Whether storing your settings inside the orchestration file, or as a separate file, the configuration format is the same. Currently, auto-archiver supports the following authentication types:
@@ -25,9 +34,10 @@ You can save your authentication information directly inside your orchestration
 
 ```{note} 
 
-The Username & Password, and API settings only work with the Generic Extractor. Other modules (like the screenshot enricher) can only use the `cookies` options. Furthermore, many sites can still detect bots and block username/password logins. Twitter/X and YouTube are two prominent ones that block username/password logging.
+Currently, the Username & Password, and API settings only work with the Generic and Antibot Extractors. Furthermore, many sites can still detect bots and block username/password logins. Twitter/X and YouTube are two prominent ones that block username/password logins.
 
-One of the 'Cookies' options is recommended for the most robust archiving.
+
+One of the 'Cookies' options is recommended for the most robust archiving, but it still isn't guaranteed to work.
 ```
 
 ```{code} yaml
@@ -43,12 +53,12 @@ authentication:
       username: myusername
       password: 123
     
-    facebook.com:
-       cookie: single_cookie
+   facebook.com:
+      cookie: single_cookie
 
-    othersite.com:
-       api_key: 123
-       api_secret: 1234
+   othersite.com:
+      api_key: 123
+      api_secret: 1234
   
 ```
 

docs/source/installation/example_scripts/ubuntu_24_server_install.md

@@ -0,0 +1,160 @@
+
+### Bash script for Ubuntu 24 Server install
+
+> NOTE: this script has not been tested by the maintainers and results from the personal experience of a user. It is meant as a guide and not an out of the box script, as you will see it's aimed at a custom branches, users, and features like the Geckodriver which are removed as of version 1.0.2. 
+
+This acts as a handy guide on all requirements. This is built and tested on the 29th of May 2025 on Ubuntu Server 24.04.2 LTS (which is the current latest LTS)
+
+```bash
+#!/bin/sh
+
+# I usually run steps manually as logged in with the user: dave
+# which the application runs under which makes debugging easier
+
+cd ~
+sudo apt update -y
+sudo apt upgrade -y
+
+# Clone only my latest branch
+git clone -b v1-test --single-branch https://github.com/djhmateer/auto-archiver
+
+mkdir ~/auto-archiver/secrets
+sudo chown -R dave ~/auto-archiver
+
+sudo apt update -y
+sudo apt upgrade -y
+
+## Python 3.12.3 comes with Ubuntu 24.04.2
+
+# Poetry install 2.1.3 on 2nd June 25
+curl -sSL https://install.python-poetry.org | python3 -
+
+# had to restart here.. 
+sudo reboot
+
+# C++ compiler so pdqhash will install next
+sudo apt install build-essential python3-dev -y
+
+cd auto-archiver
+
+poetry install
+
+# FFMpeg
+# 6.1.1-3ubuntu5 on 2nd June 25
+sudo apt install ffmpeg -y
+
+## Firefox
+# 139.0+build2-0ubuntu0.24.04.1~mt1 on 2nd Jun 25
+# 16th Jun - don't need anymore as using Chrome in antibot
+# cd ~
+# sudo add-apt-repository ppa:mozillateam/ppa -y
+
+# echo '
+# Package: *
+# Pin: release o=LP-PPA-mozillateam
+# Pin-Priority: 1001
+# ' | sudo tee /etc/apt/preferences.d/mozilla-firefox
+
+# echo 'Unattended-Upgrade::Allowed-Origins:: "LP-PPA-mozillateam:${distro_codename}";' | sudo tee /etc/apt/apt.conf.d/51unattended-upgrades-firefox
+
+# sudo apt install firefox -y
+
+wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
+
+# Chrome
+cd ~
+# got problems here - fixed below
+# 137.0.7151.103 on 16th Jun 2025
+sudo dpkg -i google-chrome-stable_current_amd64.deb
+
+# fix dependencies on install above
+sudo apt-get install -f
+
+# had to click a lot on UI to get going.
+# to test
+# google-chrome
+
+## Gecko driver
+# check version numbers for new ones
+# https://github.com/mozilla/geckodriver/releases/
+wget https://github.com/mozilla/geckodriver/releases/download/v0.36.0/geckodriver-v0.36.0-linux64.tar.gz
+tar -xvzf geckodriver*
+chmod +x geckodriver
+sudo mv geckodriver /usr/local/bin/
+rm geckodriver*
+
+# Fonts so selenium via firefox can render other languages eg Burmese
+sudo apt install fonts-noto -y
+
+# Docker
+# Add Docker's official GPG key:
+sudo apt-get update -y
+sudo apt-get install ca-certificates curl -y
+sudo install -m 0755 -d /etc/apt/keyrings
+sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
+sudo chmod a+r /etc/apt/keyrings/docker.asc
+
+# Add the repository to Apt sources:
+echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
+  $(. /etc/os-release && echo "${UBUNTU_CODENAME:-$VERSION_CODENAME}") stable" | \
+  sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+
+sudo apt-get update -y
+
+sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -y
+
+# add dave user to docker group 
+sudo usermod -aG docker $USER
+
+# reboot otherwise can't pull images
+
+# https://github.com/webrecorder/browsertrix-crawler
+# https://hub.docker.com/r/webrecorder/browsertrix-crawler/tags
+# 1.6.2 on 4th Jun 2025
+docker pull webrecorder/browsertrix-crawler:latest
+
+# exif
+sudo apt install libimage-exiftool-perl -y
+
+
+## CRON run every minute
+# the cron job running as user dave will execute the shell script
+# I have many scripts running from cron_11 upwards.
+# patch in the correct number
+sudo chmod +x ~/auto-archiver/scripts/cron_15.sh
+
+# don't want service to run until a reboot otherwise problems with Gecko driver
+sudo service cron stop
+
+# runs the script every minute
+# notice put in a # to disable so will have to manually start it.
+cat <<EOT >> run-auto-archive
+#*/1 * * * * dave /home/dave/auto-archiver/scripts/cron_15.sh
+EOT
+
+sudo mv run-auto-archive /etc/cron.d
+sudo chown root /etc/cron.d/run-auto-archive
+sudo chmod 600 /etc/cron.d/run-auto-archive
+
+# Helper alias 'c' to open the above file
+echo "alias c='sudo vim /etc/cron.d/run-auto-archive'" >> ~/.bashrc
+
+# secrets folder copy
+# I run dev from:
+# \\wsl.localhost\Ubuntu-24.04\home\dave\code\auto-archiver\secrets\
+
+# orchestration.yaml - for aa config
+# service_account - for google spreadsheet
+# anon.session - for telethon so don't have to type in phone number
+# profile.tar.gz - for wacz to have a logged in profile for facebook, x.com and instagram to get data
+
+# Youtube - POT Tokens
+# https://github.com/Brainicism/bgutil-ytdlp-pot-provider
+docker run --name bgutil-provider --restart unless-stopped -d -p 4416:4416 brainicism/bgutil-ytdlp-pot-provider
+
+
+# test run
+cd ~/auto-archiver
+
+poetry run python src/auto_archiver --config secrets/orchestration-aa-demo-main.yaml
+```

docs/source/installation/faq.md

@@ -11,7 +11,6 @@ are available on the [extractors](../modules/extractor.md) page. Some sites supp
 * Twitter
 * Instagram
 * Telegram
-* VKontact
 * Tiktok
 * Bluesky
 

docs/source/installation/installation.md

@@ -51,11 +51,14 @@ After this, you're ready to set up your [your configuration file](configurations
 If using the local installation method, you will also need to install the following dependencies locally:
 
 1.[ffmpeg](https://www.ffmpeg.org/) - for handling of downloaded videos
-2. [firefox](https://www.mozilla.org/en-US/firefox/new/) and [geckodriver](https://github.com/mozilla/geckodriver/releases) on a path folder like `/usr/local/bin` - for taking webpage screenshots with the screenshot enricher
-3. (optional) [fonts-noto](https://fonts.google.com/noto) to deal with multiple unicode characters during selenium/geckodriver's screenshots: `sudo apt install fonts-noto -y`.
+<!-- 2. [firefox](https://www.mozilla.org/en-US/firefox/new/) and [geckodriver](https://github.com/mozilla/geckodriver/releases) on a path folder like `/usr/local/bin` - for taking webpage screenshots with the screenshot enricher -->
+3. (optional) [fonts-noto](https://fonts.google.com/noto) to deal with multiple unicode characters during selenium screenshots: `sudo apt install fonts-noto -y`.
 4. [Browsertrix Crawler docker image](https://hub.docker.com/r/webrecorder/browsertrix-crawler) for the WACZ enricher/archiver
 
 
+### Custom installation scripts
+- [Ubuntu 24 Server Install by @djhmateer](example_scripts/ubuntu_24_server_install.md) - a WYSIWYG example script from a user who set up the Auto Archiver on a fresh Ubuntu 24 server.
+
 
 ## Developer Install
 

docs/source/installation/setup.md

@@ -27,8 +27,8 @@ The way you run the Auto Archiver depends on how you installed it (docker instal
 If you installed Auto Archiver using docker, open up your terminal, and copy-paste / type the following command:
 
 ```bash
-docker run -it --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver
- ```
+docker run -it --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver -- "https://example.com/1/"
+```
 
 breaking this command down:
    1. `docker run` tells docker to start a new container (an instance of the image)
@@ -42,6 +42,7 @@ breaking this command down:
        1.  `-v` same as above, this is a volume instruction
        2.  `$PWD/local_archive` is a folder `local_archive/` in case you want to archive locally and have the files accessible outside docker
        3.  `/app/local_archive` is a folder inside docker that you can reference in your orchestration.yml file 
+   6. ` -- "https://example.com/1/"` this will pass the URL to archive to the default [command line feeder](../modules/autogen/feeder/cli_feeder.md)
 
 ### Example invocations
 

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [project]
 name = "auto-archiver"
-version = "0.13.6"
+version = "1.1.0"
 description = "Automatically archive links to videos, images, and social media content from Google Sheets (and more)."
 
 requires-python = ">=3.10,<3.13"
@@ -27,7 +27,6 @@ dependencies = [
     "bs4 (>=0.0.0)",
     "loguru (>=0.0.0)",
     "ffmpeg-python (>=0.0.0)",
-    "selenium (>=0.0.0)",
     "telethon (>=0.0.0)",
     "google-api-python-client (>=0.0.0)",
     "google-auth-httplib2 (>=0.0.0)",
@@ -41,23 +40,24 @@ dependencies = [
     "instaloader (>=0.0.0)",
     "tqdm (>=0.0.0)",
     "jinja2 (>=0.0.0)",
-    "pyOpenSSL (==24.2.1)",
-    "cryptography (>=41.0.0,<42.0.0)",
     "boto3 (>=1.28.0,<2.0.0)",
     "dataclasses-json (>=0.0.0)",
-    "yt-dlp (>=2025.1.26,<2026.0.0)",
     "numpy (==2.1.3)",
-    "vk-url-scraper (>=0.0.0)",
     "requests[socks] (>=0.0.0)",
     "warcio (>=0.0.0)",
     "jsonlines (>=0.0.0)",
     "pysubs2 (>=0.0.0)",
     "retrying (>=0.0.0)",
-    "tsp-client (>=0.0.0)",
-    "certvalidator (>=0.0.0)",
     "rich-argparse (>=1.6.0,<2.0.0)",
     "ruamel-yaml (>=0.18.10,<0.19.0)",
+    "rfc3161-client (>=1.0.1,<2.0.0)",
+    "cryptography (>44.0.1,<45.0.0)",
     "opentimestamps (>=0.4.5,<0.5.0)",
+    "bgutil-ytdlp-pot-provider (>=1.0.0)",
+    "yt-dlp[curl-cffi,default] (>=2025.5.22,<2026.0.0)",
+    "secretstorage (>=3.3.3,<4.0.0)",
+    "seleniumbase (>=4.36.4,<5.0.0)",
+    "pyautogui (>=0.9.54,<0.10.0)",
 ]
 
 [tool.poetry.group.dev.dependencies]

scripts/generate_google_services.sh

@@ -0,0 +1,135 @@
+#!/usr/bin/env bash
+
+set -e  # Exit on error
+
+
+UUID=$(LC_ALL=C tr -dc a-z0-9 </dev/urandom | head -c 16) 
+PROJECT_NAME="auto-archiver-$UUID"
+ACCOUNT_NAME="autoarchiver"
+KEY_FILE="service_account-$UUID.json"
+DEST_DIR="$1"
+
+echo "====================================================="
+echo "🔧 Auto-Archiver Google Services Setup Script"
+echo "====================================================="
+echo "This script will:"
+echo "  1. Install Google Cloud SDK if needed"
+echo "  2. Create a Google Cloud project named $PROJECT_NAME"
+echo "  3. Create a service account for Auto-Archiver"
+echo "  4. Generate a key file for API access"
+echo ""
+echo "  Tip: Pass a directory path as an argument to this script to move the key file there"
+echo "  e.g. ./generate_google_services.sh /path/to/secrets"
+echo "====================================================="
+
+# Check and install Google Cloud SDK based on platform
+install_gcloud_sdk() {
+    if command -v gcloud &> /dev/null; then
+        echo "✅ Google Cloud SDK is already installed"
+        return 0
+    fi
+
+    echo "📦 Installing Google Cloud SDK..."
+    
+    # Detect OS
+    case "$(uname -s)" in
+        Darwin*)
+            if command -v brew &> /dev/null; then
+                echo "🍺 Installing via Homebrew..."
+                brew install google-cloud-sdk --cask
+            else
+                echo "📥 Downloading Google Cloud SDK for macOS..."
+                curl -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-latest-darwin-x86_64.tar.gz
+                tar -xf google-cloud-cli-latest-darwin-x86_64.tar.gz
+                ./google-cloud-sdk/install.sh --quiet
+                rm google-cloud-cli-latest-darwin-x86_64.tar.gz
+                echo "🔄 Please restart your terminal and run this script again"
+                exit 0
+            fi
+            ;;
+        Linux*)
+            echo "📥 Downloading Google Cloud SDK for Linux..."
+            curl -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-latest-linux-x86_64.tar.gz
+            tar -xf google-cloud-cli-latest-linux-x86_64.tar.gz
+            ./google-cloud-sdk/install.sh --quiet
+            rm google-cloud-cli-latest-linux-x86_64.tar.gz
+            echo "🔄 Please restart your terminal and run this script again"
+            exit 0
+            ;;
+        CYGWIN*|MINGW*|MSYS*)
+            echo "⚠️ Windows detected. Please follow manual installation instructions at:"
+            echo "https://cloud.google.com/sdk/docs/install-sdk"
+            exit 1
+            ;;
+        *)
+            echo "⚠️ Unknown operating system. Please follow manual installation instructions at:"
+            echo "https://cloud.google.com/sdk/docs/install-sdk"
+            exit 1
+            ;;
+    esac
+    
+    echo "✅ Google Cloud SDK installed"
+}
+
+# Install Google Cloud SDK if needed
+install_gcloud_sdk
+
+# Login to Google Cloud
+if gcloud auth list --filter=status:ACTIVE --format="value(account)" | grep -q "@"; then
+    echo "✅ Already authenticated with Google Cloud"
+else
+    echo "🔑 Authenticating with Google Cloud..."
+    gcloud auth login
+fi
+
+# Create project
+echo "🌟 Creating Google Cloud project: $PROJECT_NAME"
+gcloud projects create $PROJECT_NAME
+
+# Create service account
+echo "👤 Creating service account: $ACCOUNT_NAME"
+gcloud iam service-accounts create $ACCOUNT_NAME --project $PROJECT_NAME
+
+# Enable required APIs (uncomment and add APIs as needed)
+echo "⬆️ Enabling required Google APIs..."
+gcloud services enable sheets.googleapis.com --project $PROJECT_NAME
+gcloud services enable drive.googleapis.com --project $PROJECT_NAME
+
+# Get the service account email
+echo "📧 Retrieving service account email..."
+ACCOUNT_EMAIL=$(gcloud iam service-accounts list --project $PROJECT_NAME --format="value(email)")
+
+# Create and download key
+echo "🔑 Generating service account key file: $KEY_FILE"
+gcloud iam service-accounts keys create $KEY_FILE --iam-account=$ACCOUNT_EMAIL
+
+# move the file to TARGET_DIR if provided
+if [[ -n "$DEST_DIR" ]]; then
+    # Expand `~` if used
+    DEST_DIR=$(eval echo "$DEST_DIR")
+
+    # Ensure the directory exists
+    if [[ ! -d "$DEST_DIR" ]]; then
+        mkdir -p "$DEST_DIR"
+    fi
+
+    DEST_PATH="$DEST_DIR/$KEY_FILE"
+    echo "🚚 Moving key file to: $DEST_PATH"
+    mv "$KEY_FILE" "$DEST_PATH"
+    KEY_FILE="$DEST_PATH"
+fi
+
+echo "====================================================="
+echo "✅ SETUP COMPLETE!"
+echo "====================================================="
+echo "📝 Important Information:"
+echo "  • Project Name: $PROJECT_NAME"
+echo "  • Service Account: $ACCOUNT_EMAIL"
+echo "  • Key File: $KEY_FILE"
+echo ""
+echo "📋 Next Steps:"
+echo "  1. Share any Google Sheets with this email address:"
+echo "     $ACCOUNT_EMAIL"
+echo "  2. Move $KEY_FILE to your auto-archiver secrets directory"
+echo "  3. Update your auto-archiver config to use this key file (if needed)"
+echo "====================================================="

scripts/instagrapi_server/.gitignore

@@ -0,0 +1,2 @@
+secrets*
+*instagrapi_session.json

scripts/instagrapi_server/Dockerfile

@@ -0,0 +1,19 @@
+FROM python:3.12-slim
+WORKDIR /app
+
+# Install Poetry
+RUN pip install --upgrade pip
+RUN pip install poetry
+
+# Copy all source code
+COPY . .
+
+# Prevent Poetry from creating a virtual environment
+RUN poetry config virtualenvs.create false
+
+# Install dependencies
+RUN poetry install --no-root
+
+
+# Use uvicorn to run the FastAPI app
+CMD ["poetry", "run", "uvicorn", "src.instaserver:app", "--host", "0.0.0.0", "--port", "8000"]

scripts/instagrapi_server/pyproject.toml

@@ -0,0 +1,18 @@
+[project]
+name = "instaserver"
+version = "0.1.0"
+description = "A FastAPI InstagrAPI server"
+package-mode = false
+requires-python = ">=3.10"
+dependencies = [
+    "fastapi (>=0.115.12,<0.116.0)",
+    "instagrapi (>=2.1.3,<3.0.0)",
+    "uvicorn (>=0.34.0,<0.35.0)",
+    "pillow (>=11.1.0,<12.0.0)",
+    "python-dotenv (>=1.1.0,<2.0.0)"
+]
+
+
+[build-system]
+requires = ["poetry-core>=2.0.0,<3.0.0"]
+build-backend = "poetry.core.masonry.api"

scripts/instagrapi_server/run_instagrapi_server.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+#
+# run_instagrapi_server.sh
+# Usage:
+#   From repo root:   ./scripts/instagrapi_server/run_instagrapi_server.sh
+#   Or from script dir: ./run_instagrapi_server.sh
+#
+
+set -e
+
+# Step 1: cd to the script's directory (contains Dockerfile and secrets/)
+cd "$(dirname "$0")" || exit 1
+
+# Create secrets/ if it doesn't exist
+if [[ ! -d "secrets" ]]; then
+  echo "Creating secrets/ directory..."
+  mkdir secrets
+fi
+
+echo "Enter your Instagram credentials to store in secrets/.env"
+read -rp "Instagram Username: " IGUSER
+read -rsp "Instagram Password: " IGPASS
+echo ""
+
+cat <<EOF > secrets/.env
+INSTAGRAM_USERNAME=$IGUSER
+INSTAGRAM_PASSWORD=$IGPASS
+EOF
+echo "Created secrets/.env with your credentials."
+
+# Build Docker image
+IMAGE_NAME="instagrapi-server"
+echo "Building Docker image '$IMAGE_NAME'..."
+docker build -t "$IMAGE_NAME" .
+
+# Run container
+CONTAINER_NAME="ig-instasrv"
+echo "Running container '$CONTAINER_NAME'..."
+docker run -d \
+  --env-file secrets/.env \
+  -v "$(pwd)/secrets:/app/secrets" \
+  -p 8000:8000 \
+  --name "$CONTAINER_NAME" \
+  "$IMAGE_NAME"
+
+echo "Done! Instagrapi server is running on port 8000."
+echo "Use 'docker logs $CONTAINER_NAME' to view logs."
+echo "Use 'docker stop $CONTAINER_NAME' and 'docker rm $CONTAINER_NAME' to stop/remove the container."

scripts/instagrapi_server/src/instaserver.py

@@ -0,0 +1,157 @@
+"""https://subzeroid.github.io/instagrapi/
+
+Run using the following command:
+ uvicorn src.instgrapinstance.instaserver:app --host 0.0.0.0 --port 8000 --reload
+"""
+
+import logging
+import os
+import sys
+from dotenv import load_dotenv
+
+from fastapi import FastAPI, HTTPException
+from instagrapi import Client
+from instagrapi.exceptions import LoginRequired, BadCredentials
+
+load_dotenv(dotenv_path="secrets/.env")
+logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
+
+INSTAGRAM_USERNAME = os.getenv("INSTAGRAM_USERNAME")
+INSTAGRAM_PASSWORD = os.getenv("INSTAGRAM_PASSWORD")
+SESSION_FILE = "secrets/instagrapi_session.json"
+
+app = FastAPI()
+cl = Client()
+
+
+@app.on_event("startup")
+def startup_event():
+    """Login automatically when server starts"""
+    try:
+        login_instagram()
+    except RuntimeError as e:
+        logging.error(f"API failed to start: {e}")
+        sys.exit(1)
+
+
+def login_instagram():
+    """Ensures Instagrapi is logged in and session is persistent"""
+    if not INSTAGRAM_USERNAME or not INSTAGRAM_PASSWORD:
+        raise RuntimeError("Instagram credentials are missing.")
+
+    if os.path.exists(SESSION_FILE):
+        try:
+            cl.load_settings(SESSION_FILE)
+            cl.get_timeline_feed()
+            logging.info("Using saved session.")
+            return
+        except LoginRequired:
+            logging.info("Session expired. Logging in again...")
+
+    try:
+        cl.login(INSTAGRAM_USERNAME, INSTAGRAM_PASSWORD)
+        cl.dump_settings(SESSION_FILE)
+        logging.info("Login successful, session saved.")
+    except BadCredentials as bc:
+        raise RuntimeError("Incorrect Instagram username or password.") from bc
+    except Exception as e:
+        raise RuntimeError(f"Login failed: {e}") from e
+
+
+@app.get("/v1/media/by/id")
+def get_media_by_id(id: str):
+    """Fetch post details by media ID"""
+    logging.info(f"Fetching media by ID: {id}")
+    try:
+        media = cl.media_info(id)
+        return media.model_dump()
+    except Exception as e:
+        logging.warning(f"Media not found for ID {id}: {e}")
+        raise HTTPException(status_code=404, detail="Post not found") from e
+
+
+@app.get("/v1/media/by/code")
+def get_media_by_code(code: str):
+    """Fetch post details by shortcode"""
+    logging.info(f"Fetching media by shortcode: {code}")
+    try:
+        media_id = cl.media_pk_from_code(code)
+        media = cl.media_info(media_id)
+        return media.model_dump()
+    except Exception as e:
+        logging.warning(f"Media not found for code {code}: {e}")
+        raise HTTPException(status_code=404, detail="Post not found") from e
+
+
+@app.get("/v2/user/tag/medias")
+def get_user_tagged_medias(user_id: str, page_id: str = None):
+    logging.info(f"Fetching tagged medias for user_id={user_id} page_id={page_id}")
+    try:
+        # Placeholder for now
+        items, next_page_id = [], None
+        return {"response": {"items": items}, "next_page_id": next_page_id}
+    except Exception as e:
+        logging.warning(f"Tagged media not found for {user_id}: {e}")
+        raise HTTPException(status_code=404, detail="Tagged media not found") from e
+
+
+@app.get("/v1/user/highlights")
+def get_user_highlights(user_id: str):
+    logging.info(f"Fetching highlights list for user_id={user_id}")
+    try:
+        highlights = cl.user_highlights(user_id)
+        return [h.model_dump() for h in highlights]
+    except Exception as e:
+        logging.warning(f"Highlights not found for {user_id}: {e}")
+        raise HTTPException(status_code=404, detail="No highlights found") from e
+
+
+@app.get("/v2/highlight/by/id")
+def get_highlight_by_id(id: str):
+    logging.info(f"Fetching highlight details for id={id}")
+    try:
+        highlight = cl.highlight_info(id)
+        return {"response": {"reels": {f"highlight:{id}": highlight.model_dump()}}}
+    except Exception as e:
+        logging.warning(f"Highlight not found for id {id}: {e}")
+        raise HTTPException(status_code=404, detail="Highlight not found") from e
+
+
+@app.get("/v1/user/stories/by/username")
+def get_stories(username: str):
+    logging.info(f"Fetching stories for username={username}")
+    try:
+        user_id = cl.user_id_from_username(username)
+        stories = cl.user_stories(user_id)
+        return [story.model_dump() for story in stories]
+    except Exception as e:
+        logging.warning(f"Stories not found for {username}: {e}")
+        raise HTTPException(status_code=404, detail="Stories not found") from e
+
+
+@app.get("/v2/user/by/username")
+def get_user_by_username(username: str):
+    logging.info(f"Fetching user profile for username={username}")
+    try:
+        user = cl.user_info_by_username(username)
+        return {"user": user.model_dump()}
+    except Exception as e:
+        logging.warning(f"User not found: {username}: {e}")
+        raise HTTPException(status_code=404, detail="User not found") from e
+
+
+@app.get("/v1/user/medias/chunk")
+def get_user_medias(user_id: str, end_cursor: str = None):
+    logging.info(f"Fetching paginated medias for user_id={user_id}, end_cursor={end_cursor}")
+    try:
+        posts, next_cursor = cl.user_medias_paginated(user_id, end_cursor=end_cursor)
+        return [[post.model_dump() for post in posts], next_cursor]
+    except Exception as e:
+        logging.warning(f"No posts found for user_id={user_id}: {e}")
+        raise HTTPException(status_code=404, detail="No posts found") from e
+
+
+if __name__ == "__main__":
+    import uvicorn
+
+    uvicorn.run(app, host="0.0.0.0", port=8000)

scripts/settings/package.json

@@ -13,10 +13,10 @@
     "@dnd-kit/sortable": "^10.0.0",
     "@emotion/react": "latest",
     "@emotion/styled": "latest",
-    "@mui/icons-material": "^6.4.7",
+    "@mui/icons-material": "^7.1.1",
     "@mui/material": "latest",
-    "react": "19.0.0",
-    "react-dom": "19.0.0",
+    "react": "19.1.0",
+    "react-dom": "19.1.0",
     "react-markdown": "^10.0.0",
     "yaml": "^2.7.0"
   },

src/auto_archiver/core/base_module.py

@@ -71,7 +71,16 @@ class BaseModule(ABC):
         :param site: the domain of the site to get authentication information for
         :param extract_cookies: whether or not to extract cookies from the given browser/file and return the cookie jar (disabling can speed up processing if you don't actually need the cookies jar).
 
-        :returns: authdict dict of login information for the given site
+        :returns: authdict dict -> {
+            "username": str,
+            "password": str,
+            "api_key": str,
+            "api_secret": str,
+            "cookie": str,
+            "cookies_file": str,
+            "cookies_from_browser": str,
+            "cookies_jar": CookieJar
+        }
 
         **Global options:**\n
         * cookies_from_browser: str - the name of the browser to extract cookies from (e.g. 'chrome', 'firefox' - uses ytdlp under the hood to extract\n
@@ -85,15 +94,15 @@ class BaseModule(ABC):
         * cookie: str - a cookie string to use for login (specific to this site)\n
         * cookies_file: str - the path to a cookies file to use for login (specific to this site)\n
         * cookies_from_browser: str - the name of the browser to extract cookies from (specitic for this site)\n
+
         """
         # TODO: think about if/how we can deal with sites that have multiple domains (main one is x.com/twitter.com)
         # for now the user must enter them both, like "x.com,twitter.com" in their config. Maybe we just hard-code?
-
-        site = UrlUtil.domain_for_url(site).removeprefix("www.")
+        domain = UrlUtil.domain_for_url(site).removeprefix("www.")
         # add the 'www' version of the site to the list of sites to check
         authdict = {}
 
-        for to_try in [site, f"www.{site}"]:
+        for to_try in [site, domain, f"www.{domain}"]:
             if to_try in self.authentication:
                 authdict.update(self.authentication[to_try])
                 break
@@ -101,9 +110,9 @@ class BaseModule(ABC):
         # do a fuzzy string match just to print a warning - don't use it since it's insecure
         if not authdict:
             for key in self.authentication.keys():
-                if key in site or site in key:
+                if key in domain or domain in key:
                     logger.debug(
-                        f"Could not find exact authentication information for site '{site}'. \
+                        f"Could not find exact authentication information for '{domain}'. \
 did find information for '{key}' which is close, is this what you meant? \
 If so, edit your authentication settings to make sure it exactly matches."
                     )

src/auto_archiver/core/extractor.py

@@ -8,6 +8,7 @@ Factory method to initialize an extractor instance based on its name.
 
 from __future__ import annotations
 from abc import abstractmethod
+from contextlib import suppress
 import mimetypes
 import os
 import requests
@@ -16,6 +17,7 @@ from retrying import retry
 import re
 
 from auto_archiver.core import Metadata, BaseModule
+from auto_archiver.utils.url import get_media_url_best_quality
 
 
 class Extractor(BaseModule):
@@ -70,10 +72,22 @@ class Extractor(BaseModule):
         return ""
 
     @retry(wait_random_min=500, wait_random_max=3500, stop_max_attempt_number=5)
-    def download_from_url(self, url: str, to_filename: str = None, verbose=True) -> str:
+    def download_from_url(self, url: str, to_filename: str = None, verbose=True, try_best_quality=False) -> str:
         """
         downloads a URL to provided filename, or inferred from URL, returns local filename
+        Warning: if try_best_quality is True, it will return a tuple of (filename, best_quality_url) if the download was successful.
         """
+        if any(url.startswith(x) for x in ["blob:", "data:"]):
+            return None, url if try_best_quality else None
+
+        if try_best_quality:
+            with suppress(Exception):
+                # Attempt to download the original URL
+                best_quality_url = get_media_url_best_quality(url)
+                orig_download = self.download_from_url(best_quality_url, to_filename, verbose)
+                if orig_download:
+                    return orig_download, best_quality_url
+
         if not to_filename:
             to_filename = url.split("/")[-1].split("?")[0]
             if len(to_filename) > 64:
@@ -98,10 +112,14 @@ class Extractor(BaseModule):
             with open(to_filename, "wb") as f:
                 for chunk in d.iter_content(chunk_size=8192):
                     f.write(chunk)
+            if try_best_quality:
+                return to_filename, url
             return to_filename
 
         except requests.RequestException as e:
-            logger.warning(f"Failed to fetch the Media URL: {e}")
+            logger.warning(f"Failed to fetch the Media URL: {str(e)[:250]}")
+        if try_best_quality:
+            return None, url
 
     @abstractmethod
     def download(self, item: Metadata) -> Metadata | False:

src/auto_archiver/core/media.py

@@ -116,7 +116,7 @@ class Media:
         # self.is_video() should be used together with this method
         try:
             streams = ffmpeg.probe(self.filename, select_streams="v")["streams"]
-            logger.warning(f"STREAMS FOR {self.filename} {streams}")
+            logger.debug(f"STREAMS FOR {self.filename} {streams}")
             return any(s.get("duration_ts", 0) > 0 for s in streams)
         except Error:
             return False  # ffmpeg errors when reading bad files

src/auto_archiver/core/metadata.py

@@ -96,7 +96,7 @@ class Metadata:
 
     def is_empty(self) -> bool:
         meaningfull_ids = set(self.metadata.keys()) - set(
-            ["_processed_at", "url", "total_bytes", "total_size", "archive_duration_seconds"]
+            ["_processed_at", "url", "original_url", "total_bytes", "total_size", "archive_duration_seconds"]
         )
         return not self.is_success() and len(self.media) == 0 and len(meaningfull_ids) == 0
 

src/auto_archiver/core/module.py

@@ -5,6 +5,7 @@ by handling user configuration, validating the steps properties, and implementin
 """
 
 from __future__ import annotations
+import subprocess
 
 from dataclasses import dataclass
 from typing import List, TYPE_CHECKING, Type
@@ -17,7 +18,7 @@ import os
 from os.path import join
 from loguru import logger
 import auto_archiver
-from auto_archiver.core.consts import DEFAULT_MANIFEST, MANIFEST_FILE
+from auto_archiver.core.consts import DEFAULT_MANIFEST, MANIFEST_FILE, SetupError
 
 if TYPE_CHECKING:
     from .base_module import BaseModule
@@ -85,7 +86,11 @@ class ModuleFactory:
         if not available:
             message = f"Module '{module_name}' not found. Are you sure it's installed/exists?"
             if "archiver" in module_name:
-                message += f" Did you mean {module_name.replace('archiver', 'extractor')}?"
+                message += f" Did you mean '{module_name.replace('archiver', 'extractor')}'?"
+            elif "gsheet" in module_name:
+                message += " Did you mean 'gsheet_feeder_db'?"
+            elif "atlos" in module_name:
+                message += " Did you mean 'atlos_feeder_db_storage'?"
             raise IndexError(message)
         return available[0]
 
@@ -209,16 +214,13 @@ class LazyBaseModule:
 
         # check external dependencies are installed
         def check_deps(deps, check):
-            for dep in deps:
-                if not len(dep):
-                    # clear out any empty strings that a user may have erroneously added
-                    continue
-                if not check(dep):
+            for dep in filter(lambda d: len(d.strip()) > 0, deps):
+                if not check(dep.strip()):
                     logger.error(
                         f"Module '{self.name}' requires external dependency '{dep}' which is not available/setup. \
-                                 Have you installed the required dependencies for the '{self.name}' module? See the README for more information."
+                                 Have you installed the required dependencies for the '{self.name}' module? See the documentation for more information."
                     )
-                    exit(1)
+                    raise SetupError()
 
         def check_python_dep(dep):
             # first check if it's a module:
@@ -237,8 +239,22 @@ class LazyBaseModule:
 
             return find_spec(dep)
 
+        def check_bin_dep(dep):
+            dep_exists = shutil.which(dep)
+
+            if dep == "docker":
+                if os.environ.get("RUNNING_IN_DOCKER"):
+                    # this is only for the WACZ enricher, which requires docker
+                    # if we're already running in docker then we don't need docker
+                    return True
+
+                # check if docker daemon is running
+                return dep_exists and subprocess.run(["docker", "ps", "-q"]).returncode == 0
+
+            return dep_exists
+
         check_deps(self.dependencies.get("python", []), check_python_dep)
-        check_deps(self.dependencies.get("bin", []), lambda dep: shutil.which(dep))
+        check_deps(self.dependencies.get("bin", []), check_bin_dep)
 
         logger.debug(f"Loading module '{self.display_name}'...")
 
@@ -258,6 +274,9 @@ class LazyBaseModule:
         # finally, get the class instance
         instance: BaseModule = getattr(sys.modules[sub_qualname], class_name)()
 
+        # save the instance for future easy loading
+        self._instance = instance
+
         # set the name, display name and module factory
         instance.name = self.name
         instance.display_name = self.display_name
@@ -270,8 +289,6 @@ class LazyBaseModule:
         instance.config_setup(config)
         instance.setup()
 
-        # save the instance for future easy loading
-        self._instance = instance
         return instance
 
     def __repr__(self):

src/auto_archiver/core/orchestrator.py

@@ -5,6 +5,7 @@ formatting, database operations and clean up.
 """
 
 from __future__ import annotations
+from packaging import version
 from typing import Generator, Union, List, Type, TYPE_CHECKING
 import argparse
 import os
@@ -33,7 +34,7 @@ from .config import (
 from .module import ModuleFactory, LazyBaseModule
 from . import validators, Feeder, Extractor, Database, Storage, Formatter, Enricher
 from .consts import MODULE_TYPES, SetupError
-from auto_archiver.utils.url import check_url_or_raise
+from auto_archiver.utils.url import check_url_or_raise, clean
 
 if TYPE_CHECKING:
     from .base_module import BaseModule
@@ -248,7 +249,7 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
             action="store",
             dest="logging.level",
             choices=["INFO", "DEBUG", "ERROR", "WARNING"],
-            help="the logging level to use",
+            help="the logging level to use for the standard output and file logging",
             default="INFO",
             type=str.upper,
         )
@@ -263,6 +264,14 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
             default=None,
         )
 
+        parser.add_argument(
+            "--logging.each_level_in_separate_file",
+            action="store",
+            dest="logging.each_level_in_separate_file",
+            help="if set, writes each logging level to a separate file (ignores --logging.level), you must also set --logging.file. Each level will have a dedicate logs file matching your <file>.debug, <file>.info, etc.",
+            default=False,
+        )
+
     def add_individual_module_args(
         self, modules: list[LazyBaseModule] = None, parser: argparse.ArgumentParser = None
     ) -> None:
@@ -332,11 +341,24 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
 
         # add other logging info
         if self.logger_id is None:  # note - need direct comparison to None since need to consider falsy value 0
-            self.logger_id = logger.add(sys.stderr, level=logging_config["level"])
-            if log_file := logging_config["file"]:
-                logger.add(log_file) if not logging_config["rotation"] else logger.add(
-                    log_file, rotation=logging_config["rotation"]
+            use_level = logging_config["level"]
+            self.logger_id = logger.add(sys.stderr, level=use_level)
+
+            rotation = logging_config["rotation"]
+            log_file = logging_config["file"]
+
+            if logging_config.get("each_level_in_separate_file"):
+                assert logging_config["file"], (
+                    "You must set --logging.file if you want to use --logging.each_level_in_separate_file"
                 )
+                for i, level in enumerate(["DEBUG", "INFO", "SUCCESS", "WARNING", "ERROR"], start=1):
+                    logger.add(
+                        f"{log_file}.{i}_{level.lower()}",
+                        filter=lambda rec, lvl=level: rec["level"].name == lvl,
+                        rotation=rotation,
+                    )
+            elif log_file:
+                logger.add(log_file, rotation=rotation, level=use_level)
 
     def install_modules(self, modules_by_type):
         """
@@ -373,14 +395,24 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
                 if module in invalid_modules:
                     continue
 
+                # check to make sure that we're trying to load it as the correct type - i.e. make sure the user hasn't put it under the wrong 'step'
+                lazy_module: LazyBaseModule = self.module_factory.get_module_lazy(module)
+                if module_type not in lazy_module.type:
+                    types = ",".join(f"'{t}'" for t in lazy_module.type)
+                    raise SetupError(
+                        f"Configuration Error: Module '{module}' is not a {module_type}, but has the types: {types}. Please check you set this module up under the right step in your orchestration file."
+                    )
+
                 loaded_module = None
                 try:
-                    loaded_module: BaseModule = self.module_factory.get_module(module, self.config)
+                    loaded_module: BaseModule = lazy_module.load(self.config)
                 except (KeyboardInterrupt, Exception) as e:
                     if not isinstance(e, KeyboardInterrupt) and not isinstance(e, SetupError):
                         logger.error(f"Error during setup of modules: {e}\n{traceback.format_exc()}")
-                    if loaded_module and module_type == "extractor":
-                        loaded_module.cleanup()
+
+                    # access the _instance here because loaded_module may not return if there's an error
+                    if lazy_module._instance and module_type == "extractor":
+                        lazy_module._instance.cleanup()
                     raise e
 
                 if not loaded_module:
@@ -426,16 +458,19 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
 
     def check_for_updates(self):
         response = requests.get("https://pypi.org/pypi/auto-archiver/json").json()
-        latest_version = response["info"]["version"]
+        latest_version = version.parse(response["info"]["version"])
+        current_version = version.parse(__version__)
         # check version compared to current version
-        if latest_version != __version__:
+        if latest_version > current_version:
             if os.environ.get("RUNNING_IN_DOCKER"):
                 update_cmd = "`docker pull bellingcat/auto-archiver:latest`"
             else:
                 update_cmd = "`pip install --upgrade auto-archiver`"
             logger.warning("")
             logger.warning("********* IMPORTANT: UPDATE AVAILABLE ********")
-            logger.warning(f"A new version of auto-archiver is available (v{latest_version}, you have {__version__})")
+            logger.warning(
+                f"A new version of auto-archiver is available (v{latest_version}, you have v{current_version})"
+            )
             logger.warning(f"Make sure to update to the latest version using: {update_cmd}")
             logger.warning("")
 
@@ -502,7 +537,7 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
                 yield self.feed_item(item)
                 url_count += 1
 
-        logger.success(f"Processed {url_count} URL(s)")
+        logger.info(f"Processed {url_count} URL(s)")
         self.cleanup()
 
     def feed_item(self, item: Metadata) -> Metadata:
@@ -558,12 +593,13 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
             raise e
 
         # 1 - sanitize - each archiver is responsible for cleaning/expanding its own URLs
-        url = original_url
+        url = clean(original_url)
         for a in self.extractors:
             url = a.sanitize_url(url)
 
         result.set_url(url)
         if original_url != url:
+            logger.debug(f"Sanitized URL from {original_url} to {url}")
             result.set("original_url", original_url)
 
         # 2 - notify start to DBs, propagate already archived if feature enabled in DBs

src/auto_archiver/core/validators.py

@@ -4,12 +4,6 @@ import argparse
 import json
 
 
-def example_validator(value):
-    if "example" not in value:
-        raise argparse.ArgumentTypeError(f"{value} is not a valid value for this argument")
-    return value
-
-
 def positive_number(value):
     if value < 0:
         raise argparse.ArgumentTypeError(f"{value} is not a positive number")

src/auto_archiver/modules/antibot_extractor_enricher/__manifest__.py

@@ -0,0 +1,53 @@
+{
+    "name": "Antibot Extractor/Enricher",
+    "type": ["extractor", "enricher"],
+    "requires_setup": False,
+    "dependencies": {"python": ["loguru", "seleniumbase", "yt_dlp"], "bin": ["ffmpeg"]},
+    "configs": {
+        "save_to_pdf": {
+            "default": False,
+            "type": "bool",
+            "help": "save a PDF snapshot of the page.",
+        },
+        "max_download_images": {
+            "default": 50,
+            "help": "maximum number of images to download from the page (0 = no download, inf = no limit).",
+        },
+        "max_download_videos": {
+            "default": 50,
+            "help": "maximum number of videos to download from the page (0 = no download, inf = no limit).",
+        },
+        "user_data_dir": {
+            "default": "secrets/antibot_user_data",
+            "help": "Path to the user data directory for the webdriver. This is used to persist browser state, such as cookies and local storage. If you use the docker deployment, this path will be appended with `_docker` that is because the folder cannot be shared between the host and the container due to user permissions.",
+        },
+        "detect_auth_wall": {
+            "default": True,
+            "type": "bool",
+            "help": "detect if the page is behind an authentication wall (e.g. login required) and skip it. disable if you want to archive pages where logins are required.",
+        },
+        "proxy": {
+            "default": None,
+            "help": "proxy to use for the webdriver, Format: 'SERVER:PORT' or 'USER:PASS@SERVER:PORT'",
+        },
+    },
+    "autodoc_dropins": True,
+    "description": """
+    Uses a browser controlled by SeleniumBase to capture HTML, media, and screenshots/PDFs of a web page, by bypassing anti-bot measures like Cloudflare's Turnstile or Google Recaptcha.
+	
+	> ⚠️ Still in trial development, please report any issues or suggestions via [GitHub Issues](https://github.com/bellingcat/auto-archiver/issues).
+	
+    ### Features
+	- Extracts the HTML source code of the page.
+    - Takes full-page screenshots of web pages.
+	- Takes full-page PDF snapshots of web pages.
+	- Downloads images and videos from the page, excluding specified file extensions.
+
+    ### Notes
+	- Using a proxy affects Cloudflare Turnstile captcha handling, so it is recommended to use a proxy only if necessary.
+
+	### Dropins
+	This module uses sub-modules called Dropins for specific sites that allow it to handle anti-bot measures and custom Login flows. You don't need to include the dropins in your configuration, but you do need to add authentication credentials if you want to overcome login walls on those sites, see detailed instructions for each Dropin below.
+
+    """,
+}

src/auto_archiver/modules/antibot_extractor_enricher/antibot_extractor_enricher.py

@@ -0,0 +1,293 @@
+import base64
+import math
+import os
+import sys
+import traceback
+from urllib.parse import urljoin
+import glob
+import importlib.util
+
+from loguru import logger
+import selenium
+from seleniumbase import SB
+
+from auto_archiver.core import Extractor, Enricher, Metadata, Media
+from auto_archiver.modules.antibot_extractor_enricher.dropin import Dropin
+from auto_archiver.modules.antibot_extractor_enricher.dropins.default import DefaultDropin
+from auto_archiver.utils.misc import random_str
+from auto_archiver.utils.url import is_relevant_url
+
+
+class AntibotExtractorEnricher(Extractor, Enricher):
+    def setup(self) -> None:
+        self.agent = "cool"
+        if "linux" in sys.platform or "win32" in sys.platform:
+            self.agent = None  # Use the default UserAgent
+
+        # parse configuration options
+        if self.max_download_images == "inf":
+            self.max_download_images = math.inf
+        else:
+            self.max_download_images = int(self.max_download_images)
+
+        if self.max_download_videos == "inf":
+            self.max_download_videos = math.inf
+        else:
+            self.max_download_videos = int(self.max_download_videos)
+
+        self._prepare_user_data_dir()
+
+        self.dropins = self.load_dropins()
+
+    def load_dropins(self):
+        dropins = []
+
+        # TODO: add user-configurable drop-ins via config like generic_extractor
+        dropins_dir = os.path.join(os.path.dirname(__file__), "dropins")
+        for file_path in glob.glob(os.path.join(dropins_dir, "*.py")):
+            if os.path.basename(file_path).startswith("_"):
+                continue  # skip __init__.py or private modules
+            module_name = f"auto_archiver.modules.antibot_extractor_enricher.dropins.{os.path.splitext(os.path.basename(file_path))[0]}"
+            spec = importlib.util.spec_from_file_location(module_name, file_path)
+            module = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(module)
+            for attr in dir(module):
+                obj = getattr(module, attr)
+                if getattr(obj, "__module__", None) != module.__name__:
+                    continue  # Skip imported modules/classes/functions
+                if isinstance(obj, type) and issubclass(obj, Dropin):
+                    dropins.append(obj)
+        logger.debug(f"ANTIBOT loaded drop-in classes: {', '.join([d.__name__ for d in dropins])}")
+        return dropins
+
+    def sanitize_url(self, url: str) -> str:
+        for dropin in self.dropins:
+            if dropin.suitable(url):
+                return dropin.sanitize_url(url)
+        return url
+
+    def download(self, item: Metadata) -> Metadata:
+        result = Metadata()
+        result.merge(item)
+        if self.enrich(result):
+            result.status = "antibot"
+            return result
+
+    def _prepare_user_data_dir(self):
+        if self.user_data_dir:
+            in_docker = os.environ.get("RUNNING_IN_DOCKER")
+            if in_docker:
+                self.user_data_dir = self.user_data_dir.rstrip(os.path.sep) + "_docker"
+            os.makedirs(self.user_data_dir, exist_ok=True)
+
+    def enrich(self, to_enrich: Metadata, custom_data_dir: bool = True) -> bool:
+        using_user_data_dir = self.user_data_dir if custom_data_dir else None
+        url = to_enrich.get_url()
+        url_sample = url[:75]
+
+        try:
+            with SB(uc=True, agent=self.agent, headed=None, user_data_dir=using_user_data_dir, proxy=self.proxy) as sb:
+                logger.info(f"ANTIBOT selenium browser is up with agent {self.agent}, opening {url_sample}...")
+                sb.uc_open_with_reconnect(url, 4)
+
+                logger.debug(f"ANTIBOT handling CAPTCHAs for {url_sample}...")
+                sb.uc_gui_handle_cf()
+                sb.uc_gui_click_rc()  # NB: using handle instead of click breaks some sites like reddit, for now we separate here but can have dropins deciding this in the future
+
+                dropin = self._get_suitable_dropin(url, sb)
+                dropin.open_page(url)
+
+                if self.detect_auth_wall and self._hit_auth_wall(sb):
+                    logger.warning(f"ANTIBOT SKIP since auth wall or CAPTCHA was detected for {url_sample}")
+                    return False
+
+                sb.wait_for_ready_state_complete()
+                sb.sleep(1)  # margin for the page to load completely
+
+                to_enrich.set_title(sb.get_title())
+                self._enrich_html_source_code(sb, to_enrich)
+
+                self._enrich_full_page_screenshot(sb, to_enrich)
+                if self.save_to_pdf:
+                    self._enrich_full_page_pdf(sb, to_enrich)
+
+                downloaded_images, downloaded_videos = dropin.add_extra_media(to_enrich)
+
+                self._enrich_download_media(
+                    sb,
+                    to_enrich,
+                    js_css_selector=dropin.js_for_image_css_selectors(),
+                    max_media=self.max_download_images - downloaded_images,
+                )
+                self._enrich_download_media(
+                    sb,
+                    to_enrich,
+                    js_css_selector=dropin.js_for_video_css_selectors(),
+                    max_media=self.max_download_videos - downloaded_videos,
+                )
+                logger.info(f"ANTIBOT completed for {url_sample}")
+
+            return to_enrich
+        except selenium.common.exceptions.SessionNotCreatedException as e:
+            if custom_data_dir:  # the retry logic only works once
+                logger.error(
+                    f"ANTIBOT session not created error: {e}. Please remove the user_data_dir {self.user_data_dir} and try again, will retry without user data dir though."
+                )
+                return self.enrich(to_enrich, custom_data_dir=False)
+            raise e  # re-raise
+        except Exception as e:
+            logger.error(f"ANTIBOT runtime error: {e}: {traceback.format_exc()}")
+            return False
+
+    def _get_suitable_dropin(self, url: str, sb: SB):
+        """
+        Returns a suitable drop-in for the given URL.
+        This method checks if the URL is suitable for any of the registered drop-ins.
+        """
+        for dropin in self.dropins:
+            if dropin.suitable(url):
+                logger.debug(f"ANTIBOT using drop-in {dropin.__name__} for {url}")
+                return dropin(sb, self)
+
+        return DefaultDropin(sb, self)
+
+    def _hit_auth_wall(self, sb: SB) -> bool:
+        """
+        Tries to detect if the currently loaded page is an auth/login wall.
+        Returns True if login is likely required.
+        """
+        # TODO: improve this detection logic, currently it is very basic and may not cover all cases
+
+        # Common URL patterns
+        current_url = sb.get_current_url().lower()
+        if any(kw in current_url for kw in ["login", "signin", "signup", "register", "captcha"]):
+            return True
+
+        # Common visible text markers
+        login_keywords = [
+            "sign up or log in",
+            "log in to continue",
+            "sign in to continue",
+            "login required",
+            "please log in",
+            "please sign up",
+            "please sign in",
+            "login to access",
+            "sign up to access",
+            "register to access",
+            "captcha verification",
+        ]
+        for word in login_keywords + [w.capitalize() for w in login_keywords]:
+            if sb.is_text_visible(word):
+                return True
+
+        # Common title markers
+        title = sb.get_title().lower()
+        if any(
+            kw in title
+            for kw in [
+                "just a moment...",
+                "tiktok - make your day",
+                "um momento...",
+                "log in",
+                "sign in",
+                "sign up",
+                "register",
+                "captcha",
+                "verification required",
+                "access denied",
+            ]
+        ):
+            return True
+
+        # Common form fields
+        elements = [
+            "input[type='password']",
+            "input[type='email']",
+            "input[type='username']",
+            "input[type='phone']",
+            "input[name='username']",
+            "input[name='email']",
+            "input[name='password']",
+            "input[name='login']",
+        ]
+        if any(sb.is_element_visible(el) for el in elements):
+            return True
+
+        return False
+
+    @logger.catch
+    def _enrich_html_source_code(self, sb: SB, to_enrich: Metadata):
+        """
+        Enriches the HTML source code of the Metadata object.
+        This method is called by the enrich method.
+        """
+        source = sb.get_page_source()
+
+        html_filename = os.path.join(self.tmp_dir, f"source{random_str(6)}.html")
+        with open(html_filename, "w", encoding="utf-8") as f:
+            f.write(source)
+
+        to_enrich.add_media(Media(filename=html_filename), id="html_source_code")
+
+    @logger.catch
+    def _enrich_full_page_screenshot(self, sb: SB, to_enrich: Metadata):
+        """
+        Enriches the full page screenshot of the Metadata object.
+        This method is called by the enrich method.
+        """
+        start_size = sb.get_window_size()
+        w, h = start_size["width"], start_size["height"]
+
+        x = max(sb.execute_script("return document.documentElement.scrollWidth"), w)
+        y = min(max(sb.execute_script("return document.documentElement.scrollHeight"), h), 25_000)
+        logger.debug(f"Setting window size to {x}x{y} for full page screenshot.")
+        sb.set_window_size(x, y)
+
+        screen_filename = os.path.join(self.tmp_dir, f"screenshot{random_str(6)}.png")
+        sb.save_screenshot(screen_filename)
+
+        to_enrich.add_media(Media(filename=screen_filename), id="screenshot")
+
+    @logger.catch
+    def _enrich_full_page_pdf(self, sb: SB, to_enrich: Metadata):
+        """
+        Enriches the full page PDF of the Metadata object.
+        This method is called by the enrich method.
+        """
+        result = sb.driver.execute_cdp_cmd("Page.printToPDF", {"printBackground": True, "landscape": False})
+
+        pdf_data = base64.b64decode(result["data"])
+
+        pdf_filename = os.path.join(self.tmp_dir, f"pdf{random_str(6)}.pdf")
+        with open(pdf_filename, "wb") as f:
+            f.write(pdf_data)
+
+        to_enrich.add_media(Media(filename=pdf_filename), id="pdf")
+
+    @logger.catch
+    def _enrich_download_media(self, sb: SB, to_enrich: Metadata, js_css_selector: str, max_media: int):
+        """
+        Downloads media from the page and adds them to the Metadata object.
+        This method is called by the enrich method.
+        """
+        if max_media == 0:
+            return
+        url = to_enrich.get_url()
+        all_urls = set()
+
+        sources = sb.execute_script(js_css_selector)
+        # js_for_css_selectors
+        for src in sources:
+            if len(all_urls) >= max_media:
+                logger.debug(f"Reached max download limit of {max_media} images/videos.")
+                break
+            if not is_relevant_url(src):
+                continue
+            full_src = urljoin(url, src)
+            if full_src not in all_urls:
+                filename, full_src = self.download_from_url(full_src, try_best_quality=True)
+                if not filename:
+                    continue
+                all_urls.add(full_src)
+                to_enrich.add_media(Media(filename=filename, properties={"url": full_src}))

src/auto_archiver/modules/antibot_extractor_enricher/dropin.py

@@ -0,0 +1,159 @@
+import os
+from typing import Mapping
+from loguru import logger
+from seleniumbase import SB
+import yt_dlp
+
+from auto_archiver.core import Extractor, Media, Metadata
+from auto_archiver.utils.misc import ydl_entry_to_filename
+
+
+class Dropin:
+    """
+    A class to handle drop-in functionality for the antibot extractor enricher module.
+    This class is designed to be a base class for drop-ins that can handle specific websites.
+    """
+
+    @staticmethod
+    def documentation() -> Mapping[str, str]:
+        """
+        Each Dropin should auto-document itself with this method.
+        Return dictionary can include:
+        - 'name': A string representing the name of the dropin.
+        - 'description': A string describing the functionality of the dropin.
+        - 'site': A string representing the site this dropin is for.
+        - 'authentication': A dictionary with authentication example for the site.
+
+        """
+        return {}
+
+    def __init__(self, sb: SB, extractor: Extractor):
+        """
+        Initialize the Dropin with the given SeleniumBase instance.
+
+        :param sb: An instance of the SeleniumBase class that this drop-in will use.
+        :param extractor: An instance of the Extractor class that this drop-in will use.
+        """
+        self.sb: SB = sb
+        self.extractor: Extractor = extractor
+
+    @staticmethod
+    def suitable(url: str) -> bool:
+        """
+        Check if the URL is suitable for processing with this dropin.
+        :param url: The URL to check.
+        :return: True if the URL is suitable for processing, False otherwise.
+        """
+        raise NotImplementedError("This method should be implemented in the subclass")
+
+    @staticmethod
+    def sanitize_url(url: str) -> str:
+        """
+        Used to clean URLs before processing them.
+        """
+        return url
+
+    @staticmethod
+    def images_selectors() -> str:
+        """
+        CSS selector to find images in the HTML page
+        """
+        return "img"
+
+    @staticmethod
+    def video_selectors() -> str:
+        """
+        CSS selector to find videos in the HTML page.
+        """
+        return "video, source"
+
+    def js_for_image_css_selectors(self) -> str:
+        """
+        A configurable JS script that receives a css selector from the dropin itself and returns an array of Image elements according to the selection.
+
+        You can overwrite this instead of `images_selector` for more control over scraped images.
+        """
+        return f"""
+            return Array.from(document.querySelectorAll("{self.images_selectors()}")).map(el => el.src || el.href).filter(Boolean);
+        """
+
+    def js_for_video_css_selectors(self) -> str:
+        """
+        A configurable JS script that receives a css selector from the dropin itself and returns an array of Video elements according to the selection.
+
+        You can overwrite this instead of `video_selector` for more control over scraped videos.
+        """
+        return f"""
+            return Array.from(document.querySelectorAll("{self.video_selectors()}")).map(el => el.src || el.href).filter(Boolean);
+        """
+
+    def open_page(self, url) -> bool:
+        """
+        Make sure the page is opened, even if it requires authentication, captcha solving, etc.
+        :param url: The URL to open.
+        :return: True if success, False otherwise.
+        """
+        raise NotImplementedError("This method should be implemented in the subclass")
+
+    def add_extra_media(self, to_enrich: Metadata) -> tuple[int, int]:
+        """
+        Extract image and/or video data from the currently open post with SeleniumBase. Media is added to the `to_enrich` Metadata object.
+        :return: A tuple (number of Images added, number of Videos added).
+        """
+        return 0, 0
+
+    def _get_username_password(self, site) -> tuple[str, str]:
+        """
+        Get the username and password for the site from the extractor's auth data.
+        :return: A tuple (username, password).
+        """
+        auth = self.extractor.auth_for_site(site)
+        username = auth.get("username", "")
+        password = auth.get("password", "")
+        if not username or not password:
+            raise ValueError(f"{site} authentication requires a username and password.")
+        return username, password
+
+    def _download_videos_with_ytdlp(self, video_urls: list[str], to_enrich: Metadata) -> int:
+        """
+        Download videos using yt-dlp.
+        :param video_urls: List of video URLs to download.
+        :return: The number of videos downloaded.
+        """
+        if type(self.extractor.max_download_videos) is int:
+            video_urls = video_urls[: self.extractor.max_download_videos]
+
+        if not video_urls:
+            return 0
+
+        ydl_options = [
+            "-o",
+            os.path.join(self.extractor.tmp_dir, "%(id)s.%(ext)s"),
+            "--quiet",
+            "--no-playlist",
+            "--no-write-subs",
+            "--no-write-auto-subs",
+            "--postprocessor-args",
+            "ffmpeg:-bitexact",
+            "--max-filesize",
+            "1000M",  # Limit to 1GB per video
+        ]
+        *_, validated_options = yt_dlp.parse_options(ydl_options)
+        downloaded = 0
+        with yt_dlp.YoutubeDL(validated_options) as ydl:
+            for url in video_urls:
+                try:
+                    logger.debug(f"Downloading video from URL: {url}")
+                    info = ydl.extract_info(url, download=True)
+                    filename = ydl_entry_to_filename(ydl, info)
+                    if not filename:  # Failed to download video.
+                        continue
+                    media = Media(filename)
+                    for x in ["duration", "original_url", "fulltitle", "description", "upload_date"]:
+                        if x in info:
+                            media.set(x, info[x])
+                    to_enrich.add_media(media)
+                    downloaded += 1
+                except Exception as e:
+                    logger.error(f"Error downloading {url}: {e}")
+        return downloaded

src/auto_archiver/modules/antibot_extractor_enricher/dropins/default.py

@@ -0,0 +1,14 @@
+from auto_archiver.modules.antibot_extractor_enricher.dropin import Dropin
+
+
+class DefaultDropin(Dropin):
+    """
+    A default fallback drop-in class for handling generic cases in the antibot extractor enricher module.
+    """
+
+    @staticmethod
+    def suitable(url: str) -> bool:
+        return False
+
+    def open_page(self, url) -> bool:
+        return True

src/auto_archiver/modules/antibot_extractor_enricher/dropins/linkedin.py

@@ -0,0 +1,74 @@
+from typing import Mapping
+from loguru import logger
+from auto_archiver.modules.antibot_extractor_enricher.dropin import Dropin
+
+
+class LinkedinDropin(Dropin):
+    """
+    A class to handle LinkedIn drop-in functionality for the antibot extractor enricher module.
+    """
+
+    @staticmethod
+    def documentation() -> Mapping[str, str]:
+        return {
+            "name": "Linkedin Dropin",
+            "description": "Handles LinkedIn pages/posts and requires authentication to access most content but will still be useful without it. The first time you login to a new IP, LinkedIn may require an email verification code, you can do a manual login first and then it won't ask for it again.",
+            "site": "linkedin.com",
+            "authentication": {
+                "linkedin.com": {
+                    "username": "email address or phone number",
+                    "password": "password",
+                }
+            },
+        }
+
+    notifications_css_selector = 'a[href*="linkedin.com/notifications"]'
+
+    @staticmethod
+    def suitable(url: str) -> bool:
+        return "linkedin.com" in url
+
+    def js_for_image_css_selectors(self) -> str:
+        get_all_css = "main img:not([src*='profile-displayphoto']):not([src*='profile-framedphoto'])"
+        get_first_css = (
+            "main img[src*='profile-framedphoto'], main img[src*='profile-displayphoto'], main img[src*='company-logo']"
+        )
+
+        return f"""
+            const all = Array.from(document.querySelectorAll("{get_all_css}")).map(el => el.src || el.href).filter(Boolean);
+            const profile = document.querySelector("{get_first_css}");
+            return all.concat(profile?.src || profile?.href || []).filter(Boolean);
+        """
+
+    @staticmethod
+    def video_selectors() -> str:
+        # usually videos are from blob: but running the generic extractor should handle that
+        return "main video"
+
+    def open_page(self, url) -> bool:
+        if not self.sb.is_element_present(self.notifications_css_selector):
+            self._login()
+            if url != self.sb.get_current_url():
+                self.sb.open(url)
+        return True
+
+    @logger.catch
+    def _login(self) -> bool:
+        if self.sb.is_text_visible("Sign in to view more content"):
+            self.sb.click_link_text("Sign in", timeout=2)
+            self.sb.wait_for_ready_state_complete()
+        else:
+            self.sb.open("https://www.linkedin.com/login")
+            self.sb.wait_for_ready_state_complete()
+
+        username, password = self._get_username_password("linkedin.com")
+        logger.debug("LinkedinDropin Logging in to Linkedin with username: {}", username)
+        self.sb.type("#username", username)
+        self.sb.type("#password", password)
+        self.sb.click_if_visible("#password-visibility-toggle", timeout=0.5)
+        self.sb.click("button[type='submit']")
+        self.sb.wait_for_ready_state_complete()
+        # TODO: on suspicious login, LinkedIn may require an email verification code
+
+        if not self.sb.is_element_present(self.notifications_css_selector):
+            self.sb.click_if_visible('button[aria-label="Dismiss"]', timeout=0.5)

src/auto_archiver/modules/antibot_extractor_enricher/dropins/reddit.py

@@ -0,0 +1,92 @@
+from contextlib import suppress
+from typing import Mapping
+from auto_archiver.core.metadata import Metadata
+from auto_archiver.modules.antibot_extractor_enricher.dropin import Dropin
+
+from loguru import logger
+
+
+class RedditDropin(Dropin):
+    """
+    A class to handle Reddit drop-in functionality for the antibot extractor enricher module.
+    """
+
+    def documentation() -> Mapping[str, str]:
+        return {
+            "name": "Reddit Dropin",
+            "description": "Handles Reddit posts and works without authentication until Reddit flags your IP, so authentication is advised.",
+            "site": "reddit.com",
+            "authentication": {
+                "reddit.com": {
+                    "username": "email address or username",
+                    "password": "password",
+                }
+            },
+        }
+
+    @staticmethod
+    def suitable(url: str) -> bool:
+        return "reddit.com" in url
+
+    @staticmethod
+    def images_selectors() -> str:
+        return "shreddit-post img"
+
+    @staticmethod
+    def video_selectors() -> str:
+        return "shreddit-post video, shreddit-post source"
+
+    def open_page(self, url) -> bool:
+        if self.sb.is_text_visible("You've been blocked by network security."):
+            self._login()
+            if url != self.sb.get_current_url():
+                self.sb.open(url)
+        return True
+
+    @logger.catch
+    def _login(self):
+        self.sb.click_link_text("Log in")
+        self.sb.wait_for_ready_state_complete()
+        self._close_cookies_banner()
+
+        username, password = self._get_username_password("reddit.com")
+        logger.debug("RedditDropin Logging in to Reddit with username: {}", username)
+
+        self.sb.type("#login-username", username)
+        self.sb.type("#login-password", password)
+
+        elem = self.sb.find_element("button.login")
+        self.sb.execute_script("arguments[0].scrollIntoView(true);", elem)
+        self.sb.slow_click("button.login")
+        self.sb.wait_for_ready_state_complete()
+
+        if "https://www.reddit.com/login/" in self.sb.get_current_url():
+            self.sb.sleep(5)
+            self.sb.wait_for_ready_state_complete()
+
+        if self.sb.is_text_visible("You've been blocked by network security."):
+            self.sb.click_link_text("Log in")
+            self.sb.wait_for_ready_state_complete()
+            if self.sb.is_text_visible("Welcome back"):
+                logger.debug("RedditDropin Login successful")
+                self.sb.click_if_visible("this link")
+
+    def _close_cookies_banner(self):
+        with suppress(Exception):  # selenium.common.exceptions.JavascriptException
+            self.sb.execute_script("""
+                document
+                    .querySelector("reddit-cookie-banner")
+                    .shadowRoot.querySelector("faceplate-dialog")
+                    .querySelector("#accept-all-cookies-button button")
+                    .click()
+            """)
+
+    @logger.catch
+    def add_extra_media(self, to_enrich: Metadata) -> tuple[int, int]:
+        filtered_urls = self.sb.execute_script(rf"""
+            return [...document.querySelectorAll("{self.video_selectors()}")]
+            .map(el => el.src || el.href)
+            .filter(url => url && /\.(m3u8|mpd|ism)$/.test(url));
+        """)
+        logger.debug("RedditDropin Found {} video URLs", len(filtered_urls))
+        return 0, self._download_videos_with_ytdlp(filtered_urls, to_enrich)

src/auto_archiver/modules/antibot_extractor_enricher/dropins/vk.py

@@ -0,0 +1,92 @@
+import re
+from typing import Mapping
+
+from auto_archiver.core.metadata import Metadata
+from auto_archiver.modules.antibot_extractor_enricher.dropin import Dropin
+
+from loguru import logger
+
+
+class VkDropin(Dropin):
+    """
+    A class to handle VK drop-in functionality for the antibot extractor enricher module.
+    """
+
+    WALL_PATTERN = re.compile(r"(wall.{0,1}\d+_\d+)")
+    VIDEO_PATTERN = re.compile(r"(video.{0,1}\d+_\d+(?:_\w+)?)")
+    CLIP_PATTERN = re.compile(r"(clip.{0,1}\d+_\d+)")
+    PHOTO_PATTERN = re.compile(r"(photo.{0,1}\d+_\d+)")
+
+    def documentation() -> Mapping[str, str]:
+        return {
+            "name": "VKontakte Dropin",
+            "description": "Handles VKontakte posts and works without authentication for some content.",
+            "site": "vk.com",
+            "authentication": {
+                "vk.com": {
+                    "username": "phone number with country code",
+                    "password": "password",
+                }
+            },
+        }
+
+    @staticmethod
+    def suitable(url: str) -> bool:
+        return "vk.com" in url
+
+    @staticmethod
+    def sanitize_url(url: str) -> str:
+        """
+        Transforms modal URLs like 'https://vk.com/page_name?w=wall-123456_7890' to 'https://vk.com/wall-123456_7890'
+        """
+        for pattern in [VkDropin.WALL_PATTERN, VkDropin.VIDEO_PATTERN, VkDropin.CLIP_PATTERN, VkDropin.PHOTO_PATTERN]:
+            match = pattern.search(url)
+            if match:
+                return f"https://vk.com/{match.group(1)}"
+        return url
+
+    def open_page(self, url) -> bool:
+        if self.sb.is_text_visible("Sign in to VK"):
+            if self._login():
+                self.sb.open(url)
+        return True
+
+    @logger.catch
+    def _login(self) -> bool:
+        # TODO: test method, because current tests work without a login
+        self.sb.open("https://vk.com")
+        self.sb.wait_for_ready_state_complete()
+        if "/feed" in self.sb.get_current_url():
+            logger.debug("Already logged in to VK.")
+            return True
+
+        # need to login
+        username, password = self._get_username_password("vk.com")
+        logger.debug("Logging in to VK with username: {}", username)
+
+        self.sb.click('[data-testid="enter-another-way"]', timeout=10)
+        self.sb.clear('input[name="login"][type="tel"]', by="css selector", timeout=10)
+        self.sb.type('input[name="login"][type="tel"]', username, by="css selector", timeout=10)
+        self.sb.click('button[type="submit"]')
+
+        # TODO: handle captcha if it appears
+        # if sb.is_element_visible("img.vkc__CaptchaPopup__image"):
+        #     captcha_url = sb.get_attribute("img.vkc__CaptchaPopup__image", "src")
+        #     print("CAPTCHA detected:", captcha_url)
+        #     image_url = sb.get_attribute("img[alt*='captcha']", "src")
+        #     solution = solve_captcha(image_url)
+        #     sb.type("input#captcha-text, input[name='captcha']", solution)
+        #     sb.click("button[type='submit']")
+
+        self.sb.type('input[name="password"]', password, timeout=15)
+        self.sb.click('button[type="submit"]')
+        self.sb.wait_for_ready_state_complete(timeout=10)
+        self.sb.wait_for_element("body", timeout=10)
+        # self.sb.sleep(2)
+        return "/feed" in self.sb.get_current_url()
+
+    @logger.catch
+    def add_extra_media(self, to_enrich: Metadata) -> tuple[int, int]:
+        video_urls = [v.get_attribute("href") for v in self.sb.find_elements('a[href*="/video-"]')]
+
+        return 0, self._download_videos_with_ytdlp(video_urls, to_enrich)

src/auto_archiver/modules/gdrive_storage/gdrive_storage.py

@@ -93,13 +93,18 @@ class GDriveStorage(Storage):
         # upload file to gd
         logger.debug(f"uploading {filename=} to folder id {upload_to}")
         file_metadata = {"name": [filename], "parents": [upload_to]}
-        media = MediaFileUpload(media.filename, resumable=True)
-        gd_file = (
-            self.service.files()
-            .create(supportsAllDrives=True, body=file_metadata, media_body=media, fields="id")
-            .execute()
-        )
-        logger.debug(f"uploadf: uploaded file {gd_file['id']} successfully in folder={upload_to}")
+        try:
+            media = MediaFileUpload(media.filename, resumable=True)
+            gd_file = (
+                self.service.files()
+                .create(supportsAllDrives=True, body=file_metadata, media_body=media, fields="id")
+                .execute()
+            )
+            logger.debug(f"uploadf: uploaded file {gd_file['id']} successfully in folder={upload_to}")
+        except FileNotFoundError as e:
+            logger.error(f"gd uploadf: file not found {media.filename=} - {e}")
+        except Exception as e:
+            logger.error(f"gd uploadf: error uploading {media.filename=} to {upload_to} - {e}")
 
     # must be implemented even if unused
     def uploadf(self, file: IO[bytes], key: str, **kwargs: dict) -> bool:

src/auto_archiver/modules/generic_extractor/__manifest__.py

@@ -4,9 +4,7 @@
     "author": "Bellingcat",
     "type": ["extractor"],
     "requires_setup": False,
-    "dependencies": {
-        "python": ["yt_dlp", "requests", "loguru", "slugify"],
-    },
+    "dependencies": {"python": ["yt_dlp", "requests", "loguru", "slugify"], "bin": ["ffmpeg"]},
     "description": """
 This is the generic extractor used by auto-archiver, which uses `yt-dlp` under the hood.
 
@@ -32,6 +30,8 @@ For a full list of video platforms supported by `yt-dlp`, see the
 custom dropins can be created to handle additional websites and passed to the archiver
 via the command line using the `--dropins` option (TODO!).
 
+You can see all currently implemented dropins in [the source code](https://github.com/bellingcat/auto-archiver/tree/main/src/auto_archiver/modules/generic_extractor).
+
 ### Auto-Updates
 
 The Generic Extractor will also automatically check for updates to `yt-dlp` (every 5 days by default).
@@ -62,7 +62,7 @@ If you are having issues with the extractor, you can review the version of `yt-d
         },
         "end_means_success": {
             "default": True,
-            "help": "if True, any archived content will mean a 'success', if False this archiver will not return a 'success' stage; this is useful for cases when the yt-dlp will archive a video but ignore other types of content like images or text only pages that the subsequent archivers can retrieve.",
+            "help": "if True, any archived content will mean a 'success', if False this extractor will not return a 'success' stage; this is useful for cases when the yt-dlp will archive a video but ignore other types of content like images or text only pages that the subsequent extractors can retrieve.",
             "type": "bool",
         },
         "allow_playlist": {
@@ -74,6 +74,16 @@ If you are having issues with the extractor, you can review the version of `yt-d
             "default": "inf",
             "help": "Use to limit the number of videos to download when a channel or long page is being extracted. 'inf' means no limit.",
         },
+        "bguils_po_token_method": {
+            "default": "auto",
+            "help": "Set up a Proof of origin token provider. This process has additional requirements. See [authentication](https://auto-archiver.readthedocs.io/en/latest/how_to/authentication_how_to.html) for more information.",
+            "choices": ["auto", "script", "disabled"],
+        },
+        "extractor_args": {
+            "default": {},
+            "help": "Additional arguments to pass to the yt-dlp extractor. See https://github.com/yt-dlp/yt-dlp/blob/master/README.md#extractor-arguments.",
+            "type": "json_loader",
+        },
         "ytdlp_update_interval": {
             "default": 5,
             "help": "How often to check for yt-dlp updates (days). If positive, will check and update yt-dlp every [num] days. Set it to -1 to disable, or 0 to always update on every run.",

src/auto_archiver/modules/generic_extractor/dropin.py

@@ -1,3 +1,4 @@
+from typing import Type
 from yt_dlp.extractor.common import InfoExtractor
 from auto_archiver.core.metadata import Metadata
 from auto_archiver.core.extractor import Extractor
@@ -24,6 +25,8 @@ class GenericDropin:
 
     """
 
+    extractor: Type[Extractor] = None
+
     def extract_post(self, url: str, ie_instance: InfoExtractor):
         """
         This method should return the post data from the url.
@@ -55,3 +58,19 @@ class GenericDropin:
         This method should download any additional media from the post.
         """
         return metadata
+
+    def suitable(self, url, info_extractor: InfoExtractor):
+        """
+        A method to allow dropins to override their InfoExtractor's 'suitable' method.
+        Dropins should override this method and return True if the url is suitable for the extractor
+        (based on being able to parse other URLs). See the `suitable_extractors` method in the
+        `GenericExtractor` class for how this is implemented.
+
+        The default behaviour of this method is to return the result of the InfoExtractor's 'suitable' method.
+
+        ### Example: An example of where this is useful is for the FacebookIE extractor in yt-dlp. By default,
+        it's 'suitable' method only returns True for video URLs. However, we can override this method in the
+        Facebook dropin to return True for all Facebook URLs (photo/post types). This way, the Facebook dropin
+        can be used for all Facebook URLs.
+        """
+        return info_extractor.suitable(url)

src/auto_archiver/modules/generic_extractor/facebook.py

@@ -1,17 +1,154 @@
+import re
 from .dropin import GenericDropin
+from auto_archiver.core.metadata import Metadata
+from yt_dlp.extractor.facebook import FacebookIE
+
+# TODO: Remove if / when  https://github.com/yt-dlp/yt-dlp/pull/12275 is merged
+from yt_dlp.utils import (
+    clean_html,
+    get_element_by_id,
+    traverse_obj,
+    get_first,
+    merge_dicts,
+    int_or_none,
+    parse_count,
+)
+
+
+def _extract_metadata(self, webpage, video_id):
+    post_data = [
+        self._parse_json(j, video_id, fatal=False)
+        for j in re.findall(r"data-sjs>({.*?ScheduledServerJS.*?})</script>", webpage)
+    ]
+    post = (
+        traverse_obj(
+            post_data,
+            (..., "require", ..., ..., ..., "__bbox", "require", ..., ..., ..., "__bbox", "result", "data"),
+            expected_type=dict,
+        )
+        or []
+    )
+    media = traverse_obj(
+        post,
+        (
+            ...,
+            "attachments",
+            ...,
+            lambda k, v: (k == "media" and str(v["id"]) == video_id and v["__typename"] == "Video"),
+        ),
+        expected_type=dict,
+    )
+    title = get_first(media, ("title", "text"))
+    description = get_first(media, ("creation_story", "comet_sections", "message", "story", "message", "text"))
+    page_title = title or self._html_search_regex(
+        (
+            r'<h2\s+[^>]*class="uiHeaderTitle"[^>]*>(?P<content>[^<]*)</h2>',
+            r'(?s)<span class="fbPhotosPhotoCaption".*?id="fbPhotoPageCaption"><span class="hasCaption">(?P<content>.*?)</span>',
+            self._meta_regex("og:title"),
+            self._meta_regex("twitter:title"),
+            r"<title>(?P<content>.+?)</title>",
+        ),
+        webpage,
+        "title",
+        default=None,
+        group="content",
+    )
+    description = description or self._html_search_meta(
+        ["description", "og:description", "twitter:description"], webpage, "description", default=None
+    )
+    uploader_data = (
+        get_first(media, ("owner", {dict}))
+        or get_first(
+            post, ("video", "creation_story", "attachments", ..., "media", lambda k, v: k == "owner" and v["name"])
+        )
+        or get_first(post, (..., "video", lambda k, v: k == "owner" and v["name"]))
+        or get_first(post, ("node", "actors", ..., {dict}))
+        or get_first(post, ("event", "event_creator", {dict}))
+        or get_first(post, ("video", "creation_story", "short_form_video_context", "video_owner", {dict}))
+        or {}
+    )
+    uploader = uploader_data.get("name") or (
+        clean_html(get_element_by_id("fbPhotoPageAuthorName", webpage))
+        or self._search_regex(
+            (r'ownerName\s*:\s*"([^"]+)"', *self._og_regexes("title")), webpage, "uploader", fatal=False
+        )
+    )
+    timestamp = int_or_none(self._search_regex(r'<abbr[^>]+data-utime=["\'](\d+)', webpage, "timestamp", default=None))
+    thumbnail = self._html_search_meta(["og:image", "twitter:image"], webpage, "thumbnail", default=None)
+    # some webpages contain unretrievable thumbnail urls
+    # like https://lookaside.fbsbx.com/lookaside/crawler/media/?media_id=10155168902769113&get_thumbnail=1
+    # in https://www.facebook.com/yaroslav.korpan/videos/1417995061575415/
+    if thumbnail and not re.search(r"\.(?:jpg|png)", thumbnail):
+        thumbnail = None
+    info_dict = {
+        "description": description,
+        "uploader": uploader,
+        "uploader_id": uploader_data.get("id"),
+        "timestamp": timestamp,
+        "thumbnail": thumbnail,
+        "view_count": parse_count(
+            self._search_regex(
+                (r'\bviewCount\s*:\s*["\']([\d,.]+)', r'video_view_count["\']\s*:\s*(\d+)'),
+                webpage,
+                "view count",
+                default=None,
+            )
+        ),
+        "concurrent_view_count": get_first(
+            post, (("video", (..., ..., "attachments", ..., "media")), "liveViewerCount", {int_or_none})
+        ),
+        **traverse_obj(
+            post,
+            (
+                lambda _, v: video_id in v["url"],
+                "feedback",
+                {
+                    "like_count": ("likers", "count", {int}),
+                    "comment_count": ("total_comment_count", {int}),
+                    "repost_count": ("share_count_reduced", {parse_count}),
+                },
+            ),
+            get_all=False,
+        ),
+    }
+
+    info_json_ld = self._search_json_ld(webpage, video_id, default={})
+    info_json_ld["title"] = (
+        re.sub(r"\s*\|\s*Facebook$", "", title or info_json_ld.get("title") or page_title or "")
+        or (description or "").replace("\n", " ")
+        or f"Facebook video #{video_id}"
+    )
+    return merge_dicts(info_json_ld, info_dict)
 
 
 class Facebook(GenericDropin):
-    def extract_post(self, url: str, ie_instance):
-        video_id = ie_instance._match_valid_url(url).group("id")
-        ie_instance._download_webpage(url.replace("://m.facebook.com/", "://www.facebook.com/"), video_id)
-        webpage = ie_instance._download_webpage(url, ie_instance._match_valid_url(url).group("id"))
+    def extract_post(self, url: str, ie_instance: FacebookIE):
+        post_id_regex = r"(?P<id>pfbid[A-Za-z0-9]+|\d+|t\.(\d+\/\d+))"
+        post_id = re.search(post_id_regex, url).group("id")
+        webpage = ie_instance._download_webpage(url.replace("://m.facebook.com/", "://www.facebook.com/"), post_id)
 
-        # TODO: fix once https://github.com/yt-dlp/yt-dlp/pull/12275 is merged
-        post_data = ie_instance._extract_metadata(webpage)
+        # TODO: For long posts, this _extract_metadata only seems to return the first 100 or so characters, followed by ...
+
+        # TODO: If/when https://github.com/yt-dlp/yt-dlp/pull/12275 is merged, uncomment next line and delete the one after
+        # post_data = ie_instance._extract_metadata(webpage, post_id)
+        post_data = _extract_metadata(ie_instance, webpage, post_id)
         return post_data
 
-    def create_metadata(self, post: dict, ie_instance, archiver, url):
-        metadata = archiver.create_metadata(url)
-        metadata.set_title(post.get("title")).set_content(post.get("description")).set_post_data(post)
-        return metadata
+    def create_metadata(self, post: dict, ie_instance: FacebookIE, archiver, url):
+        result = Metadata()
+        result.set_content(post.get("description", ""))
+        result.set_title(post.get("title", ""))
+        result.set("author", post.get("uploader", ""))
+        result.set_url(url)
+        return result
+
+    def suitable(self, url, info_extractor: FacebookIE):
+        regex = r"(?:https?://(?:[\w-]+\.)?(?:facebook\.com||facebookwkhpilnemxj7asaniu7vnjjbiltxjqhye3mhbshg7kx5tfyd\.onion)/)"
+        return re.match(regex, url)
+
+    def skip_ytdlp_download(self, url: str, is_instance: FacebookIE):
+        """
+        Skip using the ytdlp download method for Facebook *photo* posts, they have a URL with an id of t.XXXXX/XXXXX
+        """
+        if re.search(r"/t.\d+/\d+", url):
+            return True

src/auto_archiver/modules/generic_extractor/generic_extractor.py

@@ -1,18 +1,26 @@
+import shutil
+import sys
 import datetime
 import os
 import importlib
 import subprocess
+import zipfile
 
 from typing import Generator, Type
+from urllib.request import urlretrieve
 
 import yt_dlp
 from yt_dlp.extractor.common import InfoExtractor
+from yt_dlp.utils import MaxDownloadsReached
 import pysubs2
 
 from loguru import logger
 
 from auto_archiver.core.extractor import Extractor
 from auto_archiver.core import Metadata, Media
+from auto_archiver.utils import get_datetime_from_str
+from auto_archiver.utils.misc import ydl_entry_to_filename
+from .dropin import GenericDropin
 
 
 class SkipYtdlp(Exception):
@@ -23,51 +31,154 @@ class GenericExtractor(Extractor):
     _dropins = {}
 
     def setup(self):
-        # check for file .ytdlp-update in the secrets folder
+        self.check_for_extractor_updates()
+        self.setup_po_tokens()
+        # TODO: figure out why the following is not properly recognised by yt-dlp:
+        # if "generic" not in self.extractor_args:
+        #     self.extractor_args["generic"] = "impersonate"
+
+    def check_for_extractor_updates(self):
+        """Checks whether yt-dlp or its plugins need updating and triggers a restart if so."""
         if self.ytdlp_update_interval < 0:
             return
 
-        use_secrets = os.path.exists("secrets")
-        path = os.path.join("secrets" if use_secrets else "", ".ytdlp-update")
-        next_update_check = None
-        if os.path.exists(path):
-            with open(path, "r") as f:
-                next_update_check = datetime.datetime.fromisoformat(f.read())
+        update_file = os.path.join("secrets" if os.path.exists("secrets") else "", ".ytdlp-update")
+        next_check = None
+        if os.path.exists(update_file):
+            with open(update_file, "r") as f:
+                next_check = datetime.datetime.fromisoformat(f.read())
 
-        if not next_update_check or next_update_check < datetime.datetime.now():
-            self.update_ytdlp()
+        if next_check and next_check > datetime.datetime.now():
+            return
 
-            next_update_check = datetime.datetime.now() + datetime.timedelta(days=self.ytdlp_update_interval)
-            with open(path, "w") as f:
-                f.write(next_update_check.isoformat())
+        yt_dlp_updated = self.update_package("yt-dlp")
+        bgutil_updated = self.update_package("bgutil-ytdlp-pot-provider")
 
-    def update_ytdlp(self):
-        logger.info("Checking and updating yt-dlp...")
-        logger.info(
-            f"Tip: change the 'ytdlp_update_interval' setting to control how often yt-dlp is updated. Set to -1 to disable or 0 to enable on every run. Current setting: {self.ytdlp_update_interval}"
-        )
+        # Write the new timestamp
+        with open(update_file, "w") as f:
+            next_check = datetime.datetime.now() + datetime.timedelta(days=self.ytdlp_update_interval)
+            f.write(next_check.isoformat())
+
+        if yt_dlp_updated or bgutil_updated:
+            if os.environ.get("AUTO_ARCHIVER_ALLOW_RESTART", "1") != "1":
+                logger.warning("yt-dlp or plugin was updated — please restart auto-archiver manually")
+            else:
+                logger.warning("yt-dlp or plugin was updated — restarting auto-archiver")
+                logger.warning(" ======= RESTARTING ======= ")
+                os.execv(sys.executable, [sys.executable] + sys.argv)
+
+    def update_package(self, package_name: str) -> bool:
+        logger.info(f"Checking and updating {package_name}...")
         from importlib.metadata import version as get_version
 
-        old_version = get_version("yt-dlp")
+        old_version = get_version(package_name)
         try:
-            # try and update with pip (this works inside poetry environment and in a normal virtualenv)
-            result = subprocess.run(["pip", "install", "--upgrade", "yt-dlp"], check=True, capture_output=True)
+            result = subprocess.run(["pip", "install", "--upgrade", package_name], check=True, capture_output=True)
+            if f"Successfully installed {package_name}" in result.stdout.decode():
+                new_version = importlib.metadata.version(package_name)
+                logger.info(f"{package_name} updated from {old_version} to {new_version}")
+                return True
+            logger.info(f"{package_name} already up to date")
+        except Exception as e:
+            logger.error(f"Error updating {package_name}: {e}")
+        return False
 
-            if "Successfully installed yt-dlp" in result.stdout.decode():
-                new_version = importlib.metadata.version("yt-dlp")
-                logger.info(f"yt-dlp successfully (from {old_version} to {new_version})")
-                importlib.reload(yt_dlp)
+    def setup_po_tokens(self) -> None:
+        """Setup Proof of Origin Token method conditionally.
+        Uses provider: https://github.com/Brainicism/bgutil-ytdlp-pot-provider.
+        """
+        in_docker = os.environ.get("RUNNING_IN_DOCKER")
+        if self.bguils_po_token_method == "disabled":
+            # This allows disabling of the PO Token generation script in the Docker implementation.
+            logger.warning("Proof of Origin Token generation is disabled.")
+            return
+
+        if self.bguils_po_token_method == "auto" and not in_docker:
+            logger.info(
+                "Proof of Origin Token method not explicitly set. "
+                "If you're running an external HTTP server separately, you can safely ignore this message. "
+                "To reduce the likelihood of bot detection, enable one of the methods described in the documentation: "
+                "https://auto-archiver.readthedocs.io/en/settings_page/installation/authentication.html#proof-of-origin-tokens"
+            )
+            return
+
+        # Either running in Docker, or "script" method is set beyond this point
+        self.setup_token_generation_script()
+
+    def setup_token_generation_script(self) -> None:
+        """This function sets up the Proof of Origin Token generation script method for
+        bgutil-ytdlp-pot-provider if enabled or in Docker."""
+        missing_tools = [tool for tool in ("node", "yarn", "npx") if shutil.which(tool) is None]
+        if missing_tools:
+            logger.error(
+                f"Cannot set up PO Token script; missing required tools: {', '.join(missing_tools)}. "
+                "Install these tools or run bgutils via Docker. "
+                "See: https://github.com/Brainicism/bgutil-ytdlp-pot-provider"
+            )
+            return
+        try:
+            from importlib.metadata import version as get_version
+
+            plugin_version = get_version("bgutil-ytdlp-pot-provider")
+            base_dir = os.path.expanduser("~/bgutil-ytdlp-pot-provider")
+            server_dir = os.path.join(base_dir, "server")
+            version_file = os.path.join(server_dir, ".VERSION")
+            transpiled_script = os.path.join(server_dir, "build", "generate_once.js")
+
+            # Skip setup if version is correct and transpiled script exists
+            if os.path.isfile(transpiled_script) and os.path.isfile(version_file):
+                with open(version_file) as vf:
+                    if vf.read().strip() == plugin_version:
+                        logger.info("PO Token script already set up and up to date.")
             else:
-                logger.info("yt-dlp already up to date")
+                # Remove an outdated directory and pull a new version
+                if os.path.exists(base_dir):
+                    shutil.rmtree(base_dir)
+                os.makedirs(base_dir, exist_ok=True)
+
+                zip_url = (
+                    f"https://github.com/Brainicism/bgutil-ytdlp-pot-provider/archive/refs/tags/{plugin_version}.zip"
+                )
+                zip_path = os.path.join(base_dir, f"{plugin_version}.zip")
+                logger.info(f"Downloading bgutils release zip for version {plugin_version}...")
+                urlretrieve(zip_url, zip_path)
+                with zipfile.ZipFile(zip_path, "r") as z:
+                    z.extractall(base_dir)
+                os.remove(zip_path)
+
+                extracted_root = os.path.join(base_dir, f"bgutil-ytdlp-pot-provider-{plugin_version}")
+                shutil.move(os.path.join(extracted_root, "server"), server_dir)
+                shutil.rmtree(extracted_root)
+                logger.info("Installing dependencies and transpiling PoT Generator script...")
+                subprocess.run(["yarn", "install", "--frozen-lockfile"], cwd=server_dir, check=True)
+                subprocess.run(["npx", "tsc"], cwd=server_dir, check=True)
+
+                with open(version_file, "w") as vf:
+                    vf.write(plugin_version)
+
+            script_path = os.path.join(server_dir, "build", "generate_once.js")
+            if not os.path.exists(script_path):
+                logger.error("generate_once.js not found after transpilation.")
+                return
+
+            self.extractor_args.setdefault("youtubepot-bgutilscript", {})["script_path"] = script_path
+            logger.info(f"PO Token script configured at: {script_path}")
 
         except Exception as e:
-            logger.error(f"Error updating yt-dlp: {e}")
+            logger.error(f"Failed to set up PO Token script: {e}")
 
     def suitable_extractors(self, url: str) -> Generator[str, None, None]:
         """
         Returns a list of valid extractors for the given URL"""
         for info_extractor in yt_dlp.YoutubeDL()._ies.values():
-            if info_extractor.suitable(url) and info_extractor.working():
+            if not info_extractor.working():
+                continue
+
+            # check if there's a dropin and see if that declares whether it's suitable
+            dropin: GenericDropin = self.dropin_for_name(info_extractor.ie_key())
+            if dropin and dropin.suitable(url, info_extractor):
+                yield info_extractor
+            elif info_extractor.suitable(url):
                 yield info_extractor
 
     def suitable(self, url: str) -> bool:
@@ -188,12 +299,16 @@ class GenericExtractor(Extractor):
         result = self.download_additional_media(video_data, info_extractor, result)
 
         # keep both 'title' and 'fulltitle', but prefer 'title', falling back to 'fulltitle' if it doesn't exist
-        result.set_title(video_data.pop("title", video_data.pop("fulltitle", "")))
-        result.set_url(url)
-        if "description" in video_data:
-            result.set_content(video_data["description"])
+        if not result.get_title():
+            result.set_title(video_data.pop("title", video_data.pop("fulltitle", "")))
+
+        if not result.get("url"):
+            result.set_url(url)
+
+        if "description" in video_data and not result.get("content"):
+            result.set_content(video_data.get("description"))
         # extract comments if enabled
-        if self.comments:
+        if self.comments and video_data.get("comments", []) is not None:
             result.set(
                 "comments",
                 [
@@ -207,11 +322,14 @@ class GenericExtractor(Extractor):
             )
 
         # then add the common metadata
-        if timestamp := video_data.pop("timestamp", None):
+        timestamp = video_data.pop("timestamp", None)
+        if timestamp and not result.get("timestamp"):
             timestamp = datetime.datetime.fromtimestamp(timestamp, tz=datetime.timezone.utc).isoformat()
             result.set_timestamp(timestamp)
-        if upload_date := video_data.pop("upload_date", None):
-            upload_date = datetime.datetime.strptime(upload_date, "%Y%m%d").replace(tzinfo=datetime.timezone.utc)
+
+        upload_date = video_data.pop("upload_date", None)
+        if upload_date and not result.get("upload_date"):
+            upload_date = get_datetime_from_str(upload_date, "%Y%m%d").replace(tzinfo=datetime.timezone.utc)
             result.set("upload_date", upload_date)
 
         # then clean away any keys we don't want
@@ -240,7 +358,8 @@ class GenericExtractor(Extractor):
             return False
 
         post_data = dropin.extract_post(url, ie_instance)
-        return dropin.create_metadata(post_data, ie_instance, self, url)
+        result = dropin.create_metadata(post_data, ie_instance, self, url)
+        return self.add_metadata(post_data, info_extractor, url, result)
 
     def get_metadata_for_video(
         self, data: dict, info_extractor: Type[InfoExtractor], url: str, ydl: yt_dlp.YoutubeDL
@@ -248,22 +367,29 @@ class GenericExtractor(Extractor):
         # this time download
         ydl.params["getcomments"] = self.comments
         # TODO: for playlist or long lists of videos, how to download one at a time so they can be stored before the next one is downloaded?
-        data = ydl.extract_info(url, ie_key=info_extractor.ie_key(), download=True)
+        try:
+            data = ydl.extract_info(url, ie_key=info_extractor.ie_key(), download=True)
+        except MaxDownloadsReached:  # proceed as normal once MaxDownloadsReached is raised
+            pass
+
         if "entries" in data:
             entries = data.get("entries", [])
             if not len(entries):
-                logger.warning("YoutubeDLArchiver could not find any video")
+                logger.info("YoutubeDLArchiver could not find any video")
                 return False
         else:
             entries = [data]
-
         result = Metadata()
 
         for entry in entries:
             try:
-                filename = ydl.prepare_filename(entry)
-                if not os.path.exists(filename):
-                    filename = filename.split(".")[0] + ".mkv"
+                filename = ydl_entry_to_filename(ydl, entry)
+
+                if not filename:
+                    # file was not downloaded or could not be retrieved, example: sensitive videos on YT without using cookies.
+                    continue
+
+                logger.debug(f"Using filename {filename} for entry {entry.get('id', 'unknown')}")
 
                 new_media = Media(filename)
                 for x in ["duration", "original_url", "fulltitle", "description", "upload_date"]:
@@ -282,10 +408,13 @@ class GenericExtractor(Extractor):
                 result.add_media(new_media)
             except Exception as e:
                 logger.error(f"Error processing entry {entry}: {e}")
+        if not len(result.media):
+            logger.info(f"No media found for entry {entry}, skipping.")
+            return False
 
         return self.add_metadata(data, info_extractor, url, result)
 
-    def dropin_for_name(self, dropin_name: str, additional_paths=[], package=__package__) -> Type[InfoExtractor]:
+    def dropin_for_name(self, dropin_name: str, additional_paths=[], package=__package__) -> GenericDropin:
         dropin_name = dropin_name.lower()
 
         if dropin_name == "generic":
@@ -296,6 +425,7 @@ class GenericExtractor(Extractor):
 
         def _load_dropin(dropin):
             dropin_class = getattr(dropin, dropin_class_name)()
+            dropin.extractor = self
             return self._dropins.setdefault(dropin_name, dropin_class)
 
         try:
@@ -339,18 +469,26 @@ class GenericExtractor(Extractor):
 
         dropin_submodule = self.dropin_for_name(info_extractor.ie_key())
 
+        def _helper_for_successful_extract_info(data, info_extractor, url, ydl):
+            if data.get("is_live", False) and not self.livestreams:
+                logger.warning("Livestream detected, skipping due to 'livestreams' configuration setting")
+                return False
+            # it's a valid video, that the youtubdedl can download out of the box
+            return self.get_metadata_for_video(data, info_extractor, url, ydl)
+
         try:
-            if dropin_submodule and dropin_submodule.skip_ytdlp_download(info_extractor, url):
+            if dropin_submodule and dropin_submodule.skip_ytdlp_download(url, info_extractor):
                 logger.debug(f"Skipping using ytdlp to download files for {info_extractor.ie_key()}")
                 raise SkipYtdlp()
 
             # don't download since it can be a live stream
             data = ydl.extract_info(url, ie_key=info_extractor.ie_key(), download=False)
-            if data.get("is_live", False) and not self.livestreams:
-                logger.warning("Livestream detected, skipping due to 'livestreams' configuration setting")
-                return False
-            # it's a valid video, that the youtubdedl can download out of the box
-            result = self.get_metadata_for_video(data, info_extractor, url, ydl)
+
+            result = _helper_for_successful_extract_info(data, info_extractor, url, ydl)
+
+        except MaxDownloadsReached:
+            # yt-dlp raises an error when the max downloads limit is reached, and it shouldn't for our purposes, so we consider that a success
+            result = _helper_for_successful_extract_info(data, info_extractor, url, ydl)
 
         except Exception as e:
             if info_extractor.IE_NAME == "generic":
@@ -359,7 +497,7 @@ class GenericExtractor(Extractor):
 
             if not isinstance(e, SkipYtdlp):
                 logger.debug(
-                    f'Issue using "{info_extractor.IE_NAME}" extractor to download video (error: {repr(e)}), attempting to use extractor to get post data instead'
+                    f'Issue using "{info_extractor.IE_NAME}" extractor to download video (error: {repr(e)}), attempting to use dropin to get post data instead'
                 )
 
             try:
@@ -404,16 +542,22 @@ class GenericExtractor(Extractor):
             "--write-subs" if self.subtitles else "--no-write-subs",
             "--write-auto-subs" if self.subtitles else "--no-write-auto-subs",
             "--live-from-start" if self.live_from_start else "--no-live-from-start",
-            "--proxy",
-            self.proxy if self.proxy else "",
-            f"--max-downloads {self.max_downloads}" if self.max_downloads != "inf" else "",
-            f"--playlist-end {self.max_downloads}" if self.max_downloads != "inf" else "",
+            "--postprocessor-args",
+            "ffmpeg:-bitexact",  # ensure bitexact output to avoid mismatching hashes for same video
         ]
 
+        # proxy handling
+        if self.proxy:
+            ydl_options.extend(["--proxy", self.proxy])
+
+        # max_downloads handling
+        if self.max_downloads != "inf":
+            ydl_options.extend(["--max-downloads", str(self.max_downloads)])
+            ydl_options.extend(["--playlist-end", str(self.max_downloads)])
+
         # set up auth
         auth = self.auth_for_site(url, extract_cookies=False)
-
-        # order of importance: username/pasword -> api_key -> cookie -> cookies_from_browser -> cookies_file
+        # order of importance: username/password -> api_key -> cookie -> cookies_from_browser -> cookies_file
         if auth:
             if "username" in auth and "password" in auth:
                 logger.debug(f"Using provided auth username and password for {url}")
@@ -429,6 +573,16 @@ class GenericExtractor(Extractor):
                 logger.debug(f"Using cookies from file {auth['cookies_file']} for {url}")
                 ydl_options.extend(("--cookies", auth["cookies_file"]))
 
+        # Applying user-defined extractor_args
+        if self.extractor_args:
+            for key, args in self.extractor_args.items():
+                if isinstance(args, dict):
+                    arg_str = ";".join(f"{k}={v}" for k, v in args.items())
+                else:
+                    arg_str = str(args)
+                logger.debug(f"Setting extractor_args: {key}:{arg_str}")
+                ydl_options.extend(["--extractor-args", f"{key}:{arg_str}"])
+
         if self.ytdlp_args:
             logger.debug("Adding additional ytdlp arguments: {self.ytdlp_args}")
             ydl_options += self.ytdlp_args.split(" ")

src/auto_archiver/modules/generic_extractor/tiktok.py

@@ -1,5 +1,8 @@
 import requests
 from loguru import logger
+
+from yt_dlp.extractor.tiktok import TikTokIE, TikTokLiveIE, TikTokVMIE, TikTokUserIE
+
 from auto_archiver.core import Metadata, Media
 from datetime import datetime, timezone
 from .dropin import GenericDropin
@@ -7,12 +10,17 @@ from .dropin import GenericDropin
 
 class Tiktok(GenericDropin):
     """
-    TikTok droping for the Generic Extractor that uses an unofficial API if/when ytdlp fails.
+    TikTok dropin for the Generic Extractor that uses an unofficial API if/when ytdlp fails.
     It's useful for capturing content that requires a login, like sensitive content.
     """
 
     TIKWM_ENDPOINT = "https://www.tikwm.com/api/?url={url}"
 
+    def suitable(self, url, info_extractor) -> bool:
+        """This dropin (which uses Tikvm) is suitable for *all* Tiktok type URLs - videos, lives, VMs, and users.
+        Return the 'suitable' method from the TikTokIE class."""
+        return any(extractor().suitable(url) for extractor in (TikTokIE, TikTokLiveIE, TikTokVMIE, TikTokUserIE))
+
     def extract_post(self, url: str, ie_instance):
         logger.debug(f"Using Tikwm API to attempt to download tiktok video from {url=}")
 
@@ -38,6 +46,9 @@ class Tiktok(GenericDropin):
         api_data["video_url"] = video_url
         return api_data
 
+    def keys_to_clean(self, video_data: dict, info_extractor):
+        return ["video_url", "title", "create_time", "author", "cover", "origin_cover", "ai_dynamic_cover", "duration"]
+
     def create_metadata(self, post: dict, ie_instance, archiver, url):
         # prepare result, start by downloading video
         result = Metadata()
@@ -54,17 +65,17 @@ class Tiktok(GenericDropin):
             logger.error(f"failed to download video from {video_url}")
             return False
         video_media = Media(video_downloaded)
-        if duration := post.pop("duration", None):
+        if duration := post.get("duration", None):
             video_media.set("duration", duration)
         result.add_media(video_media)
 
         # add remaining metadata
-        result.set_title(post.pop("title", ""))
+        result.set_title(post.get("title", ""))
 
-        if created_at := post.pop("create_time", None):
+        if created_at := post.get("create_time", None):
             result.set_timestamp(datetime.fromtimestamp(created_at, tz=timezone.utc))
 
-        if author := post.pop("author", None):
+        if author := post.get("author", None):
             result.set("author", author)
 
         result.set("api_data", post)

src/auto_archiver/modules/generic_extractor/twitter.py

@@ -1,16 +1,13 @@
 import re
 import mimetypes
-import json
-from datetime import datetime
 
 from loguru import logger
 from slugify import slugify
 
 from auto_archiver.core.metadata import Metadata, Media
-from auto_archiver.utils import url as UrlUtil
+from auto_archiver.utils import url as UrlUtil, get_datetime_from_str
 from auto_archiver.core.extractor import Extractor
-
-from .dropin import GenericDropin, InfoExtractor
+from auto_archiver.modules.generic_extractor.dropin import GenericDropin, InfoExtractor
 
 
 class Twitter(GenericDropin):
@@ -33,19 +30,24 @@ class Twitter(GenericDropin):
         twid = ie_instance._match_valid_url(url).group("id")
         return ie_instance._extract_status(twid=twid)
 
+    def keys_to_clean(self, video_data, info_extractor):
+        return ["user", "created_at", "entities", "favorited", "translator_type"]
+
     def create_metadata(self, tweet: dict, ie_instance: InfoExtractor, archiver: Extractor, url: str) -> Metadata:
         result = Metadata()
         try:
             if not tweet.get("user") or not tweet.get("created_at"):
                 raise ValueError("Error retreiving post. Are you sure it exists?")
-            timestamp = datetime.strptime(tweet["created_at"], "%a %b %d %H:%M:%S %z %Y")
+            timestamp = get_datetime_from_str(tweet["created_at"], "%a %b %d %H:%M:%S %z %Y")
         except (ValueError, KeyError) as ex:
             logger.warning(f"Unable to parse tweet: {str(ex)}\nRetreived tweet data: {tweet}")
             return False
 
-        result.set_title(tweet.get("full_text", "")).set_content(json.dumps(tweet, ensure_ascii=False)).set_timestamp(
-            timestamp
-        )
+        full_text = tweet.pop("full_text", "")
+        author = tweet["user"].get("name", "")
+        result.set("author", author).set_url(url)
+
+        result.set_title(f"{author} - {full_text}").set_content(full_text).set_timestamp(timestamp)
         if not tweet.get("entities", {}).get("media"):
             logger.debug("No media found, archiving tweet text only")
             result.status = "twitter-ytdl"

src/auto_archiver/modules/gsheet_feeder_db/__manifest__.py

@@ -70,10 +70,14 @@
     - Skips redundant updates for empty or invalid data fields.
 
     ### Setup
-    - Requires a Google Service Account JSON file for authentication, which should be stored in `secrets/gsheets_service_account.json`.
-    To set up a service account, follow the instructions [here](https://gspread.readthedocs.io/en/latest/oauth2.html).
-    - Define the `sheet` or `sheet_id` configuration to specify the sheet to archive.
-    - Customize the column names in your Google sheet using the `columns` configuration.
-    - The Google Sheet can be used soley as a feeder or as a feeder and database, but note you can't currently feed into the database from an alternate feeder.
+    1. Requires a Google Service Account JSON file for authentication.
+    To set up a service account, follow the instructions in the [how to](https://auto-archiver.readthedocs.io/en/latest/how_to/gsheets_setup.html),
+    or use the script:
+    ```
+    /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/bellingcat/auto-archiver/refs/heads/main/scripts/generate_google_services.sh)"
+    ```
+    2. Create a Google sheet with the required column(s) and then define the `sheet` or `sheet_id` configuration to specify this sheet.
+    3. Customize the column names in your Google sheet using the `columns` configuration.
+    4. The Google Sheet can be used solely as a feeder or as a feeder and database, but note you can't currently feed into the database from an alternate feeder.
     """,
 }

src/auto_archiver/modules/gsheet_feeder_db/gsheet_feeder_db.py

@@ -10,12 +10,13 @@ The filtered rows are processed into `Metadata` objects.
 """
 
 import os
-from typing import Tuple, Union
+from typing import Tuple, Union, Iterator
 from urllib.parse import quote
 
 import gspread
 from loguru import logger
 from slugify import slugify
+from retrying import retry
 
 from auto_archiver.core import Feeder, Database, Media
 from auto_archiver.core import Metadata
@@ -33,10 +34,10 @@ class GsheetsFeederDB(Feeder, Database):
     def open_sheet(self):
         if self.sheet:
             return self.gsheets_client.open(self.sheet)
-        else:  # self.sheet_id
+        else:
             return self.gsheets_client.open_by_key(self.sheet_id)
 
-    def __iter__(self) -> Metadata:
+    def __iter__(self) -> Iterator[Metadata]:
         sh = self.open_sheet()
         for ii, worksheet in enumerate(sh.worksheets()):
             if not self.should_process_sheet(worksheet.title):
@@ -45,14 +46,14 @@ class GsheetsFeederDB(Feeder, Database):
             logger.info(f"Opening worksheet {ii=}: {worksheet.title=} header={self.header}")
             gw = GWorksheet(worksheet, header_row=self.header, columns=self.columns)
             if len(missing_cols := self.missing_required_columns(gw)):
-                logger.warning(
+                logger.debug(
                     f"SKIPPED worksheet '{worksheet.title}' due to missing required column(s) for {missing_cols}"
                 )
                 continue
 
             # process and yield metadata here:
             yield from self._process_rows(gw)
-            logger.success(f"Finished worksheet {worksheet.title}")
+            logger.info(f"Finished worksheet {worksheet.title}")
 
     def _process_rows(self, gw: GWorksheet):
         for row in range(1 + self.header, gw.count_rows() + 1):
@@ -88,10 +89,7 @@ class GsheetsFeederDB(Feeder, Database):
         if len(self.allow_worksheets) and sheet_name not in self.allow_worksheets:
             # ALLOW rules exist AND sheet name not explicitly allowed
             return False
-        if len(self.block_worksheets) and sheet_name in self.block_worksheets:
-            # BLOCK rules exist AND sheet name is blocked
-            return False
-        return True
+        return not (self.block_worksheets and sheet_name in self.block_worksheets)
 
     def missing_required_columns(self, gw: GWorksheet) -> list:
         missing = []
@@ -101,7 +99,7 @@ class GsheetsFeederDB(Feeder, Database):
         return missing
 
     def started(self, item: Metadata) -> None:
-        logger.warning(f"STARTED {item}")
+        logger.info(f"STARTED {item}")
         gw, row = self._retrieve_gsheet(item)
         gw.set_cell(row, "status", "Archive in progress")
 
@@ -161,9 +159,8 @@ class GsheetsFeederDB(Feeder, Database):
         if (screenshot := item.get_media_by_id("screenshot")) and hasattr(screenshot, "urls"):
             batch_if_valid("screenshot", "\n".join(screenshot.urls))
 
-        if thumbnail := item.get_first_image("thumbnail"):
-            if hasattr(thumbnail, "urls"):
-                batch_if_valid("thumbnail", f'=IMAGE("{thumbnail.urls[0]}")')
+        if (thumbnail := item.get_first_image("thumbnail")) and hasattr(thumbnail, "urls"):
+            batch_if_valid("thumbnail", f'=IMAGE("{thumbnail.urls[0]}")')
 
         if browsertrix := item.get_media_by_id("browsertrix"):
             batch_if_valid("wacz", "\n".join(browsertrix.urls))
@@ -177,7 +174,16 @@ class GsheetsFeederDB(Feeder, Database):
                 ),
             )
 
-        gw.batch_set_cell(cell_updates)
+        @retry(
+            wait_incrementing_start=1000,
+            wait_incrementing_increment=3000,
+            wait_incrementing_max=20_000,
+            stop_max_attempt_number=5,
+        )
+        def batch_set_cell_with_retry(gw, cell_updates: list):
+            gw.batch_set_cell(cell_updates)
+
+        batch_set_cell_with_retry(gw, cell_updates)
 
     def _safe_status_update(self, item: Metadata, new_status: str) -> None:
         try:

src/auto_archiver/modules/html_formatter/templates/html_template.html

@@ -12,6 +12,12 @@
             font-family: 'Roboto', sans-serif;
         }
 
+        h2 {
+            white-space: normal;
+            overflow-wrap: break-word;
+            word-break: break-word;
+        }
+
         table {
             table-layout: fixed;
             width: 90%;
@@ -97,13 +103,17 @@
             background-color: #f1f1f1;
         }
 
-        .pem-certificate, .text-preview {
+        .pem-certificate,
+        .text-preview {
             text-align: left;
             font-size: small;
         }
-        .text-preview{
+
+        .text-preview {
             padding-left: 10px;
             padding-right: 10px;
+            max-height: 300px;
+            overflow: auto;
             white-space: pre-wrap;
         }
     </style>

src/auto_archiver/modules/instagram_api_extractor/__manifest__.py

@@ -31,9 +31,11 @@
         },
     },
     "description": """
-Archives various types of Instagram content using the Instagrapi API.
+Archives Instagram content using a deployment of the [Instagrapi API](https://subzeroid.github.io/instagrapi/).
 
-Requires setting up an Instagrapi API deployment and providing an access token and API endpoint.
+Requires either getting a token from using a hosted [(paid) service](https://api.instagrapi.com/docs) and setting this in the configuration file.
+Alternatively you can run your own server. We have a basic script which you can use for this which can be ran locally or using Docker.
+For more information, read the [how to guide](https://auto-archiver.readthedocs.io/en/latest/how_to/run_instagrapi_server.html) on this.
 
 ### Features
 - Connects to an Instagrapi API deployment to fetch Instagram profiles, posts, stories, highlights, reels, and tagged content.

src/auto_archiver/modules/instagram_extractor/instagram_extractor.py

@@ -29,6 +29,9 @@ class InstagramExtractor(Extractor):
     # TODO: links to stories
 
     def setup(self) -> None:
+        logger.warning("Instagram Extractor is not actively maintained, and may not work as expected.")
+        logger.warning("Please consider using the Instagram Tbot Extractor or Instagram API Extractor instead.")
+
         self.insta = instaloader.Instaloader(
             download_geotags=True,
             download_comments=True,

src/auto_archiver/modules/instagram_tbot_extractor/instagram_tbot_extractor.py

@@ -58,7 +58,7 @@ class InstagramTbotExtractor(Extractor):
                 "If you do, disable at least one of the archivers for the first-time setup of the telethon session: {e}"
             )
         with self.client.start():
-            logger.success(f"SETUP {self.name} login works.")
+            logger.info(f"SETUP {self.name} login works.")
 
     def cleanup(self) -> None:
         logger.info(f"CLEANUP {self.name}.")
@@ -88,6 +88,9 @@ class InstagramTbotExtractor(Extractor):
 
             if message:
                 result.set_content(message).set_title(message[:128])
+            elif result.is_empty():
+                logger.debug(f"No media found for link {url=} for {self.name}: {message}")
+                return False
             return result.success("insta-via-bot")
 
     def _send_url_to_bot(self, url: str):
@@ -104,13 +107,13 @@ class InstagramTbotExtractor(Extractor):
         message = ""
         time.sleep(3)
         # media is added before text by the bot so it can be used as a stop-logic mechanism
-        while attempts < max(self.timeout - 3, 3) and (not message or not len(seen_media)):
+        while attempts < max(self.timeout - 3, 15) and (not message or not len(seen_media)):
             attempts += 1
             time.sleep(1)
             for post in self.client.iter_messages(chat, min_id=since_id):
                 since_id = max(since_id, post.id)
                 # Skip known filler message:
-                if post.message == "The bot receives information through https://hikerapi.com/p/hJqpppqi":
+                if "The bot receives information through https://hikerapi.com/" in post.message:
                     continue
                 if post.media and post.id not in seen_media:
                     filename_dest = os.path.join(tmp_dir, f"{chat.id}_{post.id}")

src/auto_archiver/modules/json_enricher/__init__.py

@@ -0,0 +1 @@
+from .json_enricher import JsonEnricher

src/auto_archiver/modules/json_enricher/__manifest__.py

@@ -0,0 +1,16 @@
+{
+    "name": "JSON Enricher",
+    "type": ["enricher"],
+    "requires_setup": True,
+    "dependencies": {
+        "python": ["loguru"],
+    },
+    "configs": {},
+    "description": """
+
+    Writes all archiving process metadata to a JSON file so it can be parsed by other tools. As this is an Enricher, it will not contain the final stored URLs. 
+	
+	WARNING: The resulting JSON may reveal sensitive information about the computer and settings in which the archiving process was run. 
+
+    """,
+}

src/auto_archiver/modules/json_enricher/json_enricher.py

@@ -0,0 +1,19 @@
+import json
+from loguru import logger
+import os
+
+from auto_archiver.core import Enricher
+from auto_archiver.core import Media, Metadata
+
+
+class JsonEnricher(Enricher):
+    def enrich(self, to_enrich: Metadata) -> None:
+        url = to_enrich.get_url()
+
+        logger.debug(f"JSON Enricher for {url=}")
+
+        item_path = os.path.join(self.tmp_dir, "metadata.json")
+        with open(item_path, mode="w", encoding="utf-8") as outf:
+            json.dump(to_enrich.to_dict(), outf, indent=4, default=str, ensure_ascii=False)
+
+        to_enrich.add_media(Media(filename=item_path), id="metadata_json")

src/auto_archiver/modules/local_storage/__manifest__.py

@@ -20,7 +20,7 @@
         "save_absolute": {
             "default": False,
             "type": "bool",
-            "help": "whether the path to the stored file is absolute or relative in the output result inc. formatters (WARN: leaks the file structure)",
+            "help": "whether the path to the stored file is absolute or relative in the output result inc. formatters (Warning: saving an absolute path will show your computer's file structure)",
         },
     },
     "description": """

src/auto_archiver/modules/opentimestamps_enricher/opentimestamps_enricher.py

@@ -20,7 +20,7 @@ class OpentimestampsEnricher(Enricher):
         # Get the media files to timestamp
         media_files = [m for m in to_enrich.media if m.filename and not m.get("opentimestamps")]
         if not media_files:
-            logger.warning(f"No files found to timestamp in {url=}")
+            logger.debug(f"No files found to timestamp in {url=}")
             return
 
         timestamp_files = []
@@ -119,7 +119,7 @@ class OpentimestampsEnricher(Enricher):
         if timestamp_files:
             to_enrich.set("opentimestamped", True)
             to_enrich.set("opentimestamps_count", len(timestamp_files))
-            logger.success(f"{len(timestamp_files)} OpenTimestamps proofs created for {url=}")
+            logger.info(f"{len(timestamp_files)} OpenTimestamps proofs created for {url=}")
         else:
             to_enrich.set("opentimestamped", False)
             logger.warning(f"No successful timestamps created for {url=}")

src/auto_archiver/modules/pdq_hash_enricher/__manifest__.py

@@ -15,7 +15,7 @@
     - Skips non-image media or files unsuitable for hashing (e.g., corrupted or unsupported formats).
 
     ### Notes
-    - Best used after enrichers like `thumbnail_enricher` or `screenshot_enricher` to ensure images are available.
+    - Best used after enrichers like `thumbnail_enricher` or `antibot_extractor_enricher` (takes screenshots) to ensure images are available.
     - Uses the `pdqhash` library to compute 256-bit perceptual hashes, which are stored as hexadecimal strings.
     """,
 }

src/auto_archiver/modules/pdq_hash_enricher/pdq_hash_enricher.py

@@ -6,7 +6,7 @@ objects and calculates perceptual hashes using the PDQ hashing algorithm.
 These hashes are designed specifically for images and can be used
 for detecting duplicate or near-duplicate visual content.
 
-This enricher is typically used after thumbnail or screenshot enrichers
+This enricher is typically used after thumbnail or screenshot (antibot) enrichers
 to ensure images are available for hashing.
 
 """

src/auto_archiver/modules/s3_storage/s3_storage.py

@@ -40,6 +40,8 @@ class S3Storage(Storage):
             try:
                 if media.mimetype:
                     extra_args["ContentType"] = media.mimetype
+                    if "text" in media.mimetype:
+                        extra_args["ContentType"] += "; charset=utf-8"
             except Exception as e:
                 logger.warning(f"Unable to get mimetype for {media.key=}, error: {e}")
         self.s3.upload_fileobj(file, Bucket=self.bucket, Key=media.key, ExtraArgs=extra_args)

src/auto_archiver/modules/screenshot_enricher/__init__.py

@@ -1 +0,0 @@
-from .screenshot_enricher import ScreenshotEnricher

src/auto_archiver/modules/screenshot_enricher/__manifest__.py

@@ -1,44 +0,0 @@
-{
-    "name": "Screenshot Enricher",
-    "type": ["enricher"],
-    "requires_setup": True,
-    "dependencies": {
-        "python": ["loguru", "selenium"],
-    },
-    "configs": {
-        "width": {"default": 1280, "type": "int", "help": "width of the screenshots"},
-        "height": {"default": 1024, "type": "int", "help": "height of the screenshots"},
-        "timeout": {"default": 60, "type": "int", "help": "timeout for taking the screenshot"},
-        "sleep_before_screenshot": {
-            "default": 4,
-            "type": "int",
-            "help": "seconds to wait for the pages to load before taking screenshot",
-        },
-        "http_proxy": {
-            "default": "",
-            "help": "http proxy to use for the webdriver, eg http://proxy-user:password@proxy-ip:port",
-        },
-        "save_to_pdf": {
-            "default": False,
-            "type": "bool",
-            "help": "save the page as pdf along with the screenshot. PDF saving options can be adjusted with the 'print_options' parameter",
-        },
-        "print_options": {
-            "default": {},
-            "help": "options to pass to the pdf printer, in JSON format. See https://www.selenium.dev/documentation/webdriver/interactions/print_page/ for more information",
-            "type": "json_loader",
-        },
-    },
-    "description": """
-    Captures screenshots and optionally saves web pages as PDFs using a WebDriver.
-
-    ### Features
-    - Takes screenshots of web pages, with configurable width, height, and timeout settings.
-    - Optionally saves pages as PDFs, with additional configuration for PDF printing options.
-    - Bypasses URLs detected as authentication walls.
-    - Integrates seamlessly with the metadata enrichment pipeline, adding screenshots and PDFs as media.
-
-    ### Notes
-    - Requires a WebDriver (e.g., ChromeDriver) installed and accessible via the system's PATH.
-    """,
-}

src/auto_archiver/modules/screenshot_enricher/screenshot_enricher.py

@@ -1,52 +0,0 @@
-from loguru import logger
-import time
-import os
-import base64
-
-from selenium.common.exceptions import TimeoutException
-
-
-from auto_archiver.core import Enricher
-from auto_archiver.utils import Webdriver, url as UrlUtil, random_str
-from auto_archiver.core import Media, Metadata
-
-
-class ScreenshotEnricher(Enricher):
-    def __init__(self, webdriver_factory=None):
-        super().__init__()
-        self.webdriver_factory = webdriver_factory or Webdriver
-
-    def enrich(self, to_enrich: Metadata) -> None:
-        url = to_enrich.get_url()
-
-        if UrlUtil.is_auth_wall(url):
-            logger.debug(f"[SKIP] SCREENSHOT since url is behind AUTH WALL: {url=}")
-            return
-
-        logger.debug(f"Enriching screenshot for {url=}")
-        auth = self.auth_for_site(url)
-        with self.webdriver_factory(
-            self.width,
-            self.height,
-            self.timeout,
-            facebook_accept_cookies="facebook.com" in url,
-            http_proxy=self.http_proxy,
-            print_options=self.print_options,
-            auth=auth,
-        ) as driver:
-            try:
-                driver.get(url)
-                time.sleep(int(self.sleep_before_screenshot))
-                screenshot_file = os.path.join(self.tmp_dir, f"screenshot_{random_str(8)}.png")
-                driver.save_screenshot(screenshot_file)
-                to_enrich.add_media(Media(filename=screenshot_file), id="screenshot")
-                if self.save_to_pdf:
-                    pdf_file = os.path.join(self.tmp_dir, f"pdf_{random_str(8)}.pdf")
-                    pdf = driver.print_page(driver.print_options)
-                    with open(pdf_file, "wb") as f:
-                        f.write(base64.b64decode(pdf))
-                    to_enrich.add_media(Media(filename=pdf_file), id="pdf")
-            except TimeoutException:
-                logger.info("TimeoutException loading page for screenshot")
-            except Exception as e:
-                logger.error(f"Got error while loading webdriver for screenshot enricher: {e}")

src/auto_archiver/modules/telethon_extractor/__manifest__.py

@@ -19,7 +19,7 @@
         },
         "session_file": {
             "default": "secrets/anon",
-            "help": "optional, records the telegram login session for future usage, '.session' will be appended to the provided value.",
+            "help": "Path of the file to save the telegram login session for future usage, '.session' will be appended to the provided path.",
         },
         "join_channels": {
             "default": True,

src/auto_archiver/modules/telethon_extractor/telethon_extractor.py

@@ -1,4 +1,10 @@
+import os
 import shutil
+import re
+import time
+from pathlib import Path
+from datetime import date
+
 from telethon.sync import TelegramClient
 from telethon.errors import ChannelInvalidError
 from telethon.tl.functions.messages import ImportChatInviteRequest
@@ -8,11 +14,9 @@ from telethon.errors.rpcerrorlist import (
     InviteRequestSentError,
     InviteHashExpiredError,
 )
-from loguru import logger
+
 from tqdm import tqdm
-import re
-import time
-import os
+from loguru import logger
 
 from auto_archiver.core import Extractor
 from auto_archiver.core import Metadata, Media
@@ -31,16 +35,28 @@ class TelethonExtractor(Extractor):
         """
         logger.info(f"SETUP {self.name} checking login...")
 
+        # in case the user already added '.session' to the session_file
+        base_session_name = self.session_file.removesuffix(".session")
+        base_session_filepath = f"{base_session_name}.session"
+
+        if self.session_file and not os.path.exists(base_session_filepath):
+            logger.warning(
+                f"SETUP - Session file {base_session_filepath} does not exist for {self.name}, creating an empty one."
+            )
+            Path(base_session_filepath).touch()
+
         # make a copy of the session that is used exclusively with this archiver instance
-        new_session_file = os.path.join("secrets/", f"telethon-{time.strftime('%Y-%m-%d')}{random_str(8)}.session")
-        shutil.copy(self.session_file + ".session", new_session_file)
-        self.session_file = new_session_file.replace(".session", "")
+        self.session_file = os.path.join(
+            os.path.dirname(base_session_filepath), f"telethon-{date.today().strftime('%Y-%m-%d')}{random_str(8)}"
+        )
+        logger.debug(f"Making a copy of the session file {base_session_filepath} to {self.session_file}.session")
+        shutil.copy(base_session_filepath, f"{self.session_file}.session")
 
         # initiate the client
         self.client = TelegramClient(self.session_file, self.api_id, self.api_hash)
 
         with self.client.start():
-            logger.success(f"SETUP {self.name} login works.")
+            logger.info(f"SETUP {self.name} login works.")
 
         if self.join_channels and len(self.channel_invites):
             logger.info(f"SETUP {self.name} joining channels...")
@@ -87,8 +103,8 @@ class TelethonExtractor(Extractor):
                     pbar.update()
 
     def cleanup(self) -> None:
-        logger.info(f"CLEANUP {self.name}.")
-        session_file_name = self.session_file + ".session"
+        logger.info(f"CLEANUP {self.name} - removing session file {self.session_file}.session")
+        session_file_name = f"{self.session_file}.session"
         if os.path.exists(session_file_name):
             os.remove(session_file_name)
 
@@ -174,7 +190,7 @@ class TelethonExtractor(Extractor):
         if getattr(original_post, "grouped_id", None) is None:
             return [original_post] if getattr(original_post, "media", False) else []
 
-        search_ids = [i for i in range(original_post.id - max_amp, original_post.id + max_amp + 1)]
+        search_ids = list(range(original_post.id - max_amp, original_post.id + max_amp + 1))
         posts = self.client.get_messages(chat, ids=search_ids)
         media = []
         for post in posts:

src/auto_archiver/modules/thumbnail_enricher/thumbnail_enricher.py

@@ -35,16 +35,18 @@ class ThumbnailEnricher(Enricher):
                 logger.debug(f"generating thumbnails for {m.filename}")
                 duration = m.get("duration")
 
-                if duration is None:
-                    try:
-                        probe = ffmpeg.probe(m.filename)
-                        duration = float(
-                            next(stream for stream in probe["streams"] if stream["codec_type"] == "video")["duration"]
-                        )
-                        to_enrich.media[m_id].set("duration", duration)
-                    except Exception as e:
-                        logger.error(f"error getting duration of video {m.filename}: {e}")
-                        return
+                try:
+                    probe = ffmpeg.probe(m.filename)
+                    duration = float(
+                        next(stream for stream in probe["streams"] if stream["codec_type"] == "video")["duration"]
+                    )
+                    to_enrich.media[m_id].set("duration", duration)
+                except Exception as e:
+                    logger.warning(f"failed to get duration with FFMPEG from {m.filename}: {e}")
+
+                if not duration or type(duration) not in [float, int] or duration <= 0:
+                    logger.warning(f"cannot generate thumbnails for {m.filename} without valid duration")
+                    continue
 
                 num_thumbs = int(min(max(1, (duration / 60) * self.thumbnails_per_minute), self.max_thumbnails))
                 timestamps = [duration / (num_thumbs + 1) * i for i in range(1, num_thumbs + 1)]
@@ -57,6 +59,9 @@ class ThumbnailEnricher(Enricher):
                     ).run()
 
                     try:
+                        if not os.path.exists(output_path):
+                            logger.info(f"thumbnail {index} for media {m.filename} was not created")
+                            continue
                         thumbnails_media.append(
                             Media(filename=output_path)
                             .set("id", f"thumbnail_{index}")

src/auto_archiver/modules/timestamping_enricher/__manifest__.py

@@ -3,30 +3,38 @@
     "type": ["enricher"],
     "requires_setup": True,
     "dependencies": {
-        "python": ["loguru", "slugify", "tsp_client", "asn1crypto", "certvalidator", "certifi"],
+        "python": ["loguru", "slugify", "cryptography", "rfc3161_client", "certifi"],
     },
     "configs": {
         "tsa_urls": {
             "default": [
-                # [Adobe Approved Trust List] and [Windows Cert Store]
-                "http://timestamp.digicert.com",
-                "http://timestamp.identrust.com",
-                # "https://timestamp.entrust.net/TSS/RFC3161sha2TS", # not valid for timestamping
-                # "https://timestamp.sectigo.com", # wait 15 seconds between each request.
-                # [Adobe: European Union Trusted Lists].
-                # "https://timestamp.sectigo.com/qualified", # wait 15 seconds between each request.
-                # [Windows Cert Store]
-                "http://timestamp.globalsign.com/tsa/r6advanced1",
-                # [Adobe: European Union Trusted Lists] and [Windows Cert Store]
-                # "http://ts.quovadisglobal.com/eu", # not valid for timestamping
-                # "http://tsa.belgium.be/connect", # self-signed certificate in certificate chain
-                # "https://timestamp.aped.gov.gr/qtss", # self-signed certificate in certificate chain
-                # "http://tsa.sep.bg", # self-signed certificate in certificate chain
-                # "http://tsa.izenpe.com", #unable to get local issuer certificate
-                # "http://kstamp.keynectis.com/KSign", # unable to get local issuer certificate
-                "http://tss.accv.es:8318/tsa",
-            ],
+                # See https://github.com/trailofbits/rfc3161-client/issues/46 for a list of valid TSAs
+                # Full list of TSAs: https://gist.github.com/Manouchehri/fd754e402d98430243455713efada710
+                    "http://timestamp.identrust.com",
+                    "http://timestamp.ssl.trustwave.com",
+                    "http://zeitstempel.dfn.de",
+                    "http://ts.ssl.com",
+                    # "http://tsa.izenpe.com", # self-signed
+                    "http://tsa.lex-persona.com/tsa",
+                    # "http://ca.signfiles.com/TSAServer.aspx", # self-signed
+                    # "http://tsa.sinpe.fi.cr/tsaHttp/", # self-signed
+                    # "http://tsa.cra.ge/signserver/tsa?workerName=qtsa", # self-signed
+                    "http://tss.cnbs.gob.hn/TSS/HttpTspServer",
+                    "http://dss.nowina.lu/pki-factory/tsa/good-tsa",
+                    # "https://freetsa.org/tsr", # self-signed
+                ],
             "help": "List of RFC3161 Time Stamp Authorities to use, separate with commas if passed via the command line.",
+        },
+        "cert_authorities": {
+            "default": None,
+            "help": "Path to a file containing trusted Certificate Authorities (CAs) in PEM format. If empty, the default system authorities are used.",
+            "type": "str",
+        },
+        "allow_selfsigned": {
+            "default": False,
+            "help": "Whether or not to allow and save self-signed Timestamping certificates. This allows for a greater range of timestamping servers to be used, \
+but they are not trusted authorities",
+            "type": "bool"
         }
     },
     "description": """

src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py

@@ -1,15 +1,23 @@
 import os
-from loguru import logger
-from tsp_client import TSPSigner, SigningSettings, TSPVerifier
-from tsp_client.algorithms import DigestAlgorithm
+
 from importlib.metadata import version
-from asn1crypto.cms import ContentInfo
-from certvalidator import CertificateValidator, ValidationContext
-from asn1crypto import pem
+import hashlib
+
+from slugify import slugify
+import requests
+from loguru import logger
+
+from rfc3161_client import (decode_timestamp_response,TimestampRequestBuilder,TimeStampResponse, VerifierBuilder)
+from rfc3161_client import VerificationError as Rfc3161VerificationError
+from rfc3161_client.base import HashAlgorithm
+from rfc3161_client.tsp import SignedData
+from cryptography import x509
+from cryptography.hazmat.primitives import serialization
 import certifi
 
 from auto_archiver.core import Enricher
 from auto_archiver.core import Metadata, Media
+from auto_archiver.version import __version__
 
 
 class TimestampingEnricher(Enricher):
@@ -21,6 +29,25 @@ class TimestampingEnricher(Enricher):
     See https://gist.github.com/Manouchehri/fd754e402d98430243455713efada710 for list of timestamp authorities.
     """
 
+    session = None
+
+    def setup(self):
+        self.session = requests.Session()
+        self.session.headers.update(
+            {
+                "Content-Type": "application/timestamp-query",
+                "User-Agent": f"Auto-Archiver {__version__}",
+                "Accept": "application/timestamp-reply",
+            }
+        )
+
+    def cleaup(self) -> None:
+        """
+        Terminates the underlying network session.
+        """
+        if self.session:
+            self.session.close()
+
     def enrich(self, to_enrich: Metadata) -> None:
         url = to_enrich.get_url()
         logger.debug(f"RFC3161 timestamping existing files for {url=}")
@@ -31,11 +58,11 @@ class TimestampingEnricher(Enricher):
         ]
 
         if not len(hashes):
-            logger.warning(f"No hashes found in {url=}")
+            logger.debug(f"No hashes found in {url=}")
             return
 
-        tmp_dir = self.tmp_dir
-        hashes_fn = os.path.join(tmp_dir, "hashes.txt")
+        
+        hashes_fn = os.path.join(self.tmp_dir, "hashes.txt")
 
         data_to_sign = "\n".join(hashes)
         with open(hashes_fn, "w") as f:
@@ -43,62 +70,160 @@ class TimestampingEnricher(Enricher):
         hashes_media = Media(filename=hashes_fn)
 
         timestamp_tokens = []
-        from slugify import slugify
-
         for tsa_url in self.tsa_urls:
             try:
-                signing_settings = SigningSettings(tsp_server=tsa_url, digest_algorithm=DigestAlgorithm.SHA256)
-                signer = TSPSigner()
-                message = bytes(data_to_sign, encoding="utf8")
-                # send TSQ and get TSR from the TSA server
-                signed = signer.sign(message=message, signing_settings=signing_settings)
-                # fail if there's any issue with the certificates, uses certifi list of trusted CAs
-                TSPVerifier(certifi.where()).verify(signed, message=message)
-                # download and verify timestamping certificate
-                cert_chain = self.download_and_verify_certificate(signed)
-                # continue with saving the timestamp token
-                tst_fn = os.path.join(tmp_dir, f"timestamp_token_{slugify(tsa_url)}")
-                with open(tst_fn, "wb") as f:
-                    f.write(signed)
-                timestamp_tokens.append(Media(filename=tst_fn).set("tsa", tsa_url).set("cert_chain", cert_chain))
+                message = bytes(data_to_sign, encoding='utf8')
+
+                logger.debug(f"Timestamping {url=} with {tsa_url=}")
+                signed: TimeStampResponse = self.sign_data(tsa_url, message)
+                
+                # fail if there's any issue with the certificates, uses certifi list of trusted CAs or the user-defined `cert_authorities`
+                root_cert = self.verify_signed(signed, message)
+
+                if not root_cert:
+                    if self.allow_selfsigned:
+                        logger.warning(f"Allowing self-signed certificat from TSA {tsa_url=}")
+                    else:
+                        raise ValueError(f"No valid root certificate found for {tsa_url=}. Are you sure it's a trusted TSA? Or define an alternative trusted root with `cert_authorities`. (tried: {self.cert_authorities or certifi.where()})")
+
+                # save the timestamping certificate
+                cert_chain = self.save_certificate(signed, root_cert)
+
+                timestamp_token_path = self.save_timestamp_token(signed.time_stamp_token(), tsa_url)
+                timestamp_tokens.append(Media(filename=timestamp_token_path).set("tsa", tsa_url).set("cert_chain", cert_chain))
             except Exception as e:
                 logger.warning(f"Error while timestamping {url=} with {tsa_url=}: {e}")
 
         if len(timestamp_tokens):
             hashes_media.set("timestamp_authority_files", timestamp_tokens)
             hashes_media.set("certifi v", version("certifi"))
-            hashes_media.set("tsp_client v", version("tsp_client"))
-            hashes_media.set("certvalidator v", version("certvalidator"))
+            hashes_media.set("rfc3161-client v", version("rfc3161_client"))
+            hashes_media.set("cryptography v", version("cryptography"))
             to_enrich.add_media(hashes_media, id="timestamped_hashes")
             to_enrich.set("timestamped", True)
-            logger.success(f"{len(timestamp_tokens)} timestamp tokens created for {url=}")
+            logger.info(f"{len(timestamp_tokens)} timestamp tokens created for {url=}")
         else:
             logger.warning(f"No successful timestamps for {url=}")
 
-    def download_and_verify_certificate(self, signed: bytes) -> list[Media]:
+    def save_timestamp_token(self, timestamp_token: bytes, tsa_url: str) -> str:
+        """
+        Takes a timestamp token, and saves it to a file with the TSA URL as part of the filename.
+        """
+        tst_path = os.path.join(self.tmp_dir, f"timestamp_token_{slugify(tsa_url)}")
+        with open(tst_path, "wb") as f:
+            f.write(timestamp_token)
+        return tst_path
+
+    def verify_signed(self, timestamp_response: TimeStampResponse, message: bytes) ->  x509.Certificate:
+        """
+        Verify a Signed Timestamp Response is trusted by a known Certificate Authority.
+
+        Args:
+            timestamp_response (TimeStampResponse): The signed timestamp response.
+            message (bytes): The message that was timestamped.
+
+        Returns:
+            x509.Certificate: A valid root certificate that was used to sign the timestamp response, or None
+
+        Raises:
+            ValueError: If no valid root certificate was found in the trusted root store.
+        """
+
+        trusted_root_path = self.cert_authorities or certifi.where()
+        cert_authorities = []
+
+        with open(trusted_root_path, 'rb') as f:
+            cert_authorities = x509.load_pem_x509_certificates(f.read())
+
+        if not cert_authorities:
+            raise ValueError(f"No trusted roots found in {trusted_root_path}.")
+        
+        timestamp_certs = self.tst_certs(timestamp_response)
+        intermediate_certs = timestamp_certs[1:-1]
+
+        message_hash = None
+        hash_algorithm = timestamp_response.tst_info.message_imprint.hash_algorithm
+        if hash_algorithm == x509.ObjectIdentifier(value="2.16.840.1.101.3.4.2.3"):
+            message_hash = hashlib.sha512(message).digest()
+        elif hash_algorithm == x509.ObjectIdentifier(value="2.16.840.1.101.3.4.2.1"):
+            message_hash = hashlib.sha256(message).digest()
+        else:
+            raise ValueError(f"Unsupported hash algorithm: {hash_algorithm}")
+        
+        for certificate in cert_authorities:
+            builder = VerifierBuilder()
+            builder.add_root_certificate(certificate)
+
+            for intermediate_cert in intermediate_certs:
+                builder.add_intermediate_certificate(intermediate_cert)
+
+            verifier = builder.build()
+
+            
+            try:
+                verifier.verify(timestamp_response, message_hash)
+                return certificate
+            except Rfc3161VerificationError:
+                continue
+
+        return None
+
+    def sign_data(self, tsa_url: str, bytes_data: bytes) -> TimeStampResponse:
+        # see https://github.com/sigstore/sigstore-python/blob/99948d5b80525a5a104e904ffea58169dc6e0629/sigstore/_internal/timestamp.py#L84-L121
+
+        timestamp_request = (
+                TimestampRequestBuilder().data(bytes_data).nonce(nonce=True).build()
+            )
+        try:
+            response = self.session.post(tsa_url, data=timestamp_request.as_bytes(), timeout=10)
+            response.raise_for_status()
+        except requests.RequestException as e:
+            logger.error(f"Error while sending request to {tsa_url=}: {e}")
+            raise
+
+        # Check that we can parse the response but do not *verify* it
+        try:
+            timestamp_response = decode_timestamp_response(response.content)
+        except ValueError as e:
+            logger.error(f"Invalid timestamp response from server {tsa_url}: {e}")
+            raise
+        return timestamp_response
+    
+    def tst_certs(self, tsp_response: TimeStampResponse):
+        signed_data: SignedData = tsp_response.signed_data
+        certs = [x509.load_der_x509_certificate(c) for c in signed_data.certificates]
+        # reorder the certs to be in the correct order
+        ordered_certs = []
+        if len(certs) == 1:
+            return certs
+
+        while(len(ordered_certs) < len(certs)):
+            if len(ordered_certs) == 0:
+                for cert in certs:
+                    if not [c for c in certs if cert.subject == c.issuer]:
+                        ordered_certs.append(cert)
+                        break
+            else:
+                for cert in certs:
+                    if cert.subject == ordered_certs[-1].issuer:
+                        ordered_certs.append(cert)
+                        break
+        return ordered_certs
+
+    def save_certificate(self, tsp_response: TimeStampResponse, verified_root_cert: x509.Certificate) -> list[Media]:
         # returns the leaf certificate URL, fails if not set
-        tst = ContentInfo.load(signed)
 
-        trust_roots = []
-        with open(certifi.where(), "rb") as f:
-            for _, _, der_bytes in pem.unarmor(f.read(), multiple=True):
-                trust_roots.append(der_bytes)
-        context = ValidationContext(trust_roots=trust_roots)
+        certificates = self.tst_certs(tsp_response)
 
-        certificates = tst["content"]["certificates"]
-        first_cert = certificates[0].dump()
-        intermediate_certs = []
-        for i in range(1, len(certificates)):  # cannot use list comprehension [1:]
-            intermediate_certs.append(certificates[i].dump())
-
-        validator = CertificateValidator(first_cert, intermediate_certs=intermediate_certs, validation_context=context)
-        path = validator.validate_usage({"digital_signature"}, extended_key_usage={"time_stamping"})
+        if verified_root_cert:
+            # add the verified root certificate (if there is one - self signed certs will have None here)
+            certificates += [verified_root_cert]
 
         cert_chain = []
-        for cert in path:
-            cert_fn = os.path.join(self.tmp_dir, f"{str(cert.serial_number)[:20]}.crt")
+        for i, cert in enumerate(certificates):
+            cert_fn = os.path.join(self.tmp_dir, f"{i+1} – {str(cert.serial_number)[:20]}.crt")
             with open(cert_fn, "wb") as f:
-                f.write(cert.dump())
-            cert_chain.append(Media(filename=cert_fn).set("subject", cert.subject.native["common_name"]))
+                f.write(cert.public_bytes(encoding=serialization.Encoding.PEM))
+            cert_chain.append(Media(filename=cert_fn).set("subject", cert.subject.get_attributes_for_oid(x509.NameOID.COMMON_NAME)[0].value))
 
         return cert_chain

src/auto_archiver/modules/twitter_api_extractor/twitter_api_extractor.py

@@ -1,8 +1,8 @@
+from datetime import timezone
 import json
 import re
 import mimetypes
 import requests
-from datetime import datetime
 
 from loguru import logger
 from pytwitter import Api
@@ -10,6 +10,7 @@ from slugify import slugify
 
 from auto_archiver.core import Extractor
 from auto_archiver.core import Metadata, Media
+from auto_archiver.utils import get_datetime_from_str
 
 
 class TwitterApiExtractor(Extractor):
@@ -91,7 +92,9 @@ class TwitterApiExtractor(Extractor):
 
         result = Metadata()
         result.set_title(tweet.data.text)
-        result.set_timestamp(datetime.strptime(tweet.data.created_at, "%Y-%m-%dT%H:%M:%S.%fZ"))
+        result.set_timestamp(
+            get_datetime_from_str(tweet.data.created_at, "%Y-%m-%dT%H:%M:%S.%fZ").replace(tzinfo=timezone.utc)
+        )
 
         urls = []
         if tweet.includes:

src/auto_archiver/modules/vk_extractor/__init__.py

@@ -1 +0,0 @@
-from .vk_extractor import VkExtractor

src/auto_archiver/modules/vk_extractor/__manifest__.py

@@ -1,37 +0,0 @@
-{
-    "name": "VKontakte Extractor",
-    "type": ["extractor"],
-    "requires_setup": True,
-    "depends": ["core", "utils"],
-    "dependencies": {
-        "python": ["loguru", "vk_url_scraper"],
-    },
-    "configs": {
-        "username": {"required": True, "help": "valid VKontakte username"},
-        "password": {"required": True, "help": "valid VKontakte password"},
-        "session_file": {
-            "default": "secrets/vk_config.v2.json",
-            "help": "valid VKontakte password",
-        },
-    },
-    "description": """
-The `VkExtractor` fetches posts, text, and images from VK (VKontakte) social media pages. 
-This archiver is specialized for `/wall` posts and uses the `VkScraper` library to extract 
-and download content. Note that VK videos are handled separately by the `YTDownloader`.
-
-### Features
-- Extracts text, timestamps, and metadata from VK `/wall` posts.
-- Downloads associated images and attaches them to the resulting `Metadata` object.
-- Processes multiple segments of VK URLs that contain mixed content (e.g., wall, photo).
-- Outputs structured metadata and media using `Metadata` and `Media` objects.
-
-### Setup
-To use the `VkArchiver`, you must provide valid VKontakte login credentials and session information:
-- **Username**: A valid VKontakte account username.
-- **Password**: The corresponding password for the VKontakte account.
-- **Session File**: Optional. Path to a session configuration file (`.json`) for persistent VK login.
-
-Credentials can be set in the configuration file or directly via environment variables. Ensure you 
-have access to the VKontakte API by creating an account at [VKontakte](https://vk.com/).
-""",
-}

src/auto_archiver/modules/vk_extractor/vk_extractor.py

@@ -1,43 +0,0 @@
-from loguru import logger
-from vk_url_scraper import VkScraper
-
-from auto_archiver.utils.misc import dump_payload
-from auto_archiver.core import Extractor
-from auto_archiver.core import Metadata, Media
-
-
-class VkExtractor(Extractor):
-    """ "
-    VK videos are handled by YTDownloader, this archiver gets posts text and images.
-    Currently only works for /wall posts
-    """
-
-    def setup(self) -> None:
-        self.vks = VkScraper(self.username, self.password, session_file=self.session_file)
-
-    def download(self, item: Metadata) -> Metadata:
-        url = item.get_url()
-
-        if "vk.com" not in item.netloc:
-            return False
-
-        # some urls can contain multiple wall/photo/... parts and all will be fetched
-        vk_scrapes = self.vks.scrape(url)
-        if not len(vk_scrapes):
-            return False
-        logger.debug(f"VK: got {len(vk_scrapes)} scraped instances")
-
-        result = Metadata()
-        for scrape in vk_scrapes:
-            if not result.get_title():
-                result.set_title(scrape["text"])
-            if not result.get_timestamp():
-                result.set_timestamp(scrape["datetime"])
-
-        result.set_content(dump_payload(vk_scrapes))
-
-        filenames = self.vks.download_media(vk_scrapes, self.tmp_dir)
-        for filename in filenames:
-            result.add_media(Media(filename))
-
-        return result.success("vk")

src/auto_archiver/modules/wacz_extractor_enricher/__manifest__.py

@@ -11,7 +11,7 @@
     "configs": {
         "profile": {
             "default": None,
-            "help": "browsertrix-profile (for profile generation see https://github.com/webrecorder/browsertrix-crawler#creating-and-using-browser-profiles).",
+            "help": "browsertrix-profile (for profile generation see https://crawler.docs.browsertrix.com/user-guide/browser-profiles/).",
         },
         "docker_commands": {"default": None, "help": "if a custom docker invocation is needed"},
         "timeout": {"default": 120, "help": "timeout for WACZ generation in seconds", "type": "int"},
@@ -40,14 +40,31 @@
     Creates .WACZ archives of web pages using the `browsertrix-crawler` tool, with options for media extraction and screenshot saving.
     [Browsertrix-crawler](https://crawler.docs.browsertrix.com/user-guide/) is a headless browser-based crawler that archives web pages in WACZ format.
 
-    ### Features
+    ## Features
     - Archives web pages into .WACZ format using Docker or direct invocation of `browsertrix-crawler`.
     - Supports custom profiles for archiving private or dynamic content.
     - Extracts media (images, videos, audio) and screenshots from the archive, optionally adding them to the enrichment pipeline.
     - Generates metadata from the archived page's content and structure (e.g., titles, text).
 
-    ### Notes
-    - Requires Docker for running `browsertrix-crawler` .
-    - Configurable via parameters for timeout, media extraction, screenshots, and proxy settings.
+    ## Setup
+
+    ### Using Docker
+    If you are using the Auto Archiver [Docker image](https://auto-archiver.readthedocs.io/en/latest/installation/installation.html#installing-with-docker)
+    to run Auto Archiver (recommended), then everything is set up and you can use WACZ out of the box!
+    Otherwise, if you are using a local install of Auto Archiver (e.g. pip or dev install), then you will need to install Docker and run 
+    the docker daemon to be able to run the `browsertrix-crawler` tool.
+
+    ### Browsertrix Profiles
+    A browsertrix profile is a custom browser profile (login information, browser extensions, etc.) that can be used to archive private or dynamic content.
+    You can run the WACZ Enricher without a profile, but for more resilient archiving, it is recommended to create a profile.
+    See the [Browsertrix documentation](https://crawler.docs.browsertrix.com/user-guide/browser-profiles/) for more information on how to use the `create-login-profile` tool.
+
+
+
+    ### Docker in Docker
+    If you are running Auto Archiver within a Docker container, you will need to enable Docker in Docker to run the `browsertrix-crawler` tool.
+    This can be done by setting the `WACZ_ENABLE_DOCKER` environment variable to `1`.
+
+
     """,
 }

src/auto_archiver/modules/wacz_extractor_enricher/wacz_extractor_enricher.py

@@ -24,7 +24,8 @@ class WaczExtractorEnricher(Enricher, Extractor):
         self.use_docker = os.environ.get("WACZ_ENABLE_DOCKER") or not os.environ.get("RUNNING_IN_DOCKER")
         self.docker_in_docker = os.environ.get("WACZ_ENABLE_DOCKER") and os.environ.get("RUNNING_IN_DOCKER")
 
-        self.cwd_dind = f"/crawls/crawls{random_str(8)}"
+        self.crawl_id = random_str(8)
+        self.cwd_dind = f"/crawls/crawls{self.crawl_id}"
         self.browsertrix_home_host = os.environ.get("BROWSERTRIX_HOME_HOST")
         self.browsertrix_home_container = os.environ.get("BROWSERTRIX_HOME_CONTAINER") or self.browsertrix_home_host
         # create crawls folder if not exists, so it can be safely removed in cleanup
@@ -50,7 +51,7 @@ class WaczExtractorEnricher(Enricher, Extractor):
 
         url = to_enrich.get_url()
 
-        collection = random_str(8)
+        collection = self.crawl_id
         browsertrix_home_host = self.browsertrix_home_host or os.path.abspath(self.tmp_dir)
         browsertrix_home_container = self.browsertrix_home_container or browsertrix_home_host
 
@@ -85,6 +86,12 @@ class WaczExtractorEnricher(Enricher, Extractor):
         if self.docker_in_docker:
             cmd.extend(["--cwd", self.cwd_dind])
 
+        if self.auth_for_site(url):
+            # there's an auth for this site, but browsertrix only supports username/password auth
+            logger.warning(
+                "The WACZ enricher / Browsertrix does not support using the 'authentication' information for logging in. You should consider creating a Browser Profile for WACZ archiving. More information: https://auto-archiver.readthedocs.io/en/latest/modules/autogen/extractor/wacz_extractor_enricher.html#browsertrix-profiles"
+            )
+
         # call docker if explicitly enabled or we are running on the host (not in docker)
         if self.use_docker:
             logger.debug(f"generating WACZ in Docker for {url=}")
@@ -102,10 +109,11 @@ class WaczExtractorEnricher(Enricher, Extractor):
                 ] + cmd
 
             if self.profile:
-                profile_fn = os.path.join(browsertrix_home_container, "profile.tar.gz")
+                profile_file = f"profile-{self.crawl_id}.tar.gz"
+                profile_fn = os.path.join(browsertrix_home_container, profile_file)
                 logger.debug(f"copying {self.profile} to {profile_fn}")
                 shutil.copyfile(self.profile, profile_fn)
-                cmd.extend(["--profile", os.path.join("/crawls", "profile.tar.gz")])
+                cmd.extend(["--profile", os.path.join("/crawls", profile_file)])
 
         else:
             logger.debug(f"generating WACZ without Docker for {url=}")
@@ -186,7 +194,8 @@ class WaczExtractorEnricher(Enricher, Extractor):
                         shutil.copyfileobj(infile, outfile)
 
         # get media out of .warc
-        counter = 0
+        counter_warc_files = 0
+        counter_screenshots = 0
         seen_urls = set()
 
         with open(warc_filename, "rb") as warc_stream:
@@ -195,12 +204,12 @@ class WaczExtractorEnricher(Enricher, Extractor):
                 if (
                     record.rec_type == "resource" and record.content_type == "image/png" and self.extract_screenshot
                 ):  # screenshots
-                    fn = os.path.join(tmp_dir, f"warc-file-{counter}.png")
+                    fn = os.path.join(tmp_dir, f"browsertrix-screenshot-{counter_screenshots}.png")
                     with open(fn, "wb") as outf:
                         outf.write(record.raw_stream.read())
                     m = Media(filename=fn)
-                    to_enrich.add_media(m, "browsertrix-screenshot")
-                    counter += 1
+                    to_enrich.add_media(m, f"browsertrix-screenshot-{counter_screenshots}")
+                    counter_screenshots += 1
                 if not self.extract_media:
                     continue
 
@@ -223,7 +232,7 @@ class WaczExtractorEnricher(Enricher, Extractor):
 
                 # create local file and add media
                 ext = mimetypes.guess_extension(content_type)
-                warc_fn = f"warc-file-{counter}{ext}"
+                warc_fn = f"warc-file-{counter_warc_files}{ext}"
                 fn = os.path.join(tmp_dir, warc_fn)
 
                 record_url_best_qual = UrlUtil.twitter_best_quality_url(record_url)
@@ -248,6 +257,8 @@ class WaczExtractorEnricher(Enricher, Extractor):
                     continue
 
                 to_enrich.add_media(m, warc_fn)
-                counter += 1
+                counter_warc_files += 1
                 seen_urls.add(record_url)
-        logger.info(f"WACZ extract_media/extract_screenshot finished, found {counter} relevant media file(s)")
+        logger.info(
+            f"WACZ extract_media/extract_screenshot finished, found {counter_warc_files + counter_screenshots} relevant media file(s)"
+        )

src/auto_archiver/utils/__init__.py

@@ -2,7 +2,6 @@
 
 # we need to explicitly expose the available imports here
 from .misc import *
-from .webdriver import Webdriver
 
 # handy utils from ytdlp
 from yt_dlp.utils import clean_html, traverse_obj, strip_or_none, url_or_none

src/auto_archiver/utils/misc.py

@@ -1,5 +1,6 @@
 import hashlib
 import json
+import mimetypes
 import os
 import uuid
 from datetime import datetime, timezone
@@ -116,3 +117,26 @@ def get_timestamp(ts, utc=True, iso=True, dayfirst=True) -> str | datetime | Non
 
 def get_current_timestamp() -> str:
     return get_timestamp(datetime.now())
+
+
+def ydl_entry_to_filename(ydl, entry: dict) -> str:
+    import yt_dlp
+
+    ydl: yt_dlp.YoutubeDL
+    entry_url = entry.get("url")
+
+    filename = ydl.prepare_filename(entry)
+    if os.path.exists(filename):
+        return filename
+
+    base_filename, _ = os.path.splitext(filename)  # '/get/path/to/file' ignore '.ext'
+    directory = os.path.dirname(base_filename)  # '/get/path/to'
+    basename = os.path.basename(base_filename)  # 'file'
+    for f in os.listdir(directory):
+        if (
+            f.startswith(basename)
+            or (entry_url and os.path.splitext(f)[0] in entry_url)
+            and "video/" in (mimetypes.guess_type(f)[0] or "")
+        ):
+            return os.path.join(directory, f)
+    return False

src/auto_archiver/utils/url.py

@@ -1,11 +1,11 @@
 import re
-from urllib.parse import urlparse, urlunparse
+from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
 from ipaddress import ip_address
 
 
 AUTHWALL_URLS = [
-    re.compile(r"https:\/\/t\.me(\/c)\/(.+)\/(\d+)"),  # telegram private channels
-    re.compile(r"https:\/\/www\.instagram\.com"),  # instagram
+    re.compile(r"https?:\/\/t\.me(\/c)\/(.+)\/(\d+)"),  # telegram private channels
+    re.compile(r"https?:\/\/(www\.)?instagram\.com"),  # instagram
 ]
 
 
@@ -53,7 +53,11 @@ def domain_for_url(url: str) -> str:
 
 
 def clean(url: str) -> str:
-    return url
+    TRACKERS = {"utm_source", "utm_medium", "utm_campaign", "utm_term", "utm_content", "fbclid", "gclid"}
+
+    parsed = urlparse(url)
+    clean_qs = [(k, v) for k, v in parse_qsl(parsed.query) if k not in TRACKERS]
+    return parsed._replace(query=urlencode(clean_qs)).geturl()
 
 
 def is_auth_wall(url: str) -> bool:
@@ -78,59 +82,58 @@ def remove_get_parameters(url: str) -> str:
 def is_relevant_url(url: str) -> bool:
     """
     Detect if a detected media URL is recurring and therefore irrelevant to a specific archive. Useful, for example, for the enumeration of the media files in WARC files which include profile pictures, favicons, etc.
+
+    Assumption: URLs are relevant if they refer to files that can be downloaded with curl/requests, so excludes extensions like .m3u8.
     """
     clean_url = remove_get_parameters(url)
 
-    # favicons
-    if "favicon" in url:
-        return False
-    # ifnore icons
-    if clean_url.endswith(".ico"):
-        return False
-    # ignore SVGs
-    if remove_get_parameters(url).endswith(".svg"):
-        return False
+    IRRELEVANT_URLS = [
+        # favicons
+        ("favicon",),
+        # twitter profile pictures
+        ("twimg.com/profile_images",),
+        ("twimg.com", "default_profile_images"),
+        # instagram profile pictures
+        ("https://scontent.cdninstagram.com/", "150x150"),
+        # instagram recurring images
+        ("https://static.cdninstagram.com/rsrc.php/",),
+        # telegram
+        ("https://telegram.org/img/emoji/",),
+        # youtube
+        ("https://www.youtube.com/s/gaming/emoji/",),
+        ("https://yt3.ggpht.com", "default-user="),
+        ("https://www.youtube.com/s/search/audio/",),
+        # ok
+        ("https://ok.ru/res/i/",),
+        ("https://vk.com/emoji/",),
+        ("vk.com/images/",),
+        ("vk.com/images/reaction/",),
+        # wikipedia
+        ("wikipedia.org/static",),
+        # reddit
+        ("styles.redditmedia.com",),  # opinionated but excludes may irrelevant images like avatars and banners
+        ("emoji.redditmedia.com",),
+        # linkedin
+        ("static.licdn.com",),
+    ]
 
-    # twitter profile pictures
-    if "twimg.com/profile_images" in url:
-        return False
-    if "twimg.com" in url and "/default_profile_images" in url:
-        return False
+    # TODO: make these globally configurable
+    IRRELEVANT_ENDS_WITH = [
+        ".svg",  # ignore SVGs
+        ".ico",  # ignore icons
+        # ignore index files for videos, these should be handled by ytdlp
+        ".m3u8",
+        ".mpd",
+        ".ism",
+    ]
 
-    # instagram profile pictures
-    if "https://scontent.cdninstagram.com/" in url and "150x150" in url:
-        return False
-    # instagram recurring images
-    if "https://static.cdninstagram.com/rsrc.php/" in url:
-        return False
+    for end in IRRELEVANT_ENDS_WITH:
+        if clean_url.endswith(end):
+            return False
 
-    # telegram
-    if "https://telegram.org/img/emoji/" in url:
-        return False
-
-    # youtube
-    if "https://www.youtube.com/s/gaming/emoji/" in url:
-        return False
-    if "https://yt3.ggpht.com" in url and "default-user=" in url:
-        return False
-    if "https://www.youtube.com/s/search/audio/" in url:
-        return False
-
-    # ok
-    if " https://ok.ru/res/i/" in url:
-        return False
-
-    # vk
-    if "https://vk.com/emoji/" in url:
-        return False
-    if "vk.com/images/" in url:
-        return False
-    if "vk.com/images/reaction/" in url:
-        return False
-
-    # wikipedia
-    if "wikipedia.org/static" in url:
-        return False
+    for parts in IRRELEVANT_URLS:
+        if all(part in clean_url for part in parts):
+            return False
 
     return True
 
@@ -138,6 +141,36 @@ def is_relevant_url(url: str) -> bool:
 def twitter_best_quality_url(url: str) -> str:
     """
     some twitter image URLs point to a less-than best quality
-    this returns the URL pointing to the highest (original) quality
+    this returns the URL pointing to the highest (original) quality (with 'name=orig')
     """
-    return re.sub(r"name=(\w+)", "name=orig", url, 1)
+    parsed = urlparse(url)
+    query = parsed.query
+    if "name=" in query:
+        # Replace only the first occurrence of name=xxx with name=orig
+        new_query = re.sub(r"name=[^&]*", "name=orig", query, 1)
+        parsed = parsed._replace(query=new_query)
+        return urlunparse(parsed)
+    return url
+
+
+def get_media_url_best_quality(url: str) -> str:
+    """
+    Returns the best quality URL for the given media URL, it may not exist.
+    """
+    parsed = urlparse(url)
+
+    # twitter case
+    if any(d in parsed.netloc.replace("www", "") for d in ("twitter.com", "twimg.com", "x.com")):
+        url = twitter_best_quality_url(url)
+        parsed = urlparse(url)
+
+    # some cases https://example.com/media-1280x720.mp4 to https://example.com/media.mp4
+    basename = parsed.path.split("/")[-1]
+    match = re.match(r"(.+)-\d+x\d+(\.[a-zA-Z0-9]+)$", basename)
+    if match:
+        orig_basename = match.group(1) + match.group(2)
+        new_path = "/".join(parsed.path.split("/")[:-1] + [orig_basename])
+        parsed = parsed._replace(path=new_path)  # keep the query unchanged
+        url = urlunparse(parsed)
+
+    return url

src/auto_archiver/utils/webdriver.py

@@ -1,167 +0,0 @@
-"""This Webdriver class acts as a context manager for the selenium webdriver."""
-
-from __future__ import annotations
-
-import os
-import time
-import re
-
-# import domain_for_url
-from urllib.parse import urlparse, urlunparse
-from http.cookiejar import MozillaCookieJar
-
-from selenium import webdriver
-from selenium.webdriver.support.ui import WebDriverWait
-from selenium.webdriver.support import expected_conditions as EC
-from selenium.common import exceptions as selenium_exceptions
-from selenium.webdriver.common.print_page_options import PrintOptions
-from selenium.webdriver.common.by import By
-
-from loguru import logger
-
-
-class CookieSettingDriver(webdriver.Firefox):
-    facebook_accept_cookies: bool
-    cookies: str
-    cookiejar: MozillaCookieJar
-
-    def __init__(self, cookies, cookiejar, facebook_accept_cookies, *args, **kwargs):
-        if os.environ.get("RUNNING_IN_DOCKER"):
-            # Selenium doesn't support linux-aarch64 driver, we need to set this manually
-            kwargs["service"] = webdriver.FirefoxService(executable_path="/usr/local/bin/geckodriver")
-
-        super(CookieSettingDriver, self).__init__(*args, **kwargs)
-        self.cookies = cookies
-        self.cookiejar = cookiejar
-        self.facebook_accept_cookies = facebook_accept_cookies
-
-    def get(self, url: str):
-        if self.cookies or self.cookiejar:
-            # set up the driver to make it not 'cookie averse' (needs a context/URL)
-            # get the 'robots.txt' file which should be quick and easy
-            robots_url = urlunparse(urlparse(url)._replace(path="/robots.txt", query="", fragment=""))
-            super(CookieSettingDriver, self).get(robots_url)
-
-            if self.cookies:
-                # an explicit cookie is set for this site, use that first
-                for cookie in self.cookies.split(";"):
-                    for name, value in cookie.split("="):
-                        self.driver.add_cookie({"name": name, "value": value})
-            elif self.cookiejar:
-                domain = urlparse(url).netloc
-                regex = re.compile(f"(www)?.?{domain}$")
-                for cookie in self.cookiejar:
-                    if regex.match(cookie.domain):
-                        try:
-                            self.add_cookie(
-                                {
-                                    "name": cookie.name,
-                                    "value": cookie.value,
-                                    "path": cookie.path,
-                                    "domain": cookie.domain,
-                                    "secure": bool(cookie.secure),
-                                    "expiry": cookie.expires,
-                                }
-                            )
-                        except Exception as e:
-                            logger.warning(f"Failed to add cookie ({cookie.domain}) to webdriver for url {domain}: {e}")
-
-        super(CookieSettingDriver, self).get(url)
-        time.sleep(2)
-
-        # Try and use some common button text to reject/accept cookies
-        for text in [
-            "Refuse non-essential cookies",
-            "Decline optional cookies",
-            "Reject additional cookies",
-            "Reject all",
-            "Accept all cookies",
-        ]:
-            try:
-                xpath = f"//*[contains(translate(text(), 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz'), '{text.lower()}')]"
-                self.find_element(By.XPATH, xpath).click()
-                time.sleep(2)
-            except selenium_exceptions.NoSuchElementException:
-                pass
-
-        # now get the actual URL
-        if self.facebook_accept_cookies:
-            # try and click the 'close' button on the 'login' window to close it
-            try:
-                xpath = "//div[@role='dialog']//div[@aria-label='Close']"
-                self.find_element(By.XPATH, xpath).click()
-                time.sleep(2)
-            except selenium_exceptions.NoSuchElementException:
-                logger.warning("Unable to find the 'close' button on the facebook login window")
-                pass
-
-        else:
-            # for all other sites, try and use some common button text to reject/accept cookies
-            for text in [
-                "Refuse non-essential cookies",
-                "Decline optional cookies",
-                "Reject additional cookies",
-                "Reject all",
-                "Accept all cookies",
-            ]:
-                try:
-                    xpath = f"//*[contains(translate(text(), 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz'), '{text.lower()}')]"
-                    WebDriverWait(self, 5).until(EC.element_to_be_clickable((By.XPATH, xpath))).click()
-                    break
-                except selenium_exceptions.WebDriverException:
-                    pass
-
-
-class Webdriver:
-    def __init__(
-        self,
-        width: int,
-        height: int,
-        timeout_seconds: int,
-        facebook_accept_cookies: bool = False,
-        http_proxy: str = "",
-        print_options: dict = {},
-        auth: dict = {},
-    ) -> webdriver:
-        self.width = width
-        self.height = height
-        self.timeout_seconds = timeout_seconds
-        self.auth = auth
-        self.facebook_accept_cookies = facebook_accept_cookies
-        self.http_proxy = http_proxy
-        # create and set print options
-        self.print_options = PrintOptions()
-        for k, v in print_options.items():
-            setattr(self.print_options, k, v)
-
-    def __enter__(self) -> webdriver:
-        options = webdriver.FirefoxOptions()
-        options.add_argument("--headless")
-        options.add_argument(f"--proxy-server={self.http_proxy}")
-        options.set_preference("network.protocol-handler.external.tg", False)
-        # if facebook cookie popup is present, force the browser to English since then it's easier to click the 'Decline optional cookies' option
-        if self.facebook_accept_cookies:
-            options.add_argument("--lang=en")
-
-        try:
-            self.driver = CookieSettingDriver(
-                cookies=self.auth.get("cookies"),
-                cookiejar=self.auth.get("cookies_jar"),
-                facebook_accept_cookies=self.facebook_accept_cookies,
-                options=options,
-            )
-            self.driver.set_window_size(self.width, self.height)
-            self.driver.set_page_load_timeout(self.timeout_seconds)
-            self.driver.print_options = self.print_options
-        except selenium_exceptions.TimeoutException as e:
-            logger.error(
-                f"failed to get new webdriver, possibly due to insufficient system resources or timeout settings: {e}"
-            )
-
-        return self.driver
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        self.driver.close()
-        self.driver.quit()
-        del self.driver
-        return True

tests/.env.test.example

@@ -0,0 +1,13 @@
+# ANTIBOT reddit test credentials
+REDDIT_TEST_USERNAME=""
+REDDIT_TEST_PASSWORD=""
+
+# ANTIBOT linkedin test credentials
+LINKEDIN_TEST_USERNAME=""
+LINKEDIN_TEST_PASSWORD=""
+
+# twitter test credentials
+TWITTER_BEARER_TOKEN="TEST_KEY"
+
+# some geo/VPN IPs are blocked by truth social, disable if you have issues
+TEST_TRUTH_SOCIAL="true"

tests/conftest.py

@@ -9,15 +9,51 @@ from tempfile import TemporaryDirectory
 from typing import Dict, Tuple
 import hashlib
 
+from loguru import logger
 import pytest
-from auto_archiver.core.metadata import Metadata
+from auto_archiver.core.metadata import Metadata, Media
 from auto_archiver.core.module import ModuleFactory
 
 # Test names inserted into this list will be run last. This is useful for expensive/costly tests
 # that you only want to run if everything else succeeds (e.g. API calls). The order here is important
 # what comes first will be run first (at the end of all other tests not mentioned)
 # format is the name of the module (python file) without the .py extension
-TESTS_TO_RUN_LAST = ["test_twitter_api_archiver"]
+TESTS_TO_RUN_LAST = ["test_generic_archiver", "test_twitter_api_archiver"]
+
+
+def pytest_configure():
+    # load environment variables from .env.test file.
+    env_path = os.path.join(os.path.dirname(__file__), ".env.test")
+    if os.path.exists(env_path):
+        with open(env_path) as f:
+            for line in f:
+                line = line.strip()
+                if not line or line.startswith("#"):
+                    continue
+                if "=" in line:
+                    key, value = line.split("=", 1)
+                    os.environ[key.strip()] = value.strip().lstrip('"').rstrip('"')
+    else:
+        logger.warning(
+            f"Environment file {env_path} not found. Skipping loading environment variables, some tests may fail."
+        )
+
+
+# don't check for ytdlp updates in tests
+@pytest.fixture(autouse=True)
+def skip_check_for_update(mocker):
+    update_ytdlp = mocker.patch(
+        "auto_archiver.modules.generic_extractor.generic_extractor.GenericExtractor.update_package"
+    )
+    update_ytdlp.return_value = False
+
+
+@pytest.fixture
+def get_lazy_module():
+    def _get_lazy_module(module_name):
+        return ModuleFactory().get_module_lazy(module_name)
+
+    return _get_lazy_module
 
 
 @pytest.fixture
@@ -118,7 +154,7 @@ def pytest_runtest_setup(item):
                 pytest.xfail(f"previous test failed ({test_name})")
 
 
-@pytest.fixture()
+@pytest.fixture
 def unpickle():
     """
     Returns a helper function that unpickles a file
@@ -134,12 +170,21 @@ def unpickle():
 
 @pytest.fixture
 def mock_binary_dependencies(mocker):
+    mocker.patch("subprocess.run").return_value = mocker.Mock(returncode=0)
     mock_shutil_which = mocker.patch("shutil.which")
     # Mock all binary dependencies as available
     mock_shutil_which.return_value = "/usr/bin/fake_binary"
     return mock_shutil_which
 
 
+@pytest.fixture
+def sample_media(tmp_path) -> Media:
+    """Fixture creating a Media object with temporary source file"""
+    src_file = tmp_path / "source.txt"
+    src_file.write_text("test content")
+    return Media(_key="subdir/test.txt", filename=str(src_file))
+
+
 @pytest.fixture
 def sample_datetime():
     return datetime(2023, 1, 1, 12, 0, tzinfo=timezone.utc)

tests/data/test_modules/example_extractor/__manifest__.py

@@ -0,0 +1,11 @@
+{
+    # Display Name of your module
+    "name": "Example Extractor",
+    # Optional version number, for your own versioning purposes
+    "version": 2.0,
+    # The type of the module, must be one (or more) of the built in module types
+    "type": ["extractor"],
+    # a boolean indicating whether or not a module requires additional user setup before it can be used
+    # for example: adding API keys, installing additional software etc.
+    "requires_setup": False,
+}

tests/data/test_modules/example_extractor/example_extractor.py

@@ -0,0 +1,11 @@
+from auto_archiver.core import Extractor
+
+from loguru import logger
+
+
+class ExampleExtractor(Extractor):
+    def download(self, item):
+        logger.info("download")
+
+    def cleanup(self):
+        logger.info("cleanup")

tests/data/test_modules/example_module/example_module.py

@@ -1,27 +1,29 @@
 from auto_archiver.core import Extractor, Enricher, Feeder, Database, Storage, Formatter, Metadata
 
+from loguru import logger
+
 
 class ExampleModule(Extractor, Enricher, Feeder, Database, Storage, Formatter):
     def download(self, item):
-        print("download")
+        logger.info("download")
 
     def __iter__(self):
         yield Metadata().set_url("https://example.com")
 
     def done(self, result):
-        print("done")
+        logger.info("done")
 
     def enrich(self, to_enrich):
-        print("enrich")
+        logger.info("enrich")
 
     def get_cdn_url(self, media):
         return "nice_url"
 
     def save(self, item):
-        print("save")
+        logger.info("save")
 
     def uploadf(self, file, key, **kwargs):
-        print("uploadf")
+        logger.info("uploadf")
 
     def format(self, item):
-        print("format")
+        logger.info("format")

tests/enrichers/test_screenshot_enricher.py

@@ -1,184 +0,0 @@
-import base64
-
-import pytest
-from selenium.common.exceptions import TimeoutException
-
-from auto_archiver.core import Metadata, Media
-from auto_archiver.modules.screenshot_enricher import ScreenshotEnricher
-
-
-@pytest.fixture
-def mock_selenium_env(mocker):
-    """Patches Selenium calls and driver checks in one place."""
-
-    # Patch external dependencies
-    mock_which = mocker.patch("shutil.which")
-    mock_driver_class = mocker.patch("auto_archiver.utils.webdriver.CookieSettingDriver")
-    mock_binary_paths = mocker.patch("selenium.webdriver.common.selenium_manager.SeleniumManager.binary_paths")
-    mocker.patch("pathlib.Path.is_file", return_value=True)
-    mock_popen = mocker.patch("subprocess.Popen")
-    mocker.patch("selenium.webdriver.common.service.Service.is_connectable", return_value=True)
-    mock_firefox_options = mocker.patch("selenium.webdriver.FirefoxOptions")
-
-    # Define side effect for `shutil.which`
-    def mock_which_side_effect(dep):
-        return "/mock/geckodriver" if dep == "geckodriver" else None
-
-    mock_which.side_effect = mock_which_side_effect
-
-    # Mock binary paths
-    mock_binary_paths.return_value = {
-        "driver_path": "/mock/driver",
-        "browser_path": "/mock/browser",
-    }
-    # Mock `subprocess.Popen`
-    mock_proc = mocker.MagicMock()
-    mock_proc.poll.return_value = None
-    mock_popen.return_value = mock_proc
-    # Mock `CookieSettingDriver`
-    mock_driver = mocker.MagicMock()
-    mock_driver_class.return_value = mock_driver
-    # Mock `FirefoxOptions`
-    mock_options_instance = mocker.MagicMock()
-    mock_firefox_options.return_value = mock_options_instance
-    yield mock_driver, mock_driver_class, mock_options_instance
-
-
-@pytest.fixture
-def common_patches(tmp_path, mocker):
-    """Patches common utilities used across multiple tests."""
-    mocker.patch("auto_archiver.utils.url.is_auth_wall", return_value=False)
-    mocker.patch("os.path.join", return_value=str(tmp_path / "test.png"))
-    mocker.patch("time.sleep")
-    yield
-
-
-@pytest.fixture
-def screenshot_enricher(setup_module, mock_binary_dependencies) -> ScreenshotEnricher:
-    configs: dict = {
-        "width": 1280,
-        "height": 720,
-        "timeout": 60,
-        "sleep_before_screenshot": 4,
-        "http_proxy": "",
-        "save_to_pdf": "False",
-        "print_options": {},
-    }
-    return setup_module("screenshot_enricher", configs)
-
-
-@pytest.fixture
-def metadata_with_video():
-    m = Metadata()
-    m.set_url("https://example.com")
-    m.add_media(Media(filename="video.mp4").set("id", "video1"))
-    return m
-
-
-def test_enrich_adds_screenshot(
-    screenshot_enricher,
-    metadata_with_video,
-    mock_selenium_env,
-    common_patches,
-    tmp_path,
-):
-    mock_driver, mock_driver_class, mock_options_instance = mock_selenium_env
-    screenshot_enricher.enrich(metadata_with_video)
-    mock_driver_class.assert_called_once_with(
-        cookies=None,
-        cookiejar=None,
-        facebook_accept_cookies=False,
-        options=mock_options_instance,
-    )
-    # Verify the actual calls on the returned mock_driver
-    mock_driver.get.assert_called_once_with("https://example.com")
-    mock_driver.save_screenshot.assert_called_once_with(str(tmp_path / "test.png"))
-    # Check that the media was added (2 = original video + screenshot)
-    assert len(metadata_with_video.media) == 2
-    assert metadata_with_video.media[1].properties.get("id") == "screenshot"
-
-
-@pytest.mark.parametrize(
-    "url,is_auth",
-    [
-        ("https://example.com", False),
-        ("https://private.com", True),
-    ],
-)
-def test_enrich_auth_wall(
-    screenshot_enricher, metadata_with_video, mock_selenium_env, common_patches, url, is_auth, mocker
-):
-    # Testing with and without is_auth_wall
-    mock_driver, mock_driver_class, _ = mock_selenium_env
-    mocker.patch("auto_archiver.utils.url.is_auth_wall", return_value=is_auth)
-    metadata_with_video.set_url(url)
-    screenshot_enricher.enrich(metadata_with_video)
-
-    if is_auth:
-        mock_driver.get.assert_not_called()
-        assert len(metadata_with_video.media) == 1
-        assert metadata_with_video.media[0].properties.get("id") == "video1"
-    else:
-        mock_driver.get.assert_called_once_with(url)
-        assert len(metadata_with_video.media) == 2
-        assert metadata_with_video.media[1].properties.get("id") == "screenshot"
-
-
-def test_handle_timeout_exception(screenshot_enricher, metadata_with_video, mock_selenium_env, mocker):
-    mock_driver, mock_driver_class, mock_options_instance = mock_selenium_env
-
-    mock_driver.get.side_effect = TimeoutException
-    mock_log = mocker.patch("loguru.logger.info")
-    screenshot_enricher.enrich(metadata_with_video)
-    mock_log.assert_called_once_with("TimeoutException loading page for screenshot")
-    assert len(metadata_with_video.media) == 1
-
-
-def test_handle_general_exception(screenshot_enricher, metadata_with_video, mock_selenium_env, mocker):
-    """Test proper handling of unexpected general exceptions"""
-    mock_driver, mock_driver_class, mock_options_instance = mock_selenium_env
-    # Simulate a generic exception when save_screenshot is called
-    mock_driver.get.return_value = None
-    mock_driver.save_screenshot.side_effect = Exception("Unexpected Error")
-
-    mock_log = mocker.patch("loguru.logger.error")
-    screenshot_enricher.enrich(metadata_with_video)
-    # Verify that the exception was logged with the log
-    mock_log.assert_called_once_with("Got error while loading webdriver for screenshot enricher: Unexpected Error")
-    # And no new media was added due to the error
-    assert len(metadata_with_video.media) == 1
-
-
-def test_pdf_creation(mocker, screenshot_enricher, metadata_with_video, mock_selenium_env):
-    """Test PDF creation when save_to_pdf is enabled"""
-    mock_driver, mock_driver_class, mock_options_instance = mock_selenium_env
-    # Override the save_to_pdf option
-    screenshot_enricher.save_to_pdf = True
-    # Mock the print_page method to return base64-encoded content
-    mock_driver.print_page.return_value = base64.b64encode(b"fake_pdf_content").decode("utf-8")
-    # Patch functions with mocker
-    mocker.patch("os.path.join", side_effect=lambda *args: f"{args[-1]}")
-    mocker.patch(
-        "auto_archiver.modules.screenshot_enricher.screenshot_enricher.random_str",
-        return_value="fixed123",
-    )
-    mock_open = mocker.patch("builtins.open", new_callable=mocker.mock_open)
-
-    screenshot_enricher.enrich(metadata_with_video)
-    # Verify screenshot and PDF creation
-    mock_driver.save_screenshot.assert_called_once()
-    mock_driver.print_page.assert_called_once_with(mock_driver.print_options)
-    # Check that PDF file was opened and written
-    mock_open.assert_any_call("pdf_fixed123.pdf", "wb")
-
-    # Ensure both screenshot and PDF were added as media
-    assert len(metadata_with_video.media) == 3
-    assert metadata_with_video.media[1].properties.get("id") == "screenshot"
-    assert metadata_with_video.media[2].properties.get("id") == "pdf"
-
-
-@pytest.fixture(autouse=True)
-def cleanup_files(tmp_path):
-    yield
-    for file in tmp_path.iterdir():
-        file.unlink()

tests/enrichers/test_thumbnail_enricher.py

@@ -25,6 +25,7 @@ def mock_ffmpeg_environment(mocker):
     # Mocking all the ffmpeg calls in one place
     mock_ffmpeg_input = mocker.patch("ffmpeg.input")
     mock_makedirs = mocker.patch("os.makedirs")
+    mocker.patch("os.path.exists", return_value=True)
     (mocker.patch.object(Media, "is_video", return_value=True),)
     mock_probe = mocker.patch(
         "ffmpeg.probe",
@@ -74,12 +75,12 @@ def test_enrich_thumbnail_limits(
 def test_enrich_handles_probe_failure(thumbnail_enricher, metadata_with_video, mocker):
     mocker.patch("ffmpeg.probe", side_effect=Exception("Probe error"))
     mocker.patch("os.makedirs")
-    mock_logger = mocker.patch("loguru.logger.error")
+    mock_logger = mocker.patch("loguru.logger.warning")
     mocker.patch.object(Media, "is_video", return_value=True)
 
     thumbnail_enricher.enrich(metadata_with_video)
     # Ensure error was logged
-    mock_logger.assert_called_with("error getting duration of video video.mp4: Probe error")
+    mock_logger.assert_called_with("cannot generate thumbnails for video.mp4 without valid duration")
     # Ensure no thumbnails were created
     thumbnails = metadata_with_video.media[0].get("thumbnails")
     assert thumbnails is None
@@ -126,11 +127,14 @@ def test_enrich_handles_short_video(
     assert len(thumbnails) == expected_count
 
 
-def test_uses_existing_duration(thumbnail_enricher, metadata_with_video, mock_ffmpeg_environment):
-    metadata_with_video.media[0].set("duration", 60)
+def test_uses_existing_duration_on_exception(thumbnail_enricher, metadata_with_video, mock_ffmpeg_environment, mocker):
+    mock_logger = mocker.patch("loguru.logger.warning")
+    mock_probe = mocker.patch("ffmpeg.probe", side_effect=Exception("New probe error"))
+    metadata_with_video.media[0].set("duration", 3)
     thumbnail_enricher.enrich(metadata_with_video)
-    mock_ffmpeg_environment["mock_probe"].assert_not_called()
-    assert mock_ffmpeg_environment["mock_output"].run.call_count == 4
+    mock_probe.assert_called_once()
+    mock_logger.assert_called_with("failed to get duration with FFMPEG from video.mp4: New probe error")
+    assert mock_ffmpeg_environment["mock_output"].run.call_count == 3
 
 
 def test_enrich_metadata_structure(thumbnail_enricher, metadata_with_video, mock_ffmpeg_environment, mocker):