Merge pull request #406 from bellingcat/dev

1.2.3

.github/dependabot.yml

@@ -12,7 +12,7 @@ updates:
         patterns:
           - "*"
     schedule:
-      interval: "weekly"
+      interval: "monthly"
   
   - package-ecosystem: "github-actions"
     directory: "/"
@@ -21,7 +21,7 @@ updates:
         patterns:
           - "*"
     schedule:
-      interval: "weekly"
+      interval: "monthly"
 
   - package-ecosystem: "npm"
     directory: "/scripts/settings/"
@@ -30,11 +30,11 @@ updates:
         patterns:
           - "*"
     schedule:
-      interval: "weekly"
+      interval: "monthly"
 
   - package-ecosystem: "docker"
     # Look for a `Dockerfile` in the `root` directory
     directory: "/"
     # Check for updates once a week
     schedule:
-      interval: "weekly"
+      interval: "monthly"

.github/workflows/docker-publish.yaml

@@ -22,7 +22,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
         
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
@@ -33,14 +33,14 @@ jobs:
         uses: docker/setup-buildx-action@v3
 
       - name: Log in to Docker Hub
-        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772
+        uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804
+        uses: docker/metadata-action@c299e40c65443455700f0fdfc63efafe5b349051
         with:
           images: bellingcat/auto-archiver
       

.github/workflows/python-publish.yaml

@@ -22,10 +22,10 @@ jobs:
 
     steps:
     - name: Checkout Repository
-      uses: actions/checkout@v4
+      uses: actions/checkout@v6
 
     - name: Set up Python
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v6
       with:
         python-version-file: pyproject.toml
 

.github/workflows/ruff.yaml

@@ -20,11 +20,11 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
       - name: Install Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
-          python-version: "3.11"
+          python-version: "3.12"
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip

.github/workflows/tests-core.yaml

@@ -26,18 +26,21 @@ jobs:
         working-directory: ./
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
+
+      - name: Install ffmpeg
+        run: sudo apt-get update && sudo apt-get install -y ffmpeg
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: ${{ matrix.python-version }}
 
       - name: Install latest Poetry
         run: pipx install poetry
-
+      
       - name: Cache Poetry and pip artifacts
-        uses: actions/cache@v4
+        uses: actions/cache@v5
         with:
           path: |
             ~/.cache/pypoetry

.github/workflows/tests-download.yaml

@@ -20,10 +20,13 @@ jobs:
         working-directory: ./
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
+
+      - name: Install ffmpeg
+        run: sudo apt-get update && sudo apt-get install -y ffmpeg
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: ${{ matrix.python-version }}
 
@@ -31,7 +34,7 @@ jobs:
         run: pipx install poetry
 
       - name: Cache Poetry and pip artifacts
-        uses: actions/cache@v4
+        uses: actions/cache@v5
         with:
           path: |
             ~/.cache/pypoetry
@@ -44,4 +47,4 @@ jobs:
       - name: Run Download Tests
         run: poetry run pytest -ra -v -x -m "download"
         env:
-          TWITTER_BEARER_TOKEN: ${{ secrets.TWITTER_BEARER_TOKEN }}
+          TWITTER_BEARER_TOKEN: ${{ secrets.TWITTER_BEARER_TOKEN || '' }}

.gitignore

@@ -1,6 +1,7 @@
 tmp*/
 temp/
 .env*
+!.env*.example
 .DS_Store
 expmt/
 service_account.json
@@ -37,3 +38,7 @@ docs/source/modules/autogen/
 scripts/settings_page.html
 scripts/settings/src/schema.json 
 .vite
+downloaded_files
+latest_logs
+# for launch.json
+.vscode

.readthedocs.yaml

@@ -7,6 +7,8 @@ version: 2
 
 build:
   os: ubuntu-22.04
+  apt_packages:
+    - ffmpeg
   tools:
     python: "3.10"
     nodejs: "22"

Dockerfile

@@ -1,4 +1,4 @@
-FROM webrecorder/browsertrix-crawler:1.6.1 AS base
+FROM webrecorder/browsertrix-crawler:1.11.4 AS base
 
 ENV RUNNING_IN_DOCKER=1 \
     LANG=C.UTF-8 \
@@ -11,26 +11,8 @@ ENV RUNNING_IN_DOCKER=1 \
 ARG TARGETARCH
 
 # Installing system dependencies
-RUN add-apt-repository ppa:mozillateam/ppa && \
-	apt-get update && \
-    apt-get install -y --no-install-recommends gcc ffmpeg fonts-noto exiftool && \
-	apt-get install -y --no-install-recommends firefox-esr && \
-    ln -s /usr/bin/firefox-esr /usr/bin/firefox
-
-ARG GECKODRIVER_VERSION=0.36.0
-
-RUN if [ $(uname -m) = "aarch64" ]; then \
-        GECKODRIVER_ARCH=linux-aarch64; \
-    else \
-        GECKODRIVER_ARCH=linux64; \
-    fi && \
-    wget https://github.com/mozilla/geckodriver/releases/download/v${GECKODRIVER_VERSION}/geckodriver-v${GECKODRIVER_VERSION}-${GECKODRIVER_ARCH}.tar.gz && \
-    tar -xvzf geckodriver* -C /usr/local/bin && \
-    chmod +x /usr/local/bin/geckodriver && \
-    rm geckodriver-v* && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
-
+RUN	apt-get update && \
+    apt-get install -y --no-install-recommends gcc ffmpeg fonts-noto exiftool python3-tk 
 
 # Poetry and runtime
 FROM base AS runtime
@@ -59,11 +41,21 @@ COPY ./src/ .
 RUN /poetry-venv/bin/poetry install --only main --no-cache
 
 
+# Run as non-root user to avoid permission issues with mounted volumes (see #342)
+# The base image already has an 'ubuntu' user at UID/GID 1000.
+# Ensure directories that need write access at runtime are writable.
+RUN chown 1000:1000 /app && \
+    chown -R 1000:1000 /app/.venv/lib/python3.12/site-packages/seleniumbase/drivers/ && \
+    mkdir -p /app/local_archive /app/secrets /tmp/archive && \
+    chown -R 1000:1000 /app/local_archive /app/secrets /tmp/archive
+
 # Update PATH to include virtual environment binaries
 # Allowing entry point to run the application directly with Python
 ENV VIRTUAL_ENV=/app/.venv \
     PATH="/app/.venv/bin:$PATH"
 
+USER 1000
+
 ENTRYPOINT ["python3", "-m", "auto_archiver"]
 
 # should be executed with 2 volumes (3 if local_storage is used)

docker-compose.yaml

@@ -6,6 +6,9 @@ services:
       context: .
       dockerfile: Dockerfile
     container_name: auto-archiver
+    # Override user to match host UID/GID and avoid permission issues on volumes.
+    # Set USER_ID and GROUP_ID env vars, or defaults to 1000:1000.
+    user: "${USER_ID:-1000}:${GROUP_ID:-1000}"
     volumes:
       - ./secrets:/app/secrets
       - ./local_archive:/app/local_archive

docs/scripts/scripts.py

@@ -47,7 +47,6 @@ def generate_module_docs():
 
     for module in sorted(ModuleFactory().available_modules(), key=lambda x: (x.requires_setup, x.name)):
         # generate the markdown file from the __manifest__.py file.
-
         manifest = module.manifest
         for type in manifest["type"]:
             modules_by_type.setdefault(type, []).append(module)
@@ -64,6 +63,27 @@ def generate_module_docs():
 """
         steps_str = "\n".join(f"  {t}s:\n  - {module.name}" for t in manifest["type"])
 
+        if manifest.get("autodoc_dropins"):
+            loaded_module = module.load({})
+            dropins = loaded_module.load_dropins()
+            dropin_str = "\n##### Available Dropins\n"
+            for dropin in dropins:
+                if not (ddoc := dropin.documentation()):
+                    continue
+                dropin_str += f"\n###### {ddoc.get('name', dropin.__name__)}\n\n"
+                dropin_str += f"{ddoc.get('description')}\n\n"
+                if ddoc.get("site"):
+                    dropin_str += f"**Site**: {ddoc['site']}\n\n"
+                if dauth := ddoc.get("authentication"):
+                    dropin_str += "**YAML configuration**:\n"
+                    dropin_auth_yaml = "authentication:\n...\n"
+                    for site, creds in dauth.items():
+                        dropin_auth_yaml += f"  {site}:\n"
+                        for k, v in creds.items():
+                            dropin_auth_yaml += f'    {k}: "{v}"\n'
+                    dropin_str += f"```{{code}} yaml\n{dropin_auth_yaml}...\n```\n"
+            readme_str += dropin_str
+
         if not manifest["configs"]:
             config_string = f"# No configuration options for {module.name}.*\n"
         else:

docs/source/development/developer_guidelines.md

@@ -21,7 +21,7 @@ This allows you to run the auto-archiver without the `poetry run` prefix.
 ### Optional Development Packages
 
 Install development packages (used for unit tests etc.) using:
-`poetry install -with dev`
+`poetry install --with dev`
 
 
 ```{toctree}
@@ -33,4 +33,4 @@ docs
 release
 settings_page
 style_guide
-```
+```

docs/source/development/style_guide.md

@@ -50,7 +50,7 @@ Note not all warnings can be fixed automatically.
 
 Most fixes are safe, but some non-standard practices such as dynamic loading are not picked up by linters. Ensure you check any modifications by this before committing them.
 ```shell
-make ruff-fix
+make ruff-clean
 ```
 
 **Changing Configurations ⚙️**
@@ -67,4 +67,4 @@ One example is to extend the selected rules for linting the `pyproject.toml` fil
 extend-select = ["B"]
 ```
 
-Then re-run the `make ruff-check` command to see the new rules in action.
+Then re-run the `make ruff-check` command to see the new rules in action.

docs/source/development/testing.md

@@ -3,14 +3,14 @@
 `pytest` is used for testing. There are two main types of tests:
 
 1. 'core' tests which should be run on every change
-2. 'download' tests which hit the network. These tests will do things like make API calls (e.g. Twitter, Bluesky etc.) and should be run regularly to make sure that APIs have not changed.
+2. 'download' tests which hit the network. These tests will do things like make API calls (e.g. Twitter, Bluesky etc.) and should be run regularly to make sure that APIs have not changed, they take longer.
 
 
 ## Running Tests 
 
-1. Make sure you've installed the dev dependencies with `pytest install --with dev`
+1. Make sure you've installed the dev dependencies with `poetry install --with dev`
 2. Tests can be run as follows:
-```
+```{code} bash
 #### Command prefix of 'poetry run' removed here for simplicity
 # run core tests
 pytest -ra -v -m "not download"
@@ -18,4 +18,15 @@ pytest -ra -v -m "not download"
 pytest -ra -v -m "download"
 # run all tests
 pytest -ra -v
-```
+
+
+# run a specific test file
+pytest -ra -v tests/test_file.py
+# run a specific test function
+pytest -ra -v tests/test_file.py::test_function_name
+```
+
+3. Some tests require environment variables to be set. You can use the example `tests/.env.test.example` file as a template. Copy it to `tests/.env.test` and fill in the required values. This file will be loaded automatically by `pytest`.
+```{code} bash
+cp tests/.env.test.example tests/.env.test
+```

docs/source/flow_overview.md

@@ -8,7 +8,7 @@ The archiver archives web pages using the following workflow
 4. **Formatter** creates a report from all the archived content (HTML, PDF, ...)
 5. **Database** knows what's been archived and also stores the archive result (spreadsheet, CSV, or just the console)
 
-Each step in the workflow is handled by 'modules' that interact with the data in different ways. For example, the Twitter Extractor Module would extract information from the Twitter website. The Screenshot Enricher Module will take screenshots of the given page. See the [core modules page](core_modules.md) for an overview of all the modules that are available.
+Each step in the workflow is handled by 'modules' that interact with the data in different ways. For example, the Twitter Extractor Module would extract information from the Twitter website. The AntiBot Module will download HTML and take screenshots of the given page. See the [core modules page](core_modules.md) for an overview of all the modules that are available.
 
 Auto-archiver must have at least one module defined for each step of the workflow. This is done by setting the [configuration](installation/configurations.md) for your auto-archiver instance.
 

docs/source/how_to/03_logging.md

@@ -0,0 +1,104 @@
+# Keeping Logs
+
+Auto Archiver's logs can be helpful for debugging problematic archiving processes. This guide shows you how to use the logs configuration.
+
+## Setting up logging
+
+Logging settings can be set on the command line or using the orchestration config file ([learn more](../installation/configuration)). A special `logging` section defines the logging options.
+
+#### Enabling or Disabling Logging
+
+Logging to the console is enabled by default. If you want to globally disable Auto Archiver's logging, then you can set `enabled: false` in your `logging` config file:
+
+```{code} yaml
+:caption: orchestration.yaml
+...
+logging:
+   enabled: false
+...
+```
+
+```{note}
+This will disable all logs from Auto Archiver, but it does not disable logs for other tools that the Auto Archiver uses (for example: yt-dlp, firefox or ffmpeg). These logs will still appear in your console.
+```
+
+#### Logging Level
+
+There are 7 logging levels in total, with 5 of them used in this tool. They are: `DEBUG`, `INFO`, `SUCCESS`, `WARNING` and `ERROR`. If you select a level, only that and higher (more serious) levels will be included. `DEBUG` is the most verbose, while `ERROR` is the least verbose. 
+
+Change the warning level by setting the value in your orchestration config file:
+
+```{code} yaml
+:caption: orchestration.yaml
+
+...
+logging:
+    level: DEBUG # or INFO / WARNING / ERROR
+...
+```
+
+For normal usage, it is recommended to use the `INFO` level, or if you prefer quieter logs with less information, you can use the `WARNING` level. If you encounter issues with the archiving, then it's recommended to enable the `DEBUG` level.
+
+```{note} To learn about all logging levels, see the [loguru documentation](https://loguru.readthedocs.io/en/stable/api/logger.html)
+```
+
+### Logging Format
+By default, the console logs are formatted in a human-readable way and the file logs are formatted in JSON. This is new from version 1.1.1. If you want to change the format of the console logs to JSON too you can set the `format:` option in your logging settings. 
+
+```{code} yaml
+:caption: orchestration.yaml
+
+logging:
+    format: json
+```
+
+When the Auto Archiver is writing logs it will include context about specific tasks, so if you are archiving a URL from a Google Sheet, both the URL (and a unique `trace_id` for that URL's archiving attempt) and the Spreadsheet name and row will be included in the logs. This is useful for debugging and understanding what the Auto Archiver is doing.
+
+Using JSON allows you to easily parse the logs and extract specific information, tools like [`jq`](https://jqlang.org/) can be used to filter and search through the logs.
+
+### Logging to a file
+
+As default, auto-archiver will log to the console. But if you wish to store your logs for future reference, or you are running the auto-archiver from within code a implementation, then you may wish to enable file logging. This can be done by setting the `file:` config value in the logging settings.
+
+**Rotation:** For file logging, you can choose to 'rotate' your log files (creating new log files) so they do not get too large. Change this by setting the 'rotation' option in your logging settings. For a full list of rotation options, see the [loguru docs](https://loguru.readthedocs.io/en/stable/overview.html#easier-file-logging-with-rotation-retention-compression).
+
+```{code} yaml
+:caption: orchestration.yaml
+
+logging:
+    ...
+    file: /my/log/file.log
+    rotation: 1 day
+```
+
+### Logging each level to a different file
+If you want to log each level to a different file, you can do this by setting the `each_level_in_separate_file:` option to `true` and also setting your `file:` name, a new file will be created for each of the 5 levels used, by appending the `0_level` name to the file like so `your_file.log.1_error`. In this case the `level:` option is ignored, and all levels will be logged. 
+
+
+```{code} yaml
+:caption: orchestration.yaml
+
+logging:
+    each_level_in_separate_file: true
+    file: /my/logs/file.log 
+```
+This will create the following files:
+- `/my/logs/file.log.1_debug`
+- `/my/logs/file.log.2_info`
+- `/my/logs/file.log.3_success`
+- `/my/logs/file.log.4_warning`
+- `/my/logs/file.log.5_error`
+
+### Full logging example
+
+The below example logs only `DEBUG` logs to the console and to the file `/my/file.log`, rotating that file once per week:
+
+```{code} yaml
+:caption: orchestration.yaml
+
+logging:
+    level: DEBUG
+    format: json
+    file: /my/file.log
+    rotation: 1 week
+```

docs/source/how_to/05_upgrading_to_1_1_0.md

@@ -0,0 +1,54 @@
+# Upgrading from v1.0.1
+
+```{note} This how-to is only relevant for people who used Auto Archiver before June 2025 (versions prior to 1.1.0).
+
+If you are new to Auto Archiver, then you are already using the latest configuration format and this how-to is not relevant for you.
+```
+
+Versions 1.1.0+ of Auto Archiver has breaking changes in the configuration format, which means earlier configuration formats will not work without slight modifications.
+
+
+## Dropping `vk_extractor` module
+We have dropped the `vk_extractor` because of problems in a project we relied on. You will need to remove it from your configuration file, otherwise you will see an error like:
+
+```{code} console
+Module 'vk_extractor' not found. Are you sure it's installed/exists?
+```
+
+## Dropping `screenshot_enricher` module
+We have dropped the `screenshot_enricher` module because a new `antibot_extractor_enricher` (see below) module replaces its functionality more robustly and with less dependency hassle on geckodriver/firefox. You will need to remove it from your configuration file, otherwise you will see an error like:
+
+```{code} console
+Module 'screenshot_enricher' not found. Are you sure it's installed/exists?
+```
+
+
+## New `antibot_extractor_enricher` module and VkDropin
+We have added a new [`antibot_extractor_enricher`](../modules/autogen/extractor/antibot_extractor_enricher.md) module that uses a computer-controlled browser to extract content from websites that use anti-bot measures. You can add it to your configuration file like this:
+
+```{code} yaml
+steps:
+  extractors:
+    - antibot_extractor_enricher
+
+  # or alternatively, if you want to use it as an enricher:
+  enrichers:
+    - antibot_extractor_enricher
+```
+
+It will take a full page screenshot, a PDF capture, extract HTML source code, and any other relevant media. 
+
+It comes with Dropins that we will be adding and maintaining. 
+
+> Dropin: A module with site-specific behaviours that is loaded automatically. You don't need to add them to your configuration steps for them to run. Sometimes they need `authentication` configurations though.
+
+One such Dropin is the VkDropin which uses this automated browser to access VKontakte (VK) pages. You should add a username/password to the configuration file if you get authentication blocks from VK, to do so use the [authentication settings](authentication_how_to.md):
+
+```{code} yaml
+authentication:
+  vk.com:
+    username: your_username
+    password: your_password
+```
+
+See all available Dropins in [the source code](https://github.com/bellingcat/auto-archiver/tree/main/src/auto_archiver/modules/antibot_extractor_enricher/dropins). Usually each Dropin needs its own authentication settings, similarly to the VkDropin.

docs/source/how_to/logging.md

@@ -1,71 +0,0 @@
-# Keeping Logs
-
-Auto Archiver's logs can be helpful for debugging problematic archiving processes. This guide shows you how to use the logs to 
-
-## Setting up logging
-
-Logging settings can be set on the command line or using the orchestration config file ([learn more](../installation/configuration)). A special `logging` section defines the logging options.
-
-#### Enabling or Disabling Logging
-
-Logging to the console is enabled by default. If you want to globally disable Auto Archiver's logging, then you can set `enabled: false` in your `logging` config:
-
-```{code} yaml
-
-...
-logging:
-   enabled: false
-...
-```
-
-```{note}
-This will disable all logs from Auto Archiver, but it does not disable logs for other tools that the Auto Archiver uses (for example: yt-dlp, firefox or ffmpeg). These logs will still appear in your console.
-```
-
-#### Logging Level
-
-There are 7 logging levels in total, with 4 commonly used levels. They are: `DEBUG`, `INFO`, `WARNING` and `ERROR`.
-
-Change the warning level by setting the value in your orchestration config file:
-
-```{code} yaml
-:caption: orchestration.yaml
-
-...
-logging:
-    level: DEBUG # or INFO / WARNING / ERROR
-...
-```
-
-For normal usage, it is recommended to use the `INFO` level, or if you prefer quieter logs with less information, you can use the `WARNING` level. If you encounter issues with the archiving, then it's recommended to enable the `DEBUG` level.
-
-```{note} To learn about all logging levels, see the [loguru documentation](https://loguru.readthedocs.io/en/stable/api/logger.html)
-```
-
-### Logging to a file
-
-As default, auto-archiver will log to the console. But if you wish to store your logs for future reference, or you are running the auto-archiver from within code a implementation, then you may with to enable file logging. This can be done by setting the `file:` config value in the logging settings.
-
-**Rotation:** For file logging, you can choose to 'rotate' your log files (creating new log files) so they do not get too large. Change this by setting the 'rotation' option in your logging settings. For a full list of rotation options, see the [loguru docs](https://loguru.readthedocs.io/en/stable/overview.html#easier-file-logging-with-rotation-retention-compression).
-
-```{code} yaml
-:caption: orchestration.yaml
-
-logging:
-    ...
-    file: /my/log/file.log
-    rotation: 1 day
-```
-
-### Full logging example
-
-The below example logs only `WARNING` logs to the console and to the file `/my/file.log`, rotating that file once per week:
-
-```{code} yaml
-:caption: orchestration.yaml
-
-logging:
-    level: WARNING
-    file: /my/file.log
-    rotation: 1 week
-```

docs/source/installation/authentication.md

@@ -9,8 +9,8 @@ There are two main use cases for authentication:
 ```{note}
 
 The Authentication framework currently only works with the following modules:
-* Generic Extractor
-* Screenshot Enricher
+* [Generic Extractor](../modules/autogen/extractor/generic_extractor.md) - the main module for extracting content from websites
+* [Antibot Extractor/Enricher](../modules/autogen/extractor/antibot_extractor_enricher.md)
 
 To authenticate for WACZ archiving, see the instructions on the [](../modules/autogen/enricher/wacz_extractor_enricher.md) page.
 ```
@@ -34,7 +34,8 @@ You can save your authentication information directly inside your orchestration
 
 ```{note} 
 
-The Username & Password, and API settings only work with the Generic Extractor. Other modules (like the screenshot enricher) can only use the `cookies` options. Furthermore, many sites can still detect bots and block username/password logins. Twitter/X and YouTube are two prominent ones that block username/password logging.
+Currently, the Username & Password, and API settings only work with the Generic and Antibot Extractors. Furthermore, many sites can still detect bots and block username/password logins. Twitter/X and YouTube are two prominent ones that block username/password logins.
+
 
 One of the 'Cookies' options is recommended for the most robust archiving, but it still isn't guaranteed to work.
 ```
@@ -52,12 +53,12 @@ authentication:
       username: myusername
       password: 123
     
-    facebook.com:
-       cookie: single_cookie
+   facebook.com:
+      cookie: single_cookie
 
-    othersite.com:
-       api_key: 123
-       api_secret: 1234
+   othersite.com:
+      api_key: 123
+      api_secret: 1234
   
 ```
 

docs/source/installation/example_scripts/ubuntu_24_server_install.md

@@ -0,0 +1,160 @@
+
+### Bash script for Ubuntu 24 Server install
+
+> NOTE: this script has not been tested by the maintainers and results from the personal experience of a user. It is meant as a guide and not an out of the box script, as you will see it's aimed at a custom branches, users, and features like the Geckodriver which are removed as of version 1.0.2. 
+
+This acts as a handy guide on all requirements. This is built and tested on the 29th of May 2025 on Ubuntu Server 24.04.2 LTS (which is the current latest LTS)
+
+```bash
+#!/bin/sh
+
+# I usually run steps manually as logged in with the user: dave
+# which the application runs under which makes debugging easier
+
+cd ~
+sudo apt update -y
+sudo apt upgrade -y
+
+# Clone only my latest branch
+git clone -b v1-test --single-branch https://github.com/djhmateer/auto-archiver
+
+mkdir ~/auto-archiver/secrets
+sudo chown -R dave ~/auto-archiver
+
+sudo apt update -y
+sudo apt upgrade -y
+
+## Python 3.12.3 comes with Ubuntu 24.04.2
+
+# Poetry install 2.1.3 on 2nd June 25
+curl -sSL https://install.python-poetry.org | python3 -
+
+# had to restart here.. 
+sudo reboot
+
+# C++ compiler so pdqhash will install next
+sudo apt install build-essential python3-dev -y
+
+cd auto-archiver
+
+poetry install
+
+# FFMpeg
+# 6.1.1-3ubuntu5 on 2nd June 25
+sudo apt install ffmpeg -y
+
+## Firefox
+# 139.0+build2-0ubuntu0.24.04.1~mt1 on 2nd Jun 25
+# 16th Jun - don't need anymore as using Chrome in antibot
+# cd ~
+# sudo add-apt-repository ppa:mozillateam/ppa -y
+
+# echo '
+# Package: *
+# Pin: release o=LP-PPA-mozillateam
+# Pin-Priority: 1001
+# ' | sudo tee /etc/apt/preferences.d/mozilla-firefox
+
+# echo 'Unattended-Upgrade::Allowed-Origins:: "LP-PPA-mozillateam:${distro_codename}";' | sudo tee /etc/apt/apt.conf.d/51unattended-upgrades-firefox
+
+# sudo apt install firefox -y
+
+wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
+
+# Chrome
+cd ~
+# got problems here - fixed below
+# 137.0.7151.103 on 16th Jun 2025
+sudo dpkg -i google-chrome-stable_current_amd64.deb
+
+# fix dependencies on install above
+sudo apt-get install -f
+
+# had to click a lot on UI to get going.
+# to test
+# google-chrome
+
+## Gecko driver
+# check version numbers for new ones
+# https://github.com/mozilla/geckodriver/releases/
+wget https://github.com/mozilla/geckodriver/releases/download/v0.36.0/geckodriver-v0.36.0-linux64.tar.gz
+tar -xvzf geckodriver*
+chmod +x geckodriver
+sudo mv geckodriver /usr/local/bin/
+rm geckodriver*
+
+# Fonts so selenium via firefox can render other languages eg Burmese
+sudo apt install fonts-noto -y
+
+# Docker
+# Add Docker's official GPG key:
+sudo apt-get update -y
+sudo apt-get install ca-certificates curl -y
+sudo install -m 0755 -d /etc/apt/keyrings
+sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
+sudo chmod a+r /etc/apt/keyrings/docker.asc
+
+# Add the repository to Apt sources:
+echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
+  $(. /etc/os-release && echo "${UBUNTU_CODENAME:-$VERSION_CODENAME}") stable" | \
+  sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+
+sudo apt-get update -y
+
+sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -y
+
+# add dave user to docker group 
+sudo usermod -aG docker $USER
+
+# reboot otherwise can't pull images
+
+# https://github.com/webrecorder/browsertrix-crawler
+# https://hub.docker.com/r/webrecorder/browsertrix-crawler/tags
+# 1.6.2 on 4th Jun 2025
+docker pull webrecorder/browsertrix-crawler:latest
+
+# exif
+sudo apt install libimage-exiftool-perl -y
+
+
+## CRON run every minute
+# the cron job running as user dave will execute the shell script
+# I have many scripts running from cron_11 upwards.
+# patch in the correct number
+sudo chmod +x ~/auto-archiver/scripts/cron_15.sh
+
+# don't want service to run until a reboot otherwise problems with Gecko driver
+sudo service cron stop
+
+# runs the script every minute
+# notice put in a # to disable so will have to manually start it.
+cat <<EOT >> run-auto-archive
+#*/1 * * * * dave /home/dave/auto-archiver/scripts/cron_15.sh
+EOT
+
+sudo mv run-auto-archive /etc/cron.d
+sudo chown root /etc/cron.d/run-auto-archive
+sudo chmod 600 /etc/cron.d/run-auto-archive
+
+# Helper alias 'c' to open the above file
+echo "alias c='sudo vim /etc/cron.d/run-auto-archive'" >> ~/.bashrc
+
+# secrets folder copy
+# I run dev from:
+# \\wsl.localhost\Ubuntu-24.04\home\dave\code\auto-archiver\secrets\
+
+# orchestration.yaml - for aa config
+# service_account - for google spreadsheet
+# anon.session - for telethon so don't have to type in phone number
+# profile.tar.gz - for wacz to have a logged in profile for facebook, x.com and instagram to get data
+
+# Youtube - POT Tokens
+# https://github.com/Brainicism/bgutil-ytdlp-pot-provider
+docker run --name bgutil-provider --restart unless-stopped -d -p 4416:4416 brainicism/bgutil-ytdlp-pot-provider
+
+
+# test run
+cd ~/auto-archiver
+
+poetry run python src/auto_archiver --config secrets/orchestration-aa-demo-main.yaml
+```

docs/source/installation/installation.md

@@ -51,11 +51,14 @@ After this, you're ready to set up your [your configuration file](configurations
 If using the local installation method, you will also need to install the following dependencies locally:
 
 1.[ffmpeg](https://www.ffmpeg.org/) - for handling of downloaded videos
-2. [firefox](https://www.mozilla.org/en-US/firefox/new/) and [geckodriver](https://github.com/mozilla/geckodriver/releases) on a path folder like `/usr/local/bin` - for taking webpage screenshots with the screenshot enricher
-3. (optional) [fonts-noto](https://fonts.google.com/noto) to deal with multiple unicode characters during selenium/geckodriver's screenshots: `sudo apt install fonts-noto -y`.
+<!-- 2. [firefox](https://www.mozilla.org/en-US/firefox/new/) and [geckodriver](https://github.com/mozilla/geckodriver/releases) on a path folder like `/usr/local/bin` - for taking webpage screenshots with the screenshot enricher -->
+3. (optional) [fonts-noto](https://fonts.google.com/noto) to deal with multiple unicode characters during selenium screenshots: `sudo apt install fonts-noto -y`.
 4. [Browsertrix Crawler docker image](https://hub.docker.com/r/webrecorder/browsertrix-crawler) for the WACZ enricher/archiver
 
 
+### Custom installation scripts
+- [Ubuntu 24 Server Install by @djhmateer](example_scripts/ubuntu_24_server_install.md) - a WYSIWYG example script from a user who set up the Auto Archiver on a fresh Ubuntu 24 server.
+
 
 ## Developer Install
 

docs/source/modules/extractor.md

@@ -4,8 +4,9 @@ Extractor modules are used to extract the content of a given URL. Typically, one
 
 Extractors that are able to extract content from a wide range of websites include:
 1. Generic Extractor: parses videos and images on sites using the powerful yt-dlp library.
-2. Wayback Machine Extractor: sends pages to the Wayback machine for archiving, and stores the link.
-3. WACZ Extractor: runs a web browser to 'browse' the URL and save a copy of the page in WACZ format. 
+2. Antibot Extractor: uses a headless browser to bypass bot detection and extract content.
+3. WACZ Extractor: runs a web browser to 'browse' the URL and save a copy of the page in WACZ format.
+4. Wayback Machine Extractor: sends pages to the Wayback machine for archiving, and stores the archived link.
 
 ```{include} autogen/extractor.md
 ```

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [project]
 name = "auto-archiver"
-version = "1.0.1"
+version = "1.2.3"
 description = "Automatically archive links to videos, images, and social media content from Google Sheets (and more)."
 
 requires-python = ">=3.10,<3.13"
@@ -27,7 +27,6 @@ dependencies = [
     "bs4 (>=0.0.0)",
     "loguru (>=0.0.0)",
     "ffmpeg-python (>=0.0.0)",
-    "selenium (>=0.0.0)",
     "telethon (>=0.0.0)",
     "google-api-python-client (>=0.0.0)",
     "google-auth-httplib2 (>=0.0.0)",
@@ -51,12 +50,15 @@ dependencies = [
     "retrying (>=0.0.0)",
     "rich-argparse (>=1.6.0,<2.0.0)",
     "ruamel-yaml (>=0.18.10,<0.19.0)",
-    "rfc3161-client (>=1.0.1,<2.0.0)",
-    "cryptography (>44.0.1,<45.0.0)",
+    "rfc3161-client (>=1.0.5)",
+    "cryptography (>=46.0.3)",
     "opentimestamps (>=0.4.5,<0.5.0)",
     "bgutil-ytdlp-pot-provider (>=1.0.0)",
-    "yt-dlp[curl-cffi,default] (>=2025.5.22,<2026.0.0)",
+    "yt-dlp[curl-cffi,default] (>=2025.5.22)",
     "secretstorage (>=3.3.3,<4.0.0)",
+    "seleniumbase (>=4.36.4,<5.0.0)",
+    "pyautogui (>=0.9.54,<0.10.0)",
+    "pyperclip (>=1.9.0)", 
 ]
 
 [tool.poetry.group.dev.dependencies]
@@ -64,7 +66,7 @@ pytest = "^8.3.4"
 autopep8 = "^2.3.1"
 pytest-loguru = "^0.4.0"
 pytest-mock = "^3.14.0"
-ruff = "^0.9.10"
+ruff = "^0.15.2"
 pre-commit = "^4.1.0"
 
 [tool.poetry.group.docs.dependencies]

scripts/settings/package.json

@@ -13,10 +13,10 @@
     "@dnd-kit/sortable": "^10.0.0",
     "@emotion/react": "latest",
     "@emotion/styled": "latest",
-    "@mui/icons-material": "^6.4.7",
-    "@mui/material": "latest",
-    "react": "19.0.0",
-    "react-dom": "19.0.0",
+    "@mui/icons-material": "^7.1.1",
+    "@mui/material": "^7.1.1",
+    "react": "19.1.0",
+    "react-dom": "19.1.0",
     "react-markdown": "^10.0.0",
     "yaml": "^2.7.0"
   },

scripts/settings/src/App.tsx

@@ -31,7 +31,7 @@ import {
   Stack,
   Button,
 } from '@mui/material';
-import Grid from '@mui/material/Grid2';
+import Grid from '@mui/material/Grid';
 
 import { parseDocument, Document, YAMLSeq, YAMLMap, Scalar } from 'yaml'
 import StepCard from './StepCard';

scripts/settings/src/StepCard.tsx

@@ -25,7 +25,7 @@ import {
     Typography,
     InputAdornment,
 } from '@mui/material';
-import Grid from '@mui/material/Grid2';
+import Grid from '@mui/material/Grid';
 import DragIndicatorIcon from '@mui/icons-material/DragIndicator';
 import Visibility from '@mui/icons-material/Visibility';
 import VisibilityOff from '@mui/icons-material/VisibilityOff';

scripts/telegram_setup.py

@@ -14,7 +14,7 @@ You will need to provide your phone number and a 2FA code the first time you run
 
 import os
 from telethon.sync import TelegramClient
-from loguru import logger
+from auto_archiver.utils.custom_logger import logger
 
 
 # Create a
@@ -24,4 +24,4 @@ SESSION_FILE = "secrets/anon-insta"
 
 os.makedirs("secrets", exist_ok=True)
 with TelegramClient(SESSION_FILE, API_ID, API_HASH) as client:
-    logger.success(f"New session file created: {SESSION_FILE}.session")
+    logger.success(f"new session file created: {SESSION_FILE}.session")

src/auto_archiver/core/base_module.py

@@ -7,7 +7,7 @@ from tempfile import TemporaryDirectory
 from auto_archiver.utils import url as UrlUtil
 from auto_archiver.core.consts import MODULE_TYPES as CONF_MODULE_TYPES
 
-from loguru import logger
+from auto_archiver.utils.custom_logger import logger
 
 if TYPE_CHECKING:
     from .module import ModuleFactory
@@ -98,12 +98,11 @@ class BaseModule(ABC):
         """
         # TODO: think about if/how we can deal with sites that have multiple domains (main one is x.com/twitter.com)
         # for now the user must enter them both, like "x.com,twitter.com" in their config. Maybe we just hard-code?
-
-        site = UrlUtil.domain_for_url(site).removeprefix("www.")
+        domain = UrlUtil.domain_for_url(site).removeprefix("www.")
         # add the 'www' version of the site to the list of sites to check
         authdict = {}
 
-        for to_try in [site, f"www.{site}"]:
+        for to_try in [site, domain, f"www.{domain}"]:
             if to_try in self.authentication:
                 authdict.update(self.authentication[to_try])
                 break
@@ -111,9 +110,9 @@ class BaseModule(ABC):
         # do a fuzzy string match just to print a warning - don't use it since it's insecure
         if not authdict:
             for key in self.authentication.keys():
-                if key in site or site in key:
+                if key in domain or domain in key:
                     logger.debug(
-                        f"Could not find exact authentication information for site '{site}'. \
+                        f"Could not find exact authentication information for '{domain}'. \
 did find information for '{key}' which is close, is this what you meant? \
 If so, edit your authentication settings to make sure it exactly matches."
                     )

src/auto_archiver/core/config.py

@@ -10,7 +10,7 @@ from ruamel.yaml import YAML, CommentedMap
 import json
 import os
 
-from loguru import logger
+from auto_archiver.utils.custom_logger import logger
 
 from copy import deepcopy
 from auto_archiver.core.consts import MODULE_TYPES
@@ -118,8 +118,7 @@ class DefaultValidatingParser(argparse.ArgumentParser):
         """
         Override of error to format a nicer looking error message using logger
         """
-        logger.error("Problem with configuration file (tip: use --help to see the available options):")
-        logger.error(message)
+        logger.error(f"Problem with configuration file (tip: use --help to see the available options): \n{message}")
         self.exit(2)
 
     def parse_known_args(self, args=None, namespace=None):
@@ -136,8 +135,7 @@ class DefaultValidatingParser(argparse.ArgumentParser):
                     try:
                         self._check_value(action, action.default)
                     except argparse.ArgumentError as e:
-                        logger.error(f"You have an invalid setting in your configuration file ({action.dest}):")
-                        logger.error(e)
+                        logger.error(f"You have an invalid setting in your configuration file ({action.dest}):\n {e}")
                         exit()
 
         return super().parse_known_args(args, namespace)

src/auto_archiver/core/extractor.py

@@ -8,14 +8,16 @@ Factory method to initialize an extractor instance based on its name.
 
 from __future__ import annotations
 from abc import abstractmethod
+from contextlib import suppress
 import mimetypes
 import os
 import requests
-from loguru import logger
+from auto_archiver.utils.custom_logger import logger
 from retrying import retry
 import re
 
 from auto_archiver.core import Metadata, BaseModule
+from auto_archiver.utils.url import get_media_url_best_quality
 
 
 class Extractor(BaseModule):
@@ -70,17 +72,29 @@ class Extractor(BaseModule):
         return ""
 
     @retry(wait_random_min=500, wait_random_max=3500, stop_max_attempt_number=5)
-    def download_from_url(self, url: str, to_filename: str = None, verbose=True) -> str:
+    def download_from_url(self, url: str, to_filename: str = None, verbose=True, try_best_quality=False) -> str:
         """
         downloads a URL to provided filename, or inferred from URL, returns local filename
+        Warning: if try_best_quality is True, it will return a tuple of (filename, best_quality_url) if the download was successful.
         """
+        if any(url.startswith(x) for x in ["blob:", "data:"]):
+            return None, url if try_best_quality else None
+
+        if try_best_quality:
+            with suppress(Exception):
+                # Attempt to download the original URL
+                best_quality_url = get_media_url_best_quality(url)
+                orig_download = self.download_from_url(best_quality_url, to_filename, verbose)
+                if orig_download:
+                    return orig_download, best_quality_url
+
         if not to_filename:
             to_filename = url.split("/")[-1].split("?")[0]
             if len(to_filename) > 64:
                 to_filename = to_filename[-64:]
         to_filename = os.path.join(self.tmp_dir, to_filename)
         if verbose:
-            logger.debug(f"downloading {url[0:50]=} {to_filename=}")
+            logger.debug(f"Downloading {to_filename=}")
         headers = {
             "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36"
         }
@@ -98,10 +112,14 @@ class Extractor(BaseModule):
             with open(to_filename, "wb") as f:
                 for chunk in d.iter_content(chunk_size=8192):
                     f.write(chunk)
+            if try_best_quality:
+                return to_filename, url
             return to_filename
 
         except requests.RequestException as e:
             logger.warning(f"Failed to fetch the Media URL: {e}")
+        if try_best_quality:
+            return None, url
 
     @abstractmethod
     def download(self, item: Metadata) -> Metadata | False:

src/auto_archiver/core/media.py

@@ -11,7 +11,7 @@ from dataclasses import dataclass, field
 from dataclasses_json import dataclass_json, config
 import mimetypes
 
-from loguru import logger
+from auto_archiver.utils.custom_logger import logger
 
 
 @dataclass_json  # annotation order matters
@@ -86,7 +86,7 @@ class Media:
     @property  # getter .mimetype
     def mimetype(self) -> str:
         if not self.filename or len(self.filename) == 0:
-            logger.warning(f"cannot get mimetype from media without filename: {self}")
+            logger.warning(f"Cannot get mimetype from media without filename: {self}")
             return ""
         if not self._mimetype:
             self._mimetype = mimetypes.guess_type(self.filename)[0]
@@ -116,13 +116,12 @@ class Media:
         # self.is_video() should be used together with this method
         try:
             streams = ffmpeg.probe(self.filename, select_streams="v")["streams"]
-            logger.warning(f"STREAMS FOR {self.filename} {streams}")
+            logger.debug(f"Streams for {self.filename}: {streams}")
             return any(s.get("duration_ts", 0) > 0 for s in streams)
         except Error:
             return False  # ffmpeg errors when reading bad files
         except Exception as e:
-            logger.error(e)
-            logger.error(traceback.format_exc())
+            logger.error(f"{e}: {traceback.format_exc()}")
             try:
                 fsize = os.path.getsize(self.filename)
                 return fsize > 20_000

src/auto_archiver/core/metadata.py

@@ -17,7 +17,7 @@ from dataclasses_json import dataclass_json
 import datetime
 from urllib.parse import urlparse
 from dateutil.parser import parse as parse_dt
-from loguru import logger
+from auto_archiver.utils.custom_logger import logger
 
 from .media import Media
 
@@ -96,7 +96,7 @@ class Metadata:
 
     def is_empty(self) -> bool:
         meaningfull_ids = set(self.metadata.keys()) - set(
-            ["_processed_at", "url", "total_bytes", "total_size", "archive_duration_seconds"]
+            ["_processed_at", "url", "original_url", "total_bytes", "total_size", "archive_duration_seconds"]
         )
         return not self.is_success() and len(self.media) == 0 and len(meaningfull_ids) == 0
 

src/auto_archiver/core/module.py

@@ -16,7 +16,7 @@ import sys
 from importlib.util import find_spec
 import os
 from os.path import join
-from loguru import logger
+from auto_archiver.utils.custom_logger import logger
 import auto_archiver
 from auto_archiver.core.consts import DEFAULT_MANIFEST, MANIFEST_FILE, SetupError
 

src/auto_archiver/core/orchestrator.py

@@ -15,9 +15,11 @@ import traceback
 from copy import copy
 
 from rich_argparse import RichHelpFormatter
-from loguru import logger
+from auto_archiver.utils.custom_logger import format_for_human_readable_console, logger
 import requests
 
+from auto_archiver.utils.misc import random_str
+
 from .metadata import Metadata, Media
 from auto_archiver.version import __version__
 from .config import (
@@ -34,7 +36,7 @@ from .config import (
 from .module import ModuleFactory, LazyBaseModule
 from . import validators, Feeder, Extractor, Database, Storage, Formatter, Enricher
 from .consts import MODULE_TYPES, SetupError
-from auto_archiver.utils.url import check_url_or_raise
+from auto_archiver.utils.url import check_url_or_raise, clean
 
 if TYPE_CHECKING:
     from .base_module import BaseModule
@@ -249,7 +251,7 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
             action="store",
             dest="logging.level",
             choices=["INFO", "DEBUG", "ERROR", "WARNING"],
-            help="the logging level to use",
+            help="the logging level to use for the standard output and file logging",
             default="INFO",
             type=str.upper,
         )
@@ -264,6 +266,14 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
             default=None,
         )
 
+        parser.add_argument(
+            "--logging.each_level_in_separate_file",
+            action="store",
+            dest="logging.each_level_in_separate_file",
+            help="if set, writes each logging level to a separate file (ignores --logging.level), you must also set --logging.file. Each level will have a dedicate logs file matching your <file>.debug, <file>.info, etc.",
+            default=False,
+        )
+
     def add_individual_module_args(
         self, modules: list[LazyBaseModule] = None, parser: argparse.ArgumentParser = None
     ) -> None:
@@ -333,11 +343,32 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
 
         # add other logging info
         if self.logger_id is None:  # note - need direct comparison to None since need to consider falsy value 0
-            self.logger_id = logger.add(sys.stderr, level=logging_config["level"])
-            if log_file := logging_config["file"]:
-                logger.add(log_file) if not logging_config["rotation"] else logger.add(
-                    log_file, rotation=logging_config["rotation"]
+            use_level = logging_config["level"]
+            self.logger_id = logger.add(
+                sys.stderr,
+                level=use_level,
+                catch=True,
+                format="<level>{extra[serialized]}</level>"
+                if logging_config.get("format", "").lower() == "json"
+                else format_for_human_readable_console(),
+            )
+
+            rotation = logging_config["rotation"]
+            log_file = logging_config["file"]
+
+            if logging_config.get("each_level_in_separate_file"):
+                assert logging_config["file"], (
+                    "You must set --logging.file if you want to use --logging.each_level_in_separate_file"
                 )
+                for i, level in enumerate(["DEBUG", "INFO", "SUCCESS", "WARNING", "ERROR"], start=1):
+                    logger.add(
+                        f"{log_file}.{i}_{level.lower()}",
+                        filter=lambda rec, lvl=level: rec["level"].name == lvl,
+                        rotation=rotation,
+                        format="{extra[serialized]}",
+                    )
+            elif log_file:
+                logger.add(log_file, rotation=rotation, level=use_level, format="{extra[serialized]}")
 
     def install_modules(self, modules_by_type):
         """
@@ -445,13 +476,9 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
                 update_cmd = "`docker pull bellingcat/auto-archiver:latest`"
             else:
                 update_cmd = "`pip install --upgrade auto-archiver`"
-            logger.warning("")
-            logger.warning("********* IMPORTANT: UPDATE AVAILABLE ********")
             logger.warning(
-                f"A new version of auto-archiver is available (v{latest_version}, you have v{current_version})"
+                f"\n********* IMPORTANT: UPDATE AVAILABLE ********\nA new version of auto-archiver is available (v{latest_version}, you have v{current_version})\nMake sure to update to the latest version using: {update_cmd}\n"
             )
-            logger.warning(f"Make sure to update to the latest version using: {update_cmd}")
-            logger.warning("")
 
     def setup(self, args: list):
         """
@@ -501,7 +528,7 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
             self.setup(args)
             return self.feed()
         except Exception as e:
-            logger.error(e)
+            logger.error(f"{e}: {traceback.format_exc()}")
             exit(1)
 
     def cleanup(self) -> None:
@@ -513,10 +540,12 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
         url_count = 0
         for feeder in self.feeders:
             for item in feeder:
-                yield self.feed_item(item)
-                url_count += 1
+                with logger.contextualize(url=item.get_url(), trace=random_str(12)):
+                    logger.info("Started processing")
+                    yield self.feed_item(item)
+                    url_count += 1
 
-        logger.success(f"Processed {url_count} URL(s)")
+        logger.info(f"Processed {url_count} URL(s)")
         self.cleanup()
 
     def feed_item(self, item: Metadata) -> Metadata:
@@ -534,13 +563,13 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
             return self.archive(item)
         except KeyboardInterrupt:
             # catches keyboard interruptions to do a clean exit
-            logger.warning(f"caught interrupt on {item=}")
+            logger.warning("Caught interrupt")
             for d in self.databases:
                 d.aborted(item)
             self.cleanup()
             exit()
         except Exception as e:
-            logger.error(f"Got unexpected error on item {item}: {e}\n{traceback.format_exc()}")
+            logger.error(f"Got unexpected error: {e}\n{traceback.format_exc()}")
             for d in self.databases:
                 if isinstance(e, AssertionError):
                     d.failed(item, str(e))
@@ -568,16 +597,17 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
         try:
             check_url_or_raise(original_url)
         except ValueError as e:
-            logger.error(f"Error archiving URL {original_url}: {e}")
+            logger.error(f"Error archiving: {e}")
             raise e
 
         # 1 - sanitize - each archiver is responsible for cleaning/expanding its own URLs
-        url = original_url
+        url = clean(original_url)
         for a in self.extractors:
             url = a.sanitize_url(url)
 
         result.set_url(url)
         if original_url != url:
+            logger.debug(f"Sanitized URL to {url}")
             result.set("original_url", original_url)
 
         # 2 - notify start to DBs, propagate already archived if feature enabled in DBs
@@ -592,25 +622,25 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
                 try:
                     d.done(cached_result, cached=True)
                 except Exception as e:
-                    logger.error(f"ERROR database {d.name}: {e}: {traceback.format_exc()}")
+                    logger.error(f"Database {d.name}: {e}: {traceback.format_exc()}")
             return cached_result
 
         # 3 - call extractors until one succeeds
         for a in self.extractors:
-            logger.info(f"Trying extractor {a.name} for {url}")
+            logger.info(f"Trying extractor {a.name}")
             try:
                 result.merge(a.download(result))
                 if result.is_success():
                     break
             except Exception as e:
-                logger.error(f"ERROR archiver {a.name}: {e}: {traceback.format_exc()}")
+                logger.error(f"Extractor {a.name}: {e}: {traceback.format_exc()}")
 
         # 4 - call enrichers to work with archived content
         for e in self.enrichers:
             try:
                 e.enrich(result)
             except Exception as exc:
-                logger.error(f"ERROR enricher {e.name}: {exc}: {traceback.format_exc()}")
+                logger.error(f"Enricher {e.name}: {exc}: {traceback.format_exc()}")
 
         # 5 - store all downloaded/generated media
         result.store(storages=self.storages)
@@ -629,7 +659,7 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
             try:
                 d.done(result)
             except Exception as e:
-                logger.error(f"ERROR database {d.name}: {e}: {traceback.format_exc()}")
+                logger.error(f"Database {d.name}: {e}: {traceback.format_exc()}")
 
         return result
 

src/auto_archiver/core/storage.py

@@ -24,7 +24,7 @@ from abc import abstractmethod
 from typing import IO
 import os
 
-from loguru import logger
+from auto_archiver.utils.custom_logger import logger
 from slugify import slugify
 
 from auto_archiver.utils.misc import random_str

src/auto_archiver/modules/antibot_extractor_enricher/__manifest__.py

@@ -0,0 +1,53 @@
+{
+    "name": "Antibot Extractor/Enricher",
+    "type": ["extractor", "enricher"],
+    "requires_setup": False,
+    "dependencies": {"python": ["loguru", "seleniumbase", "yt_dlp"], "bin": ["ffmpeg"]},
+    "configs": {
+        "save_to_pdf": {
+            "default": False,
+            "type": "bool",
+            "help": "save a PDF snapshot of the page.",
+        },
+        "max_download_images": {
+            "default": 50,
+            "help": "maximum number of images to download from the page (0 = no download, inf = no limit).",
+        },
+        "max_download_videos": {
+            "default": 50,
+            "help": "maximum number of videos to download from the page (0 = no download, inf = no limit).",
+        },
+        "user_data_dir": {
+            "default": "secrets/antibot_user_data",
+            "help": "Path to the user data directory for the webdriver. This is used to persist browser state, such as cookies and local storage. If you use the docker deployment, this path will be appended with `_docker` that is because the folder cannot be shared between the host and the container due to user permissions.",
+        },
+        "detect_auth_wall": {
+            "default": True,
+            "type": "bool",
+            "help": "detect if the page is behind an authentication wall (e.g. login required) and skip it. disable if you want to archive pages where logins are required.",
+        },
+        "proxy": {
+            "default": None,
+            "help": "proxy to use for the webdriver, Format: 'SERVER:PORT' or 'USER:PASS@SERVER:PORT'",
+        },
+    },
+    "autodoc_dropins": True,
+    "description": """
+    Uses a browser controlled by SeleniumBase to capture HTML, media, and screenshots/PDFs of a web page, by bypassing anti-bot measures like Cloudflare's Turnstile or Google Recaptcha.
+	
+	> ⚠️ Still in trial development, please report any issues or suggestions via [GitHub Issues](https://github.com/bellingcat/auto-archiver/issues).
+	
+    ### Features
+	- Extracts the HTML source code of the page.
+    - Takes full-page screenshots of web pages.
+	- Takes full-page PDF snapshots of web pages.
+	- Downloads images and videos from the page, excluding specified file extensions.
+
+    ### Notes
+	- Using a proxy affects Cloudflare Turnstile captcha handling, so it is recommended to use a proxy only if necessary.
+
+	### Dropins
+	This module uses sub-modules called Dropins for specific sites that allow it to handle anti-bot measures and custom Login flows. You don't need to include the dropins in your configuration, but you do need to add authentication credentials if you want to overcome login walls on those sites, see detailed instructions for each Dropin below.
+
+    """,
+}

src/auto_archiver/modules/antibot_extractor_enricher/antibot_extractor_enricher.py

@@ -0,0 +1,329 @@
+import base64
+import math
+import os
+import sys
+import traceback
+from urllib.parse import urljoin
+import glob
+import importlib.util
+
+from auto_archiver.utils.custom_logger import logger
+import selenium
+from seleniumbase import SB
+
+from auto_archiver.core import Extractor, Enricher, Metadata, Media
+from auto_archiver.modules.antibot_extractor_enricher.dropin import Dropin
+from auto_archiver.modules.antibot_extractor_enricher.dropins.default import DefaultDropin
+from auto_archiver.utils.misc import random_str
+from auto_archiver.utils.url import is_relevant_url
+from auto_archiver.utils.deletion_detection import detect_deletion, flag_as_deleted
+
+
+class AntibotExtractorEnricher(Extractor, Enricher):
+    def setup(self) -> None:
+        self.agent = "cool"
+        if "linux" in sys.platform or "win32" in sys.platform:
+            self.agent = None  # Use the default UserAgent
+
+        # parse configuration options
+        if self.max_download_images == "inf":
+            self.max_download_images = math.inf
+        else:
+            self.max_download_images = int(self.max_download_images)
+
+        if self.max_download_videos == "inf":
+            self.max_download_videos = math.inf
+        else:
+            self.max_download_videos = int(self.max_download_videos)
+
+        self._prepare_user_data_dir()
+
+        self.dropins = self.load_dropins()
+
+    def load_dropins(self):
+        dropins = []
+
+        # TODO: add user-configurable drop-ins via config like generic_extractor
+        dropins_dir = os.path.join(os.path.dirname(__file__), "dropins")
+        for file_path in glob.glob(os.path.join(dropins_dir, "*.py")):
+            if os.path.basename(file_path).startswith("_"):
+                continue  # skip __init__.py or private modules
+            module_name = f"auto_archiver.modules.antibot_extractor_enricher.dropins.{os.path.splitext(os.path.basename(file_path))[0]}"
+            spec = importlib.util.spec_from_file_location(module_name, file_path)
+            module = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(module)
+            for attr in dir(module):
+                obj = getattr(module, attr)
+                if getattr(obj, "__module__", None) != module.__name__:
+                    continue  # Skip imported modules/classes/functions
+                if isinstance(obj, type) and issubclass(obj, Dropin):
+                    dropins.append(obj)
+        logger.debug(f"Loaded drop-in classes: {', '.join([d.__name__ for d in dropins])}")
+        return dropins
+
+    def sanitize_url(self, url: str) -> str:
+        for dropin in self.dropins:
+            if dropin.suitable(url):
+                return dropin.sanitize_url(url)
+        return url
+
+    def download(self, item: Metadata) -> Metadata:
+        result = Metadata()
+        result.merge(item)
+        if self.enrich(result):
+            result.status = "antibot"
+            return result
+
+    def _prepare_user_data_dir(self):
+        if self.user_data_dir:
+            in_docker = os.environ.get("RUNNING_IN_DOCKER")
+            if in_docker:
+                self.user_data_dir = self.user_data_dir.rstrip(os.path.sep) + "_docker"
+            os.makedirs(self.user_data_dir, exist_ok=True)
+
+    def enrich(self, to_enrich: Metadata, custom_data_dir: bool = True) -> bool:
+        if to_enrich.get_media_by_id("html_source_code"):
+            logger.info("Antibot has already been executed, skipping.")
+            return True
+        using_user_data_dir = self.user_data_dir if custom_data_dir else None
+        url = to_enrich.get_url()
+
+        # Use xvfb in Docker environments where no display is available
+        use_xvfb = bool(os.environ.get("RUNNING_IN_DOCKER"))
+
+        try:
+            with SB(
+                uc=True,
+                agent=self.agent,
+                headed=None,
+                user_data_dir=using_user_data_dir,
+                proxy=self.proxy,
+                xvfb=use_xvfb,
+            ) as sb:
+                logger.info(f"Selenium browser is up with agent {self.agent}, opening url...")
+                sb.uc_open_with_reconnect(url, 4)
+
+                logger.debug("Handling CAPTCHAs for...")
+                sb.uc_gui_handle_cf()
+                sb.uc_gui_click_rc()  # NB: using handle instead of click breaks some sites like reddit, for now we separate here but can have dropins deciding this in the future
+
+                dropin = self._get_suitable_dropin(url, sb)
+                if not dropin.open_page(url):
+                    # Check for deletion indicators
+                    page_title = sb.get_title()
+                    html_source = sb.get_page_source()
+                    deletion_info = detect_deletion(html_content=html_source, page_title=page_title, url=url)
+                    if deletion_info:
+                        flag_as_deleted(to_enrich, deletion_info)
+                        return to_enrich
+                    logger.warning("Failed to open drop-in page (not detected as deleted)")
+                    return False
+
+                if self.detect_auth_wall and (dropin.hit_auth_wall() and self._hit_auth_wall(sb)):
+                    logger.warning("Skipping since auth wall or CAPTCHA was detected")
+                    return False
+
+                sb.wait_for_ready_state_complete()
+                sb.sleep(1)  # margin for the page to load completely
+
+                page_title = sb.get_title()
+                html_source = sb.get_page_source()
+
+                # Check if the page indicates content was deleted
+                deletion_info = detect_deletion(html_content=html_source, page_title=page_title, url=url)
+                if deletion_info:
+                    flag_as_deleted(to_enrich, deletion_info)
+
+                to_enrich.set_title(page_title)
+                self._enrich_html_source_code(sb, to_enrich)
+
+                self._enrich_full_page_screenshot(sb, to_enrich)
+                if self.save_to_pdf:
+                    self._enrich_full_page_pdf(sb, to_enrich)
+
+                downloaded_images, downloaded_videos = dropin.add_extra_media(to_enrich)
+
+                self._enrich_download_media(
+                    sb,
+                    to_enrich,
+                    js_css_selector=dropin.js_for_image_css_selectors(),
+                    max_media=self.max_download_images - downloaded_images,
+                )
+                self._enrich_download_media(
+                    sb,
+                    to_enrich,
+                    js_css_selector=dropin.js_for_video_css_selectors(),
+                    max_media=self.max_download_videos - downloaded_videos,
+                )
+                logger.info("Completed")
+
+            return to_enrich
+        except selenium.common.exceptions.SessionNotCreatedException as e:
+            if custom_data_dir:  # the retry logic only works once
+                logger.error(
+                    f"Session not created error: {e}. Please remove the user_data_dir {self.user_data_dir} and try again, will retry without user data dir though."
+                )
+                return self.enrich(to_enrich, custom_data_dir=False)
+            raise e  # re-raise
+        except Exception as e:
+            logger.error(f"Runtime error: {e}: {traceback.format_exc()}")
+            return False
+
+    def _get_suitable_dropin(self, url: str, sb: SB):
+        """
+        Returns a suitable drop-in for the given URL.
+        This method checks if the URL is suitable for any of the registered drop-ins.
+        """
+        for dropin in self.dropins:
+            if dropin.suitable(url):
+                logger.debug(f"Using drop-in {dropin.__name__}")
+                return dropin(sb, self)
+
+        return DefaultDropin(sb, self)
+
+    def _hit_auth_wall(self, sb: SB) -> bool:
+        """
+        Tries to detect if the currently loaded page is an auth/login wall.
+        Returns True if login is likely required.
+        """
+        # TODO: improve this detection logic, currently it is very basic and may not cover all cases
+
+        # Common URL patterns
+        current_url = sb.get_current_url().lower()
+        if any(kw in current_url for kw in ["login", "signin", "signup", "register", "captcha"]):
+            return True
+
+        # Common visible text markers
+        login_keywords = [
+            "sign up or log in",
+            "log in to continue",
+            "sign in to continue",
+            "login required",
+            "please log in",
+            "please sign up",
+            "please sign in",
+            "login to access",
+            "sign up to access",
+            "register to access",
+            "captcha verification",
+        ]
+        for word in login_keywords + [w.capitalize() for w in login_keywords]:
+            if sb.is_text_visible(word):
+                return True
+
+        # Common title markers
+        title = sb.get_title().lower()
+        if any(
+            kw in title
+            for kw in [
+                "just a moment...",
+                "tiktok - make your day",
+                "um momento...",
+                "log in",
+                "sign in",
+                "sign up",
+                "register",
+                "captcha",
+                "verification required",
+                "access denied",
+            ]
+        ):
+            return True
+
+        # Common form fields
+        elements = [
+            "input[type='password']",
+            "input[type='email']",
+            "input[type='username']",
+            "input[type='phone']",
+            "input[name='username']",
+            "input[name='email']",
+            "input[name='password']",
+            "input[name='login']",
+        ]
+        if any(sb.is_element_visible(el) for el in elements):
+            return True
+
+        return False
+
+    @logger.catch
+    def _enrich_html_source_code(self, sb: SB, to_enrich: Metadata):
+        """
+        Enriches the HTML source code of the Metadata object.
+        This method is called by the enrich method.
+        """
+        source = sb.get_page_source()
+
+        html_filename = os.path.join(self.tmp_dir, f"source{random_str(6)}.html")
+        with open(html_filename, "w", encoding="utf-8") as f:
+            f.write(source)
+
+        to_enrich.add_media(Media(filename=html_filename), id="html_source_code")
+
+    @logger.catch
+    def _enrich_full_page_screenshot(self, sb: SB, to_enrich: Metadata):
+        """
+        Enriches the full page screenshot of the Metadata object.
+        This method is called by the enrich method.
+        """
+        start_size = sb.get_window_size()
+        w, h = start_size["width"], start_size["height"]
+
+        x = max(sb.execute_script("return document.documentElement.scrollWidth"), w)
+        y = min(max(sb.execute_script("return document.documentElement.scrollHeight"), h), 25_000)
+        logger.debug(f"Setting window size to {x}x{y} for full page screenshot.")
+        sb.set_window_size(x, y)
+
+        screen_filename = os.path.join(self.tmp_dir, f"screenshot{random_str(6)}.png")
+        sb.save_screenshot(screen_filename)
+
+        to_enrich.add_media(Media(filename=screen_filename), id="screenshot")
+
+    @logger.catch
+    def _enrich_full_page_pdf(self, sb: SB, to_enrich: Metadata):
+        """
+        Enriches the full page PDF of the Metadata object.
+        This method is called by the enrich method.
+        """
+        result = sb.driver.execute_cdp_cmd("Page.printToPDF", {"printBackground": True, "landscape": False})
+
+        pdf_data = base64.b64decode(result["data"])
+
+        pdf_filename = os.path.join(self.tmp_dir, f"pdf{random_str(6)}.pdf")
+        with open(pdf_filename, "wb") as f:
+            f.write(pdf_data)
+
+        to_enrich.add_media(Media(filename=pdf_filename), id="pdf")
+
+    @logger.catch
+    def _enrich_download_media(self, sb: SB, to_enrich: Metadata, js_css_selector: str, max_media: int):
+        """
+        Downloads media from the page and adds them to the Metadata object.
+        This method is called by the enrich method.
+        """
+        if max_media == 0:
+            return
+        url = to_enrich.get_url()
+        all_urls = set()
+        logger.debug(f"Extracting media for {js_css_selector=}")
+
+        try:
+            sources = sb.execute_script(js_css_selector)
+        except selenium.common.exceptions.JavascriptException as e:
+            logger.error(f"Error executing JavaScript selector {js_css_selector}: {e}")
+            return
+
+        # js_for_css_selectors
+        for src in sources:
+            if len(all_urls) >= max_media:
+                logger.debug(f"Reached max download limit of {max_media} images/videos.")
+                break
+            if not is_relevant_url(src):
+                continue
+            full_src = urljoin(url, src)
+            if full_src not in all_urls:
+                filename, full_src = self.download_from_url(full_src, try_best_quality=True)
+                if not filename:
+                    continue
+                all_urls.add(full_src)
+                to_enrich.add_media(Media(filename=filename, properties={"url": full_src}))

src/auto_archiver/modules/antibot_extractor_enricher/captcha_services/.gitignore

@@ -0,0 +1 @@
+*.py

src/auto_archiver/modules/antibot_extractor_enricher/dropin.py

@@ -0,0 +1,173 @@
+import json
+import os
+import traceback
+from typing import Mapping
+from auto_archiver.utils.custom_logger import logger
+from seleniumbase import SB
+import yt_dlp
+
+from auto_archiver.core import Extractor, Media, Metadata
+from auto_archiver.utils.misc import ydl_entry_to_filename
+
+
+class Dropin:
+    """
+    A class to handle drop-in functionality for the antibot extractor enricher module.
+    This class is designed to be a base class for drop-ins that can handle specific websites.
+    """
+
+    @staticmethod
+    def documentation() -> Mapping[str, str]:
+        """
+        Each Dropin should auto-document itself with this method.
+        Return dictionary can include:
+        - 'name': A string representing the name of the dropin.
+        - 'description': A string describing the functionality of the dropin.
+        - 'site': A string representing the site this dropin is for.
+        - 'authentication': A dictionary with authentication example for the site.
+
+        """
+        return {}
+
+    def __init__(self, sb: SB, extractor: Extractor):
+        """
+        Initialize the Dropin with the given SeleniumBase instance.
+
+        :param sb: An instance of the SeleniumBase class that this drop-in will use.
+        :param extractor: An instance of the Extractor class that this drop-in will use.
+        """
+        self.sb: SB = sb
+        self.extractor: Extractor = extractor
+
+    @staticmethod
+    def suitable(url: str) -> bool:
+        """
+        Check if the URL is suitable for processing with this dropin.
+        :param url: The URL to check.
+        :return: True if the URL is suitable for processing, False otherwise.
+        """
+        raise NotImplementedError("This method should be implemented in the subclass")
+
+    @staticmethod
+    def sanitize_url(url: str) -> str:
+        """
+        Used to clean URLs before processing them.
+        """
+        return url
+
+    @staticmethod
+    def images_selectors() -> str:
+        """
+        CSS selector to find images in the HTML page
+        """
+        return "img"
+
+    @staticmethod
+    def video_selectors() -> str:
+        """
+        CSS selector to find videos in the HTML page.
+        """
+        return "video, source"
+
+    def js_for_image_css_selectors(self) -> str:
+        """
+        A configurable JS script that receives a css selector from the dropin itself and returns an array of Image elements according to the selection.
+
+        You can overwrite this instead of `images_selector` for more control over scraped images.
+        """
+        if not self.images_selectors():
+            return "return [];"
+        safe_selector = json.dumps(self.images_selectors())
+        return f"""
+            return Array.from(document.querySelectorAll({safe_selector})).map(el => el.src || el.href).filter(Boolean);
+        """
+
+    def js_for_video_css_selectors(self) -> str:
+        """
+        A configurable JS script that receives a css selector from the dropin itself and returns an array of Video elements according to the selection.
+
+        You can overwrite this instead of `video_selector` for more control over scraped videos.
+        """
+        if not self.video_selectors():
+            return "return [];"
+        safe_selector = json.dumps(self.video_selectors())
+        return f"""
+            return Array.from(document.querySelectorAll({safe_selector})).map(el => el.src || el.href).filter(Boolean);
+        """
+
+    def open_page(self, url) -> bool:
+        """
+        Make sure the page is opened, even if it requires authentication, captcha solving, etc.
+        :param url: The URL to open.
+        :return: True if success, False otherwise.
+        """
+        raise NotImplementedError("This method should be implemented in the subclass")
+
+    def add_extra_media(self, to_enrich: Metadata) -> tuple[int, int]:
+        """
+        Extract image and/or video data from the currently open post with SeleniumBase. Media is added to the `to_enrich` Metadata object.
+        :return: A tuple (number of Images added, number of Videos added).
+        """
+        return 0, 0
+
+    def hit_auth_wall(self) -> bool:
+        """
+        Custom check to see if the current page is behind an authentication wall, if True is returned the default global auth wall detector is used instead. If false, no auth wall is detected and the page is considered open.
+        """
+        return True
+
+    def _get_username_password(self, site) -> tuple[str, str]:
+        """
+        Get the username and password for the site from the extractor's auth data.
+        :return: A tuple (username, password).
+        """
+        auth = self.extractor.auth_for_site(site)
+        username = auth.get("username", "")
+        password = auth.get("password", "")
+        if not username or not password:
+            raise ValueError(f"{site} authentication requires a username and password.")
+        return username, password
+
+    def _download_videos_with_ytdlp(self, video_urls: list[str], to_enrich: Metadata) -> int:
+        """
+        Download videos using yt-dlp.
+        :param video_urls: List of video URLs to download.
+        :return: The number of videos downloaded.
+        """
+        if type(self.extractor.max_download_videos) is int:
+            video_urls = video_urls[: self.extractor.max_download_videos]
+
+        if not video_urls:
+            return 0
+
+        ydl_options = [
+            "-o",
+            os.path.join(self.extractor.tmp_dir, "%(id)s.%(ext)s"),
+            "--quiet",
+            "--no-playlist",
+            "--no-write-subs",
+            "--no-write-auto-subs",
+            "--postprocessor-args",
+            "ffmpeg:-bitexact",
+            "--max-filesize",
+            "1000M",  # Limit to 1GB per video
+        ]
+        *_, validated_options = yt_dlp.parse_options(ydl_options)
+        downloaded = 0
+        with yt_dlp.YoutubeDL(validated_options) as ydl:
+            for url in video_urls:
+                try:
+                    logger.debug(f"Downloading video from url: {url}")
+                    info = ydl.extract_info(url, download=True)
+                    filename = ydl_entry_to_filename(ydl, info)
+                    if not filename:  # Failed to download video.
+                        continue
+                    media = Media(filename)
+                    for x in ["duration", "original_url", "fulltitle", "description", "upload_date"]:
+                        if x in info:
+                            media.set(x, info[x])
+                    to_enrich.add_media(media)
+                    downloaded += 1
+                except Exception as e:
+                    logger.error(f"Download failed: {e} {traceback.format_exc()}")
+        return downloaded

src/auto_archiver/modules/antibot_extractor_enricher/dropins/default.py

@@ -0,0 +1,14 @@
+from auto_archiver.modules.antibot_extractor_enricher.dropin import Dropin
+
+
+class DefaultDropin(Dropin):
+    """
+    A default fallback drop-in class for handling generic cases in the antibot extractor enricher module.
+    """
+
+    @staticmethod
+    def suitable(url: str) -> bool:
+        return False
+
+    def open_page(self, url) -> bool:
+        return True

src/auto_archiver/modules/antibot_extractor_enricher/dropins/linkedin.py

@@ -0,0 +1,74 @@
+from typing import Mapping
+from auto_archiver.utils.custom_logger import logger
+from auto_archiver.modules.antibot_extractor_enricher.dropin import Dropin
+
+
+class LinkedinDropin(Dropin):
+    """
+    A class to handle LinkedIn drop-in functionality for the antibot extractor enricher module.
+    """
+
+    @staticmethod
+    def documentation() -> Mapping[str, str]:
+        return {
+            "name": "Linkedin Dropin",
+            "description": "Handles LinkedIn pages/posts and requires authentication to access most content but will still be useful without it. The first time you login to a new IP, LinkedIn may require an email verification code, you can do a manual login first and then it won't ask for it again.",
+            "site": "linkedin.com",
+            "authentication": {
+                "linkedin.com": {
+                    "username": "email address or phone number",
+                    "password": "password",
+                }
+            },
+        }
+
+    notifications_css_selector = 'a[href*="linkedin.com/notifications"]'
+
+    @staticmethod
+    def suitable(url: str) -> bool:
+        return "linkedin.com" in url
+
+    def js_for_image_css_selectors(self) -> str:
+        get_all_css = "main img:not([src*='profile-displayphoto']):not([src*='profile-framedphoto'])"
+        get_first_css = (
+            "main img[src*='profile-framedphoto'], main img[src*='profile-displayphoto'], main img[src*='company-logo']"
+        )
+
+        return f"""
+            const all = Array.from(document.querySelectorAll("{get_all_css}")).map(el => el.src || el.href).filter(Boolean);
+            const profile = document.querySelector("{get_first_css}");
+            return all.concat(profile?.src || profile?.href || []).filter(Boolean);
+        """
+
+    @staticmethod
+    def video_selectors() -> str:
+        # usually videos are from blob: but running the generic extractor should handle that
+        return "main video"
+
+    def open_page(self, url) -> bool:
+        if not self.sb.is_element_present(self.notifications_css_selector):
+            self._login()
+            if url != self.sb.get_current_url():
+                self.sb.open(url)
+        return True
+
+    @logger.catch
+    def _login(self) -> bool:
+        if self.sb.is_text_visible("Sign in to view more content"):
+            self.sb.click_link_text("Sign in", timeout=2)
+            self.sb.wait_for_ready_state_complete()
+        else:
+            self.sb.open("https://www.linkedin.com/login")
+            self.sb.wait_for_ready_state_complete()
+
+        username, password = self._get_username_password("linkedin.com")
+        logger.debug("Logging in to Linkedin with username: {}", username)
+        self.sb.type("#username", username)
+        self.sb.type("#password", password)
+        self.sb.click_if_visible("#password-visibility-toggle", timeout=0.5)
+        self.sb.click("button[type='submit']")
+        self.sb.wait_for_ready_state_complete()
+        # TODO: on suspicious login, LinkedIn may require an email verification code
+
+        if not self.sb.is_element_present(self.notifications_css_selector):
+            self.sb.click_if_visible('button[aria-label="Dismiss"]', timeout=0.5)

src/auto_archiver/modules/antibot_extractor_enricher/dropins/reddit.py

@@ -0,0 +1,92 @@
+from contextlib import suppress
+from typing import Mapping
+from auto_archiver.core.metadata import Metadata
+from auto_archiver.modules.antibot_extractor_enricher.dropin import Dropin
+
+from auto_archiver.utils.custom_logger import logger
+
+
+class RedditDropin(Dropin):
+    """
+    A class to handle Reddit drop-in functionality for the antibot extractor enricher module.
+    """
+
+    def documentation() -> Mapping[str, str]:
+        return {
+            "name": "Reddit Dropin",
+            "description": "Handles Reddit posts and works without authentication until Reddit flags your IP, so authentication is advised.",
+            "site": "reddit.com",
+            "authentication": {
+                "reddit.com": {
+                    "username": "email address or username",
+                    "password": "password",
+                }
+            },
+        }
+
+    @staticmethod
+    def suitable(url: str) -> bool:
+        return "reddit.com" in url
+
+    @staticmethod
+    def images_selectors() -> str:
+        return "shreddit-post img"
+
+    @staticmethod
+    def video_selectors() -> str:
+        return "shreddit-post video, shreddit-post source"
+
+    def open_page(self, url) -> bool:
+        if self.sb.is_text_visible("You've been blocked by network security."):
+            self._login()
+            if url != self.sb.get_current_url():
+                self.sb.open(url)
+        return True
+
+    @logger.catch
+    def _login(self):
+        self.sb.click_link_text("Log in")
+        self.sb.wait_for_ready_state_complete()
+        self._close_cookies_banner()
+
+        username, password = self._get_username_password("reddit.com")
+        logger.debug("Logging in to Reddit with username: {}", username)
+
+        self.sb.type("#login-username", username)
+        self.sb.type("#login-password", password)
+
+        elem = self.sb.find_element("button.login")
+        self.sb.execute_script("arguments[0].scrollIntoView(true);", elem)
+        self.sb.slow_click("button.login")
+        self.sb.wait_for_ready_state_complete()
+
+        if "https://www.reddit.com/login/" in self.sb.get_current_url():
+            self.sb.sleep(5)
+            self.sb.wait_for_ready_state_complete()
+
+        if self.sb.is_text_visible("You've been blocked by network security."):
+            self.sb.click_link_text("Log in")
+            self.sb.wait_for_ready_state_complete()
+            if self.sb.is_text_visible("Welcome back"):
+                logger.debug("Login successful")
+                self.sb.click_if_visible("this link")
+
+    def _close_cookies_banner(self):
+        with suppress(Exception):  # selenium.common.exceptions.JavascriptException
+            self.sb.execute_script("""
+                document
+                    .querySelector("reddit-cookie-banner")
+                    .shadowRoot.querySelector("faceplate-dialog")
+                    .querySelector("#accept-all-cookies-button button")
+                    .click()
+            """)
+
+    @logger.catch
+    def add_extra_media(self, to_enrich: Metadata) -> tuple[int, int]:
+        filtered_urls = self.sb.execute_script(rf"""
+            return [...document.querySelectorAll("{self.video_selectors()}")]
+            .map(el => el.src || el.href)
+            .filter(url => url && /\.(m3u8|mpd|ism)$/.test(url));
+        """)
+        logger.debug("Found {} video URLs", len(filtered_urls))
+        return 0, self._download_videos_with_ytdlp(filtered_urls, to_enrich)

src/auto_archiver/modules/antibot_extractor_enricher/dropins/tiktok.py

@@ -0,0 +1,56 @@
+from contextlib import suppress
+from typing import Mapping
+
+from auto_archiver.utils.custom_logger import logger
+from auto_archiver.modules.antibot_extractor_enricher.dropin import Dropin
+
+
+class TikTokDropin(Dropin):
+    """
+    A class to handle TikTok drop-in functionality for the antibot extractor enricher module.
+
+    """
+
+    def documentation() -> Mapping[str, str]:
+        return {
+            "name": "TikTok Dropin",
+            "description": "Handles TikTok posts and works without authentication.\nNOTE: This dropin is highly susceptible to TikTok's bot detection mechanisms and may not work reliably if you reuse the same IP. The GenericExtractor is recommended for TikTok posts, as it handles video/image download more reliable. In the future we plan to implement better anti captcha measures for this dropin.",
+            "site": "tiktok.com",
+        }
+
+    @staticmethod
+    def suitable(url: str) -> bool:
+        return "tiktok.com" in url
+
+    @staticmethod
+    def images_selectors() -> str:
+        return '[data-e2e="detail-photo"] img'
+
+    @staticmethod
+    def video_selectors() -> str:
+        return None  # TikTok videos should be handled by the generic extractor
+
+    def open_page(self, url) -> bool:
+        self.sb.wait_for_ready_state_complete()
+        self._close_cookies_banner()
+        # TODO: implement login logic
+        if url != self.sb.get_current_url():
+            return False
+        if self.sb.is_text_visible("Video currently unavailable"):
+            logger.debug("Video may have been removed or is private.")
+            return False
+        return True
+
+    def hit_auth_wall(self) -> bool:
+        return False  # TikTok does not require authentication for public posts
+
+    def _close_cookies_banner(self):
+        with suppress(Exception):  # selenium.common.exceptions.JavascriptException
+            self.sb.execute_script("""
+                document
+                    .querySelector("tiktok-cookie-banner")
+                    .shadowRoot.querySelector("faceplate-dialog")
+                    .querySelector("button")
+                    .click()
+            """)
+        self.sb.click_if_visible("Skip")

src/auto_archiver/modules/antibot_extractor_enricher/dropins/vk.py

@@ -0,0 +1,92 @@
+import re
+from typing import Mapping
+
+from auto_archiver.core.metadata import Metadata
+from auto_archiver.modules.antibot_extractor_enricher.dropin import Dropin
+
+from auto_archiver.utils.custom_logger import logger
+
+
+class VkDropin(Dropin):
+    """
+    A class to handle VK drop-in functionality for the antibot extractor enricher module.
+    """
+
+    WALL_PATTERN = re.compile(r"(wall.{0,1}\d+_\d+)")
+    VIDEO_PATTERN = re.compile(r"(video.{0,1}\d+_\d+(?:_\w+)?)")
+    CLIP_PATTERN = re.compile(r"(clip.{0,1}\d+_\d+)")
+    PHOTO_PATTERN = re.compile(r"(photo.{0,1}\d+_\d+)")
+
+    def documentation() -> Mapping[str, str]:
+        return {
+            "name": "VKontakte Dropin",
+            "description": "Handles VKontakte posts and works without authentication for some content.",
+            "site": "vk.com",
+            "authentication": {
+                "vk.com": {
+                    "username": "phone number with country code",
+                    "password": "password",
+                }
+            },
+        }
+
+    @staticmethod
+    def suitable(url: str) -> bool:
+        return "vk.com" in url
+
+    @staticmethod
+    def sanitize_url(url: str) -> str:
+        """
+        Transforms modal URLs like 'https://vk.com/page_name?w=wall-123456_7890' to 'https://vk.com/wall-123456_7890'
+        """
+        for pattern in [VkDropin.WALL_PATTERN, VkDropin.VIDEO_PATTERN, VkDropin.CLIP_PATTERN, VkDropin.PHOTO_PATTERN]:
+            match = pattern.search(url)
+            if match:
+                return f"https://vk.com/{match.group(1)}"
+        return url
+
+    def open_page(self, url) -> bool:
+        if self.sb.is_text_visible("Sign in to VK"):
+            if self._login():
+                self.sb.open(url)
+        return True
+
+    @logger.catch
+    def _login(self) -> bool:
+        # TODO: test method, because current tests work without a login
+        self.sb.open("https://vk.com")
+        self.sb.wait_for_ready_state_complete()
+        if "/feed" in self.sb.get_current_url():
+            logger.debug("Already logged in to VK.")
+            return True
+
+        # need to login
+        username, password = self._get_username_password("vk.com")
+        logger.debug("Logging in to VK with username: {}", username)
+
+        self.sb.click('[data-testid="enter-another-way"]', timeout=10)
+        self.sb.clear('input[name="login"][type="tel"]', by="css selector", timeout=10)
+        self.sb.type('input[name="login"][type="tel"]', username, by="css selector", timeout=10)
+        self.sb.click('button[type="submit"]')
+
+        # TODO: handle captcha if it appears
+        # if sb.is_element_visible("img.vkc__CaptchaPopup__image"):
+        #     captcha_url = sb.get_attribute("img.vkc__CaptchaPopup__image", "src")
+        #     print("CAPTCHA detected:", captcha_url)
+        #     image_url = sb.get_attribute("img[alt*='captcha']", "src")
+        #     solution = solve_captcha(image_url)
+        #     sb.type("input#captcha-text, input[name='captcha']", solution)
+        #     sb.click("button[type='submit']")
+
+        self.sb.type('input[name="password"]', password, timeout=15)
+        self.sb.click('button[type="submit"]')
+        self.sb.wait_for_ready_state_complete(timeout=10)
+        self.sb.wait_for_element("body", timeout=10)
+        # self.sb.sleep(2)
+        return "/feed" in self.sb.get_current_url()
+
+    @logger.catch
+    def add_extra_media(self, to_enrich: Metadata) -> tuple[int, int]:
+        video_urls = [v.get_attribute("href") for v in self.sb.find_elements('a[href*="/video-"]')]
+
+        return 0, self._download_videos_with_ytdlp(video_urls, to_enrich)

src/auto_archiver/modules/api_db/api_db.py

@@ -2,7 +2,7 @@ from typing import Union
 
 import os
 import requests
-from loguru import logger
+from auto_archiver.utils.custom_logger import logger
 
 from auto_archiver.core import Database
 from auto_archiver.core import Metadata
@@ -36,9 +36,9 @@ class AAApiDb(Database):
         if not self.store_results:
             return
         if cached:
-            logger.debug(f"skipping saving archive of {item.get_url()} to the AA API because it was cached")
+            logger.debug("Skipping saving archive to AA API because it was cached")
             return
-        logger.debug(f"saving archive of {item.get_url()} to the AA API.")
+        logger.debug("Saving archive to the AA API.")
 
         payload = {
             "author_id": self.author_id,

src/auto_archiver/modules/atlos_feeder_db_storage/atlos_feeder_db_storage.py

@@ -3,7 +3,7 @@ import os
 from typing import IO, Iterator, Optional, Union
 
 import requests
-from loguru import logger
+from auto_archiver.utils.custom_logger import logger
 
 from auto_archiver.core import Database, Feeder, Media, Metadata, Storage
 from auto_archiver.utils import calculate_file_hash
@@ -66,13 +66,13 @@ class AtlosFeederDbStorage(Feeder, Database, Storage):
         """Mark an item as failed in Atlos, if the ID exists."""
         atlos_id = item.metadata.get("atlos_id")
         if not atlos_id:
-            logger.info(f"Item {item.get_url()} has no Atlos ID, skipping")
+            logger.info("No Atlos ID available, skipping")
             return
         self._post(
             f"/api/v2/source_material/metadata/{atlos_id}/auto_archiver",
             json={"metadata": {"processed": True, "status": "error", "error": reason}},
         )
-        logger.info(f"Stored failure for {item.get_url()} (ID {atlos_id}) on Atlos: {reason}")
+        logger.info(f"Stored failure ID {atlos_id} on Atlos: {reason}")
 
     def fetch(self, item: Metadata) -> Union[Metadata, bool]:
         """check and fetch if the given item has been archived already, each
@@ -88,7 +88,7 @@ class AtlosFeederDbStorage(Feeder, Database, Storage):
         """Mark an item as successfully archived in Atlos."""
         atlos_id = item.metadata.get("atlos_id")
         if not atlos_id:
-            logger.info(f"Item {item.get_url()} has no Atlos ID, skipping")
+            logger.info("Item has no Atlos ID, skipping")
             return
         self._post(
             f"/api/v2/source_material/metadata/{atlos_id}/auto_archiver",
@@ -100,7 +100,7 @@ class AtlosFeederDbStorage(Feeder, Database, Storage):
                 }
             },
         )
-        logger.info(f"Stored success for {item.get_url()} (ID {atlos_id}) on Atlos")
+        logger.info(f"Stored success ID {atlos_id} on Atlos")
 
     # ! Atlos Module - Storage Methods
 

src/auto_archiver/modules/cli_feeder/cli_feeder.py

@@ -1,5 +1,3 @@
-from loguru import logger
-
 from auto_archiver.core.feeder import Feeder
 from auto_archiver.core.metadata import Metadata
 from auto_archiver.core.consts import SetupError
@@ -16,8 +14,5 @@ class CLIFeeder(Feeder):
     def __iter__(self) -> Metadata:
         urls = self.config["urls"]
         for url in urls:
-            logger.debug(f"Processing {url}")
             m = Metadata().set_url(url)
             yield m
-
-        logger.success(f"Processed {len(urls)} URL(s)")

src/auto_archiver/modules/console_db/console_db.py

@@ -1,4 +1,4 @@
-from loguru import logger
+from auto_archiver.utils.custom_logger import logger
 
 from auto_archiver.core import Database
 from auto_archiver.core import Metadata

src/auto_archiver/modules/csv_db/csv_db.py

@@ -1,5 +1,5 @@
 import os
-from loguru import logger
+from auto_archiver.utils.custom_logger import logger
 from csv import DictWriter
 from dataclasses import asdict
 

src/auto_archiver/modules/csv_feeder/csv_feeder.py

@@ -1,4 +1,4 @@
-from loguru import logger
+from auto_archiver.utils.custom_logger import logger
 import csv
 
 from auto_archiver.core import Feeder
@@ -35,5 +35,4 @@ class CSVFeeder(Feeder):
                         logger.warning(f"Not a valid URL in row: {row}, skipping")
                         continue
                     url = row[url_column]
-                    logger.debug(f"Processing {url}")
                     yield Metadata().set_url(url)

src/auto_archiver/modules/gdrive_storage/gdrive_storage.py

@@ -8,7 +8,7 @@ from google.oauth2 import service_account
 from google.oauth2.credentials import Credentials
 from googleapiclient.discovery import build
 from googleapiclient.http import MediaFileUpload
-from loguru import logger
+from auto_archiver.utils.custom_logger import logger
 
 from auto_archiver.core import Media
 from auto_archiver.core import Storage
@@ -62,7 +62,7 @@ class GDriveStorage(Storage):
         parent_id, folder_id = self.root_folder_id, None
         path_parts = media.key.split(os.path.sep)
         filename = path_parts[-1]
-        logger.info(f"looking for folders for {path_parts[0:-1]} before getting url for {filename=}")
+        logger.info(f"Looking for folders for {path_parts[0:-1]} before getting url for {filename=}")
         for folder in path_parts[0:-1]:
             folder_id = self._get_id_from_parent_and_name(parent_id, folder, use_mime_type=True, raise_on_missing=True)
             parent_id = folder_id
@@ -70,7 +70,7 @@ class GDriveStorage(Storage):
         file_id = self._get_id_from_parent_and_name(folder_id, filename, raise_on_missing=True)
         if not file_id:
             #
-            logger.info(f"file {filename} not found in folder {folder_id}")
+            logger.info(f"File {filename} not found in folder {folder_id}")
             return None
         return f"https://drive.google.com/file/d/{file_id}/view?usp=sharing"
 
@@ -83,7 +83,7 @@ class GDriveStorage(Storage):
         parent_id, upload_to = self.root_folder_id, None
         path_parts = media.key.split(os.path.sep)
         filename = path_parts[-1]
-        logger.info(f"checking folders {path_parts[0:-1]} exist (or creating) before uploading {filename=}")
+        logger.info(f"Checking folders {path_parts[0:-1]} exist (or creating) before uploading {filename=}")
         for folder in path_parts[0:-1]:
             upload_to = self._get_id_from_parent_and_name(parent_id, folder, use_mime_type=True, raise_on_missing=False)
             if upload_to is None:
@@ -91,15 +91,20 @@ class GDriveStorage(Storage):
             parent_id = upload_to
 
         # upload file to gd
-        logger.debug(f"uploading {filename=} to folder id {upload_to}")
+        logger.debug(f"Uploading {filename=} to folder id {upload_to}")
         file_metadata = {"name": [filename], "parents": [upload_to]}
-        media = MediaFileUpload(media.filename, resumable=True)
-        gd_file = (
-            self.service.files()
-            .create(supportsAllDrives=True, body=file_metadata, media_body=media, fields="id")
-            .execute()
-        )
-        logger.debug(f"uploadf: uploaded file {gd_file['id']} successfully in folder={upload_to}")
+        try:
+            media = MediaFileUpload(media.filename, resumable=True)
+            gd_file = (
+                self.service.files()
+                .create(supportsAllDrives=True, body=file_metadata, media_body=media, fields="id")
+                .execute()
+            )
+            logger.debug(f"Uploadf: uploaded file {gd_file['id']} successfully in folder={upload_to}")
+        except FileNotFoundError as e:
+            logger.error(f"GD uploadf: file not found {media.filename=} - {e}")
+        except Exception as e:
+            logger.error(f"GD uploadf: error uploading {media.filename=} to {upload_to} - {e}")
 
     # must be implemented even if unused
     def uploadf(self, file: IO[bytes], key: str, **kwargs: dict) -> bool:
@@ -128,7 +133,7 @@ class GDriveStorage(Storage):
             self.api_cache = getattr(self, "api_cache", {})
             cache_key = f"{parent_id}_{name}_{use_mime_type}"
             if cache_key in self.api_cache:
-                logger.debug(f"cache hit for {cache_key=}")
+                logger.debug(f"Cache hit for {cache_key=}")
                 return self.api_cache[cache_key]
 
         # API logic
@@ -163,7 +168,7 @@ class GDriveStorage(Storage):
             else:
                 logger.debug(f"{debug_header} not found, attempt {attempt + 1}/{retries}.")
                 if attempt < retries - 1:
-                    logger.debug(f"sleeping for {sleep_seconds} second(s)")
+                    logger.debug(f"Sleeping for {sleep_seconds} second(s)")
                     time.sleep(sleep_seconds)
 
         if raise_on_missing:

src/auto_archiver/modules/generic_extractor/__manifest__.py

@@ -4,9 +4,7 @@
     "author": "Bellingcat",
     "type": ["extractor"],
     "requires_setup": False,
-    "dependencies": {
-        "python": ["yt_dlp", "requests", "loguru", "slugify"],
-    },
+    "dependencies": {"python": ["yt_dlp", "requests", "loguru", "slugify"], "bin": ["ffmpeg"]},
     "description": """
 This is the generic extractor used by auto-archiver, which uses `yt-dlp` under the hood.
 
@@ -32,6 +30,8 @@ For a full list of video platforms supported by `yt-dlp`, see the
 custom dropins can be created to handle additional websites and passed to the archiver
 via the command line using the `--dropins` option (TODO!).
 
+You can see all currently implemented dropins in [the source code](https://github.com/bellingcat/auto-archiver/tree/main/src/auto_archiver/modules/generic_extractor).
+
 ### Auto-Updates
 
 The Generic Extractor will also automatically check for updates to `yt-dlp` (every 5 days by default).
@@ -58,11 +58,15 @@ If you are having issues with the extractor, you can review the version of `yt-d
         },
         "proxy": {
             "default": "",
-            "help": "http/socks (https seems to not work atm) proxy to use for the webdriver, eg https://proxy-user:password@proxy-ip:port",
+            "help": "http/https/socks proxy to use for the webdriver, eg https://proxy-user:password@proxy-ip:port",
+        },
+        "proxy_on_failure_only": {
+            "default": True,
+            "help": "Applies only if a proxy is set. In that case if this setting is True, the extractor will only use the proxy if the initial request fails; if it is False, the extractor will always use the proxy.",
         },
         "end_means_success": {
             "default": True,
-            "help": "if True, any archived content will mean a 'success', if False this archiver will not return a 'success' stage; this is useful for cases when the yt-dlp will archive a video but ignore other types of content like images or text only pages that the subsequent archivers can retrieve.",
+            "help": "if True, any archived content will mean a 'success', if False this extractor will not return a 'success' stage; this is useful for cases when the yt-dlp will archive a video but ignore other types of content like images or text only pages that the subsequent extractors can retrieve.",
             "type": "bool",
         },
         "allow_playlist": {

src/auto_archiver/modules/generic_extractor/bluesky.py

@@ -1,4 +1,4 @@
-from loguru import logger
+from auto_archiver.utils.custom_logger import logger
 
 from auto_archiver.core.extractor import Extractor
 from auto_archiver.core.metadata import Metadata, Media

src/auto_archiver/modules/generic_extractor/facebook.py

@@ -34,7 +34,7 @@ def _extract_metadata(self, webpage, video_id):
             ...,
             "attachments",
             ...,
-            lambda k, v: (k == "media" and str(v["id"]) == video_id and v["__typename"] == "Video"),
+            lambda k, v: k == "media" and str(v["id"]) == video_id and v["__typename"] == "Video",
         ),
         expected_type=dict,
     )

src/auto_archiver/modules/generic_extractor/generic_extractor.py

@@ -1,10 +1,10 @@
-import mimetypes
 import shutil
 import sys
 import datetime
 import os
 import importlib
 import subprocess
+import traceback
 import zipfile
 
 from typing import Generator, Type
@@ -15,11 +15,13 @@ from yt_dlp.extractor.common import InfoExtractor
 from yt_dlp.utils import MaxDownloadsReached
 import pysubs2
 
-from loguru import logger
+from auto_archiver.utils.custom_logger import logger
 
 from auto_archiver.core.extractor import Extractor
 from auto_archiver.core import Metadata, Media
 from auto_archiver.utils import get_datetime_from_str
+from auto_archiver.utils.misc import ydl_entry_to_filename
+from auto_archiver.utils.deletion_detection import detect_deletion, flag_as_deleted
 from .dropin import GenericDropin
 
 
@@ -33,6 +35,9 @@ class GenericExtractor(Extractor):
     def setup(self):
         self.check_for_extractor_updates()
         self.setup_po_tokens()
+        # TODO: figure out why the following is not properly recognised by yt-dlp:
+        # if "generic" not in self.extractor_args:
+        #     self.extractor_args["generic"] = "impersonate"
 
     def check_for_extractor_updates(self):
         """Checks whether yt-dlp or its plugins need updating and triggers a restart if so."""
@@ -60,8 +65,7 @@ class GenericExtractor(Extractor):
             if os.environ.get("AUTO_ARCHIVER_ALLOW_RESTART", "1") != "1":
                 logger.warning("yt-dlp or plugin was updated — please restart auto-archiver manually")
             else:
-                logger.warning("yt-dlp or plugin was updated — restarting auto-archiver")
-                logger.warning(" ======= RESTARTING ======= ")
+                logger.warning("yt-dlp or plugin was updated — restarting auto-archiver\n ======= RESTARTING ======= ")
                 os.execv(sys.executable, [sys.executable] + sys.argv)
 
     def update_package(self, package_name: str) -> bool:
@@ -77,7 +81,7 @@ class GenericExtractor(Extractor):
                 return True
             logger.info(f"{package_name} already up to date")
         except Exception as e:
-            logger.error(f"Error updating {package_name}: {e}")
+            logger.error(f"Failed to update {package_name}: {e}")
         return False
 
     def setup_po_tokens(self) -> None:
@@ -203,7 +207,7 @@ class GenericExtractor(Extractor):
                 media = Media(cover_image_path)
                 metadata.add_media(media, id="cover")
             except Exception as e:
-                logger.error(f"Error downloading cover image {thumbnail_url}: {e}")
+                logger.error(f"Could not download cover image {thumbnail_url}: {e}")
 
         dropin = self.dropin_for_name(info_extractor.ie_key())
         if dropin:
@@ -305,7 +309,7 @@ class GenericExtractor(Extractor):
         if "description" in video_data and not result.get("content"):
             result.set_content(video_data.pop("description"))
         # extract comments if enabled
-        if self.comments and video_data.get("comments", []) is not None:
+        if self.comments and video_data.get("comments", None) is not None:
             result.set(
                 "comments",
                 [
@@ -351,7 +355,7 @@ class GenericExtractor(Extractor):
         if not dropin:
             # TODO: add a proper link to 'how to create your own dropin'
             logger.debug(f"""Could not find valid dropin for {info_extractor.ie_key()}.
-                     Why not try creating your own, and make sure it has a valid function called 'create_metadata'. Learn more: https://auto-archiver.readthedocs.io/en/latest/user_guidelines.html#""")
+                     Why not try creating your own, and make sure it has a valid function called 'create_metadata'. Learn more: https://auto-archiver.readthedocs.io/en/latest/modules/autogen/extractor/generic_extractor.html#dropins""")
             return False
 
         post_data = dropin.extract_post(url, ie_instance)
@@ -368,38 +372,21 @@ class GenericExtractor(Extractor):
             data = ydl.extract_info(url, ie_key=info_extractor.ie_key(), download=True)
         except MaxDownloadsReached:  # proceed as normal once MaxDownloadsReached is raised
             pass
-        logger.success(data)
 
         if "entries" in data:
             entries = data.get("entries", [])
             if not len(entries):
-                logger.warning("YoutubeDLArchiver could not find any video")
+                logger.info("GenericExtractor could not find any video")
                 return False
         else:
             entries = [data]
         result = Metadata()
 
-        def _helper_get_filename(entry: dict) -> str:
-            entry_url = entry.get("url")
-
-            filename = ydl.prepare_filename(entry)
-            base_filename, _ = os.path.splitext(filename)  # '/get/path/to/file' ignore '.ext'
-            directory = os.path.dirname(base_filename)  # '/get/path/to'
-            basename = os.path.basename(base_filename)  # 'file'
-            for f in os.listdir(directory):
-                if (
-                    f.startswith(basename)
-                    or (entry_url and os.path.splitext(f)[0] in entry_url)
-                    and "video/" in (mimetypes.guess_type(f)[0] or "")
-                ):
-                    return os.path.join(directory, f)
-            return False
-
         for entry in entries:
             try:
-                filename = _helper_get_filename(entry)
+                filename = ydl_entry_to_filename(ydl, entry)
 
-                if not filename or not os.path.exists(filename):
+                if not filename:
                     # file was not downloaded or could not be retrieved, example: sensitive videos on YT without using cookies.
                     continue
 
@@ -421,9 +408,9 @@ class GenericExtractor(Extractor):
                             logger.error(f"Error loading subtitle file {val.get('filepath')}: {e}")
                 result.add_media(new_media)
             except Exception as e:
-                logger.error(f"Error processing entry {entry}: {e}")
+                logger.error(f"Error processing entry {str(entry)[:256]}: {e} {traceback.format_exc()}")
         if not len(result.media):
-            logger.warning(f"No media found for entry {entry}, skipping.")
+            logger.info(f"No media found for entry {str(entry)[:256]}, skipping.")
             return False
 
         return self.add_metadata(data, info_extractor, url, result)
@@ -498,6 +485,13 @@ class GenericExtractor(Extractor):
             # don't download since it can be a live stream
             data = ydl.extract_info(url, ie_key=info_extractor.ie_key(), download=False)
 
+            # Check for deletion indicators in video data
+            deletion_info = detect_deletion(video_data=data, url=url)
+            if deletion_info:
+                result = Metadata()
+                flag_as_deleted(result, deletion_info)
+                return result
+
             result = _helper_for_successful_extract_info(data, info_extractor, url, ydl)
 
         except MaxDownloadsReached:
@@ -517,6 +511,16 @@ class GenericExtractor(Extractor):
             try:
                 result = self.get_metadata_for_post(info_extractor, url, ydl)
             except (yt_dlp.utils.DownloadError, yt_dlp.utils.ExtractorError) as post_e:
+                # Check if the error indicates deletion
+                deletion_info = detect_deletion(error_message=str(post_e), url=url)
+                if deletion_info:
+                    result = Metadata()
+                    flag_as_deleted(result, deletion_info)
+                    return result
+
+                if "NSFW tweet requires authentication." in str(post_e):
+                    logger.warning(str(post_e))
+                    return False
                 logger.error("Error downloading metadata for post: {error}", error=str(post_e))
                 return False
             except Exception as generic_e:
@@ -528,7 +532,7 @@ class GenericExtractor(Extractor):
                 )
                 return False
 
-        if result:
+        if result and not result.is_success():
             extractor_name = "yt-dlp"
             if info_extractor:
                 extractor_name += f"_{info_extractor.ie_key()}"
@@ -540,7 +544,7 @@ class GenericExtractor(Extractor):
 
         return result
 
-    def download(self, item: Metadata) -> Metadata:
+    def download(self, item: Metadata, skip_proxy: bool = False) -> Metadata:
         url = item.get_url()
 
         # TODO: this is a temporary hack until this issue is closed: https://github.com/yt-dlp/yt-dlp/issues/11025
@@ -548,6 +552,16 @@ class GenericExtractor(Extractor):
             url = url.replace("https://ya.ru", "https://yandex.ru")
             item.set("replaced_url", url)
 
+        # proxy_on_failure_only logic
+        if self.proxy and self.proxy_on_failure_only and not skip_proxy:
+            # when proxy_on_failure_only is True, we first try to download without a proxy and only continue with execution if that fails
+            try:
+                if without_proxy := self.download(item, skip_proxy=True):
+                    logger.info("Downloaded successfully without proxy.")
+                    return without_proxy
+            except Exception:
+                logger.debug("Download without proxy failed, trying with proxy...")
+
         ydl_options = [
             "-o",
             os.path.join(self.tmp_dir, "%(id)s.%(ext)s"),
@@ -561,7 +575,7 @@ class GenericExtractor(Extractor):
         ]
 
         # proxy handling
-        if self.proxy:
+        if self.proxy and not skip_proxy:
             ydl_options.extend(["--proxy", self.proxy])
 
         # max_downloads handling
@@ -574,31 +588,31 @@ class GenericExtractor(Extractor):
         # order of importance: username/password -> api_key -> cookie -> cookies_from_browser -> cookies_file
         if auth:
             if "username" in auth and "password" in auth:
-                logger.debug(f"Using provided auth username and password for {url}")
+                logger.debug("Using provided auth username and password")
                 ydl_options.extend(("--username", auth["username"]))
                 ydl_options.extend(("--password", auth["password"]))
             elif "cookie" in auth:
-                logger.debug(f"Using provided auth cookie for {url}")
+                logger.debug("Using provided auth cookie")
                 yt_dlp.utils.std_headers["cookie"] = auth["cookie"]
             elif "cookies_from_browser" in auth:
-                logger.debug(f"Using extracted cookies from browser {auth['cookies_from_browser']} for {url}")
+                logger.debug(f"Using extracted cookies from browser {auth['cookies_from_browser']}")
                 ydl_options.extend(("--cookies-from-browser", auth["cookies_from_browser"]))
             elif "cookies_file" in auth:
-                logger.debug(f"Using cookies from file {auth['cookies_file']} for {url}")
+                logger.debug(f"Using cookies from file {auth['cookies_file']}")
                 ydl_options.extend(("--cookies", auth["cookies_file"]))
 
         # Applying user-defined extractor_args
         if self.extractor_args:
             for key, args in self.extractor_args.items():
-                logger.debug(f"Setting extractor_args: {key}")
                 if isinstance(args, dict):
                     arg_str = ";".join(f"{k}={v}" for k, v in args.items())
                 else:
                     arg_str = str(args)
+                logger.debug(f"Setting extractor_args: {key}:{arg_str}")
                 ydl_options.extend(["--extractor-args", f"{key}:{arg_str}"])
 
         if self.ytdlp_args:
-            logger.debug("Adding additional ytdlp arguments: {self.ytdlp_args}")
+            logger.debug(f"Adding additional ytdlp arguments: {self.ytdlp_args}")
             ydl_options += self.ytdlp_args.split(" ")
 
         *_, validated_options = yt_dlp.parse_options(ydl_options)
@@ -606,9 +620,9 @@ class GenericExtractor(Extractor):
             validated_options
         )  # allsubtitles and subtitleslangs not working as expected, so default lang is always "en"
 
+        result: Metadata = None
         for info_extractor in self.suitable_extractors(url):
-            result = self.download_for_extractor(info_extractor, url, ydl)
-            if result:
-                return result
-
-        return False
+            local_result: Metadata = self.download_for_extractor(info_extractor, url, ydl)
+            if local_result:
+                result = result.merge(local_result) if result else local_result
+        return result if result else False

src/auto_archiver/modules/generic_extractor/tiktok.py

@@ -1,5 +1,6 @@
+import re
 import requests
-from loguru import logger
+from auto_archiver.utils.custom_logger import logger
 
 from yt_dlp.extractor.tiktok import TikTokIE, TikTokLiveIE, TikTokVMIE, TikTokUserIE
 
@@ -10,74 +11,113 @@ from .dropin import GenericDropin
 
 class Tiktok(GenericDropin):
     """
-    TikTok droping for the Generic Extractor that uses an unofficial API if/when ytdlp fails.
+    TikTok dropin for the Generic Extractor that uses an unofficial API if/when ytdlp fails.
     It's useful for capturing content that requires a login, like sensitive content.
     """
 
+    # Regex pattern to match TikTok photo post URLs
+    PHOTO_URL_REGEX = r"https?://(?:www\.)?tiktok\.com/@[\w\.-]+/photo/\d+"
     TIKWM_ENDPOINT = "https://www.tikwm.com/api/?url={url}"
 
     def suitable(self, url, info_extractor) -> bool:
         """This dropin (which uses Tikvm) is suitable for *all* Tiktok type URLs - videos, lives, VMs, and users.
         Return the 'suitable' method from the TikTokIE class."""
-        return any(extractor().suitable(url) for extractor in (TikTokIE, TikTokLiveIE, TikTokVMIE, TikTokUserIE))
+        return any(extractor().suitable(url) for extractor in (TikTokIE, TikTokLiveIE, TikTokVMIE, TikTokUserIE)) or (
+            re.match(self.PHOTO_URL_REGEX, url) is not None
+        )
 
     def extract_post(self, url: str, ie_instance):
-        logger.debug(f"Using Tikwm API to attempt to download tiktok video from {url=}")
+        logger.debug("Using Tikwm API to attempt to download tiktok video")
 
         endpoint = self.TIKWM_ENDPOINT.format(url=url)
 
         r = requests.get(endpoint)
         if r.status_code != 200:
-            raise ValueError(f"unexpected status code '{r.status_code}' from tikwm.com for {url=}:")
+            raise ValueError(f"Unexpected status code '{r.status_code}' from tikwm.com")
 
         try:
             json_response = r.json()
         except ValueError:
-            raise ValueError(f"failed to parse JSON response from tikwm.com for {url=}")
+            raise ValueError("Failed to parse JSON response from tikwm.com")
 
         if not json_response.get("msg") == "success" or not (api_data := json_response.get("data", {})):
-            raise ValueError(f"failed to get a valid response from tikwm.com for {url=}: {repr(json_response)}")
+            raise ValueError(f"Unable to download with tikwm.com: {repr(json_response)}")
 
         # tries to get the non-watermarked version first
-        video_url = api_data.pop("play", api_data.pop("wmplay", None))
-        if not video_url:
-            raise ValueError(f"no valid video URL found in response from tikwm.com for {url=}")
-
-        api_data["video_url"] = video_url
+        play_url = api_data.pop("play", api_data.pop("wmplay", None))
+        if play_url and "mime_type=audio" in play_url:
+            play_url = None
+        if play_url:
+            api_data["video_url"] = play_url
         return api_data
 
     def keys_to_clean(self, video_data: dict, info_extractor):
-        return ["video_url", "title", "create_time", "author", "cover", "origin_cover", "ai_dynamic_cover", "duration"]
+        return [
+            "video_url",
+            "title",
+            "create_time",
+            "author",
+            "cover",
+            "origin_cover",
+            "ai_dynamic_cover",
+            "duration",
+            "size",
+            "wm_size",
+            "music",
+            "music_info",
+            "play_count",
+            "digg_count",
+            "comment_count",
+            "share_count",
+            "download_count",
+            "collect_count",
+            "anchors",
+            "anchors_extras",
+            "is_ad",
+            "commerce_info",
+            "commercial_video_info",
+            "item_comment_settings",
+            "mentioned_users",
+        ]  # all of these will be added via api_data in a single metadata field vs individual ones in the generic extractor
 
     def create_metadata(self, post: dict, ie_instance, archiver, url):
         # prepare result, start by downloading video
         result = Metadata()
-        video_url = post.pop("video_url")
-
+        is_success = False
         # get the cover if possible
         cover_url = post.pop("origin_cover", post.pop("cover", post.pop("ai_dynamic_cover", None)))
         if cover_url and (cover_downloaded := archiver.download_from_url(cover_url)):
             result.add_media(Media(cover_downloaded))
 
-        # get the video or fail
-        video_downloaded = archiver.download_from_url(video_url, f"vid_{post.get('id', '')}")
-        if not video_downloaded:
-            logger.error(f"failed to download video from {video_url}")
-            return False
-        video_media = Media(video_downloaded)
-        if duration := post.get("duration", None):
-            video_media.set("duration", duration)
-        result.add_media(video_media)
+        for image_url in post.pop("images", []):
+            if image_downloaded := archiver.download_from_url(image_url):
+                result.add_media(Media(image_downloaded))
+                is_success = True  # this is an images post and we got it/them
+
+        # get the video if present, could be an image post
+        if video_url := post.pop("video_url", None):
+            video_downloaded = archiver.download_from_url(video_url, f"vid_{post.get('id', '')}")
+            if not video_downloaded:
+                logger.error("Failed to download video")
+                return False
+            video_media = Media(video_downloaded)
+            if duration := post.pop("duration", None):
+                video_media.set("duration", duration)
+            result.add_media(video_media)
+            is_success = True  # this is a video post and we got it
 
         # add remaining metadata
-        result.set_title(post.get("title", ""))
+        result.set_title(post.pop("title", ""))
 
-        if created_at := post.get("create_time", None):
+        if created_at := post.pop("create_time", None):
             result.set_timestamp(datetime.fromtimestamp(created_at, tz=timezone.utc))
 
-        if author := post.get("author", None):
+        if author := post.pop("author", None):
             result.set("author", author)
 
-        result.set("api_data", post)
-
+        result.set("api_data", {k: v for k, v in post.items() if v})
+        if is_success:
+            result.success("yt-dlp_TikTok")
+        else:
+            raise ValueError("Unable to download any media from TikTok post, possibly deleted or private.")
         return result

src/auto_archiver/modules/generic_extractor/twitter.py

@@ -1,14 +1,16 @@
 import re
 import mimetypes
 
-from loguru import logger
+from auto_archiver.utils.custom_logger import logger
 from slugify import slugify
 
 from auto_archiver.core.metadata import Metadata, Media
 from auto_archiver.utils import url as UrlUtil, get_datetime_from_str
 from auto_archiver.core.extractor import Extractor
-
-from .dropin import GenericDropin, InfoExtractor
+from auto_archiver.utils.deletion_detection import detect_deletion, flag_as_deleted
+from auto_archiver.modules.generic_extractor.dropin import GenericDropin, InfoExtractor
+import requests
+from retrying import retry
 
 
 class Twitter(GenericDropin):
@@ -29,7 +31,85 @@ class Twitter(GenericDropin):
 
     def extract_post(self, url: str, ie_instance: InfoExtractor):
         twid = ie_instance._match_valid_url(url).group("id")
-        return ie_instance._extract_status(twid=twid)
+        try:
+            post_data = ie_instance._extract_status(twid=twid)
+            if not post_data or not post_data.get("user") or not post_data.get("created_at"):
+                raise ValueError("Error retrieving post with twitter dropin")
+            return post_data
+        except Exception as e:
+            logger.debug(f"yt-dlp twitter extraction failed: {e}")
+            # try fxtwitter API as fallback
+            return self._fetch_fxtwitter(twid)
+
+    def _fetch_fxtwitter(self, twid: str) -> dict:
+        """Fetch tweet data from fxtwitter API and convert to expected format."""
+        fxtwitter_url = f"https://api.fxtwitter.com/status/{twid}"
+        logger.info(f"Falling back to fxtwitter API for tweet extraction: {fxtwitter_url}")
+
+        @retry(wait_random_min=500, wait_random_max=2000, stop_max_attempt_number=3)
+        def fetch_fxtwitter_data(url):
+            headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/112.0"}
+            resp = requests.get(url, headers=headers, timeout=15)
+            if resp.status_code != 200:
+                raise ValueError(f"Failed to retrieve tweet from fxtwitter API: {resp.status_code}")
+            data = resp.json()
+            if "tweet" not in data:
+                raise ValueError(f"No tweet data in fxtwitter response: {data.get('message', 'Unknown error')}")
+            return data["tweet"]
+
+        tweet = fetch_fxtwitter_data(fxtwitter_url)
+
+        # Convert fxtwitter format to expected format
+        author = tweet.get("author", {}).get("name", "")
+        created_at = tweet.get("created_at", "")  # Format: "Sun Feb 08 18:45:00 +0000 2026"
+        full_text = tweet.get("text", "") or tweet.get("raw_text", "")
+
+        # Convert media format
+        media = []
+        fx_media = tweet.get("media", {})
+
+        # Handle photos
+        for photo in fx_media.get("photos", []):
+            media.append({"type": "photo", "media_url_https": photo.get("url", "")})
+
+        # Handle videos
+        for video in fx_media.get("videos", []):
+            variants = video.get("variants", [])
+            # Convert to expected variant format
+            converted_variants = []
+            for var in variants:
+                converted_variants.append(
+                    {
+                        "url": var.get("url", ""),
+                        "content_type": var.get("content_type", "video/mp4"),
+                        "bitrate": var.get("bitrate", 0),
+                    }
+                )
+            if converted_variants:
+                media.append({"type": "video", "video_info": {"variants": converted_variants}})
+
+        # Handle animated gifs (fxtwitter may include these in videos)
+        for item in fx_media.get("all", []):
+            if item.get("type") == "gif":
+                variants = item.get("variants", [])
+                converted_variants = []
+                for var in variants:
+                    converted_variants.append(
+                        {
+                            "url": var.get("url", ""),
+                            "content_type": var.get("content_type", "video/mp4"),
+                            "bitrate": var.get("bitrate", 0),
+                        }
+                    )
+                if converted_variants:
+                    media.append({"type": "animated_gif", "video_info": {"variants": converted_variants}})
+
+        return {
+            "user": {"name": author},
+            "created_at": created_at,
+            "full_text": full_text,
+            "entities": {"media": media},
+        }
 
     def keys_to_clean(self, video_data, info_extractor):
         return ["user", "created_at", "entities", "favorited", "translator_type"]
@@ -38,7 +118,15 @@ class Twitter(GenericDropin):
         result = Metadata()
         try:
             if not tweet.get("user") or not tweet.get("created_at"):
-                raise ValueError("Error retreiving post. Are you sure it exists?")
+                # Check for deletion indicators
+                deletion_info = detect_deletion(
+                    video_data=tweet, url=url, error_message="Missing user or created_at fields"
+                )
+                if deletion_info:
+                    flag_as_deleted(result, deletion_info)
+                    return result
+
+                raise ValueError("Error retrieving post. Are you sure it exists?")
             timestamp = get_datetime_from_str(tweet["created_at"], "%a %b %d %H:%M:%S %z %Y")
         except (ValueError, KeyError) as ex:
             logger.warning(f"Unable to parse tweet: {str(ex)}\nRetreived tweet data: {tweet}")

src/auto_archiver/modules/gsheet_feeder_db/gsheet_feeder_db.py

@@ -10,12 +10,14 @@ The filtered rows are processed into `Metadata` objects.
 """
 
 import os
-from typing import Tuple, Union
+import traceback
+from typing import Tuple, Union, Iterator
 from urllib.parse import quote
 
 import gspread
-from loguru import logger
+from auto_archiver.utils.custom_logger import logger
 from slugify import slugify
+from retrying import retry
 
 from auto_archiver.core import Feeder, Database, Media
 from auto_archiver.core import Metadata
@@ -30,29 +32,40 @@ class GsheetsFeederDB(Feeder, Database):
         if not self.sheet and not self.sheet_id:
             raise ValueError("You need to define either a 'sheet' name or a 'sheet_id' in your manifest.")
 
-    def open_sheet(self):
+    @retry(
+        wait_exponential_multiplier=1,
+        stop_max_attempt_number=6,
+    )
+    def open_sheet(self) -> gspread.Spreadsheet:
         if self.sheet:
             return self.gsheets_client.open(self.sheet)
-        else:  # self.sheet_id
+        else:
             return self.gsheets_client.open_by_key(self.sheet_id)
 
-    def __iter__(self) -> Metadata:
-        sh = self.open_sheet()
-        for ii, worksheet in enumerate(sh.worksheets()):
-            if not self.should_process_sheet(worksheet.title):
-                logger.debug(f"SKIPPED worksheet '{worksheet.title}' due to allow/block rules")
-                continue
-            logger.info(f"Opening worksheet {ii=}: {worksheet.title=} header={self.header}")
-            gw = GWorksheet(worksheet, header_row=self.header, columns=self.columns)
-            if len(missing_cols := self.missing_required_columns(gw)):
-                logger.warning(
-                    f"SKIPPED worksheet '{worksheet.title}' due to missing required column(s) for {missing_cols}"
-                )
-                continue
+    @retry(
+        wait_exponential_multiplier=1,
+        stop_max_attempt_number=6,
+    )
+    def enumerate_sheets(self, sheet) -> Iterator[gspread.Worksheet]:
+        for worksheet in sheet.worksheets():
+            yield worksheet
 
-            # process and yield metadata here:
-            yield from self._process_rows(gw)
-            logger.success(f"Finished worksheet {worksheet.title}")
+    def __iter__(self) -> Iterator[Metadata]:
+        spreadsheet = self.open_sheet()
+        for worksheet in self.enumerate_sheets(spreadsheet):
+            with logger.contextualize(worksheet=f"{spreadsheet.title}:{worksheet.title}"):
+                if not self.should_process_sheet(worksheet.title):
+                    logger.debug("Skipped worksheet due to allow/block rules")
+                    continue
+                logger.info(f"Opening worksheet header={self.header}")
+                gw = GWorksheet(worksheet, header_row=self.header, columns=self.columns)
+                if len(missing_cols := self.missing_required_columns(gw)):
+                    logger.debug(f"Skipped worksheet due to missing required column(s) for {missing_cols}")
+                    continue
+
+                # process and yield metadata here:
+                yield from self._process_rows(gw)
+            logger.info(f"Finished worksheet {worksheet.title}")
 
     def _process_rows(self, gw: GWorksheet):
         for row in range(1 + self.header, gw.count_rows() + 1):
@@ -68,7 +81,9 @@ class GsheetsFeederDB(Feeder, Database):
             # All checks done - archival process starts here
             m = Metadata().set_url(url)
             self._set_context(m, gw, row)
-            yield m
+
+            with logger.contextualize(row=row):
+                yield m
 
     def _set_context(self, m: Metadata, gw: GWorksheet, row: int) -> Metadata:
         # TODO: Check folder value not being recognised
@@ -98,16 +113,16 @@ class GsheetsFeederDB(Feeder, Database):
         return missing
 
     def started(self, item: Metadata) -> None:
-        logger.info(f"STARTED {item}")
+        logger.info("STARTED")
         gw, row = self._retrieve_gsheet(item)
         gw.set_cell(row, "status", "Archive in progress")
 
     def failed(self, item: Metadata, reason: str) -> None:
-        logger.error(f"FAILED {item}")
+        logger.error("FAILED")
         self._safe_status_update(item, f"Archive failed {reason}")
 
     def aborted(self, item: Metadata) -> None:
-        logger.warning(f"ABORTED {item}")
+        logger.warning("ABORTED")
         self._safe_status_update(item, "")
 
     def fetch(self, item: Metadata) -> Union[Metadata, bool]:
@@ -116,13 +131,13 @@ class GsheetsFeederDB(Feeder, Database):
 
     def done(self, item: Metadata, cached: bool = False) -> None:
         """archival result ready - should be saved to DB"""
-        logger.success(f"DONE {item.get_url()}")
         gw, row = self._retrieve_gsheet(item)
-        # self._safe_status_update(item, 'done')
 
         cell_updates = []
         row_values = gw.get_row(row)
 
+        logger.info("DONE")
+
         def batch_if_valid(col, val, final_value=None):
             final_value = final_value or val
             try:
@@ -173,22 +188,27 @@ class GsheetsFeederDB(Feeder, Database):
                 ),
             )
 
-        gw.batch_set_cell(cell_updates)
+        @retry(
+            wait_exponential_multiplier=1,
+            stop_max_attempt_number=5,
+        )
+        def batch_set_cell_with_retry(gw, cell_updates: list):
+            gw.batch_set_cell(cell_updates)
+
+        batch_set_cell_with_retry(gw, cell_updates)
 
     def _safe_status_update(self, item: Metadata, new_status: str) -> None:
         try:
             gw, row = self._retrieve_gsheet(item)
             gw.set_cell(row, "status", new_status)
         except Exception as e:
-            logger.debug(f"Unable to update sheet: {e}")
+            logger.debug(f"Unable to update sheet: {e}: {traceback.format_exc()}")
 
     def _retrieve_gsheet(self, item: Metadata) -> Tuple[GWorksheet, int]:
         if gsheet := item.get_context("gsheet"):
             gw: GWorksheet = gsheet.get("worksheet")
             row: int = gsheet.get("row")
         elif self.sheet_id:
-            logger.error(
-                f"Unable to retrieve Gsheet for {item.get_url()}, GsheetDB must be used alongside GsheetFeeder."
-            )
+            logger.error("Unable to retrieve Gsheet, GsheetDB must be used alongside GsheetFeeder.")
 
         return gw, row

src/auto_archiver/modules/gsheet_feeder_db/gworksheet.py

@@ -1,4 +1,5 @@
 from gspread import utils
+from retrying import retry
 
 
 class GWorksheet:
@@ -26,6 +27,10 @@ class GWorksheet:
         "replaywebpage": "replaywebpage",
     }
 
+    @retry(
+        wait_exponential_multiplier=1,
+        stop_max_attempt_number=6,
+    )
     def __init__(self, worksheet, columns=COLUMN_NAMES, header_row=1):
         self.wks = worksheet
         self.columns = columns

src/auto_archiver/modules/hash_enricher/hash_enricher.py

@@ -9,7 +9,7 @@ making it suitable for handling large files efficiently.
 """
 
 import hashlib
-from loguru import logger
+from auto_archiver.utils.custom_logger import logger
 
 from auto_archiver.core import Enricher
 from auto_archiver.core import Metadata
@@ -22,8 +22,7 @@ class HashEnricher(Enricher):
     """
 
     def enrich(self, to_enrich: Metadata) -> None:
-        url = to_enrich.get_url()
-        logger.debug(f"calculating media hashes for {url=} (using {self.algorithm})")
+        logger.debug(f"Calculating media hashes with algo={self.algorithm}")
 
         for i, m in enumerate(to_enrich.media):
             if len(hd := self.calculate_hash(m.filename)):

src/auto_archiver/modules/html_formatter/html_formatter.py

@@ -4,7 +4,7 @@ import os
 import pathlib
 from jinja2 import Environment, FileSystemLoader
 from urllib.parse import quote
-from loguru import logger
+from auto_archiver.utils.custom_logger import logger
 import json
 import base64
 
@@ -35,7 +35,7 @@ class HtmlFormatter(Formatter):
     def format(self, item: Metadata) -> Media:
         url = item.get_url()
         if item.is_empty():
-            logger.debug(f"[SKIP] FORMAT there is no media or metadata to format: {url=}")
+            logger.debug("Nothing to format, skipping")
             return
 
         content = self.template.render(

src/auto_archiver/modules/html_formatter/templates/html_template.html

@@ -12,6 +12,12 @@
             font-family: 'Roboto', sans-serif;
         }
 
+        h2 {
+            white-space: normal;
+            overflow-wrap: break-word;
+            word-break: break-word;
+        }
+
         table {
             table-layout: fixed;
             width: 90%;
@@ -97,13 +103,17 @@
             background-color: #f1f1f1;
         }
 
-        .pem-certificate, .text-preview {
+        .pem-certificate,
+        .text-preview {
             text-align: left;
             font-size: small;
         }
-        .text-preview{
+
+        .text-preview {
             padding-left: 10px;
             padding-right: 10px;
+            max-height: 300px;
+            overflow: auto;
             white-space: pre-wrap;
         }
     </style>

src/auto_archiver/modules/instagram_api_extractor/__manifest__.py

@@ -22,7 +22,7 @@
         "full_profile_max_posts": {
             "default": 0,
             "type": "int",
-            "help": "Use to limit the number of posts to download when full_profile is true. 0 means no limit. limit is applied softly since posts are fetched in batch, once to: posts, tagged posts, and highlights",
+            "help": "Use to limit the number of posts to download when full_profile is true or when a URL for multiple posts is passed (like /stories /highlights ...). 0 means no limit. when full_profile is true the order of downloaded content is stories -> posts -> tagged posts -> highlights, so a value of 10 could download 2 stories, 7 posts, 1 tagged posts, and 0 highlights.",
         },
         "minimize_json_output": {
             "default": True,

src/auto_archiver/modules/instagram_api_extractor/instagram_api_extractor.py

@@ -8,11 +8,13 @@ data, reducing JSON output size, and handling large profiles.
 
 """
 
+import math
 import re
 from datetime import datetime
+import traceback
 
 import requests
-from loguru import logger
+from auto_archiver.utils.custom_logger import logger
 from retrying import retry
 from tqdm import tqdm
 
@@ -35,17 +37,19 @@ class InstagramAPIExtractor(Extractor):
     def setup(self) -> None:
         if self.api_endpoint[-1] == "/":
             self.api_endpoint = self.api_endpoint[:-1]
+        self.full_profile_max_posts = int(self.full_profile_max_posts or 0)
+        if self.full_profile_max_posts == 0:
+            self.full_profile_max_posts = math.inf
 
     def download(self, item: Metadata) -> Metadata:
         url = item.get_url()
-
         url.replace("instagr.com", "instagram.com").replace("instagr.am", "instagram.com")
         insta_matches = self.valid_url.findall(url)
-        logger.info(f"{insta_matches=}")
+
         if not len(insta_matches) or len(insta_matches[0]) != 3:
             return
         if len(insta_matches) > 1:
-            logger.warning(f"Multiple instagram matches found in {url=}, using the first one")
+            logger.debug("Multiple instagram matches found, using the first one")
             return
         g1, g2, g3 = insta_matches[0][0], insta_matches[0][1], insta_matches[0][2]
         if g1 == "":
@@ -61,13 +65,13 @@ class InstagramAPIExtractor(Extractor):
                 return self.download_post(item, id=g3, context="story")
             return self.download_stories(item, g2)
         else:
-            logger.warning(f"Unknown instagram regex group match {g1=} found in {url=}")
+            logger.warning(f"Unknown instagram regex group match {g1=}")
             return
 
     @retry(wait_random_min=1000, wait_random_max=3000, stop_max_attempt_number=5)
     def call_api(self, path: str, params: dict) -> dict:
         headers = {"accept": "application/json", "x-access-key": self.access_token}
-        logger.debug(f"calling {self.api_endpoint}/{path} with {params=}")
+        logger.debug(f"Calling {self.api_endpoint}/{path} with {params=}")
         return requests.get(f"{self.api_endpoint}/{path}", headers=headers, params=params).json()
 
     def cleanup_dict(self, d: dict | list) -> dict:
@@ -97,65 +101,84 @@ class InstagramAPIExtractor(Extractor):
             filename = self.download_from_url(pic_url)
             result.add_media(Media(filename=filename), id="profile_picture")
 
+        count_posts = 0
         if self.full_profile:
             user_id = user.get("pk")
             # download all stories
             try:
-                stories = self._download_stories_reusable(result, username)
+                stories = self._download_stories_reusable(
+                    result, username, max_to_download=self.full_profile_max_posts - count_posts
+                )
+                count_posts += len(stories)
                 result.set("#stories", len(stories))
             except Exception as e:
                 result.append("errors", f"Error downloading stories for {username}")
-                logger.error(f"Error downloading stories for {username}: {e}")
+                logger.error(f"Error downloading stories for {username}: {e} {traceback.format_exc()}")
 
             # download all posts
             try:
-                self.download_all_posts(result, user_id)
+                if count_posts < self.full_profile_max_posts:
+                    count_posts += self.download_all_posts(
+                        result, user_id, max_to_download=self.full_profile_max_posts - count_posts
+                    )
             except Exception as e:
                 result.append("errors", f"Error downloading posts for {username}")
-                logger.error(f"Error downloading posts for {username}: {e}")
+                logger.error(f"Error downloading posts for {username}: {e} {traceback.format_exc()}")
 
             # download all tagged
             try:
-                self.download_all_tagged(result, user_id)
+                if count_posts < self.full_profile_max_posts:
+                    count_posts += self.download_all_tagged(
+                        result, user_id, max_to_download=self.full_profile_max_posts - count_posts
+                    )
             except Exception as e:
                 result.append("errors", f"Error downloading tagged posts for {username}")
-                logger.error(f"Error downloading tagged posts for {username}: {e}")
+                logger.error(f"Error downloading tagged posts for {username}: {e} {traceback.format_exc()}")
 
             # download all highlights
             try:
-                self.download_all_highlights(result, username, user_id)
+                if count_posts < self.full_profile_max_posts:
+                    count_posts += self.download_all_highlights(
+                        result, username, user_id, max_to_download=self.full_profile_max_posts - count_posts
+                    )
             except Exception as e:
                 result.append("errors", f"Error downloading highlights for {username}")
-                logger.error(f"Error downloading highlights for {username}: {e}")
+                logger.error(f"Error downloading highlights for {username}: {e} {traceback.format_exc()}")
 
         result.set_url(url)  # reset as scrape_item modifies it
         return result.success("insta profile")
 
-    def download_all_highlights(self, result, username, user_id):
+    def download_all_highlights(self, result, username, user_id, max_to_download: int) -> int:
         count_highlights = 0
         highlights = self.call_api("v1/user/highlights", {"user_id": user_id})
+        highlights = highlights[: min(max_to_download, len(highlights))]  # newest to oldest
         for h in highlights:
             try:
-                h_info = self._download_highlights_reusable(result, h.get("pk"))
+                h_info = self._download_highlights_reusable(result, h.get("pk"), max_to_download=max_to_download)
                 count_highlights += len(h_info.get("items", []))
             except Exception as e:
                 result.append(
                     "errors",
                     f"Error downloading highlight id{h.get('pk')} for {username}",
                 )
-                logger.error(f"Error downloading highlight id{h.get('pk')} for {username}: {e}")
-            if self.full_profile_max_posts and count_highlights >= self.full_profile_max_posts:
-                logger.info(f"HIGHLIGHTS reached full_profile_max_posts={self.full_profile_max_posts}")
+                logger.error(
+                    f"Error downloading highlight id{h.get('pk')} for {username}: {e} {traceback.format_exc()}"
+                )
+            if count_highlights >= max_to_download:
+                logger.debug(f"HIGHLIGHTS reached max_to_download={self.full_profile_max_posts}")
                 break
         result.set("#highlights", count_highlights)
+        return count_highlights
 
-    def download_post(self, result: Metadata, code: str = None, id: str = None, context: str = None) -> Metadata:
+    def download_post(self, result: Metadata, code: str = None, id: str = None, context: str = "") -> Metadata:
         if id:
             post = self.call_api("v1/media/by/id", {"id": id})
         else:
             post = self.call_api("v1/media/by/code", {"code": code})
         assert post, f"Post {id or code} not found"
 
+        result.set(f"{context}_data", post)
+
         if caption_text := post.get("caption_text"):
             result.set_title(caption_text)
 
@@ -166,13 +189,13 @@ class InstagramAPIExtractor(Extractor):
         return result.success(f"insta {context or 'post'}")
 
     def download_highlights(self, result: Metadata, id: str) -> Metadata:
-        h_info = self._download_highlights_reusable(result, id)
+        h_info = self._download_highlights_reusable(result, id, self.full_profile_max_posts)
         items = len(h_info.get("items", []))
         del h_info["items"]
         result.set_title(h_info.get("title")).set("data", h_info).set("#reels", items)
         return result.success("insta highlights")
 
-    def _download_highlights_reusable(self, result: Metadata, id: str) -> dict:
+    def _download_highlights_reusable(self, result: Metadata, id: str, max_to_download: int) -> dict:
         full_h = self.call_api("v2/highlight/by/id", {"id": id})
         h_info = full_h.get("response", {}).get("reels", {}).get(f"highlight:{id}")
         assert h_info, f"Highlight {id} not found: {full_h=}"
@@ -182,38 +205,39 @@ class InstagramAPIExtractor(Extractor):
             result.add_media(Media(filename=filename), id=f"cover_media highlight {id}")
 
         items = h_info.get("items", [])[::-1]  # newest to oldest
+        items = items[: min(max_to_download, len(items))]
         for h in tqdm(items, desc="downloading highlights", unit="highlight"):
             try:
                 self.scrape_item(result, h, "highlight")
             except Exception as e:
                 result.append("errors", f"Error downloading highlight {h.get('id')}")
-                logger.error(f"Error downloading highlight, skipping {h.get('id')}: {e}")
+                logger.error(f"Error downloading highlight, skipping {h.get('id')}: {e} {traceback.format_exc()}")
 
         return h_info
 
     def download_stories(self, result: Metadata, username: str) -> Metadata:
         now = datetime.now().strftime("%Y-%m-%d_%H-%M")
-        stories = self._download_stories_reusable(result, username)
+        stories = self._download_stories_reusable(result, username, max_to_download=self.full_profile_max_posts)
         if stories == []:
             return result.success("insta no story")
         result.set_title(f"stories {username} at {now}").set("#stories", len(stories))
         return result.success(f"insta stories {now}")
 
-    def _download_stories_reusable(self, result: Metadata, username: str) -> list[dict]:
+    def _download_stories_reusable(self, result: Metadata, username: str, max_to_download: int) -> list[dict]:
         stories = self.call_api("v1/user/stories/by/username", {"username": username})
         if not stories or not len(stories):
             return []
-        stories = stories[::-1]  # newest to oldest
+        stories = stories[::-1][: min(max_to_download, len(stories))]  # newest to oldest
 
         for s in tqdm(stories, desc="downloading stories", unit="story"):
             try:
                 self.scrape_item(result, s, "story")
             except Exception as e:
                 result.append("errors", f"Error downloading story {s.get('id')}")
-                logger.error(f"Error downloading story, skipping {s.get('id')}: {e}")
+                logger.error(f"Error downloading story, skipping {s.get('id')}: {e} {traceback.format_exc()}")
         return stories
 
-    def download_all_posts(self, result: Metadata, user_id: str):
+    def download_all_posts(self, result: Metadata, user_id: str, max_to_download: int) -> int:
         end_cursor = None
         pbar = tqdm(desc="downloading posts")
 
@@ -223,22 +247,23 @@ class InstagramAPIExtractor(Extractor):
             if not posts or not isinstance(posts, list) or len(posts) != 2:
                 break
             posts, end_cursor = posts[0], posts[1]
-            logger.info(f"parsing {len(posts)} posts, next {end_cursor=}")
-
+            posts = posts[: min(max_to_download, len(posts))]
+            logger.info(f"Parsing {len(posts)} posts, next {end_cursor=} {post_count=} {max_to_download=}")
             for p in posts:
                 try:
                     self.scrape_item(result, p, "post")
                 except Exception as e:
                     result.append("errors", f"Error downloading post {p.get('id')}")
-                    logger.error(f"Error downloading post, skipping {p.get('id')}: {e}")
+                    logger.error(f"Error downloading post, skipping {p.get('id')}: {e} {traceback.format_exc()}")
                 pbar.update(1)
                 post_count += 1
-            if self.full_profile_max_posts and post_count >= self.full_profile_max_posts:
-                logger.info(f"POSTS reached full_profile_max_posts={self.full_profile_max_posts}")
+            if post_count >= max_to_download:
+                logger.info(f"POSTS reached max_to_download={self.full_profile_max_posts}")
                 break
         result.set("#posts", post_count)
+        return post_count
 
-    def download_all_tagged(self, result: Metadata, user_id: str):
+    def download_all_tagged(self, result: Metadata, user_id: str, max_to_download: int) -> int:
         next_page_id = ""
         pbar = tqdm(desc="downloading tagged posts")
 
@@ -250,22 +275,23 @@ class InstagramAPIExtractor(Extractor):
                 break
             next_page_id = resp.get("next_page_id")
 
-            logger.info(f"parsing {len(posts)} tagged posts, next {next_page_id=}")
-
+            logger.info(f"Parsing {len(posts)} tagged posts, next {next_page_id=}")
+            posts = posts[: min(max_to_download, len(posts))]
             for p in posts:
                 try:
                     self.scrape_item(result, p, "tagged")
                 except Exception as e:
                     result.append("errors", f"Error downloading tagged post {p.get('id')}")
-                    logger.error(f"Error downloading tagged post, skipping {p.get('id')}: {e}")
+                    logger.error(f"Error downloading tagged post, skipping {p.get('id')}: {e} {traceback.format_exc()}")
                 pbar.update(1)
                 tagged_count += 1
-            if self.full_profile_max_posts and tagged_count >= self.full_profile_max_posts:
-                logger.info(f"TAGS reached full_profile_max_posts={self.full_profile_max_posts}")
+            if tagged_count >= max_to_download:
+                logger.info(f"TAGS reached max_to_download={self.full_profile_max_posts}")
                 break
         result.set("#tagged", tagged_count)
+        return tagged_count
 
-    ### reusable parsing utils below
+    # reusable parsing utils below
 
     def scrape_item(self, result: Metadata, item: dict, context: str = None) -> dict:
         """

src/auto_archiver/modules/instagram_extractor/instagram_extractor.py

@@ -7,8 +7,9 @@ highlights, and tagged posts. Authentication is required via username/password o
 import re
 import os
 import shutil
+import traceback
 import instaloader
-from loguru import logger
+from auto_archiver.utils.custom_logger import logger
 
 from auto_archiver.core import Extractor
 from auto_archiver.core import Metadata
@@ -29,8 +30,9 @@ class InstagramExtractor(Extractor):
     # TODO: links to stories
 
     def setup(self) -> None:
-        logger.warning("Instagram Extractor is not actively maintained, and may not work as expected.")
-        logger.warning("Please consider using the Instagram Tbot Extractor or Instagram API Extractor instead.")
+        logger.warning(
+            "Instagram Extractor is not actively maintained, and may not work as expected.\nPlease consider using the Instagram Tbot Extractor or Instagram API Extractor instead."
+        )
 
         self.insta = instaloader.Instaloader(
             download_geotags=True,
@@ -43,8 +45,7 @@ class InstagramExtractor(Extractor):
             self.insta.load_session_from_file(self.username, self.session_file)
         except Exception:
             try:
-                logger.debug("Session file failed", exc_info=True)
-                logger.info("No valid session file found - Attempting login with use and password.")
+                logger.info("No valid session file found - Attempting login with username and password.")
                 self.insta.login(self.username, self.password)
                 self.insta.save_session_to_file(self.session_file)
             except Exception as e:
@@ -79,7 +80,7 @@ class InstagramExtractor(Extractor):
         return result
 
     def download_post(self, url: str, post_id: str) -> Metadata:
-        logger.debug(f"Instagram {post_id=} detected in {url=}")
+        logger.debug(f"Instagram {post_id=} detected")
 
         post = instaloader.Post.from_shortcode(self.insta.context, post_id)
         if self.insta.download_post(post, target=post.owner_username):
@@ -87,7 +88,7 @@ class InstagramExtractor(Extractor):
 
     def download_profile(self, url: str, username: str) -> Metadata:
         # gets posts, posts where username is tagged, igtv postss, stories, and highlights
-        logger.debug(f"Instagram {username=} detected in {url=}")
+        logger.debug(f"Instagram {username=} detected")
 
         profile = instaloader.Profile.from_username(self.insta.context, username)
         try:
@@ -95,27 +96,27 @@ class InstagramExtractor(Extractor):
                 try:
                     self.insta.download_post(post, target=f"profile_post_{post.owner_username}")
                 except Exception as e:
-                    logger.error(f"Failed to download post: {post.shortcode}: {e}")
+                    logger.error(f"Failed to download post: {post.shortcode}: {e} {traceback.format_exc()}")
         except Exception as e:
-            logger.error(f"Failed profile.get_posts: {e}")
+            logger.error(f"Failed profile.get_posts: {e}: {traceback.format_exc()}")
 
         try:
             for post in profile.get_tagged_posts():
                 try:
                     self.insta.download_post(post, target=f"tagged_post_{post.owner_username}")
                 except Exception as e:
-                    logger.error(f"Failed to download tagged post: {post.shortcode}: {e}")
+                    logger.error(f"Failed to download tagged post: {post.shortcode}: {e} {traceback.format_exc()}")
         except Exception as e:
-            logger.error(f"Failed profile.get_tagged_posts: {e}")
+            logger.error(f"Failed profile.get_tagged_posts: {e} {traceback.format_exc()}")
 
         try:
             for post in profile.get_igtv_posts():
                 try:
                     self.insta.download_post(post, target=f"igtv_post_{post.owner_username}")
                 except Exception as e:
-                    logger.error(f"Failed to download igtv post: {post.shortcode}: {e}")
+                    logger.error(f"Failed to download igtv post: {post.shortcode}: {e} {traceback.format_exc()}")
         except Exception as e:
-            logger.error(f"Failed profile.get_igtv_posts: {e}")
+            logger.error(f"Failed profile.get_igtv_posts: {e} {traceback.format_exc()}")
 
         try:
             for story in self.insta.get_stories([profile.userid]):
@@ -123,9 +124,9 @@ class InstagramExtractor(Extractor):
                     try:
                         self.insta.download_storyitem(item, target=f"story_item_{story.owner_username}")
                     except Exception as e:
-                        logger.error(f"Failed to download story item: {item}: {e}")
+                        logger.error(f"Failed to download story item: {item}: {e} {traceback.format_exc()}")
         except Exception as e:
-            logger.error(f"Failed get_stories: {e}")
+            logger.error(f"Failed get_stories: {e} {traceback.format_exc()}")
 
         try:
             for highlight in self.insta.get_highlights(profile.userid):
@@ -133,9 +134,9 @@ class InstagramExtractor(Extractor):
                     try:
                         self.insta.download_storyitem(item, target=f"highlight_item_{highlight.owner_username}")
                     except Exception as e:
-                        logger.error(f"Failed to download highlight item: {item}: {e}")
+                        logger.error(f"Failed to download highlight item: {item}: {e} {traceback.format_exc()}")
         except Exception as e:
-            logger.error(f"Failed get_highlights: {e}")
+            logger.error(f"Failed get_highlights: {e} {traceback.format_exc()}")
 
         return self.process_downloads(url, f"@{username}", profile._asdict(), None)
 
@@ -158,4 +159,4 @@ class InstagramExtractor(Extractor):
 
             return result.success("instagram")
         except Exception as e:
-            logger.error(f"Could not fetch instagram post {url} due to: {e}")
+            logger.error(f"Could not fetch instagram post due to: {e} {traceback.format_exc()}")

src/auto_archiver/modules/instagram_tbot_extractor/instagram_tbot_extractor.py

@@ -12,7 +12,7 @@ import shutil
 import time
 from sqlite3 import OperationalError
 
-from loguru import logger
+from auto_archiver.utils.custom_logger import logger
 from telethon.sync import TelegramClient
 
 from auto_archiver.core import Extractor
@@ -32,7 +32,7 @@ class InstagramTbotExtractor(Extractor):
         1. makes a copy of session_file that is removed in cleanup
         2. checks if the session file is valid
         """
-        logger.info(f"SETUP {self.name} checking login...")
+        logger.debug(f"SETUP {self.name} checking login...")
         self._prepare_session_file()
         self._initialize_telegram_client()
 
@@ -58,10 +58,10 @@ class InstagramTbotExtractor(Extractor):
                 "If you do, disable at least one of the archivers for the first-time setup of the telethon session: {e}"
             )
         with self.client.start():
-            logger.success(f"SETUP {self.name} login works.")
+            logger.debug(f"SETUP {self.name} login works.")
 
     def cleanup(self) -> None:
-        logger.info(f"CLEANUP {self.name}.")
+        logger.debug(f"CLEANUP {self.name}.")
         session_file_name = self.session_file + ".session"
         if os.path.exists(session_file_name):
             os.remove(session_file_name)
@@ -79,17 +79,17 @@ class InstagramTbotExtractor(Extractor):
 
             # This may be outdated and replaced by the below message, but keeping until confirmed
             if "You must enter a URL to a post" in message:
-                logger.debug(f"invalid link {url=} for {self.name}: {message}")
+                logger.debug(f"Invalid link for {self.name}: {message}")
                 return False
 
             if "Media not found or unavailable" in message:
-                logger.debug(f"No media found for link {url=} for {self.name}: {message}")
+                logger.debug(f"No media found for {self.name}: {message}")
                 return False
 
             if message:
                 result.set_content(message).set_title(message[:128])
             elif result.is_empty():
-                logger.debug(f"No media found for link {url=} for {self.name}: {message}")
+                logger.debug(f"No media found for {self.name}: {message}")
                 return False
             return result.success("insta-via-bot")
 

src/auto_archiver/modules/json_enricher/__init__.py

@@ -0,0 +1 @@
+from .json_enricher import JsonEnricher

src/auto_archiver/modules/json_enricher/__manifest__.py

@@ -0,0 +1,16 @@
+{
+    "name": "JSON Enricher",
+    "type": ["enricher"],
+    "requires_setup": True,
+    "dependencies": {
+        "python": ["loguru"],
+    },
+    "configs": {},
+    "description": """
+
+    Writes all archiving process metadata to a JSON file so it can be parsed by other tools. As this is an Enricher, it will not contain the final stored URLs. 
+	
+	WARNING: The resulting JSON may reveal sensitive information about the computer and settings in which the archiving process was run. 
+
+    """,
+}

src/auto_archiver/modules/json_enricher/json_enricher.py

@@ -0,0 +1,17 @@
+import json
+from auto_archiver.utils.custom_logger import logger
+import os
+
+from auto_archiver.core import Enricher
+from auto_archiver.core import Media, Metadata
+
+
+class JsonEnricher(Enricher):
+    def enrich(self, to_enrich: Metadata) -> None:
+        logger.debug("Enriching as JSON")
+
+        item_path = os.path.join(self.tmp_dir, "metadata.json")
+        with open(item_path, mode="w", encoding="utf-8") as outf:
+            json.dump(to_enrich.to_dict(), outf, indent=4, default=str, ensure_ascii=False)
+
+        to_enrich.add_media(Media(filename=item_path), id="metadata_json")

src/auto_archiver/modules/local_storage/local_storage.py

@@ -1,7 +1,7 @@
 import shutil
 from typing import IO
 import os
-from loguru import logger
+from auto_archiver.utils.custom_logger import logger
 
 from auto_archiver.core import Media
 from auto_archiver.core import Storage
@@ -38,8 +38,7 @@ class LocalStorage(Storage):
         os.makedirs(os.path.dirname(dest), exist_ok=True)
         logger.debug(f"[{self.__class__.__name__}] storing file {media.filename} with key {media.key} to {dest}")
 
-        res = shutil.copy2(media.filename, dest)
-        logger.info(res)
+        shutil.copy2(media.filename, dest)
         return True
 
     # must be implemented even if unused

src/auto_archiver/modules/meta_enricher/meta_enricher.py

@@ -1,6 +1,6 @@
 import datetime
 import os
-from loguru import logger
+from auto_archiver.utils.custom_logger import logger
 
 from auto_archiver.core import Enricher
 from auto_archiver.core import Metadata
@@ -12,20 +12,17 @@ class MetaEnricher(Enricher):
     """
 
     def enrich(self, to_enrich: Metadata) -> None:
-        url = to_enrich.get_url()
         if to_enrich.is_empty():
-            logger.debug(f"[SKIP] META_ENRICHER there is no media or metadata to enrich: {url=}")
+            logger.debug("[SKIP] META_ENRICHER there is no media or metadata to enrich")
             return
 
-        logger.debug(f"calculating archive metadata information for {url=}")
+        logger.debug("Calculating archive metadata information")
 
         self.enrich_file_sizes(to_enrich)
         self.enrich_archive_duration(to_enrich)
 
     def enrich_file_sizes(self, to_enrich: Metadata):
-        logger.debug(
-            f"calculating archive file sizes for url={to_enrich.get_url()} ({len(to_enrich.media)} media files)"
-        )
+        logger.debug(f"Calculating archive file sizes for {len(to_enrich.media)} media files")
         total_size = 0
         for media in to_enrich.get_all_media():
             file_stats = os.stat(media.filename)
@@ -44,7 +41,7 @@ class MetaEnricher(Enricher):
             size /= 1024
 
     def enrich_archive_duration(self, to_enrich):
-        logger.debug(f"calculating archive duration for url={to_enrich.get_url()} ")
+        logger.debug("Calculating archive duration")
 
         archive_duration = datetime.datetime.now(datetime.timezone.utc) - to_enrich.get("_processed_at")
         to_enrich.set("archive_duration_seconds", archive_duration.seconds)

src/auto_archiver/modules/metadata_enricher/__manifest__.py

@@ -3,6 +3,13 @@
     "type": ["enricher"],
     "requires_setup": True,
     "dependencies": {"python": ["loguru"], "bin": ["exiftool"]},
+    "configs": {
+        "look_for_keys": {
+            "default": [],
+            "help": "list of lowercased metadata keys that will be included in the enriched metadata. Special keys: 'author', 'datetimes', 'location' to include related metadata fields. The default empty list `[]` means all metadata will be included.",
+            "type": "list",
+        },
+    },
     "description": """
     Extracts metadata information from files using ExifTool.
 

src/auto_archiver/modules/metadata_enricher/metadata_enricher.py

@@ -1,6 +1,6 @@
 import subprocess
 import traceback
-from loguru import logger
+from auto_archiver.utils.custom_logger import logger
 
 from auto_archiver.core import Enricher
 from auto_archiver.core import Metadata
@@ -12,11 +12,12 @@ class MetadataEnricher(Enricher):
     """
 
     def enrich(self, to_enrich: Metadata) -> None:
-        url = to_enrich.get_url()
-        logger.debug(f"extracting EXIF metadata for {url=}")
+        logger.debug("Extracting EXIF metadata")
 
         for i, m in enumerate(to_enrich.media):
             if len(md := self.get_metadata(m.filename)):
+                if self.look_for_keys != []:
+                    md = self.select_metadata(md, self.look_for_keys)
                 to_enrich.media[i].set("metadata", md)
 
     def get_metadata(self, filename: str) -> dict:
@@ -24,15 +25,44 @@ class MetadataEnricher(Enricher):
             # Run ExifTool command to extract metadata from the file
             cmd = ["exiftool", filename]
             result = subprocess.run(cmd, capture_output=True, text=True)
-
             # Process the output to extract individual metadata fields
             metadata = {}
             for line in result.stdout.splitlines():
                 field, value = line.strip().split(":", 1)
                 metadata[field.strip()] = value.strip()
             return metadata
-        except FileNotFoundError:
-            logger.error("[exif_enricher] ExifTool not found. Make sure ExifTool is installed and added to PATH.")
+        except FileNotFoundError as e:
+            logger.error(f"ExifTool not found. Make sure ExifTool is installed and added to PATH. {e}")
         except Exception as e:
             logger.error(f"Error occurred: {e}: {traceback.format_exc()}")
         return {}
+
+    def select_metadata(self, all_md, requested_metadata_keys):
+        """
+        coordinates the selection of metadata from the general exiftool output to the user-specified grocery list
+        """
+        # defining the batches of metadata that get pulled for special terms
+        author_key_terms = ["author", "producer", "creator"]
+        datetime_key_terms = ["date", "time"]
+        location_key_terms = ["gps", "latitude", "longitude"]
+
+        specified_md = {}
+        for md_key in all_md.keys():
+            md_key_lower = md_key.lower()
+            # checking for special baskets within the grocery list of requested metadata
+            if ("author" in requested_metadata_keys) and any(
+                term in md_key_lower and len(all_md[md_key]) for term in author_key_terms
+            ):
+                specified_md[md_key] = all_md[md_key]
+            if ("datetime" in requested_metadata_keys) and any(
+                term in md_key_lower and len(all_md[md_key]) for term in datetime_key_terms
+            ):
+                specified_md[md_key] = all_md[md_key]
+            if ("location" in requested_metadata_keys) and any(
+                term in md_key_lower and len(all_md[md_key]) for term in location_key_terms
+            ):
+                specified_md[md_key] = all_md[md_key]
+            # if the metadata value is requested directly
+            if md_key_lower in requested_metadata_keys or md_key in requested_metadata_keys and len(all_md[md_key]):
+                specified_md[md_key] = all_md[md_key]
+        return specified_md

src/auto_archiver/modules/opentimestamps_enricher/opentimestamps_enricher.py

@@ -1,6 +1,7 @@
 import os
+import traceback
 
-from loguru import logger
+from auto_archiver.utils.custom_logger import logger
 import opentimestamps
 from opentimestamps.calendar import RemoteCalendar, DEFAULT_CALENDAR_WHITELIST
 from opentimestamps.core.timestamp import Timestamp, DetachedTimestampFile
@@ -14,13 +15,12 @@ from auto_archiver.utils.misc import get_current_timestamp
 
 class OpentimestampsEnricher(Enricher):
     def enrich(self, to_enrich: Metadata) -> None:
-        url = to_enrich.get_url()
-        logger.debug(f"OpenTimestamps timestamping files for {url=}")
+        logger.debug("OpenTimestamps timestamping files")
 
         # Get the media files to timestamp
         media_files = [m for m in to_enrich.media if m.filename and not m.get("opentimestamps")]
         if not media_files:
-            logger.warning(f"No files found to timestamp in {url=}")
+            logger.debug("No files found to timestamp")
             return
 
         timestamp_files = []
@@ -94,7 +94,7 @@ class OpentimestampsEnricher(Enricher):
                         detached_timestamp.serialize(ctx)
                         f.write(ctx.getbytes())
                 except Exception as e:
-                    logger.warning(f"Failed to serialize timestamp file: {e}")
+                    logger.warning(f"Failed to serialize timestamp file: {e} {traceback.format_exc()}")
                     continue
 
                 # Create media for the timestamp file
@@ -113,16 +113,16 @@ class OpentimestampsEnricher(Enricher):
                 media.set("opentimestamps", True)
 
             except Exception as e:
-                logger.warning(f"Error while timestamping {media.filename}: {e}")
+                logger.warning(f"Error while timestamping {media.filename}: {e} {traceback.format_exc()}")
 
         # Add timestamp files to the metadata
         if timestamp_files:
             to_enrich.set("opentimestamped", True)
             to_enrich.set("opentimestamps_count", len(timestamp_files))
-            logger.success(f"{len(timestamp_files)} OpenTimestamps proofs created for {url=}")
+            logger.info(f"{len(timestamp_files)} OpenTimestamps proofs created")
         else:
             to_enrich.set("opentimestamped", False)
-            logger.warning(f"No successful timestamps created for {url=}")
+            logger.warning("No successful timestamps created")
 
     def verify_timestamp(self, detached_timestamp):
         """

src/auto_archiver/modules/pdq_hash_enricher/__manifest__.py

@@ -15,7 +15,7 @@
     - Skips non-image media or files unsuitable for hashing (e.g., corrupted or unsupported formats).
 
     ### Notes
-    - Best used after enrichers like `thumbnail_enricher` or `screenshot_enricher` to ensure images are available.
+    - Best used after enrichers like `thumbnail_enricher` or `antibot_extractor_enricher` (takes screenshots) to ensure images are available.
     - Uses the `pdqhash` library to compute 256-bit perceptual hashes, which are stored as hexadecimal strings.
     """,
 }

src/auto_archiver/modules/pdq_hash_enricher/pdq_hash_enricher.py

@@ -6,7 +6,7 @@ objects and calculates perceptual hashes using the PDQ hashing algorithm.
 These hashes are designed specifically for images and can be used
 for detecting duplicate or near-duplicate visual content.
 
-This enricher is typically used after thumbnail or screenshot enrichers
+This enricher is typically used after thumbnail or screenshot (antibot) enrichers
 to ensure images are available for hashing.
 
 """
@@ -15,7 +15,7 @@ import traceback
 import pdqhash
 import numpy as np
 from PIL import Image, UnidentifiedImageError
-from loguru import logger
+from auto_archiver.utils.custom_logger import logger
 
 from auto_archiver.core import Enricher
 from auto_archiver.core import Metadata
@@ -28,8 +28,7 @@ class PdqHashEnricher(Enricher):
     """
 
     def enrich(self, to_enrich: Metadata) -> None:
-        url = to_enrich.get_url()
-        logger.debug(f"calculating perceptual hashes for {url=}")
+        logger.debug("Calculating perceptual hashes")
         media_with_hashes = []
 
         for m in to_enrich.media:
@@ -44,7 +43,7 @@ class PdqHashEnricher(Enricher):
                     media.set("pdq_hash", hd)
                     media_with_hashes.append(media.filename)
 
-        logger.debug(f"calculated '{len(media_with_hashes)}' perceptual hashes for {url=}: {media_with_hashes}")
+        logger.debug(f"Calculated '{len(media_with_hashes)}' perceptual hashes: {media_with_hashes}")
 
     def calculate_pdq_hash(self, filename):
         # returns a hexadecimal string with the perceptual hash for the given filename

src/auto_archiver/modules/s3_storage/s3_storage.py

@@ -2,7 +2,7 @@ from typing import IO
 
 import boto3
 import os
-from loguru import logger
+from auto_archiver.utils.custom_logger import logger
 
 from auto_archiver.core import Media
 from auto_archiver.core import Storage
@@ -40,6 +40,8 @@ class S3Storage(Storage):
             try:
                 if media.mimetype:
                     extra_args["ContentType"] = media.mimetype
+                    if "text" in media.mimetype:
+                        extra_args["ContentType"] += "; charset=utf-8"
             except Exception as e:
                 logger.warning(f"Unable to get mimetype for {media.key=}, error: {e}")
         self.s3.upload_fileobj(file, Bucket=self.bucket, Key=media.key, ExtraArgs=extra_args)
@@ -54,7 +56,7 @@ class S3Storage(Storage):
             if existing_key := self.file_in_folder(path):
                 media._key = existing_key
                 media.set("previously archived", True)
-                logger.debug(f"skipping upload of {media.filename} because it already exists in {media.key}")
+                logger.debug(f"Skipping upload of {media.filename} because it already exists in {media.key}")
                 return False
 
             _, ext = os.path.splitext(media.key)

src/auto_archiver/modules/screenshot_enricher/__init__.py

@@ -1 +0,0 @@
-from .screenshot_enricher import ScreenshotEnricher

src/auto_archiver/modules/screenshot_enricher/__manifest__.py

@@ -1,44 +0,0 @@
-{
-    "name": "Screenshot Enricher",
-    "type": ["enricher"],
-    "requires_setup": True,
-    "dependencies": {
-        "python": ["loguru", "selenium"],
-    },
-    "configs": {
-        "width": {"default": 1280, "type": "int", "help": "width of the screenshots"},
-        "height": {"default": 1024, "type": "int", "help": "height of the screenshots"},
-        "timeout": {"default": 60, "type": "int", "help": "timeout for taking the screenshot"},
-        "sleep_before_screenshot": {
-            "default": 4,
-            "type": "int",
-            "help": "seconds to wait for the pages to load before taking screenshot",
-        },
-        "http_proxy": {
-            "default": "",
-            "help": "http proxy to use for the webdriver, eg http://proxy-user:password@proxy-ip:port",
-        },
-        "save_to_pdf": {
-            "default": False,
-            "type": "bool",
-            "help": "save the page as pdf along with the screenshot. PDF saving options can be adjusted with the 'print_options' parameter",
-        },
-        "print_options": {
-            "default": {},
-            "help": "options to pass to the pdf printer, in JSON format. See https://www.selenium.dev/documentation/webdriver/interactions/print_page/ for more information",
-            "type": "json_loader",
-        },
-    },
-    "description": """
-    Captures screenshots and optionally saves web pages as PDFs using a WebDriver.
-
-    ### Features
-    - Takes screenshots of web pages, with configurable width, height, and timeout settings.
-    - Optionally saves pages as PDFs, with additional configuration for PDF printing options.
-    - Bypasses URLs detected as authentication walls.
-    - Integrates seamlessly with the metadata enrichment pipeline, adding screenshots and PDFs as media.
-
-    ### Notes
-    - Requires a WebDriver (e.g., ChromeDriver) installed and accessible via the system's PATH.
-    """,
-}

src/auto_archiver/modules/screenshot_enricher/screenshot_enricher.py

@@ -1,61 +0,0 @@
-from loguru import logger
-import time
-import os
-import base64
-
-from selenium.common.exceptions import TimeoutException
-
-
-from auto_archiver.core import Enricher
-from auto_archiver.utils import Webdriver, url as UrlUtil, random_str
-from auto_archiver.core import Media, Metadata
-
-
-class ScreenshotEnricher(Enricher):
-    def __init__(self, webdriver_factory=None):
-        super().__init__()
-        self.webdriver_factory = webdriver_factory or Webdriver
-
-    def enrich(self, to_enrich: Metadata) -> None:
-        url = to_enrich.get_url()
-
-        logger.debug(f"Enriching screenshot for {url=}")
-        auth = self.auth_for_site(url)
-
-        # screenshot enricher only supports cookie-type auth (selenium)
-        has_valid_auth = auth and (auth.get("cookies") or auth.get("cookies_jar") or auth.get("cookie"))
-
-        if UrlUtil.is_auth_wall(url) and not has_valid_auth:
-            logger.warning(f"[SKIP] SCREENSHOT since url is behind AUTH WALL and no login details provided: {url=}")
-            if any(auth.get(key) for key in ["username", "password", "api_key", "api_secret"]):
-                logger.warning(
-                    f"Screenshot enricher only supports cookie-type authentication, you have provided {auth.keys()} which are not supported.\
-                               Consider adding 'cookie', 'cookies_file' or 'cookies_from_browser' to your auth for this site."
-                )
-            return
-
-        with self.webdriver_factory(
-            self.width,
-            self.height,
-            self.timeout,
-            facebook_accept_cookies="facebook.com" in url,
-            http_proxy=self.http_proxy,
-            print_options=self.print_options,
-            auth=auth,
-        ) as driver:
-            try:
-                driver.get(url)
-                time.sleep(int(self.sleep_before_screenshot))
-                screenshot_file = os.path.join(self.tmp_dir, f"screenshot_{random_str(8)}.png")
-                driver.save_screenshot(screenshot_file)
-                to_enrich.add_media(Media(filename=screenshot_file), id="screenshot")
-                if self.save_to_pdf:
-                    pdf_file = os.path.join(self.tmp_dir, f"pdf_{random_str(8)}.pdf")
-                    pdf = driver.print_page(driver.print_options)
-                    with open(pdf_file, "wb") as f:
-                        f.write(base64.b64decode(pdf))
-                    to_enrich.add_media(Media(filename=pdf_file), id="pdf")
-            except TimeoutException:
-                logger.info("TimeoutException loading page for screenshot")
-            except Exception as e:
-                logger.error(f"Got error while loading webdriver for screenshot enricher: {e}")

src/auto_archiver/modules/ssl_enricher/ssl_enricher.py

@@ -2,7 +2,7 @@ import ssl
 import os
 from slugify import slugify
 from urllib.parse import urlparse
-from loguru import logger
+from auto_archiver.utils.custom_logger import logger
 
 from auto_archiver.core import Enricher
 from auto_archiver.core import Metadata, Media
@@ -19,10 +19,10 @@ class SSLEnricher(Enricher):
 
         url = to_enrich.get_url()
         parsed = urlparse(url)
-        assert parsed.scheme in ["https"], f"Invalid URL scheme {url=}"
+        assert parsed.scheme in ["https"], "Invalid URL scheme"
 
         domain = parsed.netloc
-        logger.debug(f"fetching SSL certificate for {domain=} in {url=}")
+        logger.debug(f"Fetching SSL certificate for {domain=}")
 
         cert = ssl.get_server_certificate((domain, 443))
         cert_fn = os.path.join(self.tmp_dir, f"{slugify(domain)}.pem")

src/auto_archiver/modules/telegram_extractor/telegram_extractor.py

@@ -2,7 +2,7 @@ import requests
 import re
 import html
 from bs4 import BeautifulSoup
-from loguru import logger
+from auto_archiver.utils.custom_logger import logger
 
 from auto_archiver.core import Extractor
 from auto_archiver.core import Metadata, Media
@@ -38,7 +38,7 @@ class TelegramExtractor(Extractor):
 
         video = s.find("video")
         if video is None:
-            logger.warning("could not find video")
+            logger.warning("Could not find video")
             image_tags = s.find_all(class_="tgme_widget_message_photo_wrap")
 
             image_urls = []

src/auto_archiver/modules/telethon_extractor/telethon_extractor.py

@@ -5,6 +5,7 @@ import time
 from pathlib import Path
 from datetime import date
 
+from telethon import functions
 from telethon.sync import TelegramClient
 from telethon.errors import ChannelInvalidError
 from telethon.tl.functions.messages import ImportChatInviteRequest
@@ -16,7 +17,7 @@ from telethon.errors.rpcerrorlist import (
 )
 
 from tqdm import tqdm
-from loguru import logger
+from auto_archiver.utils.custom_logger import logger
 
 from auto_archiver.core import Extractor
 from auto_archiver.core import Metadata, Media
@@ -24,7 +25,7 @@ from auto_archiver.utils import random_str
 
 
 class TelethonExtractor(Extractor):
-    valid_url = re.compile(r"https:\/\/t\.me(\/c){0,1}\/(.+)\/(\d+)")
+    valid_url = re.compile(r"https:\/\/t\.me(\/c){0,1}\/(.+?)(\/s){0,1}\/(\d+)")
     invite_pattern = re.compile(r"t.me(\/joinchat){0,1}\/\+?(.+)")
 
     def setup(self) -> None:
@@ -56,7 +57,7 @@ class TelethonExtractor(Extractor):
         self.client = TelegramClient(self.session_file, self.api_id, self.api_hash)
 
         with self.client.start():
-            logger.success(f"SETUP {self.name} login works.")
+            logger.info(f"SETUP {self.name} login works.")
 
         if self.join_channels and len(self.channel_invites):
             logger.info(f"SETUP {self.name} joining channels...")
@@ -64,7 +65,7 @@ class TelethonExtractor(Extractor):
                 # get currently joined channels
                 # https://docs.telethon.dev/en/stable/modules/custom.html#module-telethon.tl.custom.dialog
                 joined_channel_ids = [c.id for c in self.client.get_dialogs() if c.is_channel]
-                logger.info(f"already part of {len(joined_channel_ids)} channels")
+                logger.info(f"Already part of {len(joined_channel_ids)} channels")
 
                 i = 0
                 pbar = tqdm(desc=f"joining {len(self.channel_invites)} invite links", total=len(self.channel_invites))
@@ -79,22 +80,22 @@ class TelethonExtractor(Extractor):
                             else:
                                 ent = self.client.get_entity(invite)  # fails if not a member
                                 logger.warning(
-                                    f"please add the property id='{ent.id}' to the 'channel_invites' configuration where {invite=}, not doing so can lead to a minutes-long setup time due to telegram's rate limiting."
+                                    f"Please add the property id='{ent.id}' to the 'channel_invites' configuration where {invite=}, not doing so can lead to a minutes-long setup time due to telegram's rate limiting."
                                 )
                         except ValueError:
-                            logger.info(f"joining new channel {invite=}")
+                            logger.info(f"Joining new channel {invite=}")
                             try:
                                 self.client(ImportChatInviteRequest(match.group(2)))
                             except UserAlreadyParticipantError:
-                                logger.info(f"already joined {invite=}")
+                                logger.info(f"Already joined {invite=}")
                             except InviteRequestSentError:
-                                logger.warning(f"already sent a join request with {invite} still no answer")
+                                logger.warning(f"Already sent a join request with {invite} still no answer")
                             except InviteHashExpiredError:
                                 logger.warning(f"{invite=} has expired please find a more recent one")
                             except Exception as e:
-                                logger.error(f"could not join channel with {invite=} due to {e}")
+                                logger.error(f"Could not join channel with {invite=} due to {e}")
                         except FloodWaitError as e:
-                            logger.warning(f"got a flood error, need to wait {e.seconds} seconds")
+                            logger.warning(f"Got a flood error, need to wait {e.seconds} seconds")
                             time.sleep(e.seconds)
                             continue
                     else:
@@ -116,68 +117,91 @@ class TelethonExtractor(Extractor):
         url = item.get_url()
         # detect URLs that we definitely cannot handle
         match = self.valid_url.search(url)
-        logger.debug(f"TELETHON: {match=}")
+        logger.debug(f"Found telethon url {match=}")
         if not match:
             return False
 
         is_private = match.group(1) == "/c"
         chat = int(match.group(2)) if is_private else match.group(2)
-        post_id = int(match.group(3))
+        is_story = match.group(3) == "/s"
+        post_id = int(match.group(4))
 
         result = Metadata()
 
         # NB: not using bot_token since then private channels cannot be archived: self.client.start(bot_token=self.bot_token)
         with self.client.start():
             # with self.client.start(bot_token=self.bot_token):
-            try:
-                post = self.client.get_messages(chat, ids=post_id)
-            except ValueError as e:
-                logger.error(f"Could not fetch telegram {url} possibly it's private: {e}")
-                return False
-            except ChannelInvalidError as e:
-                logger.error(
-                    f"Could not fetch telegram {url}. This error may be fixed if you setup a bot_token in addition to api_id and api_hash (but then private channels will not be archived, we need to update this logic to handle both): {e}"
-                )
-                return False
+            if is_story:
+                try:
+                    stories = self.client(functions.stories.GetStoriesByIDRequest(peer=chat, id=[post_id]))
+                    if not stories.stories:
+                        logger.info("No stories found, possibly it's private or the story has expired.")
+                        return False
+                    story = stories.stories[0]
+                    logger.debug(f"Got story {story.id=} {story.date=} {story.expire_date=}")
+                    result.set_timestamp(story.date).set("views", story.views.to_dict()).set(
+                        "expire_date", story.expire_date
+                    )
 
-            logger.debug(f"TELETHON GOT POST {post=}")
-            if post is None:
-                return False
+                    # download the story media
+                    filename_dest = os.path.join(self.tmp_dir, f"{chat}_{post_id}", str(story.id))
+                    if filename := self.client.download_media(story.media, filename_dest):
+                        result.add_media(Media(filename))
+                except Exception as e:
+                    logger.error(f"Error fetching story {post_id} from {chat}: {e}")
+                    return False
+            else:
+                try:
+                    post = self.client.get_messages(chat, ids=post_id)
+                except ValueError as e:
+                    logger.error(f"Could not fetch telegram URL possibly it's private: {e}")
+                    return False
+                except ChannelInvalidError as e:
+                    logger.error(
+                        f"Could not fetch telegram URL. This error may be fixed if you setup a bot_token in addition to api_id and api_hash (but then private channels will not be archived, we need to update this logic to handle both): {e}"
+                    )
+                    return False
 
-            media_posts = self._get_media_posts_in_group(chat, post)
-            logger.debug(f"got {len(media_posts)=} for {url=}")
+                logger.debug(f"Got post {post=}")
+                if post is None:
+                    return False
 
-            tmp_dir = self.tmp_dir
+                media_posts = self._get_media_posts_in_group(chat, post)
+                logger.debug(f"Got {len(media_posts)=}")
 
-            group_id = post.grouped_id if post.grouped_id is not None else post.id
-            title = post.message
-            for mp in media_posts:
-                if len(mp.message) > len(title):
-                    title = mp.message  # save the longest text found (usually only 1)
+                group_id = post.grouped_id if post.grouped_id is not None else post.id
+                title = post.message
+                for mp in media_posts:
+                    if len(mp.message) > len(title):
+                        title = mp.message  # save the longest text found (usually only 1)
 
-                # media can also be in entities
-                if mp.entities:
-                    other_media_urls = [
-                        e.url
-                        for e in mp.entities
-                        if hasattr(e, "url") and e.url and self._guess_file_type(e.url) in ["video", "image", "audio"]
-                    ]
-                    if len(other_media_urls):
-                        logger.debug(f"Got {len(other_media_urls)} other media urls from {mp.id=}: {other_media_urls}")
-                    for i, om_url in enumerate(other_media_urls):
-                        filename = self.download_from_url(om_url, f"{chat}_{group_id}_{i}")
-                        result.add_media(Media(filename=filename), id=f"{group_id}_{i}")
+                    # media can also be in entities
+                    if mp.entities:
+                        other_media_urls = [
+                            e.url
+                            for e in mp.entities
+                            if hasattr(e, "url")
+                            and e.url
+                            and self._guess_file_type(e.url) in ["video", "image", "audio"]
+                        ]
+                        if len(other_media_urls):
+                            logger.debug(
+                                f"Got {len(other_media_urls)} other media urls from {mp.id=}: {other_media_urls}"
+                            )
+                        for i, om_url in enumerate(other_media_urls):
+                            filename = self.download_from_url(om_url, f"{chat}_{group_id}_{i}")
+                            result.add_media(Media(filename=filename), id=f"{group_id}_{i}")
 
-                filename_dest = os.path.join(tmp_dir, f"{chat}_{group_id}", str(mp.id))
-                filename = self.client.download_media(mp.media, filename_dest)
-                if not filename:
-                    logger.debug(f"Empty media found, skipping {str(mp)=}")
-                    continue
-                result.add_media(Media(filename))
+                    filename_dest = os.path.join(self.tmp_dir, f"{chat}_{group_id}", str(mp.id))
+                    filename = self.client.download_media(mp.media, filename_dest)
+                    if not filename:
+                        logger.debug(f"Empty media found, skipping {str(mp)=}")
+                        continue
+                    result.add_media(Media(filename))
 
-            result.set_title(title).set_timestamp(post.date).set("api_data", post.to_dict())
-            if post.message != title:
-                result.set_content(post.message)
+                result.set_title(title).set_timestamp(post.date).set("api_data", post.to_dict())
+                if post.message != title:
+                    result.set_content(post.message)
         return result.success("telethon")
 
     def _get_media_posts_in_group(self, chat, original_post, max_amp=10):

src/auto_archiver/modules/thumbnail_enricher/thumbnail_enricher.py

@@ -9,7 +9,7 @@ and identify important moments without watching the entire video.
 
 import ffmpeg
 import os
-from loguru import logger
+from auto_archiver.utils.custom_logger import logger
 
 from auto_archiver.core import Enricher
 from auto_archiver.core import Media, Metadata
@@ -27,24 +27,26 @@ class ThumbnailEnricher(Enricher):
         Calculates how many thumbnails to generate and at which timestamps based on the video duration, the number of thumbnails per minute and the max number of thumbnails.
         Thumbnails are equally distributed across the video duration.
         """
-        logger.debug(f"generating thumbnails for {to_enrich.get_url()}")
+        logger.debug("Generating thumbnails")
         for m_id, m in enumerate(to_enrich.media[::]):
             if m.is_video():
                 folder = os.path.join(self.tmp_dir, random_str(24))
                 os.makedirs(folder, exist_ok=True)
-                logger.debug(f"generating thumbnails for {m.filename}")
+                logger.debug(f"Generating thumbnails for {m.filename}")
                 duration = m.get("duration")
 
-                if duration is None:
-                    try:
-                        probe = ffmpeg.probe(m.filename)
-                        duration = float(
-                            next(stream for stream in probe["streams"] if stream["codec_type"] == "video")["duration"]
-                        )
-                        to_enrich.media[m_id].set("duration", duration)
-                    except Exception as e:
-                        logger.error(f"error getting duration of video {m.filename}: {e}")
-                        return
+                try:
+                    probe = ffmpeg.probe(m.filename)
+                    duration = float(
+                        next(stream for stream in probe["streams"] if stream["codec_type"] == "video")["duration"]
+                    )
+                    to_enrich.media[m_id].set("duration", duration)
+                except Exception as e:
+                    logger.warning(f"Failed to get duration with FFMPEG from {m.filename}: {e}")
+
+                if not duration or type(duration) not in [float, int] or duration <= 0:
+                    logger.warning(f"Cannot generate thumbnails for {m.filename} without valid duration")
+                    continue
 
                 num_thumbs = int(min(max(1, (duration / 60) * self.thumbnails_per_minute), self.max_thumbnails))
                 timestamps = [duration / (num_thumbs + 1) * i for i in range(1, num_thumbs + 1)]
@@ -57,6 +59,9 @@ class ThumbnailEnricher(Enricher):
                     ).run()
 
                     try:
+                        if not os.path.exists(output_path):
+                            logger.info(f"thumbnail {index} for media {m.filename} was not created")
+                            continue
                         thumbnails_media.append(
                             Media(filename=output_path)
                             .set("id", f"thumbnail_{index}")

src/auto_archiver/modules/timestamping_enricher/__manifest__.py

@@ -20,7 +20,7 @@
                     # "http://tsa.sinpe.fi.cr/tsaHttp/", # self-signed
                     # "http://tsa.cra.ge/signserver/tsa?workerName=qtsa", # self-signed
                     "http://tss.cnbs.gob.hn/TSS/HttpTspServer",
-                    "http://dss.nowina.lu/pki-factory/tsa/good-tsa",
+                    # "http://dss.nowina.lu/pki-factory/tsa/good-tsa",
                     # "https://freetsa.org/tsr", # self-signed
                 ],
             "help": "List of RFC3161 Time Stamp Authorities to use, separate with commas if passed via the command line.",

src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py

@@ -4,12 +4,12 @@ from importlib.metadata import version
 import hashlib
 
 from slugify import slugify
+from retrying import retry
 import requests
-from loguru import logger
+from auto_archiver.utils.custom_logger import logger
 
-from rfc3161_client import (decode_timestamp_response,TimestampRequestBuilder,TimeStampResponse, VerifierBuilder)
+from rfc3161_client import (decode_timestamp_response, TimestampRequestBuilder, TimeStampResponse, VerifierBuilder)
 from rfc3161_client import VerificationError as Rfc3161VerificationError
-from rfc3161_client.base import HashAlgorithm
 from rfc3161_client.tsp import SignedData
 from cryptography import x509
 from cryptography.hazmat.primitives import serialization
@@ -49,8 +49,7 @@ class TimestampingEnricher(Enricher):
             self.session.close()
 
     def enrich(self, to_enrich: Metadata) -> None:
-        url = to_enrich.get_url()
-        logger.debug(f"RFC3161 timestamping existing files for {url=}")
+        logger.debug(f"RFC3161 timestamping existing files")
 
         # create a new text file with the existing media hashes
         hashes = [
@@ -58,10 +57,9 @@ class TimestampingEnricher(Enricher):
         ]
 
         if not len(hashes):
-            logger.warning(f"No hashes found in {url=}")
+            logger.debug(f"No hashes found")
             return
 
-        
         hashes_fn = os.path.join(self.tmp_dir, "hashes.txt")
 
         data_to_sign = "\n".join(hashes)
@@ -74,9 +72,9 @@ class TimestampingEnricher(Enricher):
             try:
                 message = bytes(data_to_sign, encoding='utf8')
 
-                logger.debug(f"Timestamping {url=} with {tsa_url=}")
+                logger.debug(f"Timestamping with {tsa_url=}")
                 signed: TimeStampResponse = self.sign_data(tsa_url, message)
-                
+
                 # fail if there's any issue with the certificates, uses certifi list of trusted CAs or the user-defined `cert_authorities`
                 root_cert = self.verify_signed(signed, message)
 
@@ -92,7 +90,7 @@ class TimestampingEnricher(Enricher):
                 timestamp_token_path = self.save_timestamp_token(signed.time_stamp_token(), tsa_url)
                 timestamp_tokens.append(Media(filename=timestamp_token_path).set("tsa", tsa_url).set("cert_chain", cert_chain))
             except Exception as e:
-                logger.warning(f"Error while timestamping {url=} with {tsa_url=}: {e}")
+                logger.warning(f"Error while timestamping with {tsa_url=}: {e}")
 
         if len(timestamp_tokens):
             hashes_media.set("timestamp_authority_files", timestamp_tokens)
@@ -101,9 +99,9 @@ class TimestampingEnricher(Enricher):
             hashes_media.set("cryptography v", version("cryptography"))
             to_enrich.add_media(hashes_media, id="timestamped_hashes")
             to_enrich.set("timestamped", True)
-            logger.success(f"{len(timestamp_tokens)} timestamp tokens created for {url=}")
+            logger.info(f"{len(timestamp_tokens)} timestamp tokens created")
         else:
-            logger.warning(f"No successful timestamps for {url=}")
+            logger.warning(f"No successful timestamps found")
 
     def save_timestamp_token(self, timestamp_token: bytes, tsa_url: str) -> str:
         """
@@ -114,7 +112,7 @@ class TimestampingEnricher(Enricher):
             f.write(timestamp_token)
         return tst_path
 
-    def verify_signed(self, timestamp_response: TimeStampResponse, message: bytes) ->  x509.Certificate:
+    def verify_signed(self, timestamp_response: TimeStampResponse, message: bytes) -> x509.Certificate:
         """
         Verify a Signed Timestamp Response is trusted by a known Certificate Authority.
 
@@ -137,7 +135,7 @@ class TimestampingEnricher(Enricher):
 
         if not cert_authorities:
             raise ValueError(f"No trusted roots found in {trusted_root_path}.")
-        
+
         timestamp_certs = self.tst_certs(timestamp_response)
         intermediate_certs = timestamp_certs[1:-1]
 
@@ -149,7 +147,7 @@ class TimestampingEnricher(Enricher):
             message_hash = hashlib.sha256(message).digest()
         else:
             raise ValueError(f"Unsupported hash algorithm: {hash_algorithm}")
-        
+
         for certificate in cert_authorities:
             builder = VerifierBuilder()
             builder.add_root_certificate(certificate)
@@ -159,7 +157,6 @@ class TimestampingEnricher(Enricher):
 
             verifier = builder.build()
 
-            
             try:
                 verifier.verify(timestamp_response, message_hash)
                 return certificate
@@ -172,23 +169,38 @@ class TimestampingEnricher(Enricher):
         # see https://github.com/sigstore/sigstore-python/blob/99948d5b80525a5a104e904ffea58169dc6e0629/sigstore/_internal/timestamp.py#L84-L121
 
         timestamp_request = (
-                TimestampRequestBuilder().data(bytes_data).nonce(nonce=True).build()
-            )
-        try:
+            TimestampRequestBuilder().data(bytes_data).nonce(nonce=True).build()
+        )
+
+        @retry(
+            wait_exponential_multiplier=1,
+            stop_max_attempt_number=2,
+        )
+        def sign_with_retry():
             response = self.session.post(tsa_url, data=timestamp_request.as_bytes(), timeout=10)
             response.raise_for_status()
+            return response
+
+        try:
+            response = sign_with_retry()
         except requests.RequestException as e:
             logger.error(f"Error while sending request to {tsa_url=}: {e}")
             raise
 
+        @retry(
+            wait_exponential_multiplier=1,
+            stop_max_attempt_number=2,
+        )
+        def decode_with_retry(response):
+            return decode_timestamp_response(response.content)
         # Check that we can parse the response but do not *verify* it
         try:
-            timestamp_response = decode_timestamp_response(response.content)
+            timestamp_response = decode_with_retry(response)
         except ValueError as e:
             logger.error(f"Invalid timestamp response from server {tsa_url}: {e}")
             raise
         return timestamp_response
-    
+
     def tst_certs(self, tsp_response: TimeStampResponse):
         signed_data: SignedData = tsp_response.signed_data
         certs = [x509.load_der_x509_certificate(c) for c in signed_data.certificates]
@@ -197,7 +209,7 @@ class TimestampingEnricher(Enricher):
         if len(certs) == 1:
             return certs
 
-        while(len(ordered_certs) < len(certs)):
+        while (len(ordered_certs) < len(certs)):
             if len(ordered_certs) == 0:
                 for cert in certs:
                     if not [c for c in certs if cert.subject == c.issuer]:
@@ -221,7 +233,7 @@ class TimestampingEnricher(Enricher):
 
         cert_chain = []
         for i, cert in enumerate(certificates):
-            cert_fn = os.path.join(self.tmp_dir, f"{i+1} – {str(cert.serial_number)[:20]}.crt")
+            cert_fn = os.path.join(self.tmp_dir, f"{i + 1} – {str(cert.serial_number)[:20]}.crt")
             with open(cert_fn, "wb") as f:
                 f.write(cert.public_bytes(encoding=serialization.Encoding.PEM))
             cert_chain.append(Media(filename=cert_fn).set("subject", cert.subject.get_attributes_for_oid(x509.NameOID.COMMON_NAME)[0].value))

src/auto_archiver/modules/twitter_api_extractor/twitter_api_extractor.py

@@ -1,9 +1,10 @@
+from datetime import timezone
 import json
 import re
 import mimetypes
 import requests
 
-from loguru import logger
+from auto_archiver.utils.custom_logger import logger
 from pytwitter import Api
 from slugify import slugify
 
@@ -44,10 +45,9 @@ class TwitterApiExtractor(Extractor):
         if "https://t.co/" in url:
             try:
                 r = requests.get(url, timeout=30)
-                logger.debug(f"Expanded url {url} to {r.url}")
                 url = r.url
-            except Exception:
-                logger.error(f"Failed to expand url {url}")
+            except Exception as e:
+                logger.error(f"Failed to expand Twitter URL: {e}")
         return url
 
     def download(self, item: Metadata) -> Metadata:
@@ -66,7 +66,7 @@ class TwitterApiExtractor(Extractor):
             return False, False
 
         username, tweet_id = matches[0]  # only one URL supported
-        logger.debug(f"Found {username=} and {tweet_id=} in {url=}")
+        logger.debug(f"Found {username=} and {tweet_id=}")
 
         return username, tweet_id
 
@@ -84,14 +84,16 @@ class TwitterApiExtractor(Extractor):
                 media_fields=["type", "duration_ms", "url", "variants"],
                 tweet_fields=["attachments", "author_id", "created_at", "entities", "id", "text", "possibly_sensitive"],
             )
-            logger.debug(tweet)
+            logger.debug(f"Got {tweet=}")
         except Exception as e:
             logger.error(f"Could not get tweet: {e}")
             return False
 
         result = Metadata()
         result.set_title(tweet.data.text)
-        result.set_timestamp(get_datetime_from_str(tweet.data.created_at, "%Y-%m-%dT%H:%M:%S.%fZ"))
+        result.set_timestamp(
+            get_datetime_from_str(tweet.data.created_at, "%Y-%m-%dT%H:%M:%S.%fZ").replace(tzinfo=timezone.utc)
+        )
 
         urls = []
         if tweet.includes:

src/auto_archiver/modules/wacz_extractor_enricher/wacz_extractor_enricher.py

@@ -4,7 +4,7 @@ import os
 import shutil
 import subprocess
 from zipfile import ZipFile
-from loguru import logger
+from auto_archiver.utils.custom_logger import logger
 from warcio.archiveiterator import ArchiveIterator
 
 from auto_archiver.core import Media, Metadata
@@ -24,8 +24,7 @@ class WaczExtractorEnricher(Enricher, Extractor):
         self.use_docker = os.environ.get("WACZ_ENABLE_DOCKER") or not os.environ.get("RUNNING_IN_DOCKER")
         self.docker_in_docker = os.environ.get("WACZ_ENABLE_DOCKER") and os.environ.get("RUNNING_IN_DOCKER")
 
-        self.crawl_id = random_str(8)
-        self.cwd_dind = f"/crawls/crawls{self.crawl_id}"
+        self.cwd_dind = f"/crawls/crawls{random_str(8)}"
         self.browsertrix_home_host = os.environ.get("BROWSERTRIX_HOME_HOST")
         self.browsertrix_home_container = os.environ.get("BROWSERTRIX_HOME_CONTAINER") or self.browsertrix_home_host
         # create crawls folder if not exists, so it can be safely removed in cleanup
@@ -51,7 +50,8 @@ class WaczExtractorEnricher(Enricher, Extractor):
 
         url = to_enrich.get_url()
 
-        collection = self.crawl_id
+        crawl_id = random_str(8)
+        collection = crawl_id
         browsertrix_home_host = self.browsertrix_home_host or os.path.abspath(self.tmp_dir)
         browsertrix_home_container = self.browsertrix_home_container or browsertrix_home_host
 
@@ -83,8 +83,10 @@ class WaczExtractorEnricher(Enricher, Extractor):
             # "--blockAds" # note: this has been known to cause issues on cloudflare protected sites
         ]
 
+        crawl_cwd_dind = os.path.join(self.cwd_dind, crawl_id)
         if self.docker_in_docker:
-            cmd.extend(["--cwd", self.cwd_dind])
+            os.makedirs(crawl_cwd_dind, exist_ok=True)
+            cmd.extend(["--cwd", crawl_cwd_dind])
 
         if self.auth_for_site(url):
             # there's an auth for this site, but browsertrix only supports username/password auth
@@ -94,7 +96,7 @@ class WaczExtractorEnricher(Enricher, Extractor):
 
         # call docker if explicitly enabled or we are running on the host (not in docker)
         if self.use_docker:
-            logger.debug(f"generating WACZ in Docker for {url=}")
+            logger.debug("Generating WACZ in Docker")
             logger.debug(f"{browsertrix_home_host=} {browsertrix_home_container=}")
             if self.docker_commands:
                 cmd = self.docker_commands + cmd
@@ -109,14 +111,14 @@ class WaczExtractorEnricher(Enricher, Extractor):
                 ] + cmd
 
             if self.profile:
-                profile_file = f"profile-{self.crawl_id}.tar.gz"
+                profile_file = f"profile-{crawl_id}.tar.gz"
                 profile_fn = os.path.join(browsertrix_home_container, profile_file)
-                logger.debug(f"copying {self.profile} to {profile_fn}")
+                logger.debug(f"Copying {self.profile} to {profile_fn}")
                 shutil.copyfile(self.profile, profile_fn)
                 cmd.extend(["--profile", os.path.join("/crawls", profile_file)])
 
         else:
-            logger.debug(f"generating WACZ without Docker for {url=}")
+            logger.debug("Generating WACZ without Docker")
 
             if self.profile:
                 cmd.extend(["--profile", os.path.join("/app", str(self.profile))])
@@ -137,7 +139,7 @@ class WaczExtractorEnricher(Enricher, Extractor):
             return False
 
         if self.docker_in_docker:
-            wacz_fn = os.path.join(self.cwd_dind, "collections", collection, f"{collection}.wacz")
+            wacz_fn = os.path.join(crawl_cwd_dind, "collections", collection, f"{collection}.wacz")
         elif self.use_docker:
             wacz_fn = os.path.join(browsertrix_home_container, "collections", collection, f"{collection}.wacz")
         else:
@@ -152,7 +154,7 @@ class WaczExtractorEnricher(Enricher, Extractor):
             self.extract_media_from_wacz(to_enrich, wacz_fn)
 
         if self.docker_in_docker:
-            jsonl_fn = os.path.join(self.cwd_dind, "collections", collection, "pages", "pages.jsonl")
+            jsonl_fn = os.path.join(crawl_cwd_dind, "collections", collection, "pages", "pages.jsonl")
         elif self.use_docker:
             jsonl_fn = os.path.join(browsertrix_home_container, "collections", collection, "pages", "pages.jsonl")
         else:
@@ -204,7 +206,7 @@ class WaczExtractorEnricher(Enricher, Extractor):
                 if (
                     record.rec_type == "resource" and record.content_type == "image/png" and self.extract_screenshot
                 ):  # screenshots
-                    fn = os.path.join(tmp_dir, f"warc-file-{counter_screenshots}.png")
+                    fn = os.path.join(tmp_dir, f"browsertrix-screenshot-{counter_screenshots}.png")
                     with open(fn, "wb") as outf:
                         outf.write(record.raw_stream.read())
                     m = Media(filename=fn)
@@ -232,7 +234,7 @@ class WaczExtractorEnricher(Enricher, Extractor):
 
                 # create local file and add media
                 ext = mimetypes.guess_extension(content_type)
-                warc_fn = f"warc-file-{counter_screenshots}{ext}"
+                warc_fn = f"warc-file-{counter_warc_files}{ext}"
                 fn = os.path.join(tmp_dir, warc_fn)
 
                 record_url_best_qual = UrlUtil.twitter_best_quality_url(record_url)

src/auto_archiver/modules/wayback_extractor_enricher/wayback_extractor_enricher.py

@@ -1,8 +1,8 @@
 import json
-from loguru import logger
+from auto_archiver.utils.custom_logger import logger
 import time
 import requests
-
+from urllib3.exceptions import MaxRetryError
 from auto_archiver.core import Extractor, Enricher
 from auto_archiver.utils import url as UrlUtil
 from auto_archiver.core import Metadata
@@ -31,21 +31,28 @@ class WaybackExtractorEnricher(Enricher, Extractor):
 
         url = to_enrich.get_url()
         if UrlUtil.is_auth_wall(url):
-            logger.debug(f"[SKIP] WAYBACK since url is behind AUTH WALL: {url=}")
+            logger.debug("[SKIP] WAYBACK since url is behind AUTH WALL")
             return
 
-        logger.debug(f"calling wayback for {url=}")
-
         if to_enrich.get("wayback"):
             logger.info(f"Wayback enricher had already been executed: {to_enrich.get('wayback')}")
             return True
 
+        logger.debug("Calling Wayback")
+
         ia_headers = {"Accept": "application/json", "Authorization": f"LOW {self.key}:{self.secret}"}
         post_data = {"url": url}
         if self.if_not_archived_within:
             post_data["if_not_archived_within"] = self.if_not_archived_within
         # see https://docs.google.com/document/d/1Nsv52MvSjbLb2PCpHlat0gkzw0EvtSgpKHu4mk0MnrA for more options
-        r = requests.post("https://web.archive.org/save/", headers=ia_headers, data=post_data, proxies=proxies)
+        try:
+            r = requests.post("https://web.archive.org/save/", headers=ia_headers, data=post_data, proxies=proxies)
+        except MaxRetryError as e:
+            logger.warning(
+                f"MaxRetryError during Wayback POST call to /save, this may be do to a high number of calls leading to rate limiting: {e}"
+            )
+            to_enrich.set("wayback", "failed: possible rate limit")
+            return False
 
         if r.status_code != 200:
             logger.error(em := f"Internet archive failed with status of {r.status_code}: {r.json()}")
@@ -68,7 +75,7 @@ class WaybackExtractorEnricher(Enricher, Extractor):
         attempt = 1
         while not wayback_url and time.time() - start_time <= self.timeout:
             try:
-                logger.debug(f"GETting status for {job_id=} on {url=} ({attempt=})")
+                logger.debug(f"GETting status for {job_id=} ({attempt=})")
                 r_status = requests.get(
                     f"https://web.archive.org/save/status/{job_id}", headers=ia_headers, proxies=proxies
                 )
@@ -76,16 +83,19 @@ class WaybackExtractorEnricher(Enricher, Extractor):
                 if r_status.status_code == 200 and r_json["status"] == "success":
                     wayback_url = f"https://web.archive.org/web/{r_json['timestamp']}/{r_json['original_url']}"
                 elif r_status.status_code != 200 or r_json["status"] != "pending":
+                    if r_json.get("status_ext") in ["error:blocked-url", "error:unauthorized"]:
+                        logger.warning("Wayback cannot currently archive the URL, skipping.")
+                        to_enrich.set("wayback", r_json.get("status_ext"))
                     logger.error(f"Wayback failed with {r_json}")
                     return False
             except requests.exceptions.RequestException as e:
-                logger.warning(f"RequestException: fetching status for {url=} due to: {e}")
+                logger.warning(f"RequestException: fetching status due to: {e}")
                 break
             except json.decoder.JSONDecodeError:
-                logger.error(f"Expected a JSON from Wayback and got {r.text} for {url=}")
+                logger.error(f"Expected a JSON from Wayback and got {r.text}")
                 break
             except Exception as e:
-                logger.warning(f"error fetching status for {url=} due to: {e}")
+                logger.warning(f"error fetching status due to: {e}")
             if not wayback_url:
                 attempt += 1
                 time.sleep(1)  # TODO: can be improved with exponential backoff

src/auto_archiver/modules/whisper_enricher/whisper_enricher.py

@@ -1,7 +1,7 @@
 import traceback
 import requests
 import time
-from loguru import logger
+from auto_archiver.utils.custom_logger import logger
 
 from auto_archiver.core import Enricher
 from auto_archiver.core import Metadata, Media
@@ -25,7 +25,7 @@ class WhisperEnricher(Enricher):
 
     def enrich(self, to_enrich: Metadata) -> None:
         url = to_enrich.get_url()
-        logger.debug(f"WHISPER[{self.action}]: iterating media items for {url=}.")
+        logger.debug(f"WHISPER[{self.action}]: iterating media items")
 
         job_results = {}
         for i, m in enumerate(to_enrich.media):
@@ -35,7 +35,7 @@ class WhisperEnricher(Enricher):
                 try:
                     job_id = self.submit_job(m)
                     job_results[job_id] = False
-                    logger.debug(f"JOB SUBMITTED: {job_id=} for {m.key=}")
+                    logger.debug(f"Job submitted: {job_id=} for {m.key=}")
                     to_enrich.media[i].set("whisper_model", {"job_id": job_id})
                 except Exception as e:
                     logger.error(
@@ -72,14 +72,14 @@ class WhisperEnricher(Enricher):
             "type": self.action,
             # "language": "string" # may be a config
         }
-        logger.debug(f"calling API with {payload=}")
+        logger.debug(f"Calling API with {payload=}")
         response = requests.post(
             f"{self.api_endpoint}/jobs", json=payload, headers={"Authorization": f"Bearer {self.api_key}"}
         )
         assert response.status_code == 201, (
             f"calling the whisper api {self.api_endpoint} returned a non-success code: {response.status_code}"
         )
-        logger.debug(response.json())
+        logger.debug(f"Response from whisper API: {response.json()}")
         return response.json()["id"]
 
     def check_jobs(self, job_results: dict):
@@ -115,7 +115,7 @@ class WhisperEnricher(Enricher):
             assert r_res.status_code == 200, (
                 f"Job artifacts did not respond with 200, instead with: {r_res.status_code}"
             )
-            logger.success(r_res.json())
+            logger.info(f"Job {job_id} completed successfully:{r_res.json()}")
             result = {}
             for art_id, artifact in enumerate(r_res.json()):
                 subtitle = []

src/auto_archiver/utils/__init__.py

@@ -2,7 +2,6 @@
 
 # we need to explicitly expose the available imports here
 from .misc import *
-from .webdriver import Webdriver
 
 # handy utils from ytdlp
 from yt_dlp.utils import clean_html, traverse_obj, strip_or_none, url_or_none

src/auto_archiver/utils/custom_logger.py

@@ -0,0 +1,66 @@
+from loguru import logger
+import json
+
+
+def type_serializer(obj):
+    """Fallback function for objects json can't handle."""
+    if isinstance(obj, type):
+        return obj.__name__
+    return str(obj)
+
+
+def extract_location(record, short=False):
+    """Extracts the file name, function name, and line number from the log record."""
+    if short:
+        return f"{record['file'].name}:{record['line']}"
+    return f"{record['file'].name}:{record['function']}:{record['line']}"
+
+
+def extract_log_data(record):
+    subset = {
+        "level": record["level"].name,
+        "time": record["time"].isoformat(timespec="seconds"),
+    }
+    subset["loc"] = extract_location(record)
+
+    # This is where logger.contextualize() parameters can be added to the output
+    for extra_key in ["trace", "url", "worksheet", "row"]:
+        if extra_val := record.get("extra", {}).get(extra_key):
+            subset[extra_key] = extra_val
+
+    subset["message"] = record["message"]
+    if exception := record.get("exception"):
+        subset["exception"] = exception
+    return subset
+
+
+def serialize_for_console(record):
+    subset = extract_log_data(record)
+    subset.pop("message", None)
+    subset.pop("level", None)
+    subset.pop("loc", None)
+    subset.pop("time", None)
+    if not subset:
+        return ""
+    return json.dumps(subset, ensure_ascii=False, default=type_serializer)
+
+
+def serialize(record):
+    return json.dumps(extract_log_data(record), ensure_ascii=False, default=type_serializer)
+
+
+def patching(record):
+    record["extra"]["serialized"] = serialize(record)
+    record["extra"]["serialize_for_console"] = serialize_for_console(record)
+
+
+def format_for_human_readable_console():
+    return (
+        "<green>{time:YYYY-MM-DD HH:mm:ss.SSS}</green> | "
+        "<level>{level: <8}</level> | "
+        "<cyan>{file}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> | "
+        "{extra[serialize_for_console]} <level>{message}</level>"
+    )
+
+
+logger = logger.patch(patching)

src/auto_archiver/utils/deletion_detection.py

@@ -0,0 +1,273 @@
+"""
+Deletion Detection Utilities
+
+Provides a best-effort detection of deleted, missing, or unavailable content
+across various social media platforms based on presence of expected keywords.
+
+This module helps identify removed content, helps to:
+- Document content that existed but was deleted
+- Track patterns of content removal
+- Preserve metadata about missing content
+"""
+
+from typing import Optional, Dict, List
+from auto_archiver.utils.custom_logger import logger
+from urllib.parse import urlparse
+
+
+class DeletionIndicators:
+    """
+    Platform-specific indicators that content has been deleted or is unavailable, alongside generic indicators.
+    """
+
+    # Twitter/X deletion indicators
+    TWITTER = [
+        "Hmm...this page doesn't exist",
+        "Try searching for something else",
+        "This Tweet is unavailable",
+        "This account doesn't exist",
+        "This Tweet has been deleted",
+        "This account has been suspended",
+        "Sorry, that page doesn't exist",
+        "The Tweet you're looking for isn't available",
+    ]
+
+    # Facebook deletion indicators
+    FACEBOOK = [
+        "This content isn't available",
+        "Sorry, this content isn't available",
+        "This content is no longer available",
+        "The link you followed may be broken",
+        "Page Not Found",
+        "Content Not Found",
+        "This content is no longer on Facebook",
+    ]
+
+    # Instagram deletion indicators
+    INSTAGRAM = [
+        "Sorry, this page isn't available",
+        "The link you followed may be broken",
+        "Media not found or unavailable",
+        "This post is no longer available",
+        "This account is private",
+    ]
+
+    # TikTok deletion indicators
+    TIKTOK = [
+        "Couldn't find this account",
+        "This video is no longer available",
+        "This video is currently unavailable",
+        "Video not found",
+        "This video may have been deleted",
+    ]
+
+    # YouTube deletion indicators
+    YOUTUBE = [
+        "This video isn't available anymore",
+        "Video unavailable",
+        "This video has been removed",
+        "This video is no longer available",
+        "This video is private",
+        "This video has been removed by the uploader",
+        "This video has been deleted",
+    ]
+
+    # Reddit deletion indicators
+    REDDIT = [
+        "this post has been removed",
+        "this comment has been removed",
+        "[removed]",
+        "[deleted]",
+        "page not found",
+        "there doesn't seem to be anything here",
+    ]
+
+    # VK deletion indicators
+    VK = [
+        "Post deleted",
+        "Page not found",
+        "Content unavailable",
+        "Access denied",
+    ]
+
+    # Telegram deletion indicators
+    TELEGRAM = [
+        "Message not found",
+        "Deleted message",
+        "Channel is private",
+    ]
+
+    # Generic indicators (work across platforms)
+    GENERIC = [
+        "has been removed",
+        "no longer available",
+        "content removed",
+        "access denied",
+        "page not found",
+    ]
+
+    @classmethod
+    def all_indicators(cls) -> List[str]:
+        """Returns all deletion indicators from all platforms."""
+        return (
+            cls.TWITTER
+            + cls.FACEBOOK
+            + cls.INSTAGRAM
+            + cls.TIKTOK
+            + cls.YOUTUBE
+            + cls.REDDIT
+            + cls.VK
+            + cls.TELEGRAM
+            + cls.GENERIC
+        )
+
+    @classmethod
+    def for_url(cls, url: str) -> List[str]:
+        """Returns platform-specific indicators based on URL domain."""
+        platform = _extract_platform(url)
+
+        indicators_map = {
+            "twitter": cls.TWITTER + cls.GENERIC,
+            "facebook": cls.FACEBOOK + cls.GENERIC,
+            "instagram": cls.INSTAGRAM + cls.GENERIC,
+            "tiktok": cls.TIKTOK + cls.GENERIC,
+            "youtube": cls.YOUTUBE + cls.GENERIC,
+            "reddit": cls.REDDIT + cls.GENERIC,
+            "vk": cls.VK + cls.GENERIC,
+            "telegram": cls.TELEGRAM + cls.GENERIC,
+        }
+        return indicators_map.get(platform, cls.GENERIC)
+
+
+def detect_deletion(
+    html_content: str = None,
+    page_title: str = None,
+    error_message: str = None,
+    url: str = None,
+    video_data: dict = None,
+) -> Optional[Dict[str, any]]:
+    """
+    Best-effort deletion detection across multiple signals.
+
+    Checks HTML content, page titles, error messages, and video metadata for
+    indicators that content has been deleted or is unavailable.
+
+    Args:
+        html_content: Raw HTML source of the page
+        page_title: Browser page title
+        error_message: Any error message from the extractor
+        url: The URL being archived (for platform-specific detection)
+        video_data: Video metadata from yt-dlp or other extractors
+
+    Returns:
+        Dictionary with deletion details if detected, None otherwise.
+        Format: {
+            "is_deleted": True,
+            "indicator": "specific text that was found",
+            "source": "html|title|error|metadata",
+            "platform": "twitter|facebook|etc"
+        }
+    """
+
+    # Determine indicators to check based on URL
+    if url:
+        indicators = DeletionIndicators.for_url(url)
+        platform = _extract_platform(url)
+    else:
+        indicators = DeletionIndicators.all_indicators()
+        platform = "unknown"
+
+    # Check HTML content
+    if html_content:
+        for indicator in indicators:
+            if indicator.lower() in html_content.lower():
+                logger.info(f"Deletion detected in HTML: '{indicator}' found for {url}")
+                return {"is_deleted": True, "indicator": indicator, "source": "html_content", "platform": platform}
+
+    # Check page title
+    if page_title:
+        for indicator in indicators:
+            if indicator.lower() in page_title.lower():
+                logger.info(f"Deletion detected in page title: '{indicator}' found for {url}")
+                return {"is_deleted": True, "indicator": indicator, "source": "page_title", "platform": platform}
+
+    # Check error messages
+    if error_message:
+        for indicator in indicators:
+            if indicator.lower() in str(error_message).lower():
+                logger.info(f"Deletion detected in error: '{indicator}' found for {url}")
+                return {"is_deleted": True, "indicator": indicator, "source": "error_message", "platform": platform}
+
+    # Check video metadata (from yt-dlp)
+    if video_data:
+        # Check if yt-dlp flagged it as unavailable
+        if video_data.get("availability") in ["unavailable", "private", "deleted"]:
+            logger.info(f"Deletion detected in metadata: availability={video_data.get('availability')}")
+            return {
+                "is_deleted": True,
+                "indicator": f"availability: {video_data.get('availability')}",
+                "source": "video_metadata",
+                "platform": platform,
+            }
+
+        # Check description/title for deletion indicators
+        for key in ["title", "description", "fulltitle"]:
+            if key in video_data:
+                for indicator in indicators:
+                    if indicator.lower() in str(video_data[key]).lower():
+                        logger.info(f"Deletion detected in {key}: '{indicator}'")
+                        return {
+                            "is_deleted": True,
+                            "indicator": indicator,
+                            "source": f"video_metadata_{key}",
+                            "platform": platform,
+                        }
+
+    return None
+
+
+def _extract_platform(url: str) -> str:
+    """Extracts platform name from URL."""
+    parsed = urlparse(url)
+    domain = parsed.netloc
+
+    if "twitter.com" in domain or "x.com" in domain:
+        return "twitter"
+    elif "facebook.com" in domain or "fb.com" in domain:
+        return "facebook"
+    elif "instagram.com" in domain:
+        return "instagram"
+    elif "tiktok.com" in domain:
+        return "tiktok"
+    elif "youtube.com" in domain or "youtu.be" in domain:
+        return "youtube"
+    elif "reddit.com" in domain:
+        return "reddit"
+    elif "vk.com" in domain:
+        return "vk"
+    elif "t.me" in domain:
+        return "telegram"
+    return "unknown"
+
+
+def flag_as_deleted(metadata, deletion_info: Dict[str, any]) -> None:
+    """
+    Flags metadata object as deleted/unavailable.
+    Adds tentative deletion information to the metadata object.
+
+    Args:
+        metadata: Metadata object to update
+        deletion_info: Dictionary from detect_deletion()
+    """
+    metadata.set("deletion_detected", True)
+    metadata.set("deletion_indicator", deletion_info.get("indicator"))
+    metadata.set("deletion_source", deletion_info.get("source"))
+    metadata.set("deletion_platform", deletion_info.get("platform"))
+    metadata.status = "deleted_or_unavailable"
+
+    logger.debug(
+        f"Content marked as deleted/unavailable: "
+        f"platform={deletion_info.get('platform')}, "
+        f"indicator='{deletion_info.get('indicator')}', "
+        f"source={deletion_info.get('source')}"
+    )