bellingcat/auto-archiver
1
0
Fork 0
mirror of https://github.com/bellingcat/auto-archiver.git synced 2026-06-08 03:18:28 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #312 from bellingcat/dev v1.1.0

Browse Source 
v1.1.0 WIP
This commit is contained in:
Miguel Sozinho Ramalho
2025-06-17 23:57:22 +01:00
committed by GitHub
parent 6735fa890b 918cb220be
commit 3869ea73d7
71 changed files with 2911 additions and 972 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
.github/dependabot.yml vendored
View File

@@ -12,7 +12,7 @@ updates:
patterns:
- "*"
schedule:
interval: "weekly"
interval: "monthly"
- package-ecosystem: "github-actions"
directory: "/"
@@ -21,7 +21,7 @@ updates:
patterns:
- "*"
schedule:
interval: "weekly"
interval: "monthly"
- package-ecosystem: "npm"
directory: "/scripts/settings/"
@@ -30,11 +30,11 @@ updates:
patterns:
- "*"
schedule:
interval: "weekly"
interval: "monthly"
- package-ecosystem: "docker"
# Look for a `Dockerfile` in the `root` directory
directory: "/"
# Check for updates once a week
schedule:
interval: "weekly"
interval: "monthly"

5
.github/workflows/tests-core.yaml vendored
View File

@@ -28,6 +28,9 @@ jobs:
steps:
- uses: actions/checkout@v4
- name: Install ffmpeg
run: sudo apt-get update && sudo apt-get install -y ffmpeg
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
@@ -35,7 +38,7 @@ jobs:
- name: Install latest Poetry
run: pipx install poetry
- name: Cache Poetry and pip artifacts
uses: actions/cache@v4
with:

3
.github/workflows/tests-download.yaml vendored
View File

@@ -22,6 +22,9 @@ jobs:
steps:
- uses: actions/checkout@v4
- name: Install ffmpeg
run: sudo apt-get update && sudo apt-get install -y ffmpeg
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:

5
.gitignore vendored
View File

@@ -1,6 +1,7 @@
tmp*/
temp/
.env*
!.env*.example
.DS_Store
expmt/
service_account.json
@@ -37,3 +38,7 @@ docs/source/modules/autogen/
scripts/settings_page.html
scripts/settings/src/schema.json
.vite
downloaded_files
latest_logs
# for launch.json
.vscode

24
Dockerfile
View File

@@ -1,4 +1,4 @@
FROM webrecorder/browsertrix-crawler:1.6.1 AS base
FROM webrecorder/browsertrix-crawler:1.6.3 AS base
ENV RUNNING_IN_DOCKER=1 \
LANG=C.UTF-8 \
@@ -11,26 +11,8 @@ ENV RUNNING_IN_DOCKER=1 \
ARG TARGETARCH
# Installing system dependencies
RUN add-apt-repository ppa:mozillateam/ppa && \
apt-get update && \
apt-get install -y --no-install-recommends gcc ffmpeg fonts-noto exiftool && \
apt-get install -y --no-install-recommends firefox-esr && \
ln -s /usr/bin/firefox-esr /usr/bin/firefox
ARG GECKODRIVER_VERSION=0.36.0
RUN if [ $(uname -m) = "aarch64" ]; then \
GECKODRIVER_ARCH=linux-aarch64; \
else \
GECKODRIVER_ARCH=linux64; \
fi && \
wget https://github.com/mozilla/geckodriver/releases/download/v${GECKODRIVER_VERSION}/geckodriver-v${GECKODRIVER_VERSION}-${GECKODRIVER_ARCH}.tar.gz && \
tar -xvzf geckodriver* -C /usr/local/bin && \
chmod +x /usr/local/bin/geckodriver && \
rm geckodriver-v* && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
RUN apt-get update && \
apt-get install -y --no-install-recommends gcc ffmpeg fonts-noto exiftool python3-tk
# Poetry and runtime
FROM base AS runtime

22
docs/scripts/scripts.py
View File

@@ -47,7 +47,6 @@ def generate_module_docs():
for module in sorted(ModuleFactory().available_modules(), key=lambda x: (x.requires_setup, x.name)):
# generate the markdown file from the __manifest__.py file.
manifest = module.manifest
for type in manifest["type"]:
modules_by_type.setdefault(type, []).append(module)
@@ -64,6 +63,27 @@ def generate_module_docs():
"""
steps_str = "\n".join(f" {t}s:\n - {module.name}" for t in manifest["type"])
if manifest.get("autodoc_dropins"):
loaded_module = module.load({})
dropins = loaded_module.load_dropins()
dropin_str = "\n##### Available Dropins\n"
for dropin in dropins:
if not (ddoc := dropin.documentation()):
continue
dropin_str += f"\n###### {ddoc.get('name', dropin.__name__)}\n\n"
dropin_str += f"{ddoc.get('description')}\n\n"
if ddoc.get("site"):
dropin_str += f"**Site**: {ddoc['site']}\n\n"
if dauth := ddoc.get("authentication"):
dropin_str += "**YAML configuration**:\n"
dropin_auth_yaml = "authentication:\n...\n"
for site, creds in dauth.items():
dropin_auth_yaml += f" {site}:\n"
for k, v in creds.items():
dropin_auth_yaml += f' {k}: "{v}"\n'
dropin_str += f"```{{code}} yaml\n{dropin_auth_yaml}...\n```\n"
readme_str += dropin_str
if not manifest["configs"]:
config_string = f"# No configuration options for {module.name}.*\n"
else:

15
docs/source/development/testing.md
View File

@@ -3,14 +3,14 @@
`pytest` is used for testing. There are two main types of tests:
1. 'core' tests which should be run on every change
2. 'download' tests which hit the network. These tests will do things like make API calls (e.g. Twitter, Bluesky etc.) and should be run regularly to make sure that APIs have not changed.
2. 'download' tests which hit the network. These tests will do things like make API calls (e.g. Twitter, Bluesky etc.) and should be run regularly to make sure that APIs have not changed, they take longer.
## Running Tests
1. Make sure you've installed the dev dependencies with `pytest install --with dev`
2. Tests can be run as follows:
```
```{code} bash
#### Command prefix of 'poetry run' removed here for simplicity
# run core tests
pytest -ra -v -m "not download"
@@ -18,4 +18,15 @@ pytest -ra -v -m "not download"
pytest -ra -v -m "download"
# run all tests
pytest -ra -v
# run a specific test file
pytest -ra -v tests/test_file.py
# run a specific test function
pytest -ra -v tests/test_file.py::test_function_name
```
3. Some tests require environment variables to be set. You can use the example `.env.test.example` file as a template. Copy it to `.env.test` and fill in the required values. This file will be loaded automatically by `pytest`.
```{code} bash
cp .env.test.example .env.test
```

2
docs/source/flow_overview.md
View File

@@ -8,7 +8,7 @@ The archiver archives web pages using the following workflow
4. **Formatter** creates a report from all the archived content (HTML, PDF, ...)
5. **Database** knows what's been archived and also stores the archive result (spreadsheet, CSV, or just the console)
Each step in the workflow is handled by 'modules' that interact with the data in different ways. For example, the Twitter Extractor Module would extract information from the Twitter website. The Screenshot Enricher Module will take screenshots of the given page. See the [core modules page](core_modules.md) for an overview of all the modules that are available.
Each step in the workflow is handled by 'modules' that interact with the data in different ways. For example, the Twitter Extractor Module would extract information from the Twitter website. The AntiBot Module will download HTML and take screenshots of the given page. See the [core modules page](core_modules.md) for an overview of all the modules that are available.
Auto-archiver must have at least one module defined for each step of the workflow. This is done by setting the [configuration](installation/configurations.md) for your auto-archiver instance.

0
docs/source/how_to/authentication_how_to.md → docs/source/how_to/01_authentication_how_to.md
View File

0
docs/source/how_to/gsheets_setup.md → docs/source/how_to/02_gsheets_setup.md
View File

34
docs/source/how_to/logging.md → docs/source/how_to/03_logging.md
View File

@@ -1,6 +1,6 @@
# Keeping Logs
Auto Archiver's logs can be helpful for debugging problematic archiving processes. This guide shows you how to use the logs to
Auto Archiver's logs can be helpful for debugging problematic archiving processes. This guide shows you how to use the logs configuration.
## Setting up logging
@@ -8,10 +8,10 @@ Logging settings can be set on the command line or using the orchestration confi
#### Enabling or Disabling Logging
Logging to the console is enabled by default. If you want to globally disable Auto Archiver's logging, then you can set `enabled: false` in your `logging` config:
Logging to the console is enabled by default. If you want to globally disable Auto Archiver's logging, then you can set `enabled: false` in your `logging` config file:
```{code} yaml
:caption: orchestration.yaml
...
logging:
enabled: false
@@ -24,7 +24,7 @@ This will disable all logs from Auto Archiver, but it does not disable logs for
#### Logging Level
There are 7 logging levels in total, with 4 commonly used levels. They are: `DEBUG`, `INFO`, `WARNING` and `ERROR`.
There are 7 logging levels in total, with 5 of them used in this tool. They are: `DEBUG`, `INFO`, `SUCCESS`, `WARNING` and `ERROR`.
Change the warning level by setting the value in your orchestration config file:
@@ -44,7 +44,7 @@ For normal usage, it is recommended to use the `INFO` level, or if you prefer qu
### Logging to a file
As default, auto-archiver will log to the console. But if you wish to store your logs for future reference, or you are running the auto-archiver from within code a implementation, then you may with to enable file logging. This can be done by setting the `file:` config value in the logging settings.
As default, auto-archiver will log to the console. But if you wish to store your logs for future reference, or you are running the auto-archiver from within code a implementation, then you may wish to enable file logging. This can be done by setting the `file:` config value in the logging settings.
**Rotation:** For file logging, you can choose to 'rotate' your log files (creating new log files) so they do not get too large. Change this by setting the 'rotation' option in your logging settings. For a full list of rotation options, see the [loguru docs](https://loguru.readthedocs.io/en/stable/overview.html#easier-file-logging-with-rotation-retention-compression).
@@ -57,15 +57,33 @@ logging:
rotation: 1 day
```
### Full logging example
### Logging each level to a different file
If you want to log each level to a different file, you can do this by setting the `each_level_in_separate_file:` option to `true` and also setting your `file:` name, a new file will be created for each of the 5 levels used, by appending the `0_level` name to the file like so `your_file.log.1_error`. In this case the `level:` option is ignored, and all levels will be logged.
The below example logs only `WARNING` logs to the console and to the file `/my/file.log`, rotating that file once per week:
```{code} yaml
:caption: orchestration.yaml
logging:
level: WARNING
each_level_in_separate_file: true
file: /my/logs/file.log
```
This will create the following files:
- `/my/logs/file.log.1_debug`
- `/my/logs/file.log.2_info`
- `/my/logs/file.log.3_success`
- `/my/logs/file.log.4_warning`
- `/my/logs/file.log.5_error`
### Full logging example
The below example logs only `DEBUG` logs to the console and to the file `/my/file.log`, rotating that file once per week:
```{code} yaml
:caption: orchestration.yaml
logging:
level: DEBUG
file: /my/file.log
rotation: 1 week
```

0
docs/source/how_to/run_instagrapi_server.md → docs/source/how_to/04_run_instagrapi_server.md
View File

54
docs/source/how_to/05_upgrading_to_1_1_0.md Normal file
View File

@@ -0,0 +1,54 @@
# Upgrading from v1.0.1
```{note} This how-to is only relevant for people who used Auto Archiver before June 2025 (versions prior to 1.1.0).
If you are new to Auto Archiver, then you are already using the latest configuration format and this how-to is not relevant for you.
```
Versions 1.1.0+ of Auto Archiver has breaking changes in the configuration format, which means earlier configuration formats will not work without slight modifications.
## Dropping `vk_extractor` module
We have dropped the `vk_extractor` because of problems in a project we relied on. You will need to remove it from your configuration file, otherwise you will see an error like:
```{code} console
Module 'vk_extractor' not found. Are you sure it's installed/exists?
```
## Dropping `screenshot_enricher` module
We have dropped the `screenshot_enricher` module because a new `antibot_extractor_enricher` (see below) module replaces its functionality more robustly and with less dependency hassle on geckodriver/firefox. You will need to remove it from your configuration file, otherwise you will see an error like:
```{code} console
Module 'screenshot_enricher' not found. Are you sure it's installed/exists?
```
## New `antibot_extractor_enricher` module and VkDropin
We have added a new [`antibot_extractor_enricher`](../modules/autogen/extractor/antibot_extractor_enricher.md) module that uses a computer-controlled browser to extract content from websites that use anti-bot measures. You can add it to your configuration file like this:
```{code} yaml
steps:
extractors:
- antibot_extractor_enricher
# or alternatively, if you want to use it as an enricher:
enrichers:
- antibot_extractor_enricher
```
It will take a full page screenshot, a PDF capture, extract HTML source code, and any other relevant media.
It comes with Dropins that we will be adding and maintaining.
> Dropin: A module with site-specific behaviours that is loaded automatically. You don't need to add them to your configuration steps for them to run. Sometimes they need `authentication` configurations though.
One such Dropin is the VkDropin which uses this automated browser to access VKontakte (VK) pages. You should add a username/password to the configuration file if you get authentication blocks from VK, to do so use the [authentication settings](authentication_how_to.md):
```{code} yaml
authentication:
vk.com:
username: your_username
password: your_password
```
See all available Dropins in [the source code](https://github.com/bellingcat/auto-archiver/tree/main/src/auto_archiver/modules/antibot_extractor_enricher/dropins). Usually each Dropin needs its own authentication settings, similarly to the VkDropin.

0
docs/source/how_to/new_config_format.md → docs/source/how_to/06_new_config_format.md
View File

17
docs/source/installation/authentication.md
View File

@@ -9,8 +9,8 @@ There are two main use cases for authentication:
```{note}
The Authentication framework currently only works with the following modules:
* Generic Extractor
* Screenshot Enricher
* [Generic Extractor](../modules/autogen/extractor/generic_extractor.md) - the main module for extracting content from websites
* [Antibot Extractor/Enricher](../modules/autogen/extractor/antibot_extractor_enricher.md)
To authenticate for WACZ archiving, see the instructions on the [](../modules/autogen/enricher/wacz_extractor_enricher.md) page.
```
@@ -34,7 +34,8 @@ You can save your authentication information directly inside your orchestration
```{note}
The Username & Password, and API settings only work with the Generic Extractor. Other modules (like the screenshot enricher) can only use the `cookies` options. Furthermore, many sites can still detect bots and block username/password logins. Twitter/X and YouTube are two prominent ones that block username/password logging.
Currently, the Username & Password, and API settings only work with the Generic and Antibot Extractors. Furthermore, many sites can still detect bots and block username/password logins. Twitter/X and YouTube are two prominent ones that block username/password logins.
One of the 'Cookies' options is recommended for the most robust archiving, but it still isn't guaranteed to work.
```
@@ -52,12 +53,12 @@ authentication:
username: myusername
password: 123
facebook.com:
cookie: single_cookie
facebook.com:
cookie: single_cookie
othersite.com:
api_key: 123
api_secret: 1234
othersite.com:
api_key: 123
api_secret: 1234
```

160
docs/source/installation/example_scripts/ubuntu_24_server_install.md Normal file
View File

@@ -0,0 +1,160 @@
### Bash script for Ubuntu 24 Server install
> NOTE: this script has not been tested by the maintainers and results from the personal experience of a user. It is meant as a guide and not an out of the box script, as you will see it's aimed at a custom branches, users, and features like the Geckodriver which are removed as of version 1.0.2.
This acts as a handy guide on all requirements. This is built and tested on the 29th of May 2025 on Ubuntu Server 24.04.2 LTS (which is the current latest LTS)
```bash
#!/bin/sh
# I usually run steps manually as logged in with the user: dave
# which the application runs under which makes debugging easier
cd ~
sudo apt update -y
sudo apt upgrade -y
# Clone only my latest branch
git clone -b v1-test --single-branch https://github.com/djhmateer/auto-archiver
mkdir ~/auto-archiver/secrets
sudo chown -R dave ~/auto-archiver
sudo apt update -y
sudo apt upgrade -y
## Python 3.12.3 comes with Ubuntu 24.04.2
# Poetry install 2.1.3 on 2nd June 25
curl -sSL https://install.python-poetry.org | python3 -
# had to restart here..
sudo reboot
# C++ compiler so pdqhash will install next
sudo apt install build-essential python3-dev -y
cd auto-archiver
poetry install
# FFMpeg
# 6.1.1-3ubuntu5 on 2nd June 25
sudo apt install ffmpeg -y
## Firefox
# 139.0+build2-0ubuntu0.24.04.1~mt1 on 2nd Jun 25
# 16th Jun - don't need anymore as using Chrome in antibot
# cd ~
# sudo add-apt-repository ppa:mozillateam/ppa -y
# echo '
# Package: *
# Pin: release o=LP-PPA-mozillateam
# Pin-Priority: 1001
# ' | sudo tee /etc/apt/preferences.d/mozilla-firefox
# echo 'Unattended-Upgrade::Allowed-Origins:: "LP-PPA-mozillateam:${distro_codename}";' | sudo tee /etc/apt/apt.conf.d/51unattended-upgrades-firefox
# sudo apt install firefox -y
wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
# Chrome
cd ~
# got problems here - fixed below
# 137.0.7151.103 on 16th Jun 2025
sudo dpkg -i google-chrome-stable_current_amd64.deb
# fix dependencies on install above
sudo apt-get install -f
# had to click a lot on UI to get going.
# to test
# google-chrome
## Gecko driver
# check version numbers for new ones
# https://github.com/mozilla/geckodriver/releases/
wget https://github.com/mozilla/geckodriver/releases/download/v0.36.0/geckodriver-v0.36.0-linux64.tar.gz
tar -xvzf geckodriver*
chmod +x geckodriver
sudo mv geckodriver /usr/local/bin/
rm geckodriver*
# Fonts so selenium via firefox can render other languages eg Burmese
sudo apt install fonts-noto -y
# Docker
# Add Docker's official GPG key:
sudo apt-get update -y
sudo apt-get install ca-certificates curl -y
sudo install -m 0755 -d /etc/apt/keyrings
sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
sudo chmod a+r /etc/apt/keyrings/docker.asc
# Add the repository to Apt sources:
echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \
$(. /etc/os-release && echo "${UBUNTU_CODENAME:-$VERSION_CODENAME}") stable" | \
sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
sudo apt-get update -y
sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -y
# add dave user to docker group
sudo usermod -aG docker $USER
# reboot otherwise can't pull images
# https://github.com/webrecorder/browsertrix-crawler
# https://hub.docker.com/r/webrecorder/browsertrix-crawler/tags
# 1.6.2 on 4th Jun 2025
docker pull webrecorder/browsertrix-crawler:latest
# exif
sudo apt install libimage-exiftool-perl -y
## CRON run every minute
# the cron job running as user dave will execute the shell script
# I have many scripts running from cron_11 upwards.
# patch in the correct number
sudo chmod +x ~/auto-archiver/scripts/cron_15.sh
# don't want service to run until a reboot otherwise problems with Gecko driver
sudo service cron stop
# runs the script every minute
# notice put in a # to disable so will have to manually start it.
cat <<EOT >> run-auto-archive
#*/1 * * * * dave /home/dave/auto-archiver/scripts/cron_15.sh
EOT
sudo mv run-auto-archive /etc/cron.d
sudo chown root /etc/cron.d/run-auto-archive
sudo chmod 600 /etc/cron.d/run-auto-archive
# Helper alias 'c' to open the above file
echo "alias c='sudo vim /etc/cron.d/run-auto-archive'" >> ~/.bashrc
# secrets folder copy
# I run dev from:
# \\wsl.localhost\Ubuntu-24.04\home\dave\code\auto-archiver\secrets\
# orchestration.yaml - for aa config
# service_account - for google spreadsheet
# anon.session - for telethon so don't have to type in phone number
# profile.tar.gz - for wacz to have a logged in profile for facebook, x.com and instagram to get data
# Youtube - POT Tokens
# https://github.com/Brainicism/bgutil-ytdlp-pot-provider
docker run --name bgutil-provider --restart unless-stopped -d -p 4416:4416 brainicism/bgutil-ytdlp-pot-provider
# test run
cd ~/auto-archiver
poetry run python src/auto_archiver --config secrets/orchestration-aa-demo-main.yaml
```

7
docs/source/installation/installation.md
View File

@@ -51,11 +51,14 @@ After this, you're ready to set up your [your configuration file](configurations
If using the local installation method, you will also need to install the following dependencies locally:
1.[ffmpeg](https://www.ffmpeg.org/) - for handling of downloaded videos
2. [firefox](https://www.mozilla.org/en-US/firefox/new/) and [geckodriver](https://github.com/mozilla/geckodriver/releases) on a path folder like `/usr/local/bin` - for taking webpage screenshots with the screenshot enricher
3. (optional) [fonts-noto](https://fonts.google.com/noto) to deal with multiple unicode characters during selenium/geckodriver's screenshots: `sudo apt install fonts-noto -y`.
<!-- 2. [firefox](https://www.mozilla.org/en-US/firefox/new/) and [geckodriver](https://github.com/mozilla/geckodriver/releases) on a path folder like `/usr/local/bin` - for taking webpage screenshots with the screenshot enricher -->
3. (optional) [fonts-noto](https://fonts.google.com/noto) to deal with multiple unicode characters during selenium screenshots: `sudo apt install fonts-noto -y`.
4. [Browsertrix Crawler docker image](https://hub.docker.com/r/webrecorder/browsertrix-crawler) for the WACZ enricher/archiver
### Custom installation scripts
- [Ubuntu 24 Server Install by @djhmateer](example_scripts/ubuntu_24_server_install.md) - a WYSIWYG example script from a user who set up the Auto Archiver on a fresh Ubuntu 24 server.
## Developer Install

981
poetry.lock generated
View File

File diff suppressed because it is too large Load Diff

5
pyproject.toml
View File

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
[project]
name = "auto-archiver"
version = "1.0.1"
version = "1.1.0"
description = "Automatically archive links to videos, images, and social media content from Google Sheets (and more)."
requires-python = ">=3.10,<3.13"
@@ -27,7 +27,6 @@ dependencies = [
"bs4 (>=0.0.0)",
"loguru (>=0.0.0)",
"ffmpeg-python (>=0.0.0)",
"selenium (>=0.0.0)",
"telethon (>=0.0.0)",
"google-api-python-client (>=0.0.0)",
"google-auth-httplib2 (>=0.0.0)",
@@ -57,6 +56,8 @@ dependencies = [
"bgutil-ytdlp-pot-provider (>=1.0.0)",
"yt-dlp[curl-cffi,default] (>=2025.5.22,<2026.0.0)",
"secretstorage (>=3.3.3,<4.0.0)",
"seleniumbase (>=4.36.4,<5.0.0)",
"pyautogui (>=0.9.54,<0.10.0)",
]
[tool.poetry.group.dev.dependencies]

396
scripts/settings/package-lock.json generated
View File

@@ -12,10 +12,10 @@
"@dnd-kit/sortable": "^10.0.0",
"@emotion/react": "latest",
"@emotion/styled": "latest",
"@mui/icons-material": "^6.4.7",
"@mui/icons-material": "^7.1.1",
"@mui/material": "latest",
"react": "19.0.0",
"react-dom": "19.0.0",
"react": "19.1.0",
"react-dom": "19.1.0",
"react-markdown": "^10.0.0",
"yaml": "^2.7.0"
},
@@ -57,9 +57,9 @@
}
},
"node_modules/@babel/compat-data": {
"version": "7.27.3",
"resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.27.3.tgz",
"integrity": "sha512-V42wFfx1ymFte+ecf6iXghnnP8kWTO+ZLXIyZq+1LAXHHvTZdVxicn4yiVYdYMGaCO3tmqub11AorKkv+iodqw==",
"version": "7.27.5",
"resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.27.5.tgz",
"integrity": "sha512-KiRAp/VoJaWkkte84TvUd9qjdbZAdiqyvMxrGl1N6vzFogKmaLgoM3L1kgtLicp2HP5fBJS8JrZKLVIZGVJAVg==",
"dev": true,
"license": "MIT",
"engines": {
@@ -105,12 +105,12 @@
"license": "MIT"
},
"node_modules/@babel/generator": {
"version": "7.27.3",
"resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.27.3.tgz",
"integrity": "sha512-xnlJYj5zepml8NXtjkG0WquFUv8RskFqyFcVgTBp5k+NaA/8uw/K+OSVf8AMGw5e9HKP2ETd5xpK5MLZQD6b4Q==",
"version": "7.27.5",
"resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.27.5.tgz",
"integrity": "sha512-ZGhA37l0e/g2s1Cnzdix0O3aLYm66eF8aufiVteOgnwxgnRP8GoyMj7VWsgWnQbVKXyge7hqrFh2K2TQM6t1Hw==",
"license": "MIT",
"dependencies": {
"@babel/parser": "^7.27.3",
"@babel/parser": "^7.27.5",
"@babel/types": "^7.27.3",
"@jridgewell/gen-mapping": "^0.3.5",
"@jridgewell/trace-mapping": "^0.3.25",
@@ -207,23 +207,23 @@
}
},
"node_modules/@babel/helpers": {
"version": "7.27.4",
"resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.27.4.tgz",
"integrity": "sha512-Y+bO6U+I7ZKaM5G5rDUZiYfUvQPUibYmAFe7EnKdnKBbVXDZxvp+MWOH5gYciY0EPk4EScsuFMQBbEfpdRKSCQ==",
"version": "7.27.6",
"resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.27.6.tgz",
"integrity": "sha512-muE8Tt8M22638HU31A3CgfSUciwz1fhATfoVai05aPXGor//CdWDCbnlY1yvBPo07njuVOCNGCSp/GTt12lIug==",
"dev": true,
"license": "MIT",
"dependencies": {
"@babel/template": "^7.27.2",
"@babel/types": "^7.27.3"
"@babel/types": "^7.27.6"
},
"engines": {
"node": ">=6.9.0"
}
},
"node_modules/@babel/parser": {
"version": "7.27.4",
"resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.27.4.tgz",
"integrity": "sha512-BRmLHGwpUqLFR2jzx9orBuX/ABDkj2jLKOXrHDTN2aOKL+jFDDKaRNo9nyYsIl9h/UE/7lMKdDjKQQyxKKDZ7g==",
"version": "7.27.5",
"resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.27.5.tgz",
"integrity": "sha512-OsQd175SxWkGlzbny8J3K8TnnDD0N3lrIUtB92xwyRpzaenGZhxDvxN/JgU00U3CDZNj9tPuDJ5H0WS4Nt3vKg==",
"license": "MIT",
"dependencies": {
"@babel/types": "^7.27.3"
@@ -268,9 +268,9 @@
}
},
"node_modules/@babel/runtime": {
"version": "7.27.4",
"resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.27.4.tgz",
"integrity": "sha512-t3yaEOuGu9NlIZ+hIeGbBjFtZT7j2cb2tg0fuaJKeGotchRjjLfrBA9Kwf8quhpP1EUuxModQg04q/mBwyg8uA==",
"version": "7.27.6",
"resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.27.6.tgz",
"integrity": "sha512-vbavdySgbTTrmFE+EsiqUTzlOr5bzlnJtUv9PynGCAKvfQqjIXbvFdumPM/GxMDfyuGMJaJAU6TO4zc1Jf1i8Q==",
"license": "MIT",
"engines": {
"node": ">=6.9.0"
@@ -309,9 +309,9 @@
}
},
"node_modules/@babel/types": {
"version": "7.27.3",
"resolved": "https://registry.npmjs.org/@babel/types/-/types-7.27.3.tgz",
"integrity": "sha512-Y1GkI4ktrtvmawoSq+4FCVHNryea6uR+qUQy0AGxLSsjCX0nVmkYQMBLHDkXZuo5hGx7eYdnIaslsdBFm7zbUw==",
"version": "7.27.6",
"resolved": "https://registry.npmjs.org/@babel/types/-/types-7.27.6.tgz",
"integrity": "sha512-ETyHEk2VHHvl9b9jZP5IHPavHYk57EhanlRRuae9XCpb/j5bDCbPPMOBfCWhnl/7EDJz0jEMCi/RhccCE8r1+Q==",
"license": "MIT",
"dependencies": {
"@babel/helper-string-parser": "^7.27.1",
@@ -994,9 +994,9 @@
}
},
"node_modules/@mui/core-downloads-tracker": {
"version": "6.4.12",
"resolved": "https://registry.npmjs.org/@mui/core-downloads-tracker/-/core-downloads-tracker-6.4.12.tgz",
"integrity": "sha512-M7IkG4LqSJfkY+thlQQHNkcS5NdmMDwLq/2RKoW40XR0mv/2BYb6X8fRnyaxP4zGdPD2M4MQdbzKihSVormJ7Q==",
"version": "7.1.1",
"resolved": "https://registry.npmjs.org/@mui/core-downloads-tracker/-/core-downloads-tracker-7.1.1.tgz",
"integrity": "sha512-yBckQs4aQ8mqukLnPC6ivIRv6guhaXi8snVl00VtyojBbm+l6VbVhyTSZ68Abcx7Ah8B+GZhrB7BOli+e+9LkQ==",
"license": "MIT",
"funding": {
"type": "opencollective",
@@ -1004,12 +1004,12 @@
}
},
"node_modules/@mui/icons-material": {
"version": "6.4.12",
"resolved": "https://registry.npmjs.org/@mui/icons-material/-/icons-material-6.4.12.tgz",
"integrity": "sha512-ILTe3A2te0+Vb9TG4P1AZVmZFOjDDCV/b2nBmV1rNOmSu3Q/xkHghW+yMhMffwHcXklMlcajMlc4iFSkPbrTKw==",
"version": "7.1.1",
"resolved": "https://registry.npmjs.org/@mui/icons-material/-/icons-material-7.1.1.tgz",
"integrity": "sha512-X37+Yc8QpEnl0sYmz+WcLFy2dWgNRzbswDzLPXG7QU1XDVlP5TPp1HXjdmCupOWLL/I9m1fyhcyZl8/HPpp/Cg==",
"license": "MIT",
"dependencies": {
"@babel/runtime": "^7.26.0"
"@babel/runtime": "^7.27.1"
},
"engines": {
"node": ">=14.0.0"
@@ -1019,7 +1019,7 @@
"url": "https://opencollective.com/mui-org"
},
"peerDependencies": {
"@mui/material": "^6.4.12",
"@mui/material": "^7.1.1",
"@types/react": "^17.0.0 || ^18.0.0 || ^19.0.0",
"react": "^17.0.0 || ^18.0.0 || ^19.0.0"
},
@@ -1030,22 +1030,22 @@
}
},
"node_modules/@mui/material": {
"version": "6.4.12",
"resolved": "https://registry.npmjs.org/@mui/material/-/material-6.4.12.tgz",
"integrity": "sha512-VqoLNS5UaNqoS1FybezZR/PaAvzbTmRe0Mx//afXbolIah43eozpX2FckaFffLvMoiSIyxx1+AMHyENTr2Es0Q==",
"version": "7.1.1",
"resolved": "https://registry.npmjs.org/@mui/material/-/material-7.1.1.tgz",
"integrity": "sha512-mTpdmdZCaHCGOH3SrYM41+XKvNL0iQfM9KlYgpSjgadXx/fEKhhvOktxm8++Xw6FFeOHoOiV+lzOI8X1rsv71A==",
"license": "MIT",
"dependencies": {
"@babel/runtime": "^7.26.0",
"@mui/core-downloads-tracker": "^6.4.12",
"@mui/system": "^6.4.12",
"@mui/types": "~7.2.24",
"@mui/utils": "^6.4.9",
"@babel/runtime": "^7.27.1",
"@mui/core-downloads-tracker": "^7.1.1",
"@mui/system": "^7.1.1",
"@mui/types": "^7.4.3",
"@mui/utils": "^7.1.1",
"@popperjs/core": "^2.11.8",
"@types/react-transition-group": "^4.4.12",
"clsx": "^2.1.1",
"csstype": "^3.1.3",
"prop-types": "^15.8.1",
"react-is": "^19.0.0",
"react-is": "^19.1.0",
"react-transition-group": "^4.4.5"
},
"engines": {
@@ -1058,7 +1058,7 @@
"peerDependencies": {
"@emotion/react": "^11.5.0",
"@emotion/styled": "^11.3.0",
"@mui/material-pigment-css": "^6.4.12",
"@mui/material-pigment-css": "^7.1.1",
"@types/react": "^17.0.0 || ^18.0.0 || ^19.0.0",
"react": "^17.0.0 || ^18.0.0 || ^19.0.0",
"react-dom": "^17.0.0 || ^18.0.0 || ^19.0.0"
@@ -1079,13 +1079,13 @@
}
},
"node_modules/@mui/private-theming": {
"version": "6.4.9",
"resolved": "https://registry.npmjs.org/@mui/private-theming/-/private-theming-6.4.9.tgz",
"integrity": "sha512-LktcVmI5X17/Q5SkwjCcdOLBzt1hXuc14jYa7NPShog0GBDCDvKtcnP0V7a2s6EiVRlv7BzbWEJzH6+l/zaCxw==",
"version": "7.1.1",
"resolved": "https://registry.npmjs.org/@mui/private-theming/-/private-theming-7.1.1.tgz",
"integrity": "sha512-M8NbLUx+armk2ZuaxBkkMk11ultnWmrPlN0Xe3jUEaBChg/mcxa5HWIWS1EE4DF36WRACaAHVAvyekWlDQf0PQ==",
"license": "MIT",
"dependencies": {
"@babel/runtime": "^7.26.0",
"@mui/utils": "^6.4.9",
"@babel/runtime": "^7.27.1",
"@mui/utils": "^7.1.1",
"prop-types": "^15.8.1"
},
"engines": {
@@ -1106,12 +1106,12 @@
}
},
"node_modules/@mui/styled-engine": {
"version": "6.4.11",
"resolved": "https://registry.npmjs.org/@mui/styled-engine/-/styled-engine-6.4.11.tgz",
"integrity": "sha512-74AUmlHXaGNbyUqdK/+NwDJOZqgRQw6BcNvhoWYLq3LGbLTkE+khaJ7soz6cIabE4CPYqO2/QAIU1Z/HEjjpcw==",
"version": "7.1.1",
"resolved": "https://registry.npmjs.org/@mui/styled-engine/-/styled-engine-7.1.1.tgz",
"integrity": "sha512-R2wpzmSN127j26HrCPYVQ53vvMcT5DaKLoWkrfwUYq3cYytL6TQrCH8JBH3z79B6g4nMZZVoaXrxO757AlShaw==",
"license": "MIT",
"dependencies": {
"@babel/runtime": "^7.26.0",
"@babel/runtime": "^7.27.1",
"@emotion/cache": "^11.13.5",
"@emotion/serialize": "^1.3.3",
"@emotion/sheet": "^1.4.0",
@@ -1140,16 +1140,16 @@
}
},
"node_modules/@mui/system": {
"version": "6.4.12",
"resolved": "https://registry.npmjs.org/@mui/system/-/system-6.4.12.tgz",
"integrity": "sha512-fgEfm1qxpKCztndESeL1L0sLwA2c7josZ2w42D8OM3pbLee4bH2twEjoMo6qf7z2rNw1Uc9EU9haXeMoq0oTdQ==",
"version": "7.1.1",
"resolved": "https://registry.npmjs.org/@mui/system/-/system-7.1.1.tgz",
"integrity": "sha512-Kj1uhiqnj4Zo7PDjAOghtXJtNABunWvhcRU0O7RQJ7WOxeynoH6wXPcilphV8QTFtkKaip8EiNJRiCD+B3eROA==",
"license": "MIT",
"dependencies": {
"@babel/runtime": "^7.26.0",
"@mui/private-theming": "^6.4.9",
"@mui/styled-engine": "^6.4.11",
"@mui/types": "~7.2.24",
"@mui/utils": "^6.4.9",
"@babel/runtime": "^7.27.1",
"@mui/private-theming": "^7.1.1",
"@mui/styled-engine": "^7.1.1",
"@mui/types": "^7.4.3",
"@mui/utils": "^7.1.1",
"clsx": "^2.1.1",
"csstype": "^3.1.3",
"prop-types": "^15.8.1"
@@ -1180,10 +1180,13 @@
}
},
"node_modules/@mui/types": {
"version": "7.2.24",
"resolved": "https://registry.npmjs.org/@mui/types/-/types-7.2.24.tgz",
"integrity": "sha512-3c8tRt/CbWZ+pEg7QpSwbdxOk36EfmhbKf6AGZsD1EcLDLTSZoxxJ86FVtcjxvjuhdyBiWKSTGZFaXCnidO2kw==",
"version": "7.4.3",
"resolved": "https://registry.npmjs.org/@mui/types/-/types-7.4.3.tgz",
"integrity": "sha512-2UCEiK29vtiZTeLdS2d4GndBKacVyxGvReznGXGr+CzW/YhjIX+OHUdCIczZjzcRAgKBGmE9zCIgoV9FleuyRQ==",
"license": "MIT",
"dependencies": {
"@babel/runtime": "^7.27.1"
},
"peerDependencies": {
"@types/react": "^17.0.0 || ^18.0.0 || ^19.0.0"
},
@@ -1194,17 +1197,17 @@
}
},
"node_modules/@mui/utils": {
"version": "6.4.9",
"resolved": "https://registry.npmjs.org/@mui/utils/-/utils-6.4.9.tgz",
"integrity": "sha512-Y12Q9hbK9g+ZY0T3Rxrx9m2m10gaphDuUMgWxyV5kNJevVxXYCLclYUCC9vXaIk1/NdNDTcW2Yfr2OGvNFNmHg==",
"version": "7.1.1",
"resolved": "https://registry.npmjs.org/@mui/utils/-/utils-7.1.1.tgz",
"integrity": "sha512-BkOt2q7MBYl7pweY2JWwfrlahhp+uGLR8S+EhiyRaofeRYUWL2YKbSGQvN4hgSN1i8poN0PaUiii1kEMrchvzg==",
"license": "MIT",
"dependencies": {
"@babel/runtime": "^7.26.0",
"@mui/types": "~7.2.24",
"@babel/runtime": "^7.27.1",
"@mui/types": "^7.4.3",
"@types/prop-types": "^15.7.14",
"clsx": "^2.1.1",
"prop-types": "^15.8.1",
"react-is": "^19.0.0"
"react-is": "^19.1.0"
},
"engines": {
"node": ">=14.0.0"
@@ -1234,16 +1237,16 @@
}
},
"node_modules/@rolldown/pluginutils": {
"version": "1.0.0-beta.9",
"resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-beta.9.tgz",
"integrity": "sha512-e9MeMtVWo186sgvFFJOPGy7/d2j2mZhLJIdVW0C/xDluuOvymEATqz6zKsP0ZmXGzQtqlyjz5sC1sYQUoJG98w==",
"version": "1.0.0-beta.11",
"resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-beta.11.tgz",
"integrity": "sha512-L/gAA/hyCSuzTF1ftlzUSI/IKr2POHsv1Dd78GfqkR83KMNuswWD61JxGV2L7nRwBBBSDr6R1gCkdTmoN7W4ag==",
"dev": true,
"license": "MIT"
},
"node_modules/@rollup/rollup-android-arm-eabi": {
"version": "4.41.1",
"resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.41.1.tgz",
"integrity": "sha512-NELNvyEWZ6R9QMkiytB4/L4zSEaBC03KIXEghptLGLZWJ6VPrL63ooZQCOnlx36aQPGhzuOMwDerC1Eb2VmrLw==",
"version": "4.42.0",
"resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.42.0.tgz",
"integrity": "sha512-gldmAyS9hpj+H6LpRNlcjQWbuKUtb94lodB9uCz71Jm+7BxK1VIOo7y62tZZwxhA7j1ylv/yQz080L5WkS+LoQ==",
"cpu": [
"arm"
],
@@ -1255,9 +1258,9 @@
]
},
"node_modules/@rollup/rollup-android-arm64": {
"version": "4.41.1",
"resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.41.1.tgz",
"integrity": "sha512-DXdQe1BJ6TK47ukAoZLehRHhfKnKg9BjnQYUu9gzhI8Mwa1d2fzxA1aw2JixHVl403bwp1+/o/NhhHtxWJBgEA==",
"version": "4.42.0",
"resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.42.0.tgz",
"integrity": "sha512-bpRipfTgmGFdCZDFLRvIkSNO1/3RGS74aWkJJTFJBH7h3MRV4UijkaEUeOMbi9wxtxYmtAbVcnMtHTPBhLEkaw==",
"cpu": [
"arm64"
],
@@ -1269,9 +1272,9 @@
]
},
"node_modules/@rollup/rollup-darwin-arm64": {
"version": "4.41.1",
"resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.41.1.tgz",
"integrity": "sha512-5afxvwszzdulsU2w8JKWwY8/sJOLPzf0e1bFuvcW5h9zsEg+RQAojdW0ux2zyYAz7R8HvvzKCjLNJhVq965U7w==",
"version": "4.42.0",
"resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.42.0.tgz",
"integrity": "sha512-JxHtA081izPBVCHLKnl6GEA0w3920mlJPLh89NojpU2GsBSB6ypu4erFg/Wx1qbpUbepn0jY4dVWMGZM8gplgA==",
"cpu": [
"arm64"
],
@@ -1283,9 +1286,9 @@
]
},
"node_modules/@rollup/rollup-darwin-x64": {
"version": "4.41.1",
"resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.41.1.tgz",
"integrity": "sha512-egpJACny8QOdHNNMZKf8xY0Is6gIMz+tuqXlusxquWu3F833DcMwmGM7WlvCO9sB3OsPjdC4U0wHw5FabzCGZg==",
"version": "4.42.0",
"resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.42.0.tgz",
"integrity": "sha512-rv5UZaWVIJTDMyQ3dCEK+m0SAn6G7H3PRc2AZmExvbDvtaDc+qXkei0knQWcI3+c9tEs7iL/4I4pTQoPbNL2SA==",
"cpu": [
"x64"
],
@@ -1297,9 +1300,9 @@
]
},
"node_modules/@rollup/rollup-freebsd-arm64": {
"version": "4.41.1",
"resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.41.1.tgz",
"integrity": "sha512-DBVMZH5vbjgRk3r0OzgjS38z+atlupJ7xfKIDJdZZL6sM6wjfDNo64aowcLPKIx7LMQi8vybB56uh1Ftck/Atg==",
"version": "4.42.0",
"resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.42.0.tgz",
"integrity": "sha512-fJcN4uSGPWdpVmvLuMtALUFwCHgb2XiQjuECkHT3lWLZhSQ3MBQ9pq+WoWeJq2PrNxr9rPM1Qx+IjyGj8/c6zQ==",
"cpu": [
"arm64"
],
@@ -1311,9 +1314,9 @@
]
},
"node_modules/@rollup/rollup-freebsd-x64": {
"version": "4.41.1",
"resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.41.1.tgz",
"integrity": "sha512-3FkydeohozEskBxNWEIbPfOE0aqQgB6ttTkJ159uWOFn42VLyfAiyD9UK5mhu+ItWzft60DycIN1Xdgiy8o/SA==",
"version": "4.42.0",
"resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.42.0.tgz",
"integrity": "sha512-CziHfyzpp8hJpCVE/ZdTizw58gr+m7Y2Xq5VOuCSrZR++th2xWAz4Nqk52MoIIrV3JHtVBhbBsJcAxs6NammOQ==",
"cpu": [
"x64"
],
@@ -1325,9 +1328,9 @@
]
},
"node_modules/@rollup/rollup-linux-arm-gnueabihf": {
"version": "4.41.1",
"resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.41.1.tgz",
"integrity": "sha512-wC53ZNDgt0pqx5xCAgNunkTzFE8GTgdZ9EwYGVcg+jEjJdZGtq9xPjDnFgfFozQI/Xm1mh+D9YlYtl+ueswNEg==",
"version": "4.42.0",
"resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.42.0.tgz",
"integrity": "sha512-UsQD5fyLWm2Fe5CDM7VPYAo+UC7+2Px4Y+N3AcPh/LdZu23YcuGPegQly++XEVaC8XUTFVPscl5y5Cl1twEI4A==",
"cpu": [
"arm"
],
@@ -1339,9 +1342,9 @@
]
},
"node_modules/@rollup/rollup-linux-arm-musleabihf": {
"version": "4.41.1",
"resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.41.1.tgz",
"integrity": "sha512-jwKCca1gbZkZLhLRtsrka5N8sFAaxrGz/7wRJ8Wwvq3jug7toO21vWlViihG85ei7uJTpzbXZRcORotE+xyrLA==",
"version": "4.42.0",
"resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.42.0.tgz",
"integrity": "sha512-/i8NIrlgc/+4n1lnoWl1zgH7Uo0XK5xK3EDqVTf38KvyYgCU/Rm04+o1VvvzJZnVS5/cWSd07owkzcVasgfIkQ==",
"cpu": [
"arm"
],
@@ -1353,9 +1356,9 @@
]
},
"node_modules/@rollup/rollup-linux-arm64-gnu": {
"version": "4.41.1",
"resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.41.1.tgz",
"integrity": "sha512-g0UBcNknsmmNQ8V2d/zD2P7WWfJKU0F1nu0k5pW4rvdb+BIqMm8ToluW/eeRmxCared5dD76lS04uL4UaNgpNA==",
"version": "4.42.0",
"resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.42.0.tgz",
"integrity": "sha512-eoujJFOvoIBjZEi9hJnXAbWg+Vo1Ov8n/0IKZZcPZ7JhBzxh2A+2NFyeMZIRkY9iwBvSjloKgcvnjTbGKHE44Q==",
"cpu": [
"arm64"
],
@@ -1367,9 +1370,9 @@
]
},
"node_modules/@rollup/rollup-linux-arm64-musl": {
"version": "4.41.1",
"resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.41.1.tgz",
"integrity": "sha512-XZpeGB5TKEZWzIrj7sXr+BEaSgo/ma/kCgrZgL0oo5qdB1JlTzIYQKel/RmhT6vMAvOdM2teYlAaOGJpJ9lahg==",
"version": "4.42.0",
"resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.42.0.tgz",
"integrity": "sha512-/3NrcOWFSR7RQUQIuZQChLND36aTU9IYE4j+TB40VU78S+RA0IiqHR30oSh6P1S9f9/wVOenHQnacs/Byb824g==",
"cpu": [
"arm64"
],
@@ -1381,9 +1384,9 @@
]
},
"node_modules/@rollup/rollup-linux-loongarch64-gnu": {
"version": "4.41.1",
"resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loongarch64-gnu/-/rollup-linux-loongarch64-gnu-4.41.1.tgz",
"integrity": "sha512-bkCfDJ4qzWfFRCNt5RVV4DOw6KEgFTUZi2r2RuYhGWC8WhCA8lCAJhDeAmrM/fdiAH54m0mA0Vk2FGRPyzI+tw==",
"version": "4.42.0",
"resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loongarch64-gnu/-/rollup-linux-loongarch64-gnu-4.42.0.tgz",
"integrity": "sha512-O8AplvIeavK5ABmZlKBq9/STdZlnQo7Sle0LLhVA7QT+CiGpNVe197/t8Aph9bhJqbDVGCHpY2i7QyfEDDStDg==",
"cpu": [
"loong64"
],
@@ -1395,9 +1398,9 @@
]
},
"node_modules/@rollup/rollup-linux-powerpc64le-gnu": {
"version": "4.41.1",
"resolved": "https://registry.npmjs.org/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.41.1.tgz",
"integrity": "sha512-3mr3Xm+gvMX+/8EKogIZSIEF0WUu0HL9di+YWlJpO8CQBnoLAEL/roTCxuLncEdgcfJcvA4UMOf+2dnjl4Ut1A==",
"version": "4.42.0",
"resolved": "https://registry.npmjs.org/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.42.0.tgz",
"integrity": "sha512-6Qb66tbKVN7VyQrekhEzbHRxXXFFD8QKiFAwX5v9Xt6FiJ3BnCVBuyBxa2fkFGqxOCSGGYNejxd8ht+q5SnmtA==",
"cpu": [
"ppc64"
],
@@ -1409,9 +1412,9 @@
]
},
"node_modules/@rollup/rollup-linux-riscv64-gnu": {
"version": "4.41.1",
"resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.41.1.tgz",
"integrity": "sha512-3rwCIh6MQ1LGrvKJitQjZFuQnT2wxfU+ivhNBzmxXTXPllewOF7JR1s2vMX/tWtUYFgphygxjqMl76q4aMotGw==",
"version": "4.42.0",
"resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.42.0.tgz",
"integrity": "sha512-KQETDSEBamQFvg/d8jajtRwLNBlGc3aKpaGiP/LvEbnmVUKlFta1vqJqTrvPtsYsfbE/DLg5CC9zyXRX3fnBiA==",
"cpu": [
"riscv64"
],
@@ -1423,9 +1426,9 @@
]
},
"node_modules/@rollup/rollup-linux-riscv64-musl": {
"version": "4.41.1",
"resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.41.1.tgz",
"integrity": "sha512-LdIUOb3gvfmpkgFZuccNa2uYiqtgZAz3PTzjuM5bH3nvuy9ty6RGc/Q0+HDFrHrizJGVpjnTZ1yS5TNNjFlklw==",
"version": "4.42.0",
"resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.42.0.tgz",
"integrity": "sha512-qMvnyjcU37sCo/tuC+JqeDKSuukGAd+pVlRl/oyDbkvPJ3awk6G6ua7tyum02O3lI+fio+eM5wsVd66X0jQtxw==",
"cpu": [
"riscv64"
],
@@ -1437,9 +1440,9 @@
]
},
"node_modules/@rollup/rollup-linux-s390x-gnu": {
"version": "4.41.1",
"resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.41.1.tgz",
"integrity": "sha512-oIE6M8WC9ma6xYqjvPhzZYk6NbobIURvP/lEbh7FWplcMO6gn7MM2yHKA1eC/GvYwzNKK/1LYgqzdkZ8YFxR8g==",
"version": "4.42.0",
"resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.42.0.tgz",
"integrity": "sha512-I2Y1ZUgTgU2RLddUHXTIgyrdOwljjkmcZ/VilvaEumtS3Fkuhbw4p4hgHc39Ypwvo2o7sBFNl2MquNvGCa55Iw==",
"cpu": [
"s390x"
],
@@ -1451,9 +1454,9 @@
]
},
"node_modules/@rollup/rollup-linux-x64-gnu": {
"version": "4.41.1",
"resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.41.1.tgz",
"integrity": "sha512-cWBOvayNvA+SyeQMp79BHPK8ws6sHSsYnK5zDcsC3Hsxr1dgTABKjMnMslPq1DvZIp6uO7kIWhiGwaTdR4Og9A==",
"version": "4.42.0",
"resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.42.0.tgz",
"integrity": "sha512-Gfm6cV6mj3hCUY8TqWa63DB8Mx3NADoFwiJrMpoZ1uESbK8FQV3LXkhfry+8bOniq9pqY1OdsjFWNsSbfjPugw==",
"cpu": [
"x64"
],
@@ -1465,9 +1468,9 @@
]
},
"node_modules/@rollup/rollup-linux-x64-musl": {
"version": "4.41.1",
"resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.41.1.tgz",
"integrity": "sha512-y5CbN44M+pUCdGDlZFzGGBSKCA4A/J2ZH4edTYSSxFg7ce1Xt3GtydbVKWLlzL+INfFIZAEg1ZV6hh9+QQf9YQ==",
"version": "4.42.0",
"resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.42.0.tgz",
"integrity": "sha512-g86PF8YZ9GRqkdi0VoGlcDUb4rYtQKyTD1IVtxxN4Hpe7YqLBShA7oHMKU6oKTCi3uxwW4VkIGnOaH/El8de3w==",
"cpu": [
"x64"
],
@@ -1479,9 +1482,9 @@
]
},
"node_modules/@rollup/rollup-win32-arm64-msvc": {
"version": "4.41.1",
"resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.41.1.tgz",
"integrity": "sha512-lZkCxIrjlJlMt1dLO/FbpZbzt6J/A8p4DnqzSa4PWqPEUUUnzXLeki/iyPLfV0BmHItlYgHUqJe+3KiyydmiNQ==",
"version": "4.42.0",
"resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.42.0.tgz",
"integrity": "sha512-+axkdyDGSp6hjyzQ5m1pgcvQScfHnMCcsXkx8pTgy/6qBmWVhtRVlgxjWwDp67wEXXUr0x+vD6tp5W4x6V7u1A==",
"cpu": [
"arm64"
],
@@ -1493,9 +1496,9 @@
]
},
"node_modules/@rollup/rollup-win32-ia32-msvc": {
"version": "4.41.1",
"resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.41.1.tgz",
"integrity": "sha512-+psFT9+pIh2iuGsxFYYa/LhS5MFKmuivRsx9iPJWNSGbh2XVEjk90fmpUEjCnILPEPJnikAU6SFDiEUyOv90Pg==",
"version": "4.42.0",
"resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.42.0.tgz",
"integrity": "sha512-F+5J9pelstXKwRSDq92J0TEBXn2nfUrQGg+HK1+Tk7VOL09e0gBqUHugZv7SW4MGrYj41oNCUe3IKCDGVlis2g==",
"cpu": [
"ia32"
],
@@ -1507,9 +1510,9 @@
]
},
"node_modules/@rollup/rollup-win32-x64-msvc": {
"version": "4.41.1",
"resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.41.1.tgz",
"integrity": "sha512-Wq2zpapRYLfi4aKxf2Xff0tN+7slj2d4R87WEzqw7ZLsVvO5zwYCIuEGSZYiK41+GlwUo1HiR+GdkLEJnCKTCw==",
"version": "4.42.0",
"resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.42.0.tgz",
"integrity": "sha512-LpHiJRwkaVz/LqjHjK8LCi8osq7elmpwujwbXKNW88bM8eeGxavJIKKjkjpMHAh/2xfnrt1ZSnhTv41WYUHYmA==",
"cpu": [
"x64"
],
@@ -1575,9 +1578,9 @@
}
},
"node_modules/@types/estree": {
"version": "1.0.7",
"resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.7.tgz",
"integrity": "sha512-w28IoSUCJpidD/TGviZwwMJckNESJZXFu7NBZ5YJ4mEUnNraUn9Pm8HSZm/jDF1pDWYKspWE7oVphigUPRakIQ==",
"version": "1.0.8",
"resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.8.tgz",
"integrity": "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==",
"license": "MIT"
},
"node_modules/@types/estree-jsx": {
@@ -1620,24 +1623,24 @@
"license": "MIT"
},
"node_modules/@types/prop-types": {
"version": "15.7.14",
"resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.14.tgz",
"integrity": "sha512-gNMvNH49DJ7OJYv+KAKn0Xp45p8PLl6zo2YnvDIbTd4J6MER2BmWN49TG7n9LvkyihINxeKW8+3bfS2yDC9dzQ==",
"version": "15.7.15",
"resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.15.tgz",
"integrity": "sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw==",
"license": "MIT"
},
"node_modules/@types/react": {
"version": "19.1.6",
"resolved": "https://registry.npmjs.org/@types/react/-/react-19.1.6.tgz",
"integrity": "sha512-JeG0rEWak0N6Itr6QUx+X60uQmN+5t3j9r/OVDtWzFXKaj6kD1BwJzOksD0FF6iWxZlbE1kB0q9vtnU2ekqa1Q==",
"version": "19.1.7",
"resolved": "https://registry.npmjs.org/@types/react/-/react-19.1.7.tgz",
"integrity": "sha512-BnsPLV43ddr05N71gaGzyZ5hzkCmGwhMvYc8zmvI8Ci1bRkkDSzDDVfAXfN2tk748OwI7ediiPX6PfT9p0QGVg==",
"license": "MIT",
"dependencies": {
"csstype": "^3.0.2"
}
},
"node_modules/@types/react-dom": {
"version": "19.1.5",
"resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.1.5.tgz",
"integrity": "sha512-CMCjrWucUBZvohgZxkjd6S9h0nZxXjzus6yDfUb+xLxYM7VvjKNH1tQrE9GWLql1XoOP4/Ds3bwFqShHUYraGg==",
"version": "19.1.6",
"resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.1.6.tgz",
"integrity": "sha512-4hOiT/dwO8Ko0gV1m/TJZYk3y0KBnY9vzDh7W+DH17b2HFSOGgdj33dhihPeuy3l0q23+4e+hoXHV6hCC4dCXw==",
"dev": true,
"license": "MIT",
"peerDependencies": {
@@ -1666,16 +1669,16 @@
"license": "ISC"
},
"node_modules/@vitejs/plugin-react": {
"version": "4.5.0",
"resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-4.5.0.tgz",
"integrity": "sha512-JuLWaEqypaJmOJPLWwO335Ig6jSgC1FTONCWAxnqcQthLTK/Yc9aH6hr9z/87xciejbQcnP3GnA1FWUSWeXaeg==",
"version": "4.5.2",
"resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-4.5.2.tgz",
"integrity": "sha512-QNVT3/Lxx99nMQWJWF7K4N6apUEuT0KlZA3mx/mVaoGj3smm/8rc8ezz15J1pcbcjDK0V15rpHetVfya08r76Q==",
"dev": true,
"license": "MIT",
"dependencies": {
"@babel/core": "^7.26.10",
"@babel/plugin-transform-react-jsx-self": "^7.25.9",
"@babel/plugin-transform-react-jsx-source": "^7.25.9",
"@rolldown/pluginutils": "1.0.0-beta.9",
"@babel/core": "^7.27.4",
"@babel/plugin-transform-react-jsx-self": "^7.27.1",
"@babel/plugin-transform-react-jsx-source": "^7.27.1",
"@rolldown/pluginutils": "1.0.0-beta.11",
"@types/babel__core": "^7.20.5",
"react-refresh": "^0.17.0"
},
@@ -1683,7 +1686,7 @@
"node": "^14.18.0 || >=16.0.0"
},
"peerDependencies": {
"vite": "^4.2.0 || ^5.0.0 || ^6.0.0"
"vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0-beta.0"
}
},
"node_modules/babel-plugin-macros": {
@@ -1767,9 +1770,9 @@
}
},
"node_modules/caniuse-lite": {
"version": "1.0.30001720",
"resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001720.tgz",
"integrity": "sha512-Ec/2yV2nNPwb4DnTANEV99ZWwm3ZWfdlfkQbWSDDt+PsXEVYwlhPH8tdMaPunYTKKmz7AnHi2oNEi1GcmKCD8g==",
"version": "1.0.30001721",
"resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001721.tgz",
"integrity": "sha512-cOuvmUVtKrtEaoKiO0rSc29jcjwMwX5tOHDy4MgVFEWiUXj4uBMJkwI8MDySkgXidpMiHUcviogAvFi4pA2hDQ==",
"dev": true,
"funding": [
{
@@ -1956,9 +1959,9 @@
}
},
"node_modules/electron-to-chromium": {
"version": "1.5.161",
"resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.161.tgz",
"integrity": "sha512-hwtetwfKNZo/UlwHIVBlKZVdy7o8bIZxxKs0Mv/ROPiQQQmDgdm5a+KvKtBsxM8ZjFzTaCeLoodZ8jiBE3o9rA==",
"version": "1.5.166",
"resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.166.tgz",
"integrity": "sha512-QPWqHL0BglzPYyJJ1zSSmwFFL6MFXhbACOCcsCdUMCkzPdS9/OIBVxg516X/Ado2qwAq8k0nJJ7phQPCqiaFAw==",
"dev": true,
"license": "ISC"
},
@@ -2051,9 +2054,9 @@
"license": "MIT"
},
"node_modules/fdir": {
"version": "6.4.5",
"resolved": "https://registry.npmjs.org/fdir/-/fdir-6.4.5.tgz",
"integrity": "sha512-4BG7puHpVsIYxZUbiUE3RqGloLaSSwzYie5jvasC4LWuBWzZawynvYouhjbQKw2JuIGYdm0DzIxl8iVidKlUEw==",
"version": "6.4.6",
"resolved": "https://registry.npmjs.org/fdir/-/fdir-6.4.6.tgz",
"integrity": "sha512-hiFoqpyZcfNm1yc4u8oWCf9A2c4D3QjCrks3zmoVKVxpQRzmPNar1hUJcBG2RQHvEVGDN+Jm81ZheVLAQMK6+w==",
"dev": true,
"license": "MIT",
"peerDependencies": {
@@ -3197,24 +3200,24 @@
}
},
"node_modules/react": {
"version": "19.0.0",
"resolved": "https://registry.npmjs.org/react/-/react-19.0.0.tgz",
"integrity": "sha512-V8AVnmPIICiWpGfm6GLzCR/W5FXLchHop40W4nXBmdlEceh16rCN8O8LNWm5bh5XUX91fh7KpA+W0TgMKmgTpQ==",
"version": "19.1.0",
"resolved": "https://registry.npmjs.org/react/-/react-19.1.0.tgz",
"integrity": "sha512-FS+XFBNvn3GTAWq26joslQgWNoFu08F4kl0J4CgdNKADkdSGXQyTCnKteIAJy96Br6YbpEU1LSzV5dYtjMkMDg==",
"license": "MIT",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/react-dom": {
"version": "19.0.0",
"resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.0.0.tgz",
"integrity": "sha512-4GV5sHFG0e/0AD4X+ySy6UJd3jVl1iNsNHdpad0qhABJ11twS3TTBnseqsKurKcsNqCEFeGL3uLpVChpIO3QfQ==",
"version": "19.1.0",
"resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.1.0.tgz",
"integrity": "sha512-Xs1hdnE+DyKgeHJeJznQmYMIBG3TKIHJJT95Q58nHLSrElKlGQqDTR2HQ9fx5CN/Gk6Vh/kupBTDLU11/nDk/g==",
"license": "MIT",
"dependencies": {
"scheduler": "^0.25.0"
"scheduler": "^0.26.0"
},
"peerDependencies": {
"react": "^19.0.0"
"react": "^19.1.0"
}
},
"node_modules/react-is": {
@@ -3339,9 +3342,9 @@
}
},
"node_modules/rollup": {
"version": "4.41.1",
"resolved": "https://registry.npmjs.org/rollup/-/rollup-4.41.1.tgz",
"integrity": "sha512-cPmwD3FnFv8rKMBc1MxWCwVQFxwf1JEmSX3iQXrRVVG15zerAIXRjMFVWnd5Q5QvgKF7Aj+5ykXFhUl+QGnyOw==",
"version": "4.42.0",
"resolved": "https://registry.npmjs.org/rollup/-/rollup-4.42.0.tgz",
"integrity": "sha512-LW+Vse3BJPyGJGAJt1j8pWDKPd73QM8cRXYK1IxOBgL2AGLu7Xd2YOW0M2sLUBCkF5MshXXtMApyEAEzMVMsnw==",
"dev": true,
"license": "MIT",
"dependencies": {
@@ -3355,33 +3358,40 @@
"npm": ">=8.0.0"
},
"optionalDependencies": {
"@rollup/rollup-android-arm-eabi": "4.41.1",
"@rollup/rollup-android-arm64": "4.41.1",
"@rollup/rollup-darwin-arm64": "4.41.1",
"@rollup/rollup-darwin-x64": "4.41.1",
"@rollup/rollup-freebsd-arm64": "4.41.1",
"@rollup/rollup-freebsd-x64": "4.41.1",
"@rollup/rollup-linux-arm-gnueabihf": "4.41.1",
"@rollup/rollup-linux-arm-musleabihf": "4.41.1",
"@rollup/rollup-linux-arm64-gnu": "4.41.1",
"@rollup/rollup-linux-arm64-musl": "4.41.1",
"@rollup/rollup-linux-loongarch64-gnu": "4.41.1",
"@rollup/rollup-linux-powerpc64le-gnu": "4.41.1",
"@rollup/rollup-linux-riscv64-gnu": "4.41.1",
"@rollup/rollup-linux-riscv64-musl": "4.41.1",
"@rollup/rollup-linux-s390x-gnu": "4.41.1",
"@rollup/rollup-linux-x64-gnu": "4.41.1",
"@rollup/rollup-linux-x64-musl": "4.41.1",
"@rollup/rollup-win32-arm64-msvc": "4.41.1",
"@rollup/rollup-win32-ia32-msvc": "4.41.1",
"@rollup/rollup-win32-x64-msvc": "4.41.1",
"@rollup/rollup-android-arm-eabi": "4.42.0",
"@rollup/rollup-android-arm64": "4.42.0",
"@rollup/rollup-darwin-arm64": "4.42.0",
"@rollup/rollup-darwin-x64": "4.42.0",
"@rollup/rollup-freebsd-arm64": "4.42.0",
"@rollup/rollup-freebsd-x64": "4.42.0",
"@rollup/rollup-linux-arm-gnueabihf": "4.42.0",
"@rollup/rollup-linux-arm-musleabihf": "4.42.0",
"@rollup/rollup-linux-arm64-gnu": "4.42.0",
"@rollup/rollup-linux-arm64-musl": "4.42.0",
"@rollup/rollup-linux-loongarch64-gnu": "4.42.0",
"@rollup/rollup-linux-powerpc64le-gnu": "4.42.0",
"@rollup/rollup-linux-riscv64-gnu": "4.42.0",
"@rollup/rollup-linux-riscv64-musl": "4.42.0",
"@rollup/rollup-linux-s390x-gnu": "4.42.0",
"@rollup/rollup-linux-x64-gnu": "4.42.0",
"@rollup/rollup-linux-x64-musl": "4.42.0",
"@rollup/rollup-win32-arm64-msvc": "4.42.0",
"@rollup/rollup-win32-ia32-msvc": "4.42.0",
"@rollup/rollup-win32-x64-msvc": "4.42.0",
"fsevents": "~2.3.2"
}
},
"node_modules/rollup/node_modules/@types/estree": {
"version": "1.0.7",
"resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.7.tgz",
"integrity": "sha512-w28IoSUCJpidD/TGviZwwMJckNESJZXFu7NBZ5YJ4mEUnNraUn9Pm8HSZm/jDF1pDWYKspWE7oVphigUPRakIQ==",
"dev": true,
"license": "MIT"
},
"node_modules/scheduler": {
"version": "0.25.0",
"resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.25.0.tgz",
"integrity": "sha512-xFVuu11jh+xcO7JOAGJNOXld8/TcEHK/4CituBUeUb5hqxJLj9YuemAEuvm9gQ/+pgXYfbQuqAkiYu+u7YEsNA==",
"version": "0.26.0",
"resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.26.0.tgz",
"integrity": "sha512-NlHwttCI/l5gCPR3D1nNXtWABUmBwvZpEQiD4IXSbIDq8BzLIK/7Ir5gTFSGZDUu37K5cMNp0hFtzO38sC7gWA==",
"license": "MIT"
},
"node_modules/semver": {

6
scripts/settings/package.json
View File

@@ -13,10 +13,10 @@
"@dnd-kit/sortable": "^10.0.0",
"@emotion/react": "latest",
"@emotion/styled": "latest",
"@mui/icons-material": "^6.4.7",
"@mui/icons-material": "^7.1.1",
"@mui/material": "latest",
"react": "19.0.0",
"react-dom": "19.0.0",
"react": "19.1.0",
"react-dom": "19.1.0",
"react-markdown": "^10.0.0",
"yaml": "^2.7.0"
},

9
src/auto_archiver/core/base_module.py
View File

@@ -98,12 +98,11 @@ class BaseModule(ABC):
"""
# TODO: think about if/how we can deal with sites that have multiple domains (main one is x.com/twitter.com)
# for now the user must enter them both, like "x.com,twitter.com" in their config. Maybe we just hard-code?
site = UrlUtil.domain_for_url(site).removeprefix("www.")
domain = UrlUtil.domain_for_url(site).removeprefix("www.")
# add the 'www' version of the site to the list of sites to check
authdict = {}
for to_try in [site, f"www.{site}"]:
for to_try in [site, domain, f"www.{domain}"]:
if to_try in self.authentication:
authdict.update(self.authentication[to_try])
break
@@ -111,9 +110,9 @@ class BaseModule(ABC):
# do a fuzzy string match just to print a warning - don't use it since it's insecure
if not authdict:
for key in self.authentication.keys():
if key in site or site in key:
if key in domain or domain in key:
logger.debug(
f"Could not find exact authentication information for site '{site}'. \
f"Could not find exact authentication information for '{domain}'. \
did find information for '{key}' which is close, is this what you meant? \
If so, edit your authentication settings to make sure it exactly matches."
)

22
src/auto_archiver/core/extractor.py
View File

@@ -8,6 +8,7 @@ Factory method to initialize an extractor instance based on its name.
from __future__ import annotations
from abc import abstractmethod
from contextlib import suppress
import mimetypes
import os
import requests
@@ -16,6 +17,7 @@ from retrying import retry
import re
from auto_archiver.core import Metadata, BaseModule
from auto_archiver.utils.url import get_media_url_best_quality
class Extractor(BaseModule):
@@ -70,10 +72,22 @@ class Extractor(BaseModule):
return ""
@retry(wait_random_min=500, wait_random_max=3500, stop_max_attempt_number=5)
def download_from_url(self, url: str, to_filename: str = None, verbose=True) -> str:
def download_from_url(self, url: str, to_filename: str = None, verbose=True, try_best_quality=False) -> str:
"""
downloads a URL to provided filename, or inferred from URL, returns local filename
Warning: if try_best_quality is True, it will return a tuple of (filename, best_quality_url) if the download was successful.
"""
if any(url.startswith(x) for x in ["blob:", "data:"]):
return None, url if try_best_quality else None
if try_best_quality:
with suppress(Exception):
# Attempt to download the original URL
best_quality_url = get_media_url_best_quality(url)
orig_download = self.download_from_url(best_quality_url, to_filename, verbose)
if orig_download:
return orig_download, best_quality_url
if not to_filename:
to_filename = url.split("/")[-1].split("?")[0]
if len(to_filename) > 64:
@@ -98,10 +112,14 @@ class Extractor(BaseModule):
with open(to_filename, "wb") as f:
for chunk in d.iter_content(chunk_size=8192):
f.write(chunk)
if try_best_quality:
return to_filename, url
return to_filename
except requests.RequestException as e:
logger.warning(f"Failed to fetch the Media URL: {e}")
logger.warning(f"Failed to fetch the Media URL: {str(e)[:250]}")
if try_best_quality:
return None, url
@abstractmethod
def download(self, item: Metadata) -> Metadata | False:

2
src/auto_archiver/core/media.py
View File

@@ -116,7 +116,7 @@ class Media:
# self.is_video() should be used together with this method
try:
streams = ffmpeg.probe(self.filename, select_streams="v")["streams"]
logger.warning(f"STREAMS FOR {self.filename} {streams}")
logger.debug(f"STREAMS FOR {self.filename} {streams}")
return any(s.get("duration_ts", 0) > 0 for s in streams)
except Error:
return False # ffmpeg errors when reading bad files

2
src/auto_archiver/core/metadata.py
View File

@@ -96,7 +96,7 @@ class Metadata:
def is_empty(self) -> bool:
meaningfull_ids = set(self.metadata.keys()) - set(
["_processed_at", "url", "total_bytes", "total_size", "archive_duration_seconds"]
["_processed_at", "url", "original_url", "total_bytes", "total_size", "archive_duration_seconds"]
)
return not self.is_success() and len(self.media) == 0 and len(meaningfull_ids) == 0

38
src/auto_archiver/core/orchestrator.py
View File

@@ -34,7 +34,7 @@ from .config import (
from .module import ModuleFactory, LazyBaseModule
from . import validators, Feeder, Extractor, Database, Storage, Formatter, Enricher
from .consts import MODULE_TYPES, SetupError
from auto_archiver.utils.url import check_url_or_raise
from auto_archiver.utils.url import check_url_or_raise, clean
if TYPE_CHECKING:
from .base_module import BaseModule
@@ -249,7 +249,7 @@ Here's how that would look: \n\nsteps:\n extractors:\n - [your_extractor_name_
action="store",
dest="logging.level",
choices=["INFO", "DEBUG", "ERROR", "WARNING"],
help="the logging level to use",
help="the logging level to use for the standard output and file logging",
default="INFO",
type=str.upper,
)
@@ -264,6 +264,14 @@ Here's how that would look: \n\nsteps:\n extractors:\n - [your_extractor_name_
default=None,
)
parser.add_argument(
"--logging.each_level_in_separate_file",
action="store",
dest="logging.each_level_in_separate_file",
help="if set, writes each logging level to a separate file (ignores --logging.level), you must also set --logging.file. Each level will have a dedicate logs file matching your <file>.debug, <file>.info, etc.",
default=False,
)
def add_individual_module_args(
self, modules: list[LazyBaseModule] = None, parser: argparse.ArgumentParser = None
) -> None:
@@ -333,11 +341,24 @@ Here's how that would look: \n\nsteps:\n extractors:\n - [your_extractor_name_
# add other logging info
if self.logger_id is None: # note - need direct comparison to None since need to consider falsy value 0
self.logger_id = logger.add(sys.stderr, level=logging_config["level"])
if log_file := logging_config["file"]:
logger.add(log_file) if not logging_config["rotation"] else logger.add(
log_file, rotation=logging_config["rotation"]
use_level = logging_config["level"]
self.logger_id = logger.add(sys.stderr, level=use_level)
rotation = logging_config["rotation"]
log_file = logging_config["file"]
if logging_config.get("each_level_in_separate_file"):
assert logging_config["file"], (
"You must set --logging.file if you want to use --logging.each_level_in_separate_file"
)
for i, level in enumerate(["DEBUG", "INFO", "SUCCESS", "WARNING", "ERROR"], start=1):
logger.add(
f"{log_file}.{i}_{level.lower()}",
filter=lambda rec, lvl=level: rec["level"].name == lvl,
rotation=rotation,
)
elif log_file:
logger.add(log_file, rotation=rotation, level=use_level)
def install_modules(self, modules_by_type):
"""
@@ -516,7 +537,7 @@ Here's how that would look: \n\nsteps:\n extractors:\n - [your_extractor_name_
yield self.feed_item(item)
url_count += 1
logger.success(f"Processed {url_count} URL(s)")
logger.info(f"Processed {url_count} URL(s)")
self.cleanup()
def feed_item(self, item: Metadata) -> Metadata:
@@ -572,12 +593,13 @@ Here's how that would look: \n\nsteps:\n extractors:\n - [your_extractor_name_
raise e
# 1 - sanitize - each archiver is responsible for cleaning/expanding its own URLs
url = original_url
url = clean(original_url)
for a in self.extractors:
url = a.sanitize_url(url)
result.set_url(url)
if original_url != url:
logger.debug(f"Sanitized URL from {original_url} to {url}")
result.set("original_url", original_url)
# 2 - notify start to DBs, propagate already archived if feature enabled in DBs

0
src/auto_archiver/modules/antibot_extractor_enricher/__init__.py Normal file
View File

53
src/auto_archiver/modules/antibot_extractor_enricher/__manifest__.py Normal file
View File

@@ -0,0 +1,53 @@
{
"name": "Antibot Extractor/Enricher",
"type": ["extractor", "enricher"],
"requires_setup": False,
"dependencies": {"python": ["loguru", "seleniumbase", "yt_dlp"], "bin": ["ffmpeg"]},
"configs": {
"save_to_pdf": {
"default": False,
"type": "bool",
"help": "save a PDF snapshot of the page.",
},
"max_download_images": {
"default": 50,
"help": "maximum number of images to download from the page (0 = no download, inf = no limit).",
},
"max_download_videos": {
"default": 50,
"help": "maximum number of videos to download from the page (0 = no download, inf = no limit).",
},
"user_data_dir": {
"default": "secrets/antibot_user_data",
"help": "Path to the user data directory for the webdriver. This is used to persist browser state, such as cookies and local storage. If you use the docker deployment, this path will be appended with `_docker` that is because the folder cannot be shared between the host and the container due to user permissions.",
},
"detect_auth_wall": {
"default": True,
"type": "bool",
"help": "detect if the page is behind an authentication wall (e.g. login required) and skip it. disable if you want to archive pages where logins are required.",
},
"proxy": {
"default": None,
"help": "proxy to use for the webdriver, Format: 'SERVER:PORT' or 'USER:PASS@SERVER:PORT'",
},
},
"autodoc_dropins": True,
"description": """
Uses a browser controlled by SeleniumBase to capture HTML, media, and screenshots/PDFs of a web page, by bypassing anti-bot measures like Cloudflare's Turnstile or Google Recaptcha.
> ⚠️ Still in trial development, please report any issues or suggestions via [GitHub Issues](https://github.com/bellingcat/auto-archiver/issues).
### Features
- Extracts the HTML source code of the page.
- Takes full-page screenshots of web pages.
- Takes full-page PDF snapshots of web pages.
- Downloads images and videos from the page, excluding specified file extensions.
### Notes
- Using a proxy affects Cloudflare Turnstile captcha handling, so it is recommended to use a proxy only if necessary.
### Dropins
This module uses sub-modules called Dropins for specific sites that allow it to handle anti-bot measures and custom Login flows. You don't need to include the dropins in your configuration, but you do need to add authentication credentials if you want to overcome login walls on those sites, see detailed instructions for each Dropin below.
""",
}

293
src/auto_archiver/modules/antibot_extractor_enricher/antibot_extractor_enricher.py Normal file
View File

@@ -0,0 +1,293 @@
import base64
import math
import os
import sys
import traceback
from urllib.parse import urljoin
import glob
import importlib.util
from loguru import logger
import selenium
from seleniumbase import SB
from auto_archiver.core import Extractor, Enricher, Metadata, Media
from auto_archiver.modules.antibot_extractor_enricher.dropin import Dropin
from auto_archiver.modules.antibot_extractor_enricher.dropins.default import DefaultDropin
from auto_archiver.utils.misc import random_str
from auto_archiver.utils.url import is_relevant_url
class AntibotExtractorEnricher(Extractor, Enricher):
def setup(self) -> None:
self.agent = "cool"
if "linux" in sys.platform or "win32" in sys.platform:
self.agent = None # Use the default UserAgent
# parse configuration options
if self.max_download_images == "inf":
self.max_download_images = math.inf
else:
self.max_download_images = int(self.max_download_images)
if self.max_download_videos == "inf":
self.max_download_videos = math.inf
else:
self.max_download_videos = int(self.max_download_videos)
self._prepare_user_data_dir()
self.dropins = self.load_dropins()
def load_dropins(self):
dropins = []
# TODO: add user-configurable drop-ins via config like generic_extractor
dropins_dir = os.path.join(os.path.dirname(__file__), "dropins")
for file_path in glob.glob(os.path.join(dropins_dir, "*.py")):
if os.path.basename(file_path).startswith("_"):
continue # skip __init__.py or private modules
module_name = f"auto_archiver.modules.antibot_extractor_enricher.dropins.{os.path.splitext(os.path.basename(file_path))[0]}"
spec = importlib.util.spec_from_file_location(module_name, file_path)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
for attr in dir(module):
obj = getattr(module, attr)
if getattr(obj, "__module__", None) != module.__name__:
continue # Skip imported modules/classes/functions
if isinstance(obj, type) and issubclass(obj, Dropin):
dropins.append(obj)
logger.debug(f"ANTIBOT loaded drop-in classes: {', '.join([d.__name__ for d in dropins])}")
return dropins
def sanitize_url(self, url: str) -> str:
for dropin in self.dropins:
if dropin.suitable(url):
return dropin.sanitize_url(url)
return url
def download(self, item: Metadata) -> Metadata:
result = Metadata()
result.merge(item)
if self.enrich(result):
result.status = "antibot"
return result
def _prepare_user_data_dir(self):
if self.user_data_dir:
in_docker = os.environ.get("RUNNING_IN_DOCKER")
if in_docker:
self.user_data_dir = self.user_data_dir.rstrip(os.path.sep) + "_docker"
os.makedirs(self.user_data_dir, exist_ok=True)
def enrich(self, to_enrich: Metadata, custom_data_dir: bool = True) -> bool:
using_user_data_dir = self.user_data_dir if custom_data_dir else None
url = to_enrich.get_url()
url_sample = url[:75]
try:
with SB(uc=True, agent=self.agent, headed=None, user_data_dir=using_user_data_dir, proxy=self.proxy) as sb:
logger.info(f"ANTIBOT selenium browser is up with agent {self.agent}, opening {url_sample}...")
sb.uc_open_with_reconnect(url, 4)
logger.debug(f"ANTIBOT handling CAPTCHAs for {url_sample}...")
sb.uc_gui_handle_cf()
sb.uc_gui_click_rc() # NB: using handle instead of click breaks some sites like reddit, for now we separate here but can have dropins deciding this in the future
dropin = self._get_suitable_dropin(url, sb)
dropin.open_page(url)
if self.detect_auth_wall and self._hit_auth_wall(sb):
logger.warning(f"ANTIBOT SKIP since auth wall or CAPTCHA was detected for {url_sample}")
return False
sb.wait_for_ready_state_complete()
sb.sleep(1) # margin for the page to load completely
to_enrich.set_title(sb.get_title())
self._enrich_html_source_code(sb, to_enrich)
self._enrich_full_page_screenshot(sb, to_enrich)
if self.save_to_pdf:
self._enrich_full_page_pdf(sb, to_enrich)
downloaded_images, downloaded_videos = dropin.add_extra_media(to_enrich)
self._enrich_download_media(
sb,
to_enrich,
js_css_selector=dropin.js_for_image_css_selectors(),
max_media=self.max_download_images - downloaded_images,
)
self._enrich_download_media(
sb,
to_enrich,
js_css_selector=dropin.js_for_video_css_selectors(),
max_media=self.max_download_videos - downloaded_videos,
)
logger.info(f"ANTIBOT completed for {url_sample}")
return to_enrich
except selenium.common.exceptions.SessionNotCreatedException as e:
if custom_data_dir: # the retry logic only works once
logger.error(
f"ANTIBOT session not created error: {e}. Please remove the user_data_dir {self.user_data_dir} and try again, will retry without user data dir though."
)
return self.enrich(to_enrich, custom_data_dir=False)
raise e # re-raise
except Exception as e:
logger.error(f"ANTIBOT runtime error: {e}: {traceback.format_exc()}")
return False
def _get_suitable_dropin(self, url: str, sb: SB):
"""
Returns a suitable drop-in for the given URL.
This method checks if the URL is suitable for any of the registered drop-ins.
"""
for dropin in self.dropins:
if dropin.suitable(url):
logger.debug(f"ANTIBOT using drop-in {dropin.__name__} for {url}")
return dropin(sb, self)
return DefaultDropin(sb, self)
def _hit_auth_wall(self, sb: SB) -> bool:
"""
Tries to detect if the currently loaded page is an auth/login wall.
Returns True if login is likely required.
"""
# TODO: improve this detection logic, currently it is very basic and may not cover all cases
# Common URL patterns
current_url = sb.get_current_url().lower()
if any(kw in current_url for kw in ["login", "signin", "signup", "register", "captcha"]):
return True
# Common visible text markers
login_keywords = [
"sign up or log in",
"log in to continue",
"sign in to continue",
"login required",
"please log in",
"please sign up",
"please sign in",
"login to access",
"sign up to access",
"register to access",
"captcha verification",
]
for word in login_keywords + [w.capitalize() for w in login_keywords]:
if sb.is_text_visible(word):
return True
# Common title markers
title = sb.get_title().lower()
if any(
kw in title
for kw in [
"just a moment...",
"tiktok - make your day",
"um momento...",
"log in",
"sign in",
"sign up",
"register",
"captcha",
"verification required",
"access denied",
]
):
return True
# Common form fields
elements = [
"input[type='password']",
"input[type='email']",
"input[type='username']",
"input[type='phone']",
"input[name='username']",
"input[name='email']",
"input[name='password']",
"input[name='login']",
]
if any(sb.is_element_visible(el) for el in elements):
return True
return False
@logger.catch
def _enrich_html_source_code(self, sb: SB, to_enrich: Metadata):
"""
Enriches the HTML source code of the Metadata object.
This method is called by the enrich method.
"""
source = sb.get_page_source()
html_filename = os.path.join(self.tmp_dir, f"source{random_str(6)}.html")
with open(html_filename, "w", encoding="utf-8") as f:
f.write(source)
to_enrich.add_media(Media(filename=html_filename), id="html_source_code")
@logger.catch
def _enrich_full_page_screenshot(self, sb: SB, to_enrich: Metadata):
"""
Enriches the full page screenshot of the Metadata object.
This method is called by the enrich method.
"""
start_size = sb.get_window_size()
w, h = start_size["width"], start_size["height"]
x = max(sb.execute_script("return document.documentElement.scrollWidth"), w)
y = min(max(sb.execute_script("return document.documentElement.scrollHeight"), h), 25_000)
logger.debug(f"Setting window size to {x}x{y} for full page screenshot.")
sb.set_window_size(x, y)
screen_filename = os.path.join(self.tmp_dir, f"screenshot{random_str(6)}.png")
sb.save_screenshot(screen_filename)
to_enrich.add_media(Media(filename=screen_filename), id="screenshot")
@logger.catch
def _enrich_full_page_pdf(self, sb: SB, to_enrich: Metadata):
"""
Enriches the full page PDF of the Metadata object.
This method is called by the enrich method.
"""
result = sb.driver.execute_cdp_cmd("Page.printToPDF", {"printBackground": True, "landscape": False})
pdf_data = base64.b64decode(result["data"])
pdf_filename = os.path.join(self.tmp_dir, f"pdf{random_str(6)}.pdf")
with open(pdf_filename, "wb") as f:
f.write(pdf_data)
to_enrich.add_media(Media(filename=pdf_filename), id="pdf")
@logger.catch
def _enrich_download_media(self, sb: SB, to_enrich: Metadata, js_css_selector: str, max_media: int):
"""
Downloads media from the page and adds them to the Metadata object.
This method is called by the enrich method.
"""
if max_media == 0:
return
url = to_enrich.get_url()
all_urls = set()
sources = sb.execute_script(js_css_selector)
# js_for_css_selectors
for src in sources:
if len(all_urls) >= max_media:
logger.debug(f"Reached max download limit of {max_media} images/videos.")
break
if not is_relevant_url(src):
continue
full_src = urljoin(url, src)
if full_src not in all_urls:
filename, full_src = self.download_from_url(full_src, try_best_quality=True)
if not filename:
continue
all_urls.add(full_src)
to_enrich.add_media(Media(filename=filename, properties={"url": full_src}))

159
src/auto_archiver/modules/antibot_extractor_enricher/dropin.py Normal file
View File

@@ -0,0 +1,159 @@
import os
from typing import Mapping
from loguru import logger
from seleniumbase import SB
import yt_dlp
from auto_archiver.core import Extractor, Media, Metadata
from auto_archiver.utils.misc import ydl_entry_to_filename
class Dropin:
"""
A class to handle drop-in functionality for the antibot extractor enricher module.
This class is designed to be a base class for drop-ins that can handle specific websites.
"""
@staticmethod
def documentation() -> Mapping[str, str]:
"""
Each Dropin should auto-document itself with this method.
Return dictionary can include:
- 'name': A string representing the name of the dropin.
- 'description': A string describing the functionality of the dropin.
- 'site': A string representing the site this dropin is for.
- 'authentication': A dictionary with authentication example for the site.
"""
return {}
def __init__(self, sb: SB, extractor: Extractor):
"""
Initialize the Dropin with the given SeleniumBase instance.
:param sb: An instance of the SeleniumBase class that this drop-in will use.
:param extractor: An instance of the Extractor class that this drop-in will use.
"""
self.sb: SB = sb
self.extractor: Extractor = extractor
@staticmethod
def suitable(url: str) -> bool:
"""
Check if the URL is suitable for processing with this dropin.
:param url: The URL to check.
:return: True if the URL is suitable for processing, False otherwise.
"""
raise NotImplementedError("This method should be implemented in the subclass")
@staticmethod
def sanitize_url(url: str) -> str:
"""
Used to clean URLs before processing them.
"""
return url
@staticmethod
def images_selectors() -> str:
"""
CSS selector to find images in the HTML page
"""
return "img"
@staticmethod
def video_selectors() -> str:
"""
CSS selector to find videos in the HTML page.
"""
return "video, source"
def js_for_image_css_selectors(self) -> str:
"""
A configurable JS script that receives a css selector from the dropin itself and returns an array of Image elements according to the selection.
You can overwrite this instead of `images_selector` for more control over scraped images.
"""
return f"""
return Array.from(document.querySelectorAll("{self.images_selectors()}")).map(el => el.src || el.href).filter(Boolean);
"""
def js_for_video_css_selectors(self) -> str:
"""
A configurable JS script that receives a css selector from the dropin itself and returns an array of Video elements according to the selection.
You can overwrite this instead of `video_selector` for more control over scraped videos.
"""
return f"""
return Array.from(document.querySelectorAll("{self.video_selectors()}")).map(el => el.src || el.href).filter(Boolean);
"""
def open_page(self, url) -> bool:
"""
Make sure the page is opened, even if it requires authentication, captcha solving, etc.
:param url: The URL to open.
:return: True if success, False otherwise.
"""
raise NotImplementedError("This method should be implemented in the subclass")
def add_extra_media(self, to_enrich: Metadata) -> tuple[int, int]:
"""
Extract image and/or video data from the currently open post with SeleniumBase. Media is added to the `to_enrich` Metadata object.
:return: A tuple (number of Images added, number of Videos added).
"""
return 0, 0
def _get_username_password(self, site) -> tuple[str, str]:
"""
Get the username and password for the site from the extractor's auth data.
:return: A tuple (username, password).
"""
auth = self.extractor.auth_for_site(site)
username = auth.get("username", "")
password = auth.get("password", "")
if not username or not password:
raise ValueError(f"{site} authentication requires a username and password.")
return username, password
def _download_videos_with_ytdlp(self, video_urls: list[str], to_enrich: Metadata) -> int:
"""
Download videos using yt-dlp.
:param video_urls: List of video URLs to download.
:return: The number of videos downloaded.
"""
if type(self.extractor.max_download_videos) is int:
video_urls = video_urls[: self.extractor.max_download_videos]
if not video_urls:
return 0
ydl_options = [
"-o",
os.path.join(self.extractor.tmp_dir, "%(id)s.%(ext)s"),
"--quiet",
"--no-playlist",
"--no-write-subs",
"--no-write-auto-subs",
"--postprocessor-args",
"ffmpeg:-bitexact",
"--max-filesize",
"1000M", # Limit to 1GB per video
]
*_, validated_options = yt_dlp.parse_options(ydl_options)
downloaded = 0
with yt_dlp.YoutubeDL(validated_options) as ydl:
for url in video_urls:
try:
logger.debug(f"Downloading video from URL: {url}")
info = ydl.extract_info(url, download=True)
filename = ydl_entry_to_filename(ydl, info)
if not filename: # Failed to download video.
continue
media = Media(filename)
for x in ["duration", "original_url", "fulltitle", "description", "upload_date"]:
if x in info:
media.set(x, info[x])
to_enrich.add_media(media)
downloaded += 1
except Exception as e:
logger.error(f"Error downloading {url}: {e}")
return downloaded

14
src/auto_archiver/modules/antibot_extractor_enricher/dropins/default.py Normal file
View File

@@ -0,0 +1,14 @@
from auto_archiver.modules.antibot_extractor_enricher.dropin import Dropin
class DefaultDropin(Dropin):
"""
A default fallback drop-in class for handling generic cases in the antibot extractor enricher module.
"""
@staticmethod
def suitable(url: str) -> bool:
return False
def open_page(self, url) -> bool:
return True

74
src/auto_archiver/modules/antibot_extractor_enricher/dropins/linkedin.py Normal file
View File

@@ -0,0 +1,74 @@
from typing import Mapping
from loguru import logger
from auto_archiver.modules.antibot_extractor_enricher.dropin import Dropin
class LinkedinDropin(Dropin):
"""
A class to handle LinkedIn drop-in functionality for the antibot extractor enricher module.
"""
@staticmethod
def documentation() -> Mapping[str, str]:
return {
"name": "Linkedin Dropin",
"description": "Handles LinkedIn pages/posts and requires authentication to access most content but will still be useful without it. The first time you login to a new IP, LinkedIn may require an email verification code, you can do a manual login first and then it won't ask for it again.",
"site": "linkedin.com",
"authentication": {
"linkedin.com": {
"username": "email address or phone number",
"password": "password",
}
},
}
notifications_css_selector = 'a[href*="linkedin.com/notifications"]'
@staticmethod
def suitable(url: str) -> bool:
return "linkedin.com" in url
def js_for_image_css_selectors(self) -> str:
get_all_css = "main img:not([src*='profile-displayphoto']):not([src*='profile-framedphoto'])"
get_first_css = (
"main img[src*='profile-framedphoto'], main img[src*='profile-displayphoto'], main img[src*='company-logo']"
)
return f"""
const all = Array.from(document.querySelectorAll("{get_all_css}")).map(el => el.src || el.href).filter(Boolean);
const profile = document.querySelector("{get_first_css}");
return all.concat(profile?.src || profile?.href || []).filter(Boolean);
"""
@staticmethod
def video_selectors() -> str:
# usually videos are from blob: but running the generic extractor should handle that
return "main video"
def open_page(self, url) -> bool:
if not self.sb.is_element_present(self.notifications_css_selector):
self._login()
if url != self.sb.get_current_url():
self.sb.open(url)
return True
@logger.catch
def _login(self) -> bool:
if self.sb.is_text_visible("Sign in to view more content"):
self.sb.click_link_text("Sign in", timeout=2)
self.sb.wait_for_ready_state_complete()
else:
self.sb.open("https://www.linkedin.com/login")
self.sb.wait_for_ready_state_complete()
username, password = self._get_username_password("linkedin.com")
logger.debug("LinkedinDropin Logging in to Linkedin with username: {}", username)
self.sb.type("#username", username)
self.sb.type("#password", password)
self.sb.click_if_visible("#password-visibility-toggle", timeout=0.5)
self.sb.click("button[type='submit']")
self.sb.wait_for_ready_state_complete()
# TODO: on suspicious login, LinkedIn may require an email verification code
if not self.sb.is_element_present(self.notifications_css_selector):
self.sb.click_if_visible('button[aria-label="Dismiss"]', timeout=0.5)

92
src/auto_archiver/modules/antibot_extractor_enricher/dropins/reddit.py Normal file
View File

@@ -0,0 +1,92 @@
from contextlib import suppress
from typing import Mapping
from auto_archiver.core.metadata import Metadata
from auto_archiver.modules.antibot_extractor_enricher.dropin import Dropin
from loguru import logger
class RedditDropin(Dropin):
"""
A class to handle Reddit drop-in functionality for the antibot extractor enricher module.
"""
def documentation() -> Mapping[str, str]:
return {
"name": "Reddit Dropin",
"description": "Handles Reddit posts and works without authentication until Reddit flags your IP, so authentication is advised.",
"site": "reddit.com",
"authentication": {
"reddit.com": {
"username": "email address or username",
"password": "password",
}
},
}
@staticmethod
def suitable(url: str) -> bool:
return "reddit.com" in url
@staticmethod
def images_selectors() -> str:
return "shreddit-post img"
@staticmethod
def video_selectors() -> str:
return "shreddit-post video, shreddit-post source"
def open_page(self, url) -> bool:
if self.sb.is_text_visible("You've been blocked by network security."):
self._login()
if url != self.sb.get_current_url():
self.sb.open(url)
return True
@logger.catch
def _login(self):
self.sb.click_link_text("Log in")
self.sb.wait_for_ready_state_complete()
self._close_cookies_banner()
username, password = self._get_username_password("reddit.com")
logger.debug("RedditDropin Logging in to Reddit with username: {}", username)
self.sb.type("#login-username", username)
self.sb.type("#login-password", password)
elem = self.sb.find_element("button.login")
self.sb.execute_script("arguments[0].scrollIntoView(true);", elem)
self.sb.slow_click("button.login")
self.sb.wait_for_ready_state_complete()
if "https://www.reddit.com/login/" in self.sb.get_current_url():
self.sb.sleep(5)
self.sb.wait_for_ready_state_complete()
if self.sb.is_text_visible("You've been blocked by network security."):
self.sb.click_link_text("Log in")
self.sb.wait_for_ready_state_complete()
if self.sb.is_text_visible("Welcome back"):
logger.debug("RedditDropin Login successful")
self.sb.click_if_visible("this link")
def _close_cookies_banner(self):
with suppress(Exception): # selenium.common.exceptions.JavascriptException
self.sb.execute_script("""
document
.querySelector("reddit-cookie-banner")
.shadowRoot.querySelector("faceplate-dialog")
.querySelector("#accept-all-cookies-button button")
.click()
""")
@logger.catch
def add_extra_media(self, to_enrich: Metadata) -> tuple[int, int]:
filtered_urls = self.sb.execute_script(rf"""
return [...document.querySelectorAll("{self.video_selectors()}")]
.map(el => el.src || el.href)
.filter(url => url && /\.(m3u8|mpd|ism)$/.test(url));
""")
logger.debug("RedditDropin Found {} video URLs", len(filtered_urls))
return 0, self._download_videos_with_ytdlp(filtered_urls, to_enrich)

92
src/auto_archiver/modules/antibot_extractor_enricher/dropins/vk.py Normal file
View File

@@ -0,0 +1,92 @@
import re
from typing import Mapping
from auto_archiver.core.metadata import Metadata
from auto_archiver.modules.antibot_extractor_enricher.dropin import Dropin
from loguru import logger
class VkDropin(Dropin):
"""
A class to handle VK drop-in functionality for the antibot extractor enricher module.
"""
WALL_PATTERN = re.compile(r"(wall.{0,1}\d+_\d+)")
VIDEO_PATTERN = re.compile(r"(video.{0,1}\d+_\d+(?:_\w+)?)")
CLIP_PATTERN = re.compile(r"(clip.{0,1}\d+_\d+)")
PHOTO_PATTERN = re.compile(r"(photo.{0,1}\d+_\d+)")
def documentation() -> Mapping[str, str]:
return {
"name": "VKontakte Dropin",
"description": "Handles VKontakte posts and works without authentication for some content.",
"site": "vk.com",
"authentication": {
"vk.com": {
"username": "phone number with country code",
"password": "password",
}
},
}
@staticmethod
def suitable(url: str) -> bool:
return "vk.com" in url
@staticmethod
def sanitize_url(url: str) -> str:
"""
Transforms modal URLs like 'https://vk.com/page_name?w=wall-123456_7890' to 'https://vk.com/wall-123456_7890'
"""
for pattern in [VkDropin.WALL_PATTERN, VkDropin.VIDEO_PATTERN, VkDropin.CLIP_PATTERN, VkDropin.PHOTO_PATTERN]:
match = pattern.search(url)
if match:
return f"https://vk.com/{match.group(1)}"
return url
def open_page(self, url) -> bool:
if self.sb.is_text_visible("Sign in to VK"):
if self._login():
self.sb.open(url)
return True
@logger.catch
def _login(self) -> bool:
# TODO: test method, because current tests work without a login
self.sb.open("https://vk.com")
self.sb.wait_for_ready_state_complete()
if "/feed" in self.sb.get_current_url():
logger.debug("Already logged in to VK.")
return True
# need to login
username, password = self._get_username_password("vk.com")
logger.debug("Logging in to VK with username: {}", username)
self.sb.click('[data-testid="enter-another-way"]', timeout=10)
self.sb.clear('input[name="login"][type="tel"]', by="css selector", timeout=10)
self.sb.type('input[name="login"][type="tel"]', username, by="css selector", timeout=10)
self.sb.click('button[type="submit"]')
# TODO: handle captcha if it appears
# if sb.is_element_visible("img.vkc__CaptchaPopup__image"):
# captcha_url = sb.get_attribute("img.vkc__CaptchaPopup__image", "src")
# print("CAPTCHA detected:", captcha_url)
# image_url = sb.get_attribute("img[alt*='captcha']", "src")
# solution = solve_captcha(image_url)
# sb.type("input#captcha-text, input[name='captcha']", solution)
# sb.click("button[type='submit']")
self.sb.type('input[name="password"]', password, timeout=15)
self.sb.click('button[type="submit"]')
self.sb.wait_for_ready_state_complete(timeout=10)
self.sb.wait_for_element("body", timeout=10)
# self.sb.sleep(2)
return "/feed" in self.sb.get_current_url()
@logger.catch
def add_extra_media(self, to_enrich: Metadata) -> tuple[int, int]:
video_urls = [v.get_attribute("href") for v in self.sb.find_elements('a[href*="/video-"]')]
return 0, self._download_videos_with_ytdlp(video_urls, to_enrich)

19
src/auto_archiver/modules/gdrive_storage/gdrive_storage.py
View File

@@ -93,13 +93,18 @@ class GDriveStorage(Storage):
# upload file to gd
logger.debug(f"uploading {filename=} to folder id {upload_to}")
file_metadata = {"name": [filename], "parents": [upload_to]}
media = MediaFileUpload(media.filename, resumable=True)
gd_file = (
self.service.files()
.create(supportsAllDrives=True, body=file_metadata, media_body=media, fields="id")
.execute()
)
logger.debug(f"uploadf: uploaded file {gd_file['id']} successfully in folder={upload_to}")
try:
media = MediaFileUpload(media.filename, resumable=True)
gd_file = (
self.service.files()
.create(supportsAllDrives=True, body=file_metadata, media_body=media, fields="id")
.execute()
)
logger.debug(f"uploadf: uploaded file {gd_file['id']} successfully in folder={upload_to}")
except FileNotFoundError as e:
logger.error(f"gd uploadf: file not found {media.filename=} - {e}")
except Exception as e:
logger.error(f"gd uploadf: error uploading {media.filename=} to {upload_to} - {e}")
# must be implemented even if unused
def uploadf(self, file: IO[bytes], key: str, **kwargs: dict) -> bool:

8
src/auto_archiver/modules/generic_extractor/__manifest__.py
View File

@@ -4,9 +4,7 @@
"author": "Bellingcat",
"type": ["extractor"],
"requires_setup": False,
"dependencies": {
"python": ["yt_dlp", "requests", "loguru", "slugify"],
},
"dependencies": {"python": ["yt_dlp", "requests", "loguru", "slugify"], "bin": ["ffmpeg"]},
"description": """
This is the generic extractor used by auto-archiver, which uses `yt-dlp` under the hood.
@@ -32,6 +30,8 @@ For a full list of video platforms supported by `yt-dlp`, see the
custom dropins can be created to handle additional websites and passed to the archiver
via the command line using the `--dropins` option (TODO!).
You can see all currently implemented dropins in [the source code](https://github.com/bellingcat/auto-archiver/tree/main/src/auto_archiver/modules/generic_extractor).
### Auto-Updates
The Generic Extractor will also automatically check for updates to `yt-dlp` (every 5 days by default).
@@ -62,7 +62,7 @@ If you are having issues with the extractor, you can review the version of `yt-d
},
"end_means_success": {
"default": True,
"help": "if True, any archived content will mean a 'success', if False this archiver will not return a 'success' stage; this is useful for cases when the yt-dlp will archive a video but ignore other types of content like images or text only pages that the subsequent archivers can retrieve.",
"help": "if True, any archived content will mean a 'success', if False this extractor will not return a 'success' stage; this is useful for cases when the yt-dlp will archive a video but ignore other types of content like images or text only pages that the subsequent extractors can retrieve.",
"type": "bool",
},
"allow_playlist": {

34
src/auto_archiver/modules/generic_extractor/generic_extractor.py
View File

@@ -1,4 +1,3 @@
import mimetypes
import shutil
import sys
import datetime
@@ -20,6 +19,7 @@ from loguru import logger
from auto_archiver.core.extractor import Extractor
from auto_archiver.core import Metadata, Media
from auto_archiver.utils import get_datetime_from_str
from auto_archiver.utils.misc import ydl_entry_to_filename
from .dropin import GenericDropin
@@ -33,6 +33,9 @@ class GenericExtractor(Extractor):
def setup(self):
self.check_for_extractor_updates()
self.setup_po_tokens()
# TODO: figure out why the following is not properly recognised by yt-dlp:
# if "generic" not in self.extractor_args:
# self.extractor_args["generic"] = "impersonate"
def check_for_extractor_updates(self):
"""Checks whether yt-dlp or its plugins need updating and triggers a restart if so."""
@@ -303,7 +306,7 @@ class GenericExtractor(Extractor):
result.set_url(url)
if "description" in video_data and not result.get("content"):
result.set_content(video_data.pop("description"))
result.set_content(video_data.get("description"))
# extract comments if enabled
if self.comments and video_data.get("comments", []) is not None:
result.set(
@@ -368,38 +371,21 @@ class GenericExtractor(Extractor):
data = ydl.extract_info(url, ie_key=info_extractor.ie_key(), download=True)
except MaxDownloadsReached: # proceed as normal once MaxDownloadsReached is raised
pass
logger.success(data)
if "entries" in data:
entries = data.get("entries", [])
if not len(entries):
logger.warning("YoutubeDLArchiver could not find any video")
logger.info("YoutubeDLArchiver could not find any video")
return False
else:
entries = [data]
result = Metadata()
def _helper_get_filename(entry: dict) -> str:
entry_url = entry.get("url")
filename = ydl.prepare_filename(entry)
base_filename, _ = os.path.splitext(filename) # '/get/path/to/file' ignore '.ext'
directory = os.path.dirname(base_filename) # '/get/path/to'
basename = os.path.basename(base_filename) # 'file'
for f in os.listdir(directory):
if (
f.startswith(basename)
or (entry_url and os.path.splitext(f)[0] in entry_url)
and "video/" in (mimetypes.guess_type(f)[0] or "")
):
return os.path.join(directory, f)
return False
for entry in entries:
try:
filename = _helper_get_filename(entry)
filename = ydl_entry_to_filename(ydl, entry)
if not filename or not os.path.exists(filename):
if not filename:
# file was not downloaded or could not be retrieved, example: sensitive videos on YT without using cookies.
continue
@@ -423,7 +409,7 @@ class GenericExtractor(Extractor):
except Exception as e:
logger.error(f"Error processing entry {entry}: {e}")
if not len(result.media):
logger.warning(f"No media found for entry {entry}, skipping.")
logger.info(f"No media found for entry {entry}, skipping.")
return False
return self.add_metadata(data, info_extractor, url, result)
@@ -590,11 +576,11 @@ class GenericExtractor(Extractor):
# Applying user-defined extractor_args
if self.extractor_args:
for key, args in self.extractor_args.items():
logger.debug(f"Setting extractor_args: {key}")
if isinstance(args, dict):
arg_str = ";".join(f"{k}={v}" for k, v in args.items())
else:
arg_str = str(args)
logger.debug(f"Setting extractor_args: {key}:{arg_str}")
ydl_options.extend(["--extractor-args", f"{key}:{arg_str}"])
if self.ytdlp_args:

2
src/auto_archiver/modules/generic_extractor/tiktok.py
View File

@@ -10,7 +10,7 @@ from .dropin import GenericDropin
class Tiktok(GenericDropin):
"""
TikTok droping for the Generic Extractor that uses an unofficial API if/when ytdlp fails.
TikTok dropin for the Generic Extractor that uses an unofficial API if/when ytdlp fails.
It's useful for capturing content that requires a login, like sensitive content.
"""

3
src/auto_archiver/modules/generic_extractor/twitter.py
View File

@@ -7,8 +7,7 @@ from slugify import slugify
from auto_archiver.core.metadata import Metadata, Media
from auto_archiver.utils import url as UrlUtil, get_datetime_from_str
from auto_archiver.core.extractor import Extractor
from .dropin import GenericDropin, InfoExtractor
from auto_archiver.modules.generic_extractor.dropin import GenericDropin, InfoExtractor
class Twitter(GenericDropin):

22
src/auto_archiver/modules/gsheet_feeder_db/gsheet_feeder_db.py
View File

@@ -10,12 +10,13 @@ The filtered rows are processed into `Metadata` objects.
"""
import os
from typing import Tuple, Union
from typing import Tuple, Union, Iterator
from urllib.parse import quote
import gspread
from loguru import logger
from slugify import slugify
from retrying import retry
from auto_archiver.core import Feeder, Database, Media
from auto_archiver.core import Metadata
@@ -33,10 +34,10 @@ class GsheetsFeederDB(Feeder, Database):
def open_sheet(self):
if self.sheet:
return self.gsheets_client.open(self.sheet)
else: # self.sheet_id
else:
return self.gsheets_client.open_by_key(self.sheet_id)
def __iter__(self) -> Metadata:
def __iter__(self) -> Iterator[Metadata]:
sh = self.open_sheet()
for ii, worksheet in enumerate(sh.worksheets()):
if not self.should_process_sheet(worksheet.title):
@@ -45,14 +46,14 @@ class GsheetsFeederDB(Feeder, Database):
logger.info(f"Opening worksheet {ii=}: {worksheet.title=} header={self.header}")
gw = GWorksheet(worksheet, header_row=self.header, columns=self.columns)
if len(missing_cols := self.missing_required_columns(gw)):
logger.warning(
logger.debug(
f"SKIPPED worksheet '{worksheet.title}' due to missing required column(s) for {missing_cols}"
)
continue
# process and yield metadata here:
yield from self._process_rows(gw)
logger.success(f"Finished worksheet {worksheet.title}")
logger.info(f"Finished worksheet {worksheet.title}")
def _process_rows(self, gw: GWorksheet):
for row in range(1 + self.header, gw.count_rows() + 1):
@@ -173,7 +174,16 @@ class GsheetsFeederDB(Feeder, Database):
),
)
gw.batch_set_cell(cell_updates)
@retry(
wait_incrementing_start=1000,
wait_incrementing_increment=3000,
wait_incrementing_max=20_000,
stop_max_attempt_number=5,
)
def batch_set_cell_with_retry(gw, cell_updates: list):
gw.batch_set_cell(cell_updates)
batch_set_cell_with_retry(gw, cell_updates)
def _safe_status_update(self, item: Metadata, new_status: str) -> None:
try:

14
src/auto_archiver/modules/html_formatter/templates/html_template.html
View File

@@ -12,6 +12,12 @@
font-family: 'Roboto', sans-serif;
}
h2 {
white-space: normal;
overflow-wrap: break-word;
word-break: break-word;
}
table {
table-layout: fixed;
width: 90%;
@@ -97,13 +103,17 @@
background-color: #f1f1f1;
}
.pem-certificate, .text-preview {
.pem-certificate,
.text-preview {
text-align: left;
font-size: small;
}
.text-preview{
.text-preview {
padding-left: 10px;
padding-right: 10px;
max-height: 300px;
overflow: auto;
white-space: pre-wrap;
}
</style>

2
src/auto_archiver/modules/instagram_tbot_extractor/instagram_tbot_extractor.py
View File

@@ -58,7 +58,7 @@ class InstagramTbotExtractor(Extractor):
"If you do, disable at least one of the archivers for the first-time setup of the telethon session: {e}"
)
with self.client.start():
logger.success(f"SETUP {self.name} login works.")
logger.info(f"SETUP {self.name} login works.")
def cleanup(self) -> None:
logger.info(f"CLEANUP {self.name}.")

1
src/auto_archiver/modules/json_enricher/__init__.py Normal file
View File

@@ -0,0 +1 @@
from .json_enricher import JsonEnricher

16
src/auto_archiver/modules/json_enricher/__manifest__.py Normal file
View File

@@ -0,0 +1,16 @@
{
"name": "JSON Enricher",
"type": ["enricher"],
"requires_setup": True,
"dependencies": {
"python": ["loguru"],
},
"configs": {},
"description": """
Writes all archiving process metadata to a JSON file so it can be parsed by other tools. As this is an Enricher, it will not contain the final stored URLs.
WARNING: The resulting JSON may reveal sensitive information about the computer and settings in which the archiving process was run.
""",
}

19
src/auto_archiver/modules/json_enricher/json_enricher.py Normal file
View File

@@ -0,0 +1,19 @@
import json
from loguru import logger
import os
from auto_archiver.core import Enricher
from auto_archiver.core import Media, Metadata
class JsonEnricher(Enricher):
def enrich(self, to_enrich: Metadata) -> None:
url = to_enrich.get_url()
logger.debug(f"JSON Enricher for {url=}")
item_path = os.path.join(self.tmp_dir, "metadata.json")
with open(item_path, mode="w", encoding="utf-8") as outf:
json.dump(to_enrich.to_dict(), outf, indent=4, default=str, ensure_ascii=False)
to_enrich.add_media(Media(filename=item_path), id="metadata_json")

4
src/auto_archiver/modules/opentimestamps_enricher/opentimestamps_enricher.py
View File

@@ -20,7 +20,7 @@ class OpentimestampsEnricher(Enricher):
# Get the media files to timestamp
media_files = [m for m in to_enrich.media if m.filename and not m.get("opentimestamps")]
if not media_files:
logger.warning(f"No files found to timestamp in {url=}")
logger.debug(f"No files found to timestamp in {url=}")
return
timestamp_files = []
@@ -119,7 +119,7 @@ class OpentimestampsEnricher(Enricher):
if timestamp_files:
to_enrich.set("opentimestamped", True)
to_enrich.set("opentimestamps_count", len(timestamp_files))
logger.success(f"{len(timestamp_files)} OpenTimestamps proofs created for {url=}")
logger.info(f"{len(timestamp_files)} OpenTimestamps proofs created for {url=}")
else:
to_enrich.set("opentimestamped", False)
logger.warning(f"No successful timestamps created for {url=}")

2
src/auto_archiver/modules/pdq_hash_enricher/__manifest__.py
View File

@@ -15,7 +15,7 @@
- Skips non-image media or files unsuitable for hashing (e.g., corrupted or unsupported formats).
### Notes
- Best used after enrichers like `thumbnail_enricher` or `screenshot_enricher` to ensure images are available.
- Best used after enrichers like `thumbnail_enricher` or `antibot_extractor_enricher` (takes screenshots) to ensure images are available.
- Uses the `pdqhash` library to compute 256-bit perceptual hashes, which are stored as hexadecimal strings.
""",
}

2
src/auto_archiver/modules/pdq_hash_enricher/pdq_hash_enricher.py
View File

@@ -6,7 +6,7 @@ objects and calculates perceptual hashes using the PDQ hashing algorithm.
These hashes are designed specifically for images and can be used
for detecting duplicate or near-duplicate visual content.
This enricher is typically used after thumbnail or screenshot enrichers
This enricher is typically used after thumbnail or screenshot (antibot) enrichers
to ensure images are available for hashing.
"""

2
src/auto_archiver/modules/s3_storage/s3_storage.py
View File

@@ -40,6 +40,8 @@ class S3Storage(Storage):
try:
if media.mimetype:
extra_args["ContentType"] = media.mimetype
if "text" in media.mimetype:
extra_args["ContentType"] += "; charset=utf-8"
except Exception as e:
logger.warning(f"Unable to get mimetype for {media.key=}, error: {e}")
self.s3.upload_fileobj(file, Bucket=self.bucket, Key=media.key, ExtraArgs=extra_args)

1
src/auto_archiver/modules/screenshot_enricher/__init__.py
View File

@@ -1 +0,0 @@
from .screenshot_enricher import ScreenshotEnricher

44
src/auto_archiver/modules/screenshot_enricher/__manifest__.py
View File

@@ -1,44 +0,0 @@
{
"name": "Screenshot Enricher",
"type": ["enricher"],
"requires_setup": True,
"dependencies": {
"python": ["loguru", "selenium"],
},
"configs": {
"width": {"default": 1280, "type": "int", "help": "width of the screenshots"},
"height": {"default": 1024, "type": "int", "help": "height of the screenshots"},
"timeout": {"default": 60, "type": "int", "help": "timeout for taking the screenshot"},
"sleep_before_screenshot": {
"default": 4,
"type": "int",
"help": "seconds to wait for the pages to load before taking screenshot",
},
"http_proxy": {
"default": "",
"help": "http proxy to use for the webdriver, eg http://proxy-user:password@proxy-ip:port",
},
"save_to_pdf": {
"default": False,
"type": "bool",
"help": "save the page as pdf along with the screenshot. PDF saving options can be adjusted with the 'print_options' parameter",
},
"print_options": {
"default": {},
"help": "options to pass to the pdf printer, in JSON format. See https://www.selenium.dev/documentation/webdriver/interactions/print_page/ for more information",
"type": "json_loader",
},
},
"description": """
Captures screenshots and optionally saves web pages as PDFs using a WebDriver.
### Features
- Takes screenshots of web pages, with configurable width, height, and timeout settings.
- Optionally saves pages as PDFs, with additional configuration for PDF printing options.
- Bypasses URLs detected as authentication walls.
- Integrates seamlessly with the metadata enrichment pipeline, adding screenshots and PDFs as media.
### Notes
- Requires a WebDriver (e.g., ChromeDriver) installed and accessible via the system's PATH.
""",
}

61
src/auto_archiver/modules/screenshot_enricher/screenshot_enricher.py
View File

@@ -1,61 +0,0 @@
from loguru import logger
import time
import os
import base64
from selenium.common.exceptions import TimeoutException
from auto_archiver.core import Enricher
from auto_archiver.utils import Webdriver, url as UrlUtil, random_str
from auto_archiver.core import Media, Metadata
class ScreenshotEnricher(Enricher):
def __init__(self, webdriver_factory=None):
super().__init__()
self.webdriver_factory = webdriver_factory or Webdriver
def enrich(self, to_enrich: Metadata) -> None:
url = to_enrich.get_url()
logger.debug(f"Enriching screenshot for {url=}")
auth = self.auth_for_site(url)
# screenshot enricher only supports cookie-type auth (selenium)
has_valid_auth = auth and (auth.get("cookies") or auth.get("cookies_jar") or auth.get("cookie"))
if UrlUtil.is_auth_wall(url) and not has_valid_auth:
logger.warning(f"[SKIP] SCREENSHOT since url is behind AUTH WALL and no login details provided: {url=}")
if any(auth.get(key) for key in ["username", "password", "api_key", "api_secret"]):
logger.warning(
f"Screenshot enricher only supports cookie-type authentication, you have provided {auth.keys()} which are not supported.\
Consider adding 'cookie', 'cookies_file' or 'cookies_from_browser' to your auth for this site."
)
return
with self.webdriver_factory(
self.width,
self.height,
self.timeout,
facebook_accept_cookies="facebook.com" in url,
http_proxy=self.http_proxy,
print_options=self.print_options,
auth=auth,
) as driver:
try:
driver.get(url)
time.sleep(int(self.sleep_before_screenshot))
screenshot_file = os.path.join(self.tmp_dir, f"screenshot_{random_str(8)}.png")
driver.save_screenshot(screenshot_file)
to_enrich.add_media(Media(filename=screenshot_file), id="screenshot")
if self.save_to_pdf:
pdf_file = os.path.join(self.tmp_dir, f"pdf_{random_str(8)}.pdf")
pdf = driver.print_page(driver.print_options)
with open(pdf_file, "wb") as f:
f.write(base64.b64decode(pdf))
to_enrich.add_media(Media(filename=pdf_file), id="pdf")
except TimeoutException:
logger.info("TimeoutException loading page for screenshot")
except Exception as e:
logger.error(f"Got error while loading webdriver for screenshot enricher: {e}")

2
src/auto_archiver/modules/telethon_extractor/telethon_extractor.py
View File

@@ -56,7 +56,7 @@ class TelethonExtractor(Extractor):
self.client = TelegramClient(self.session_file, self.api_id, self.api_hash)
with self.client.start():
logger.success(f"SETUP {self.name} login works.")
logger.info(f"SETUP {self.name} login works.")
if self.join_channels and len(self.channel_invites):
logger.info(f"SETUP {self.name} joining channels...")

25
src/auto_archiver/modules/thumbnail_enricher/thumbnail_enricher.py
View File

@@ -35,16 +35,18 @@ class ThumbnailEnricher(Enricher):
logger.debug(f"generating thumbnails for {m.filename}")
duration = m.get("duration")
if duration is None:
try:
probe = ffmpeg.probe(m.filename)
duration = float(
next(stream for stream in probe["streams"] if stream["codec_type"] == "video")["duration"]
)
to_enrich.media[m_id].set("duration", duration)
except Exception as e:
logger.error(f"error getting duration of video {m.filename}: {e}")
return
try:
probe = ffmpeg.probe(m.filename)
duration = float(
next(stream for stream in probe["streams"] if stream["codec_type"] == "video")["duration"]
)
to_enrich.media[m_id].set("duration", duration)
except Exception as e:
logger.warning(f"failed to get duration with FFMPEG from {m.filename}: {e}")
if not duration or type(duration) not in [float, int] or duration <= 0:
logger.warning(f"cannot generate thumbnails for {m.filename} without valid duration")
continue
num_thumbs = int(min(max(1, (duration / 60) * self.thumbnails_per_minute), self.max_thumbnails))
timestamps = [duration / (num_thumbs + 1) * i for i in range(1, num_thumbs + 1)]
@@ -57,6 +59,9 @@ class ThumbnailEnricher(Enricher):
).run()
try:
if not os.path.exists(output_path):
logger.info(f"thumbnail {index} for media {m.filename} was not created")
continue
thumbnails_media.append(
Media(filename=output_path)
.set("id", f"thumbnail_{index}")

4
src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py
View File

@@ -58,7 +58,7 @@ class TimestampingEnricher(Enricher):
]
if not len(hashes):
logger.warning(f"No hashes found in {url=}")
logger.debug(f"No hashes found in {url=}")
return
@@ -101,7 +101,7 @@ class TimestampingEnricher(Enricher):
hashes_media.set("cryptography v", version("cryptography"))
to_enrich.add_media(hashes_media, id="timestamped_hashes")
to_enrich.set("timestamped", True)
logger.success(f"{len(timestamp_tokens)} timestamp tokens created for {url=}")
logger.info(f"{len(timestamp_tokens)} timestamp tokens created for {url=}")
else:
logger.warning(f"No successful timestamps for {url=}")

5
src/auto_archiver/modules/twitter_api_extractor/twitter_api_extractor.py
View File

@@ -1,3 +1,4 @@
from datetime import timezone
import json
import re
import mimetypes
@@ -91,7 +92,9 @@ class TwitterApiExtractor(Extractor):
result = Metadata()
result.set_title(tweet.data.text)
result.set_timestamp(get_datetime_from_str(tweet.data.created_at, "%Y-%m-%dT%H:%M:%S.%fZ"))
result.set_timestamp(
get_datetime_from_str(tweet.data.created_at, "%Y-%m-%dT%H:%M:%S.%fZ").replace(tzinfo=timezone.utc)
)
urls = []
if tweet.includes:

4
src/auto_archiver/modules/wacz_extractor_enricher/wacz_extractor_enricher.py
View File

@@ -204,7 +204,7 @@ class WaczExtractorEnricher(Enricher, Extractor):
if (
record.rec_type == "resource" and record.content_type == "image/png" and self.extract_screenshot
): # screenshots
fn = os.path.join(tmp_dir, f"warc-file-{counter_screenshots}.png")
fn = os.path.join(tmp_dir, f"browsertrix-screenshot-{counter_screenshots}.png")
with open(fn, "wb") as outf:
outf.write(record.raw_stream.read())
m = Media(filename=fn)
@@ -232,7 +232,7 @@ class WaczExtractorEnricher(Enricher, Extractor):
# create local file and add media
ext = mimetypes.guess_extension(content_type)
warc_fn = f"warc-file-{counter_screenshots}{ext}"
warc_fn = f"warc-file-{counter_warc_files}{ext}"
fn = os.path.join(tmp_dir, warc_fn)
record_url_best_qual = UrlUtil.twitter_best_quality_url(record_url)

1
src/auto_archiver/utils/__init__.py
View File

@@ -2,7 +2,6 @@
# we need to explicitly expose the available imports here
from .misc import *
from .webdriver import Webdriver
# handy utils from ytdlp
from yt_dlp.utils import clean_html, traverse_obj, strip_or_none, url_or_none

24
src/auto_archiver/utils/misc.py
View File

@@ -1,5 +1,6 @@
import hashlib
import json
import mimetypes
import os
import uuid
from datetime import datetime, timezone
@@ -116,3 +117,26 @@ def get_timestamp(ts, utc=True, iso=True, dayfirst=True) -> str | datetime | Non
def get_current_timestamp() -> str:
return get_timestamp(datetime.now())
def ydl_entry_to_filename(ydl, entry: dict) -> str:
import yt_dlp
ydl: yt_dlp.YoutubeDL
entry_url = entry.get("url")
filename = ydl.prepare_filename(entry)
if os.path.exists(filename):
return filename
base_filename, _ = os.path.splitext(filename) # '/get/path/to/file' ignore '.ext'
directory = os.path.dirname(base_filename) # '/get/path/to'
basename = os.path.basename(base_filename) # 'file'
for f in os.listdir(directory):
if (
f.startswith(basename)
or (entry_url and os.path.splitext(f)[0] in entry_url)
and "video/" in (mimetypes.guess_type(f)[0] or "")
):
return os.path.join(directory, f)
return False

54
src/auto_archiver/utils/url.py
View File

@@ -1,5 +1,5 @@
import re
from urllib.parse import urlparse, urlunparse
from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
from ipaddress import ip_address
@@ -53,7 +53,11 @@ def domain_for_url(url: str) -> str:
def clean(url: str) -> str:
return url
TRACKERS = {"utm_source", "utm_medium", "utm_campaign", "utm_term", "utm_content", "fbclid", "gclid"}
parsed = urlparse(url)
clean_qs = [(k, v) for k, v in parse_qsl(parsed.query) if k not in TRACKERS]
return parsed._replace(query=urlencode(clean_qs)).geturl()
def is_auth_wall(url: str) -> bool:
@@ -78,6 +82,8 @@ def remove_get_parameters(url: str) -> str:
def is_relevant_url(url: str) -> bool:
"""
Detect if a detected media URL is recurring and therefore irrelevant to a specific archive. Useful, for example, for the enumeration of the media files in WARC files which include profile pictures, favicons, etc.
Assumption: URLs are relevant if they refer to files that can be downloaded with curl/requests, so excludes extensions like .m3u8.
"""
clean_url = remove_get_parameters(url)
@@ -104,11 +110,21 @@ def is_relevant_url(url: str) -> bool:
("vk.com/images/reaction/",),
# wikipedia
("wikipedia.org/static",),
# reddit
("styles.redditmedia.com",), # opinionated but excludes may irrelevant images like avatars and banners
("emoji.redditmedia.com",),
# linkedin
("static.licdn.com",),
]
# TODO: make these globally configurable
IRRELEVANT_ENDS_WITH = [
".svg", # ignore SVGs
".ico", # ignore icons
# ignore index files for videos, these should be handled by ytdlp
".m3u8",
".mpd",
".ism",
]
for end in IRRELEVANT_ENDS_WITH:
@@ -125,6 +141,36 @@ def is_relevant_url(url: str) -> bool:
def twitter_best_quality_url(url: str) -> str:
"""
some twitter image URLs point to a less-than best quality
this returns the URL pointing to the highest (original) quality
this returns the URL pointing to the highest (original) quality (with 'name=orig')
"""
return re.sub(r"name=(\w+)", "name=orig", url, 1)
parsed = urlparse(url)
query = parsed.query
if "name=" in query:
# Replace only the first occurrence of name=xxx with name=orig
new_query = re.sub(r"name=[^&]*", "name=orig", query, 1)
parsed = parsed._replace(query=new_query)
return urlunparse(parsed)
return url
def get_media_url_best_quality(url: str) -> str:
"""
Returns the best quality URL for the given media URL, it may not exist.
"""
parsed = urlparse(url)
# twitter case
if any(d in parsed.netloc.replace("www", "") for d in ("twitter.com", "twimg.com", "x.com")):
url = twitter_best_quality_url(url)
parsed = urlparse(url)
# some cases https://example.com/media-1280x720.mp4 to https://example.com/media.mp4
basename = parsed.path.split("/")[-1]
match = re.match(r"(.+)-\d+x\d+(\.[a-zA-Z0-9]+)$", basename)
if match:
orig_basename = match.group(1) + match.group(2)
new_path = "/".join(parsed.path.split("/")[:-1] + [orig_basename])
parsed = parsed._replace(path=new_path) # keep the query unchanged
url = urlunparse(parsed)
return url

167
src/auto_archiver/utils/webdriver.py
View File

@@ -1,167 +0,0 @@
"""This Webdriver class acts as a context manager for the selenium webdriver."""
from __future__ import annotations
import os
import time
import re
# import domain_for_url
from urllib.parse import urlparse, urlunparse
from http.cookiejar import MozillaCookieJar
from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common import exceptions as selenium_exceptions
from selenium.webdriver.common.print_page_options import PrintOptions
from selenium.webdriver.common.by import By
from loguru import logger
class CookieSettingDriver(webdriver.Firefox):
facebook_accept_cookies: bool
cookie: str
cookie_jar: MozillaCookieJar
def __init__(self, cookie, cookie_jar, facebook_accept_cookies, *args, **kwargs):
if os.environ.get("RUNNING_IN_DOCKER"):
# Selenium doesn't support linux-aarch64 driver, we need to set this manually
kwargs["service"] = webdriver.FirefoxService(executable_path="/usr/local/bin/geckodriver")
super(CookieSettingDriver, self).__init__(*args, **kwargs)
self.cookie = cookie
self.cookie_jar = cookie_jar
self.facebook_accept_cookies = facebook_accept_cookies
def get(self, url: str):
if self.cookie_jar or self.cookie:
# set up the driver to make it not 'cookie averse' (needs a context/URL)
# get the 'robots.txt' file which should be quick and easy
robots_url = urlunparse(urlparse(url)._replace(path="/robots.txt", query="", fragment=""))
super(CookieSettingDriver, self).get(robots_url)
if self.cookie:
# an explicit cookie is set for this site, use that first
for cookie in self.cookies.split(";"):
for name, value in cookie.split("="):
self.driver.add_cookie({"name": name, "value": value})
elif self.cookie_jar:
domain = urlparse(url).netloc.removeprefix("www.")
regex = re.compile(f"(www)?.?{domain}$")
for cookie in self.cookie_jar:
if regex.match(cookie.domain):
try:
self.add_cookie(
{
"name": cookie.name,
"value": cookie.value,
"path": cookie.path,
"domain": cookie.domain,
"secure": bool(cookie.secure),
"expiry": cookie.expires,
}
)
except Exception as e:
logger.warning(f"Failed to add cookie ({cookie.domain}) to webdriver for url {domain}: {e}")
super(CookieSettingDriver, self).get(url)
time.sleep(2)
# Try and use some common button text to reject/accept cookies
for text in [
"Refuse non-essential cookies",
"Decline optional cookies",
"Reject additional cookies",
"Reject all",
"Accept all cookies",
]:
try:
xpath = f"//*[contains(translate(text(), 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz'), '{text.lower()}')]"
self.find_element(By.XPATH, xpath).click()
time.sleep(2)
except selenium_exceptions.NoSuchElementException:
pass
# now get the actual URL
if self.facebook_accept_cookies:
# try and click the 'close' button on the 'login' window to close it
try:
xpath = "//div[@role='dialog']//div[@aria-label='Close']"
self.find_element(By.XPATH, xpath).click()
time.sleep(2)
except selenium_exceptions.NoSuchElementException:
logger.warning("Unable to find the 'close' button on the facebook login window")
pass
else:
# for all other sites, try and use some common button text to reject/accept cookies
for text in [
"Refuse non-essential cookies",
"Decline optional cookies",
"Reject additional cookies",
"Reject all",
"Accept all cookies",
]:
try:
xpath = f"//*[contains(translate(text(), 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz'), '{text.lower()}')]"
WebDriverWait(self, 5).until(EC.element_to_be_clickable((By.XPATH, xpath))).click()
break
except selenium_exceptions.WebDriverException:
pass
class Webdriver:
def __init__(
self,
width: int,
height: int,
timeout_seconds: int,
facebook_accept_cookies: bool = False,
http_proxy: str = "",
print_options: dict = {},
auth: dict = {},
) -> webdriver:
self.width = width
self.height = height
self.timeout_seconds = timeout_seconds
self.auth = auth
self.facebook_accept_cookies = facebook_accept_cookies
self.http_proxy = http_proxy
# create and set print options
self.print_options = PrintOptions()
for k, v in print_options.items():
setattr(self.print_options, k, v)
def __enter__(self) -> webdriver:
options = webdriver.FirefoxOptions()
options.add_argument("--headless")
options.add_argument(f"--proxy-server={self.http_proxy}")
options.set_preference("network.protocol-handler.external.tg", False)
# if facebook cookie popup is present, force the browser to English since then it's easier to click the 'Decline optional cookies' option
if self.facebook_accept_cookies:
options.add_argument("--lang=en")
try:
self.driver = CookieSettingDriver(
cookie=self.auth.get("cookie"),
cookie_jar=self.auth.get("cookies_jar"),
facebook_accept_cookies=self.facebook_accept_cookies,
options=options,
)
self.driver.set_window_size(self.width, self.height)
self.driver.set_page_load_timeout(self.timeout_seconds)
self.driver.print_options = self.print_options
except selenium_exceptions.TimeoutException as e:
logger.error(
f"failed to get new webdriver, possibly due to insufficient system resources or timeout settings: {e}"
)
return self.driver
def __exit__(self, exc_type, exc_val, exc_tb):
self.driver.close()
self.driver.quit()
del self.driver
return True

13
tests/.env.test.example Normal file
View File

@@ -0,0 +1,13 @@
# ANTIBOT reddit test credentials
REDDIT_TEST_USERNAME=""
REDDIT_TEST_PASSWORD=""
# ANTIBOT linkedin test credentials
LINKEDIN_TEST_USERNAME=""
LINKEDIN_TEST_PASSWORD=""
# twitter test credentials
TWITTER_BEARER_TOKEN="TEST_KEY"
# some geo/VPN IPs are blocked by truth social, disable if you have issues
TEST_TRUTH_SOCIAL="true"

19
tests/conftest.py
View File

@@ -9,6 +9,7 @@ from tempfile import TemporaryDirectory
from typing import Dict, Tuple
import hashlib
from loguru import logger
import pytest
from auto_archiver.core.metadata import Metadata, Media
from auto_archiver.core.module import ModuleFactory
@@ -20,6 +21,24 @@ from auto_archiver.core.module import ModuleFactory
TESTS_TO_RUN_LAST = ["test_generic_archiver", "test_twitter_api_archiver"]
def pytest_configure():
# load environment variables from .env.test file.
env_path = os.path.join(os.path.dirname(__file__), ".env.test")
if os.path.exists(env_path):
with open(env_path) as f:
for line in f:
line = line.strip()
if not line or line.startswith("#"):
continue
if "=" in line:
key, value = line.split("=", 1)
os.environ[key.strip()] = value.strip().lstrip('"').rstrip('"')
else:
logger.warning(
f"Environment file {env_path} not found. Skipping loading environment variables, some tests may fail."
)
# don't check for ytdlp updates in tests
@pytest.fixture(autouse=True)
def skip_check_for_update(mocker):

216
tests/enrichers/test_screenshot_enricher.py
View File

@@ -1,216 +0,0 @@
import base64
import pytest
from selenium.common.exceptions import TimeoutException
from auto_archiver.core import Metadata, Media
from auto_archiver.modules.screenshot_enricher import ScreenshotEnricher
@pytest.fixture
def mock_selenium_env(mocker):
"""Patches Selenium calls and driver checks in one place."""
# Patch external dependencies
mock_which = mocker.patch("shutil.which")
mock_driver_class = mocker.patch("auto_archiver.utils.webdriver.CookieSettingDriver")
mock_binary_paths = mocker.patch("selenium.webdriver.common.selenium_manager.SeleniumManager.binary_paths")
mocker.patch("pathlib.Path.is_file", return_value=True)
mock_popen = mocker.patch("subprocess.Popen")
mocker.patch("selenium.webdriver.common.service.Service.is_connectable", return_value=True)
mock_firefox_options = mocker.patch("selenium.webdriver.FirefoxOptions")
# Define side effect for `shutil.which`
def mock_which_side_effect(dep):
return "/mock/geckodriver" if dep == "geckodriver" else None
mock_which.side_effect = mock_which_side_effect
# Mock binary paths
mock_binary_paths.return_value = {
"driver_path": "/mock/driver",
"browser_path": "/mock/browser",
}
# Mock `subprocess.Popen`
mock_proc = mocker.MagicMock()
mock_proc.poll.return_value = None
mock_popen.return_value = mock_proc
# Mock `CookieSettingDriver`
mock_driver = mocker.MagicMock()
mock_driver_class.return_value = mock_driver
# Mock `FirefoxOptions`
mock_options_instance = mocker.MagicMock()
mock_firefox_options.return_value = mock_options_instance
yield mock_driver, mock_driver_class, mock_options_instance
@pytest.fixture
def common_patches(tmp_path, mocker):
"""Patches common utilities used across multiple tests."""
mocker.patch("auto_archiver.utils.url.is_auth_wall", return_value=False)
mocker.patch("os.path.join", return_value=str(tmp_path / "test.png"))
mocker.patch("time.sleep")
yield
@pytest.fixture
def screenshot_enricher(setup_module, mock_binary_dependencies) -> ScreenshotEnricher:
configs: dict = {
"width": 1280,
"height": 720,
"timeout": 60,
"sleep_before_screenshot": 4,
"http_proxy": "",
"save_to_pdf": "False",
"print_options": {},
}
return setup_module("screenshot_enricher", configs)
@pytest.fixture
def metadata_with_video():
m = Metadata()
m.set_url("https://example.com")
m.add_media(Media(filename="video.mp4").set("id", "video1"))
return m
def test_enrich_adds_screenshot(
screenshot_enricher,
metadata_with_video,
mock_selenium_env,
common_patches,
tmp_path,
):
mock_driver, mock_driver_class, mock_options_instance = mock_selenium_env
screenshot_enricher.enrich(metadata_with_video)
mock_driver_class.assert_called_once_with(
cookie=None,
cookie_jar=None,
facebook_accept_cookies=False,
options=mock_options_instance,
)
# Verify the actual calls on the returned mock_driver
mock_driver.get.assert_called_once_with("https://example.com")
mock_driver.save_screenshot.assert_called_once_with(str(tmp_path / "test.png"))
# Check that the media was added (2 = original video + screenshot)
assert len(metadata_with_video.media) == 2
assert metadata_with_video.media[1].properties.get("id") == "screenshot"
@pytest.mark.parametrize(
"url,is_auth",
[
("https://example.com", False),
("https://private.com", True),
],
)
def test_enrich_auth_wall(
screenshot_enricher, metadata_with_video, mock_selenium_env, common_patches, url, is_auth, mocker
):
# Testing with and without is_auth_wall
mock_driver, mock_driver_class, _ = mock_selenium_env
mocker.patch("auto_archiver.utils.url.is_auth_wall", return_value=is_auth)
metadata_with_video.set_url(url)
screenshot_enricher.enrich(metadata_with_video)
if is_auth:
mock_driver.get.assert_not_called()
assert len(metadata_with_video.media) == 1
assert metadata_with_video.media[0].properties.get("id") == "video1"
else:
mock_driver.get.assert_called_once_with(url)
assert len(metadata_with_video.media) == 2
assert metadata_with_video.media[1].properties.get("id") == "screenshot"
def test_skip_authwall_no_cookies(screenshot_enricher, caplog):
with caplog.at_level("WARNING"):
screenshot_enricher.enrich(Metadata().set_url("https://instagram.com"))
assert "[SKIP] SCREENSHOT since url" in caplog.text
@pytest.mark.parametrize(
"auth",
[
{"cookie": "cookie"},
{"cookies_jar": "cookie"},
],
)
def test_dont_skip_authwall_with_cookies(screenshot_enricher, caplog, mocker, mock_selenium_env, auth):
mocker.patch("auto_archiver.utils.url.is_auth_wall", return_value=True)
# patch the authentication dict:
screenshot_enricher.authentication = {"example.com": auth}
with caplog.at_level("WARNING"):
screenshot_enricher.enrich(Metadata().set_url("https://example.com"))
assert "[SKIP] SCREENSHOT since url" not in caplog.text
def test_show_warning_wrong_auth_type(screenshot_enricher, caplog, mocker, mock_selenium_env):
mock_driver, mock_driver_class, _ = mock_selenium_env
mocker.patch("auto_archiver.utils.url.is_auth_wall", return_value=True)
screenshot_enricher.authentication = {"example.com": {"username": "user", "password": "pass"}}
with caplog.at_level("WARNING"):
screenshot_enricher.enrich(Metadata().set_url("https://example.com"))
assert "Screenshot enricher only supports cookie-type authentication" in caplog.text
def test_handle_timeout_exception(screenshot_enricher, metadata_with_video, mock_selenium_env, mocker):
mock_driver, mock_driver_class, mock_options_instance = mock_selenium_env
mock_driver.get.side_effect = TimeoutException
mock_log = mocker.patch("loguru.logger.info")
screenshot_enricher.enrich(metadata_with_video)
mock_log.assert_called_once_with("TimeoutException loading page for screenshot")
assert len(metadata_with_video.media) == 1
def test_handle_general_exception(screenshot_enricher, metadata_with_video, mock_selenium_env, mocker):
"""Test proper handling of unexpected general exceptions"""
mock_driver, mock_driver_class, mock_options_instance = mock_selenium_env
# Simulate a generic exception when save_screenshot is called
mock_driver.get.return_value = None
mock_driver.save_screenshot.side_effect = Exception("Unexpected Error")
mock_log = mocker.patch("loguru.logger.error")
screenshot_enricher.enrich(metadata_with_video)
# Verify that the exception was logged with the log
mock_log.assert_called_once_with("Got error while loading webdriver for screenshot enricher: Unexpected Error")
# And no new media was added due to the error
assert len(metadata_with_video.media) == 1
def test_pdf_creation(mocker, screenshot_enricher, metadata_with_video, mock_selenium_env):
"""Test PDF creation when save_to_pdf is enabled"""
mock_driver, mock_driver_class, mock_options_instance = mock_selenium_env
# Override the save_to_pdf option
screenshot_enricher.save_to_pdf = True
# Mock the print_page method to return base64-encoded content
mock_driver.print_page.return_value = base64.b64encode(b"fake_pdf_content").decode("utf-8")
# Patch functions with mocker
mocker.patch("os.path.join", side_effect=lambda *args: f"{args[-1]}")
mocker.patch(
"auto_archiver.modules.screenshot_enricher.screenshot_enricher.random_str",
return_value="fixed123",
)
mock_open = mocker.patch("builtins.open", new_callable=mocker.mock_open)
screenshot_enricher.enrich(metadata_with_video)
# Verify screenshot and PDF creation
mock_driver.save_screenshot.assert_called_once()
mock_driver.print_page.assert_called_once_with(mock_driver.print_options)
# Check that PDF file was opened and written
mock_open.assert_any_call("pdf_fixed123.pdf", "wb")
# Ensure both screenshot and PDF were added as media
assert len(metadata_with_video.media) == 3
assert metadata_with_video.media[1].properties.get("id") == "screenshot"
assert metadata_with_video.media[2].properties.get("id") == "pdf"
@pytest.fixture(autouse=True)
def cleanup_files(tmp_path):
yield
for file in tmp_path.iterdir():
file.unlink()

16
tests/enrichers/test_thumbnail_enricher.py
View File

@@ -25,6 +25,7 @@ def mock_ffmpeg_environment(mocker):
# Mocking all the ffmpeg calls in one place
mock_ffmpeg_input = mocker.patch("ffmpeg.input")
mock_makedirs = mocker.patch("os.makedirs")
mocker.patch("os.path.exists", return_value=True)
(mocker.patch.object(Media, "is_video", return_value=True),)
mock_probe = mocker.patch(
"ffmpeg.probe",
@@ -74,12 +75,12 @@ def test_enrich_thumbnail_limits(
def test_enrich_handles_probe_failure(thumbnail_enricher, metadata_with_video, mocker):
mocker.patch("ffmpeg.probe", side_effect=Exception("Probe error"))
mocker.patch("os.makedirs")
mock_logger = mocker.patch("loguru.logger.error")
mock_logger = mocker.patch("loguru.logger.warning")
mocker.patch.object(Media, "is_video", return_value=True)
thumbnail_enricher.enrich(metadata_with_video)
# Ensure error was logged
mock_logger.assert_called_with("error getting duration of video video.mp4: Probe error")
mock_logger.assert_called_with("cannot generate thumbnails for video.mp4 without valid duration")
# Ensure no thumbnails were created
thumbnails = metadata_with_video.media[0].get("thumbnails")
assert thumbnails is None
@@ -126,11 +127,14 @@ def test_enrich_handles_short_video(
assert len(thumbnails) == expected_count
def test_uses_existing_duration(thumbnail_enricher, metadata_with_video, mock_ffmpeg_environment):
metadata_with_video.media[0].set("duration", 60)
def test_uses_existing_duration_on_exception(thumbnail_enricher, metadata_with_video, mock_ffmpeg_environment, mocker):
mock_logger = mocker.patch("loguru.logger.warning")
mock_probe = mocker.patch("ffmpeg.probe", side_effect=Exception("New probe error"))
metadata_with_video.media[0].set("duration", 3)
thumbnail_enricher.enrich(metadata_with_video)
mock_ffmpeg_environment["mock_probe"].assert_not_called()
assert mock_ffmpeg_environment["mock_output"].run.call_count == 4
mock_probe.assert_called_once()
mock_logger.assert_called_with("failed to get duration with FFMPEG from video.mp4: New probe error")
assert mock_ffmpeg_environment["mock_output"].run.call_count == 3
def test_enrich_metadata_structure(thumbnail_enricher, metadata_with_video, mock_ffmpeg_environment, mocker):

101
tests/extractors/test_antibot_dropin_vk.py Normal file
View File

@@ -0,0 +1,101 @@
import pytest
from auto_archiver.modules.antibot_extractor_enricher.dropins.vk import VkDropin
@pytest.mark.parametrize(
"input_url,expected",
[
# Unrelated URL, should return unchanged
(
"https://vk.com/id123456",
"https://vk.com/id123456",
),
(
"https://example.com/",
"https://example.com/",
),
# Wall post modal URL
(
"https://vk.com/somepage?w=wall-123456_7890",
"https://vk.com/wall-123456_7890",
),
# Wall post modal URL with no dash
(
"https://vk.com/somepage?w=wall123456_7890",
"https://vk.com/wall123456_7890",
),
# Photo modal URL
(
"https://vk.com/somepage?w=photo-654321_9876",
"https://vk.com/photo-654321_9876",
),
# Photo modal URL with no dash
(
"https://vk.com/somepage?w=photo654321_9876",
"https://vk.com/photo654321_9876",
),
# Video modal URL
(
"https://vk.com/somepage?w=video-111222_3334",
"https://vk.com/video-111222_3334",
),
# Video modal URL with extra part
(
"https://vk.com/somepage?w=video-111222_3334_ABC",
"https://vk.com/video-111222_3334_ABC",
),
# Video modal URL with no dash
(
"https://vk.com/somepage?w=video111222_3334",
"https://vk.com/video111222_3334",
),
# No modal, should return unchanged
(
"https://vk.com/wall-123456_7890",
"https://vk.com/wall-123456_7890",
),
(
"https://vk.com/photo-654321_9876",
"https://vk.com/photo-654321_9876",
),
(
"https://vk.com/video-111222_3334",
"https://vk.com/video-111222_3334",
),
# Clip modal URL
(
"https://vk.com/somepage?w=clip-555666_7778",
"https://vk.com/clip-555666_7778",
),
# Clip modal URL with no dash
(
"https://vk.com/somepage?w=clip555666_7778",
"https://vk.com/clip555666_7778",
),
# Clip modal URL with extra part
(
"https://vk.com/somepage?w=clip-555666_7778_ABC",
"https://vk.com/clip-555666_7778",
),
# No modal, should return unchanged (clip)
(
"https://vk.com/clip-555666_7778",
"https://vk.com/clip-555666_7778",
),
# Modal with multiple params, should still work with right priority
(
"https://vk.com/somepage?z=photo-654321_9876&w=wall-123456_7890",
"https://vk.com/wall-123456_7890",
),
(
"https://vk.com/somepage?z=photo-654321_9876&w=video-111222_3334",
"https://vk.com/video-111222_3334",
),
(
"https://vk.com/somepage?z=video-111222_3334&w=wall-654321_9876",
"https://vk.com/wall-654321_9876",
),
],
)
def test_sanitize_url(input_url, expected):
assert VkDropin.sanitize_url(input_url) == expected

264
tests/extractors/test_antibot_extractor_enricher.py Normal file
View File

@@ -0,0 +1,264 @@
import os
import pytest
from auto_archiver.modules.antibot_extractor_enricher.antibot_extractor_enricher import AntibotExtractorEnricher
from .test_extractor_base import TestExtractorBase
class DummySB:
def __init__(self, url="", title="", visible_texts=None, visible_elements=None):
self._url = url
self._title = title
self._visible_texts = visible_texts or set()
self._visible_elements = visible_elements or set()
def get_current_url(self):
return self._url
def get_title(self):
return self._title
def is_text_visible(self, text):
return text in self._visible_texts
def is_element_visible(self, selector):
return selector in self._visible_elements
class TestAntibotExtractorEnricher(TestExtractorBase):
"""Tests Antibot Extractor/Enricher"""
extractor_module = "antibot_extractor_enricher"
extractor: AntibotExtractorEnricher
config = {
"save_to_pdf": False,
"max_download_images": 0,
"max_download_videos": 0,
"user_data_dir": "./tests/tmp/user_data",
"proxy": None,
"authentication": {
"reddit.com": {
"username": os.environ.get("REDDIT_TEST_USERNAME"),
"password": os.environ.get("REDDIT_TEST_PASSWORD"),
},
"linkedin.com": {
"username": os.environ.get("LINKEDIN_TEST_USERNAME"),
"password": os.environ.get("LINKEDIN_TEST_PASSWORD"),
},
},
}
@pytest.mark.download
@pytest.mark.parametrize(
"url,in_title,in_text,image_count,video_count",
[
(
"https://en.wikipedia.org/wiki/Western_barn_owl",
"western barn owl",
"Tyto alba",
5,
0,
),
(
"https://www.bellingcat.com/news/2025/04/29/open-sources-show-myanmar-junta-airstrike-damages-despite-post-earthquake-ceasefire/",
"open sources show myanmar",
"Bellingcat has geolocated",
5,
0,
),
(
"https://www.bellingcat.com/news/2025/03/27/gaza-israel-palestine-shot-killed-injured-destroyed-dangerous-drone-journalists-in-gaza/",
"shot from above",
"continued the work of Gazan journalists",
5,
1,
),
(
"https://www.bellingcat.com/about/general-information",
"general information",
"Stichting Bellingcat",
0, # SVGs are ignored
0,
),
(
"https://vk.com/wikipedia?from=search&w=wall-36156673_20451",
"Hounds of Love",
"16 сентября 1985 года лейблом EMI Records.",
5,
0,
),
],
)
def test_download_pages_with_media(self, setup_module, make_item, url, in_title, in_text, image_count, video_count):
"""
Test downloading pages with media.
"""
self.extractor = setup_module(
self.extractor_module,
self.config
| {
"save_to_pdf": True,
"max_download_images": 5,
"max_download_videos": "inf",
},
)
url = self.extractor.sanitize_url(url)
item = make_item(url)
result = self.extractor.download(item)
assert result.status == "antibot", "Expected status to be 'antibot'"
# Check title contains all required words (case-insensitive)
page_title = result.get_title() or ""
assert in_title.lower() in page_title.lower(), f"Expected title to contain '{in_title}', got '{page_title}'"
# Check text contains all required words (case-insensitive)
with open(result.get_media_by_id("html_source_code").filename, "r", encoding="utf-8") as f:
html_content = f.read()
assert in_text.lower() in html_content.lower(), (
f"Expected HTML to contain '{in_text}', got '{html_content}'"
)
image_media = [m for m in result.media if m.is_image() and not m.get("id") == "screenshot"]
assert len(image_media) == image_count, f"Expected {image_count} image items, got {len(image_media)}"
video_media = [m for m in result.media if m.is_video()]
assert len(video_media) == video_count, f"Expected {video_count} video items, got {len(video_media)}"
for expected_id in ["screenshot", "pdf", "html_source_code"]:
assert any(m.get("id") == expected_id for m in result.media), (
f"Expected media with id '{expected_id}' not found"
)
@pytest.mark.skipif(
not os.environ.get("REDDIT_TEST_USERNAME") or not os.environ.get("REDDIT_TEST_PASSWORD"),
reason="No Reddit test credentials provided",
)
@pytest.mark.download
@pytest.mark.parametrize(
"url,in_title,in_text,image_count,video_count",
[
(
"https://www.reddit.com/r/BeAmazed/comments/1l6b1n4/duy_tran_is_the_owner_and_prime_wood_work_artist/",
"Duy tran is the owner and prime wood work artist",
" Created Jan 26, 2015",
4,
0,
),
],
)
def test_reddit_download_with_login(
self, setup_module, make_item, url, in_title, in_text, image_count, video_count
):
self.test_download_pages_with_media(setup_module, make_item, url, in_title, in_text, image_count, video_count)
@pytest.mark.skipif(
not os.environ.get("LINKEDIN_TEST_USERNAME") or not os.environ.get("LINKEDIN_TEST_PASSWORD"),
reason="No LinkedIn test credentials provided",
)
@pytest.mark.download
@pytest.mark.parametrize(
"url,in_title,in_text,image_count,video_count",
[
(
"https://www.linkedin.com/posts/bellingcat_live-podcast-bellingcat-activity-7331725631799398400-xocM/",
"Post",
"It takes time to go from hunch to reporting...",
2,
0,
),
],
)
def test_linkedin_download_with_login(
self, setup_module, make_item, url, in_title, in_text, image_count, video_count
):
self.test_download_pages_with_media(setup_module, make_item, url, in_title, in_text, image_count, video_count)
@pytest.mark.download
@pytest.mark.parametrize(
"url,in_html",
[
(
"https://myrotvorets.center/about/",
"Центр «Миротворець»",
),
(
"https://seleniumbase.io/apps/turnstile",
'<img id="captcha-success" src="https://seleniumbase.io/cdn/img/green_check.png" style="" width="180">',
),
(
"https://seleniumbase.io/apps/form_turnstile",
'<img id="captcha-success" src="https://seleniumbase.io/cdn/img/green_check.png" width="120" style="">',
),
(
"https://gitlab.com/users/sign_in",
"Password",
),
],
)
def test_overcome_cloudflare_turnstile(self, setup_module, make_item, url, in_html):
"""
Test downloading a page with Cloudflare Turnstile captcha.
"""
self.extractor = setup_module(
self.extractor_module,
{
"save_to_pdf": True,
"detect_auth_wall": False,
"max_download_images": 5,
"max_download_videos": "inf",
},
)
item = make_item(url)
self.extractor.enrich(item)
assert item.status != "antibot", "Expected status not to be 'antibot' after handling Cloudflare Turnstile"
html_media = item.get_media_by_id("html_source_code")
with open(html_media.filename, "r", encoding="utf-8") as f:
html_content = f.read()
assert in_html.lower() in html_content.lower(), f"Expected HTML to contain '{in_html}'"
@pytest.mark.parametrize(
"url,title,visible_texts,visible_elements,expected",
[
# URL triggers
("https://example.com/login", "Welcome", set(), set(), True),
("https://example.com/somepage", "Just a moment...", set(), set(), True),
("https://example.com/", "Welcome", {"Please log in"}, set(), True),
("https://example.com/", "Welcome", set(), {"input[type='password']"}, True),
("https://example.com/", "Welcome", set("No issue here"), set(), False),
# Title triggers
("https://example.com/", "Log in", set(), set(), True),
("https://example.com/", "Verification required", set(), set(), True),
# Text triggers (case-insensitive)
("https://example.com/", "Welcome", {"Sign up or log in"}, set(), True),
("https://example.com/", "Welcome", {"sign up or log in"}, set(), True),
# Element triggers
("https://example.com/", "Welcome", set(), {"input[name='email']"}, True),
# No triggers
("https://example.com/", "Welcome", set(), set(), False),
],
)
def test_hit_auth_wall(self, url, title, visible_texts, visible_elements, expected):
extractor = AntibotExtractorEnricher()
sb = DummySB(url=url, title=title, visible_texts=visible_texts, visible_elements=visible_elements)
assert extractor._hit_auth_wall(sb) == expected
def test_enrich_handles_sb_exception(self, make_item, mocker):
"""
Test that enrich returns False and logs error if SB raises an exception.
"""
# Patch SB to raise an exception on context enter
mock_sb = mocker.patch("auto_archiver.modules.antibot_extractor_enricher.antibot_extractor_enricher.SB")
mock_logger = mocker.patch("auto_archiver.modules.antibot_extractor_enricher.antibot_extractor_enricher.logger")
mock_sb.side_effect = Exception("SB failed")
item = make_item("https://example.com/")
result = self.extractor.enrich(item)
assert result is False
mock_logger.error.assert_called()

9
tests/extractors/test_generic_extractor.py
View File

@@ -10,6 +10,7 @@ from auto_archiver.modules.generic_extractor.generic_extractor import GenericExt
from .test_extractor_base import TestExtractorBase
CI = os.getenv("GITHUB_ACTIONS", "") == "true"
TEST_TRUTH_SOCIAL = os.getenv("TEST_TRUTH_SOCIAL", "") == "true"
class TestGenericExtractor(TestExtractorBase):
@@ -97,7 +98,7 @@ class TestGenericExtractor(TestExtractorBase):
)
def test_download_nonexistent_media(self, make_item, url):
"""
Test to make sure that the extractor doesn't break on non-existend posts/media
Test to make sure that the extractor doesn't break on non-existent posts/media
It should return 'False'
"""
@@ -149,6 +150,7 @@ class TestGenericExtractor(TestExtractorBase):
result = self.extractor.download(item)
assert result is not False
@pytest.mark.skipif(not TEST_TRUTH_SOCIAL, reason="Truth social download tests disabled in environment variables.")
@pytest.mark.skipif(CI, reason="Truth social blocks GH actions.")
@pytest.mark.download
def test_truthsocial_download_video(self, make_item):
@@ -157,6 +159,7 @@ class TestGenericExtractor(TestExtractorBase):
assert len(result.media) == 1
assert result is not False
@pytest.mark.skipif(not TEST_TRUTH_SOCIAL, reason="Truth social download tests disabled in environment variables.")
@pytest.mark.skipif(CI, reason="Truth social blocks GH actions.")
@pytest.mark.download
def test_truthsocial_download_no_media(self, make_item):
@@ -164,6 +167,7 @@ class TestGenericExtractor(TestExtractorBase):
result = self.extractor.download(item)
assert result is not False
@pytest.mark.skipif(not TEST_TRUTH_SOCIAL, reason="Truth social download tests disabled in environment variables.")
@pytest.mark.skipif(CI, reason="Truth social blocks GH actions.")
@pytest.mark.download
def test_truthsocial_download_poll(self, make_item):
@@ -171,6 +175,7 @@ class TestGenericExtractor(TestExtractorBase):
result = self.extractor.download(item)
assert result is not False
@pytest.mark.skipif(not TEST_TRUTH_SOCIAL, reason="Truth social download tests disabled in environment variables.")
@pytest.mark.skipif(CI, reason="Truth social blocks GH actions.")
@pytest.mark.download
def test_truthsocial_download_single_image(self, make_item):
@@ -179,6 +184,7 @@ class TestGenericExtractor(TestExtractorBase):
assert len(result.media) == 1
assert result is not False
@pytest.mark.skipif(not TEST_TRUTH_SOCIAL, reason="Truth social download tests disabled in environment variables.")
@pytest.mark.skipif(CI, reason="Truth social blocks GH actions.")
@pytest.mark.download
def test_truthsocial_download_multiple_images(self, make_item):
@@ -276,6 +282,7 @@ class TestGenericExtractor(TestExtractorBase):
assert "Bellingchat Premium is with Kolina Koltai" in post.get_title()
@pytest.mark.skip(reason="Newer yt-dlp versions don't support image download.")
@pytest.mark.download
def test_download_facebook_image(self, make_item):
post = self.extractor.download(

8
tests/extractors/test_twitter_api_extractor.py
View File

@@ -140,22 +140,22 @@ class TestTwitterApiExtractor(TestExtractorBase):
(
"https://x.com/SozinhoRamalho/status/1876710769913450647",
"ignore tweet, testing sensitivity warning nudity https://t.co/t3u0hQsSB1",
datetime.datetime(2024, 12, 31, 14, 18, 33, tzinfo=datetime.timezone.utc),
datetime.datetime(2025, 1, 7, 19, 21, 29, tzinfo=datetime.timezone.utc),
),
(
"https://x.com/SozinhoRamalho/status/1876710875475681357",
"ignore tweet, testing sensitivity warning violence https://t.co/syYDSkpjZD",
datetime.datetime(2024, 12, 31, 14, 18, 33, tzinfo=datetime.timezone.utc),
datetime.datetime(2025, 1, 7, 19, 21, 54, tzinfo=datetime.timezone.utc),
),
(
"https://x.com/SozinhoRamalho/status/1876711053813227618",
"ignore tweet, testing sensitivity warning sensitive https://t.co/XE7cRdjzYq",
datetime.datetime(2024, 12, 31, 14, 18, 33, tzinfo=datetime.timezone.utc),
datetime.datetime(2025, 1, 7, 19, 22, 37, tzinfo=datetime.timezone.utc),
),
(
"https://x.com/SozinhoRamalho/status/1876711141314801937",
"ignore tweet, testing sensitivity warning nudity, violence, sensitivity https://t.co/YxCFbbhYE3",
datetime.datetime(2024, 12, 31, 14, 18, 33, tzinfo=datetime.timezone.utc),
datetime.datetime(2025, 1, 7, 19, 22, 58, tzinfo=datetime.timezone.utc),
),
],
)

15
tests/storages/test_S3_storage.py
View File

@@ -45,6 +45,19 @@ class TestS3Storage:
assert self.storage.get_cdn_url(media) == "https://cdn.example.com/another/path.jpg"
def test_uploadf_sets_acl_public(self, mocker):
media = Media("test.png")
mock_file = mocker.MagicMock()
mock_s3_upload = mocker.patch.object(self.storage.s3, "upload_fileobj")
mocker.patch.object(self.storage, "is_upload_needed", return_value=True)
self.storage.uploadf(mock_file, media)
mock_s3_upload.assert_called_once_with(
mock_file,
Bucket="test-bucket",
Key=media.key,
ExtraArgs={"ACL": "public-read", "ContentType": "image/png"},
)
def test_uploadf_detects_charset_for_text_files(self, mocker):
media = Media("test.txt")
mock_file = mocker.MagicMock()
mock_s3_upload = mocker.patch.object(self.storage.s3, "upload_fileobj")
@@ -54,7 +67,7 @@ class TestS3Storage:
mock_file,
Bucket="test-bucket",
Key=media.key,
ExtraArgs={"ACL": "public-read", "ContentType": "text/plain"},
ExtraArgs={"ACL": "public-read", "ContentType": "text/plain; charset=utf-8"},
)
def test_upload_decision_logic(self, mocker):

86
tests/utils/test_urls.py
View File

@@ -1,11 +1,13 @@
import pytest
from auto_archiver.utils.url import (
clean,
is_auth_wall,
check_url_or_raise,
domain_for_url,
is_relevant_url,
remove_get_parameters,
twitter_best_quality_url,
get_media_url_best_quality,
)
@@ -95,6 +97,11 @@ def test_remove_get_parameters(url, without_get):
("https://example.com/150x150.jpg", True),
("https://example.com/rsrc.php/", True),
("https://example.com/img/emoji/", True),
("https://styles.redditmedia.com/123", False),
("https://emoji.redditmedia.com/abc.jpg", False),
("https://example.com/rsrc.m3u8?asdasd=10", False),
("https://example.com/rsrc.mpd", False),
("https://example.com/rsrc.ism?vid=12", False),
],
)
def test_is_relevant_url(url, relevant):
@@ -104,10 +111,87 @@ def test_is_relevant_url(url, relevant):
@pytest.mark.parametrize(
"url, best_quality",
[
("https://twitter.com/some_image.jpg?name=small", "https://twitter.com/some_image.jpg?name=orig"),
(
"https://twitter.com/some_image.jpg?name=small&this_is_another=145",
"https://twitter.com/some_image.jpg?name=orig&this_is_another=145",
),
("https://twitter.com/some_image.jpg", "https://twitter.com/some_image.jpg"),
("https://twitter.com/some_image.jpg?name=orig", "https://twitter.com/some_image.jpg?name=orig"),
],
)
def test_twitter_best_quality_url(url, best_quality):
assert twitter_best_quality_url(url) == best_quality
@pytest.mark.parametrize(
"input_url,expected_url",
[
# Twitter: add/replace name= to name=orig
(
"https://pbs.twimg.com/media/abc123?format=jpg&name=small",
"https://pbs.twimg.com/media/abc123?format=jpg&name=orig",
),
("https://pbs.twimg.com/media/abc123?name=large", "https://pbs.twimg.com/media/abc123?name=orig"),
("https://pbs.twimg.com/media/abc123?format=jpg", "https://pbs.twimg.com/media/abc123?format=jpg"),
# Twitter: already orig
(
"https://pbs.twimg.com/media/abc123?format=jpg&name=orig",
"https://pbs.twimg.com/media/abc123?format=jpg&name=orig",
),
# X.com domain
("https://x.com/media/abc123?name=medium", "https://x.com/media/abc123?name=orig"),
# twimg.com domain
("https://twimg.com/media/abc123?name=thumb", "https://twimg.com/media/abc123?name=orig"),
# Non-twitter domain, no change
("https://example.com/media/file.mp4", "https://example.com/media/file.mp4"),
# Remove -WxH from basename
("https://example.com/media/file-1280x720.mp4", "https://example.com/media/file.mp4"),
("https://example.com/media/file-1920x1080.jpg?foo=bar", "https://example.com/media/file.jpg?foo=bar"),
# Both twitter and -WxH
("https://pbs.twimg.com/media/abc-1280x720.jpg?name=small", "https://pbs.twimg.com/media/abc.jpg?name=orig"),
# No match for -WxH, no change
("https://example.com/media/file.mp4?foo=bar", "https://example.com/media/file.mp4?foo=bar"),
# Path with multiple directories
("https://example.com/a/b/c/file-640x480.png", "https://example.com/a/b/c/file.png"),
# -WxH in directory, not basename (should not change)
("https://example.com/media-1280x720/file.mp4", "https://example.com/media-1280x720/file.mp4"),
],
)
def test_get_media_url_best_quality(input_url, expected_url):
assert get_media_url_best_quality(input_url) == expected_url
@pytest.mark.parametrize(
"input_url,expected_url",
[
# No trackers present
("https://example.com/page?foo=bar&baz=qux", "https://example.com/page?foo=bar&baz=qux"),
# Single tracker present
("https://example.com/page?utm_source=google&foo=bar", "https://example.com/page?foo=bar"),
# Multiple trackers present
("https://example.com/page?utm_source=google&utm_medium=email&utm_campaign=spring", "https://example.com/page"),
# Trackers mixed with other params
(
"https://example.com/page?foo=bar&utm_content=abc&baz=qux&gclid=123",
"https://example.com/page?foo=bar&baz=qux",
),
# Only trackers present
("https://example.com/page?utm_source=google&gclid=123", "https://example.com/page"),
# No query string
("https://example.com/page", "https://example.com/page"),
# Trackers in fragment (should not be removed)
("https://example.com/page#utm_source=google", "https://example.com/page#utm_source=google"),
# Trackers after fragment
("https://example.com/page?utm_source=google#section-1", "https://example.com/page#section-1"),
# Trackers with empty value
("https://example.com/page?utm_source=&foo=bar", "https://example.com/page?foo=bar"),
# Trackers with multiple values
("https://example.com/page?utm_source=google&utm_source=bing&foo=bar", "https://example.com/page?foo=bar"),
# Trackers with encoded values
("https://example.com/page?utm_source=google%20ads&foo=bar", "https://example.com/page?foo=bar"),
# Unrelated param with similar name
("https://example.com/page?utm_sourc=keepme&foo=bar", "https://example.com/page?utm_sourc=keepme&foo=bar"),
],
)
def test_clean_removes_trackers(input_url, expected_url):
assert clean(input_url) == expected_url