bellingcat/auto-archiver
1
0
Fork 0
mirror of https://github.com/bellingcat/auto-archiver.git synced 2026-06-11 04:38:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge main into timestamping_enricher

Browse Source
This commit is contained in:
Patrick Robertson
2025-03-24 15:09:29 +04:00
parent 89ee6f19b6 7b454baa02
commit dfde6f1995
219 changed files with 11049 additions and 2933 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
.github/dependabot.yml vendored Normal file
View File

@@ -0,0 +1,12 @@
# To get started with Dependabot version updates, you'll need to specify which
# package ecosystems to update and where the package manifests are located.
# Please see the documentation for all configuration options:
# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
version: 2
updates:
- package-ecosystem: "" # See documentation for possible values
directory: "/" # Location of package manifests
schedule:
interval: "weekly"

6
.github/workflows/docker-publish.yaml vendored
View File

@@ -11,7 +11,7 @@ on:
env:
# Use docker.io for Docker Hub if empty
REGISTRY: ghcr.io
REGISTRY: docker.io
# github.repository as <account>/<repo>
IMAGE_NAME: ${{ github.repository }}
@@ -45,10 +45,12 @@ jobs:
images: bellingcat/auto-archiver
- name: Build and push Docker image
uses: docker/build-push-action@v2
uses: docker/build-push-action@v6
with:
context: .
platforms: linux/amd64,linux/arm64
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:cache
cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:cache,mode=max

24
.github/workflows/ruff.yaml vendored Normal file
View File

@@ -0,0 +1,24 @@
name: Ruff Formatting & Linting
on:
push:
branches: [ main ]
pull_request:
branches: [ main ]
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Install Python
uses: actions/setup-python@v5
with:
python-version: "3.11"
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install ruff
- name: Run Ruff
run: ruff check --output-format=github . && ruff format --check

4
.github/workflows/tests-core.yaml vendored
View File

@@ -5,9 +5,13 @@ on:
branches: [ main ]
paths:
- src/**
- poetry.lock
- pyproject.toml
pull_request:
paths:
- src/**
- poetry.lock
- pyproject.toml
jobs:
tests:

4
.gitignore vendored
View File

@@ -4,6 +4,7 @@ temp/
.DS_Store
expmt/
service_account.json
service_account-*.json
__pycache__/
._*
anu.html
@@ -33,3 +34,6 @@ dist*
docs/_build/
docs/source/autoapi/
docs/source/modules/autogen/
scripts/settings_page.html
scripts/settings/src/schema.json
.vite

7
.pre-commit-config.yaml Normal file
View File

@@ -0,0 +1,7 @@
# Run Ruff formatter on commits.
repos:
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.9.10
hooks:
- id: ruff
- id: ruff-format

6
.readthedocs.yaml
View File

@@ -9,6 +9,7 @@ build:
os: ubuntu-22.04
tools:
python: "3.10"
nodejs: "22"
jobs:
post_install:
- pip install poetry
@@ -17,6 +18,11 @@ build:
# See https://github.com/readthedocs/readthedocs.org/pull/11152/
- VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH poetry install --with docs
# generate the config editor page. Schema then HTML
- VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH poetry run python scripts/generate_settings_schema.py
# install node dependencies and build the settings
- cd scripts/settings && npm install && npm run build && yes | cp -v dist/index.html ../../docs/source/installation/settings.html && cd ../..
sphinx:
configuration: docs/source/conf.py

15
Dockerfile
View File

@@ -7,13 +7,24 @@ ENV RUNNING_IN_DOCKER=1 \
PYTHONFAULTHANDLER=1 \
PATH="/root/.local/bin:$PATH"
ARG TARGETARCH
# Installing system dependencies
RUN add-apt-repository ppa:mozillateam/ppa && \
apt-get update && \
apt-get install -y --no-install-recommends gcc ffmpeg fonts-noto exiftool && \
apt-get install -y --no-install-recommends firefox-esr && \
ln -s /usr/bin/firefox-esr /usr/bin/firefox && \
wget https://github.com/mozilla/geckodriver/releases/download/v0.35.0/geckodriver-v0.35.0-linux64.tar.gz && \
ln -s /usr/bin/firefox-esr /usr/bin/firefox
ARG GECKODRIVER_VERSION=0.36.0
RUN if [ $(uname -m) = "aarch64" ]; then \
GECKODRIVER_ARCH=linux-aarch64; \
else \
GECKODRIVER_ARCH=linux64; \
fi && \
wget https://github.com/mozilla/geckodriver/releases/download/v${GECKODRIVER_VERSION}/geckodriver-v${GECKODRIVER_VERSION}-${GECKODRIVER_ARCH}.tar.gz && \
tar -xvzf geckodriver* -C /usr/local/bin && \
chmod +x /usr/local/bin/geckodriver && \
rm geckodriver-v* && \

79
Makefile Normal file
View File

@@ -0,0 +1,79 @@
# Variables
SPHINXOPTS ?=
SPHINXBUILD ?= sphinx-build
SOURCEDIR = docs/source
BUILDDIR = docs/_build
.PHONY: help
help:
@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
@echo "Additional Commands:"
@echo " make test - Run all tests in 'tests/' with pytest"
@echo " make ruff-check - Run Ruff linting and formatting checks (safe)"
@echo " make ruff-clean - Auto-fix Ruff linting and formatting issues"
@echo " make docs - Generate documentation (same as 'make html')"
@echo " make clean-docs - Remove generated docs"
@echo " make docker-build - Build the Auto Archiver Docker image"
@echo " make docker-compose - Run Auto Archiver with Docker Compose"
@echo " make docker-compose-rebuild - Rebuild and run Auto Archiver with Docker Compose"
@echo " make show-docs - Build and open the documentation in a browser"
.PHONY: test
test:
@echo "Running tests..."
@pytest tests --disable-warnings
.PHONY: ruff-check
ruff-check:
@echo "Checking code style with Ruff (safe)..."
@ruff check .
.PHONY: ruff-clean
ruff-clean:
@echo "Fixing lint and formatting issues with Ruff..."
@ruff check . --fix
@ruff format .
.PHONY: docs
docs:
@echo "Building documentation..."
@$(SPHINXBUILD) -M html "$(SOURCEDIR)" "$(BUILDDIR)"
.PHONY: clean-docs
clean-docs:
@echo "Cleaning up generated documentation files..."
@$(SPHINXBUILD) -M clean "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
@rm -rf "$(SOURCEDIR)/autoapi/" "$(SOURCEDIR)/modules/autogen/"
@echo "Cleanup complete."
.PHONY: show-docs
show-docs:
@echo "Opening documentation in browser..."
@open "$(BUILDDIR)/html/index.html"
.PHONY: docker-build
docker-build:
@echo "Building local Auto Archiver Docker image..."
@docker compose build # Uses the same build context as docker-compose.yml
.PHONY: docker-compose
docker-compose:
@echo "Running Auto Archiver with Docker Compose..."
@docker compose up
.PHONY: docker-compose-rebuild
docker-compose-rebuild:
@echo "Rebuilding and running Auto Archiver with Docker Compose..."
@docker compose up --build
# Catch-all for Sphinx commands
.PHONY: Makefile
%: Makefile
@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

4
README.md
View File

@@ -10,7 +10,7 @@
Auto Archiver is a Python tool to automatically archive content on the web in a secure and verifiable way. It takes URLs from different sources (e.g. a CSV file, Google Sheets, command line etc.) and archives the content of each one. It can archive social media posts, videos, images and webpages. Content can enriched, then saved either locally or remotely (S3 bucket, Google Drive). The status of the archiving process can be appended to a CSV report, or if using Google Sheets – back to the original sheet.
Auto Archiver is a Python tool to automatically archive content on the web in a secure and verifiable way. It takes URLs from different sources (e.g. a CSV file, Google Sheets, command line etc.) and archives the content of each one. It can archive social media posts, videos, images and webpages. Content can be enriched, then saved either locally or remotely (S3 bucket, Google Drive). The status of the archiving process can be appended to a CSV report, or if using Google Sheets – back to the original sheet.
<div class="hidden_rtd">
@@ -29,7 +29,7 @@ View the [Installation Guide](https://auto-archiver.readthedocs.io/en/latest/ins
To get started quickly using Docker:
`docker pull bellingcat/auto-archiver && docker run --rm -v secrets:/app/secrets bellingcat/auto-archiver --config secrets/orchestration.yaml`
`docker pull bellingcat/auto-archiver && docker run -it --rm -v secrets:/app/secrets bellingcat/auto-archiver --config secrets/orchestration.yaml`
Or pip:

4
docker-compose.yaml
View File

@@ -1,4 +1,3 @@
version: '3.8'
services:
auto-archiver:
@@ -10,7 +9,4 @@ services:
volumes:
- ./secrets:/app/secrets
- ./local_archive:/app/local_archive
environment:
- WACZ_ENABLE_DOCKER=true
- RUNNING_IN_DOCKER=true
command: --config secrets/orchestration.yaml

20
docs/Makefile
View File

@@ -1,20 +0,0 @@
# Minimal makefile for Sphinx documentation
#
# You can set these variables from the command line, and also
# from the environment for the first two.
SPHINXOPTS ?=
SPHINXBUILD ?= sphinx-build
SOURCEDIR = source
BUILDDIR = _build
# Put it first so that "make" without argument is like "make help".
help:
@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
.PHONY: help Makefile
# Catch-all target: route all unknown targets to Sphinx using the new
# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
%: Makefile
@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

2
docs/scripts/__init__.py
View File

@@ -1 +1 @@
from scripts import generate_module_docs
from scripts import generate_module_docs

67
docs/scripts/scripts.py
View File

@@ -10,12 +10,12 @@ MODULES_FOLDER = Path(__file__).parent.parent.parent.parent / "src" / "auto_arch
SAVE_FOLDER = Path(__file__).parent.parent / "source" / "modules" / "autogen"
type_color = {
'feeder': "<span style='color: #FFA500'>[feeder](/core_modules.md#feeder-modules)</a></span>",
'extractor': "<span style='color: #00FF00'>[extractor](/core_modules.md#extractor-modules)</a></span>",
'enricher': "<span style='color: #0000FF'>[enricher](/core_modules.md#enricher-modules)</a></span>",
'database': "<span style='color: #FF00FF'>[database](/core_modules.md#database-modules)</a></span>",
'storage': "<span style='color: #FFFF00'>[storage](/core_modules.md#storage-modules)</a></span>",
'formatter': "<span style='color: #00FFFF'>[formatter](/core_modules.md#formatter-modules)</a></span>",
"feeder": "<span style='color: #FFA500'>[feeder](/core_modules.md#feeder-modules)</a></span>",
"extractor": "<span style='color: #00FF00'>[extractor](/core_modules.md#extractor-modules)</a></span>",
"enricher": "<span style='color: #0000FF'>[enricher](/core_modules.md#enricher-modules)</a></span>",
"database": "<span style='color: #FF00FF'>[database](/core_modules.md#database-modules)</a></span>",
"storage": "<span style='color: #FFFF00'>[storage](/core_modules.md#storage-modules)</a></span>",
"formatter": "<span style='color: #00FFFF'>[formatter](/core_modules.md#formatter-modules)</a></span>",
}
TABLE_HEADER = ("Option", "Description", "Default", "Type")
@@ -34,6 +34,7 @@ steps:
"""
def generate_module_docs():
yaml = YAML()
SAVE_FOLDER.mkdir(exist_ok=True)
@@ -48,49 +49,49 @@ def generate_module_docs():
# generate the markdown file from the __manifest__.py file.
manifest = module.manifest
for type in manifest['type']:
for type in manifest["type"]:
modules_by_type.setdefault(type, []).append(module)
description = "\n".join(l.lstrip() for l in manifest['description'].split("\n"))
types = ", ".join(type_color[t] for t in manifest['type'])
description = "\n".join(line.lstrip() for line in manifest["description"].split("\n"))
types = ", ".join(type_color[t] for t in manifest["type"])
readme_str = f"""
# {manifest['name']}
# {manifest["name"]}
```{{admonition}} Module type
{types}
```
{description}
"""
steps_str = "\n".join(f" {t}s:\n - {module.name}" for t in manifest['type'])
"""
steps_str = "\n".join(f" {t}s:\n - {module.name}" for t in manifest["type"])
if not manifest['configs']:
if not manifest["configs"]:
config_string = f"# No configuration options for {module.name}.*\n"
else:
config_table = header_row
config_yaml = {}
global_yaml[module.name] = CommentedMap()
global_yaml.yaml_set_comment_before_after_key(module.name, f"\n\n{module.display_name} configuration options")
global_yaml.yaml_set_comment_before_after_key(
module.name, f"\n\n{module.display_name} configuration options"
)
for key, value in manifest['configs'].items():
type = value.get('type', 'string')
if type == 'json_loader':
value['type'] = 'json'
elif type == 'str':
for key, value in manifest["configs"].items():
type = value.get("type", "string")
if type == "json_loader":
value["type"] = "json"
elif type == "str":
type = "string"
default = value.get('default', '')
default = value.get("default", "")
config_yaml[key] = default
global_yaml[module.name][key] = default
if value.get('help', ''):
global_yaml[module.name].yaml_add_eol_comment(value.get('help', ''), key)
if value.get("help", ""):
global_yaml[module.name].yaml_add_eol_comment(value.get("help", ""), key)
help = "**Required**. " if value.get('required', False) else "Optional. "
help += value.get('help', '')
help = "**Required**. " if value.get("required", False) else "Optional. "
help += value.get("help", "")
config_table += f"| `{module.name}.{key}` | {help} | {value.get('default', '')} | {type} |\n"
global_table += f"| `{module.name}.{key}` | {help} | {default} | {type} |\n"
readme_str += "\n## Configuration Options\n"
@@ -98,18 +99,18 @@ def generate_module_docs():
config_string = io.BytesIO()
yaml.dump({module.name: config_yaml}, config_string)
config_string = config_string.getvalue().decode('utf-8')
config_string = config_string.getvalue().decode("utf-8")
yaml_string = EXAMPLE_YAML.format(steps_str=steps_str, config_string=config_string)
readme_str += f"```{{code}} yaml\n{yaml_string}\n```\n"
if manifest['configs']:
if manifest["configs"]:
readme_str += "\n### Command Line:\n"
readme_str += config_table
# add a link to the autodoc refs
readme_str += f"\n[API Reference](../../../autoapi/{module.name}/index)\n"
# create the module.type folder, use the first type just for where to store the file
for type in manifest['type']:
for type in manifest["type"]:
type_folder = SAVE_FOLDER / type
type_folder.mkdir(exist_ok=True)
with open(type_folder / f"{module.name}.md", "w") as f:
@@ -117,10 +118,10 @@ def generate_module_docs():
f.write(readme_str)
generate_index(modules_by_type)
del global_yaml['placeholder']
del global_yaml["placeholder"]
global_string = io.BytesIO()
global_yaml = yaml.dump(global_yaml, global_string)
global_string = global_string.getvalue().decode('utf-8')
global_string = global_string.getvalue().decode("utf-8")
global_yaml = f"```yaml\n{global_string}\n```"
with open(SAVE_FOLDER / "configs_cheatsheet.md", "w") as f:
f.write("### Configuration File\n" + global_yaml + "\n### Command Line\n" + global_table)
@@ -144,4 +145,4 @@ def generate_index(modules_by_type):
if __name__ == "__main__":
generate_module_docs()
generate_module_docs()

47
docs/source/conf.py
View File

@@ -5,7 +5,7 @@ import os
from importlib.metadata import metadata
from datetime import datetime
sys.path.append(os.path.abspath('../scripts'))
sys.path.append(os.path.abspath("../scripts"))
from scripts import generate_module_docs
from auto_archiver.version import __version__
@@ -20,33 +20,35 @@ project = package_metadata["name"]
copyright = str(datetime.now().year)
author = "Bellingcat"
release = package_metadata["version"]
language = 'en'
language = "en"
# -- General configuration ---------------------------------------------------
extensions = [
"myst_parser", # Markdown support
"autoapi.extension", # Generate API documentation from docstrings
"sphinxcontrib.mermaid", # Mermaid diagrams
"sphinx.ext.viewcode", # Source code links
"myst_parser", # Markdown support
"autoapi.extension", # Generate API documentation from docstrings
"sphinxcontrib.mermaid", # Mermaid diagrams
"sphinx.ext.viewcode", # Source code links
"sphinx_copybutton",
"sphinx.ext.napoleon", # Google-style and NumPy-style docstrings
"sphinx.ext.napoleon", # Google-style and NumPy-style docstrings
"sphinx.ext.autosectionlabel",
# 'sphinx.ext.autosummary', # Summarize module/class/function docs
]
templates_path = ['_templates']
templates_path = ["_templates"]
exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", ""]
# -- AutoAPI Configuration ---------------------------------------------------
autoapi_type = 'python'
autoapi_type = "python"
autoapi_dirs = ["../../src/auto_archiver/core/", "../../src/auto_archiver/utils/"]
# get all the modules and add them to the autoapi_dirs
autoapi_dirs.extend([f"../../src/auto_archiver/modules/{m}" for m in os.listdir("../../src/auto_archiver/modules")])
autodoc_typehints = "signature" # Include type hints in the signature
autoapi_ignore = ["*/version.py", ] # Ignore specific modules
autoapi_keep_files = True # Option to retain intermediate JSON files for debugging
autoapi_add_toctree_entry = True # Include API docs in the TOC
autodoc_typehints = "signature" # Include type hints in the signature
autoapi_ignore = [
"*/version.py",
] # Ignore specific modules
autoapi_keep_files = True # Option to retain intermediate JSON files for debugging
autoapi_add_toctree_entry = True # Include API docs in the TOC
autoapi_python_use_implicit_namespaces = True
autoapi_template_dir = "../_templates/autoapi"
autoapi_options = [
@@ -59,13 +61,13 @@ autoapi_options = [
# -- Markdown Support --------------------------------------------------------
myst_enable_extensions = [
"deflist", # Definition lists
"html_admonition", # HTML-style admonitions
"html_image", # Inline HTML images
"replacements", # Substitutions like (C)
"smartquotes", # Smart quotes
"linkify", # Auto-detect links
"substitution", # Text substitutions
"deflist", # Definition lists
"html_admonition", # HTML-style admonitions
"html_image", # Inline HTML images
"replacements", # Substitutions like (C)
"smartquotes", # Smart quotes
"linkify", # Auto-detect links
"substitution", # Text substitutions
]
myst_heading_anchors = 2
myst_fence_as_directive = ["mermaid"]
@@ -76,7 +78,7 @@ source_suffix = {
}
# -- Options for HTML output -------------------------------------------------
html_theme = 'sphinx_book_theme'
html_theme = "sphinx_book_theme"
html_static_path = ["../_static"]
html_css_files = ["custom.css"]
html_title = f"Auto Archiver v{__version__}"
@@ -87,7 +89,6 @@ html_theme_options = {
}
copybutton_prompt_text = r">>> |\.\.\."
copybutton_prompt_is_regexp = True
copybutton_only_copy_prompt_lines = False
copybutton_only_copy_prompt_lines = False

2
docs/source/development/developer_guidelines.md
View File

@@ -31,4 +31,6 @@ docker_development
testing
docs
release
settings_page
style_guide
```

9
docs/source/development/docs.md
View File

@@ -36,3 +36,12 @@ open docs/_build/html/index.html
sphinx-autobuild docs/source docs/_build/html
```
### Managing Readthedocs (RTD) Versions
Version management is done at [https://app.readthedocs.org/projects/auto-archiver/](https://app.readthedocs.org/projects/auto-archiver/)
(login required). Once logged in, you can create new versions, delete old versions or change visibility of versions. More info on
[RTD](https://docs.readthedocs.com/platform/stable/versions.html).
Currently, the Auto Archiver project is set up to automatically create a new docs version for each `vX.Y.Z` release. For more on this,
see the RTD [instructions on automation](https://docs.readthedocs.com/platform/stable/guides/automation-rules.html) or edit the existing automation rule in the project settings.

26
docs/source/development/release.md
View File

@@ -2,14 +2,32 @@
```{note} This is a work in progress.
```
### Update the project version
1. Update the version number in [version.py](src/auto_archiver/version.py)
2. Go to github releases > new release > use `vx.y.z` for matching version notation
1. package is automatically updated in pypi
2. docker image is automatically pushed to dockerhub
Update the version number in the project file: [pyproject.toml](../../pyproject.toml) following SemVer:
```toml
[project]
name = "auto-archiver"
version = "0.1.1"
```
Then commit and push the changes.
* The package version is automatically updated in PyPi using the workflow [python-publish.yml](../../.github/workflows/python-publish.yml)
* A Docker image is automatically pushed with the git tag to dockerhub using the workflow [docker-publish.yml](../../.github/workflows/docker-publish.yml)
### Create the release on Git
The release needs a git tag which should match the project version number, prefixed with a 'v'. For example, if the project version is `0.1.1`, the git tag should be `v0.1.1`.
This can be done the usual way, or created within the Github UI when you create the release.
Go to GitHub releases > new release > create the release with the new tag and the release notes.
manual release to docker hub
* `docker image tag auto-archiver bellingcat/auto-archiver:latest`
* `docker push bellingcat/auto-archiver`
### Building the Settings Page
The Settings page is built as part of the python-publish workflow and packaged within the app.

31
docs/source/development/settings_page.md Normal file
View File

@@ -0,0 +1,31 @@
# Configuration Editor
The [configuration editor](../installation/config_editor.md), is an easy-to-use UI for users to edit their auto-archiver settings.
The single-file app is built using React and vite. To get started developing the package, follow these steps:
1. Make sure you have Node v22 installed.
```{note} Tip: if you don't have node installed:
Use `nvm` to manage your node installations. Use:
`curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash` to install `nvm` and then `nvm i 22` to install Node v22
```
2. Generate the `schema.json` file for the currently installed modules using `python scripts/generate_settings_schema.py`
3. Go to the settings folder `cd scripts/settings/` and build your environment with `npm i`
4. Run a development version of the page with `npm run dev` and then open localhost:5173.
5. Build a release version of the page with `npm run build`
A release version creates a single-file app called `dist/index.html`. This file should be copied to `docs/source/installation/settings_base.html` so that it can be integrated into the sphinx docs.
```{note}
The single-file app dist/index.html does not include any `<html>` or `<head>` tags as it is designed to be built into a RTD docs page. Edit `index.html` in the settings folder if you wish to modify the built page.
```
## Readthedocs Integration
The configuration editor is built as part of the RTD deployment (see `.readthedocs.yaml` file). This command is run every time RTD is built:
`cd scripts/settings && npm install && npm run build && yes | cp dist/index.html ../../docs/source/installation/settings_base.html && cd ../..`

70
docs/source/development/style_guide.md Normal file
View File

@@ -0,0 +1,70 @@
# Style Guide
The project uses [Ruff](https://docs.astral.sh/ruff/) for linting and formatting.
Our style configurations are set in the `pyproject.toml` file. If needed, you can modify them there.
### **Formatting (Auto-Run Before Commit) 🛠️**
We have a pre-commit hook to run the formatter before you commit.
This requires you to set it up once locally, then it will run automatically when you commit changes.
```shell
poetry run pre-commit install
```
Ruff can also be to run automatically.
Alternative: Ruff can also be [integrated with most editors](https://docs.astral.sh/ruff/editors/setup/) for real-time formatting.
If you wish to disable the pre-commit hook (for example, if you want to commit some WIP code) you can use the `--no-verify` flag when you commit.
For example: `git commit -m "WIP Code" --no-verify`
### **Linting (Check Before Pushing) 🔍**
We recommend you also run the linter before pushing code.
We have [Makefile](../../../Makefile) commands to run common tasks.
Tip: if you're on Windows you might need to install `make` first, or alternatively you can use ruff commands directly.
**Lint Check:** This outputs a report of any issues found, without attempting to fix them:
```shell
make ruff-check
```
Tip: To see a more detailed linting report, you can remove the following line from the `pyproject.toml` file:
```toml
[tool.ruff]
# Remove this for a more detailed lint report
output-format = "concise"
```
**Lint Fix:** This command will attempt to fix some of the issues it picked up with the lint check.
Note not all warnings can be fixed automatically.
⚠️ Warning: This can cause breaking changes. ⚠️
Most fixes are safe, but some non-standard practices such as dynamic loading are not picked up by linters. Ensure you check any modifications by this before committing them.
```shell
make ruff-fix
```
**Changing Configurations ⚙️**
Our rules are quite lenient for general usage, but if you want to run more rigorous checks you can then run checks with additional rules to see more nuanced errors which you can review manually.
Check out the [ruff documentation](https://docs.astral.sh/ruff/configuration/) for the full list of rules.
One example is to extend the selected rules for linting the `pyproject.toml` file:
```toml
[tool.ruff.lint]
# Extend the rules to check for by adding them to this option:
# See documentation for more details: https://docs.astral.sh/ruff/rules/
extend-select = ["B"]
```
Then re-run the `make ruff-check` command to see the new rules in action.

2
docs/source/how_to/authentication_how_to.md
View File

@@ -46,7 +46,7 @@ First, we need to install an extension in our browser to export the cookies for
**2. Export the cookies**
```{note} See the note [here](../installation/authentication.md#recommendations-for-authentication) on why you shouldn't use your own personal account for achiving.
```{note} See the note [here](../installation/authentication.md#recommendations-for-authentication) on why you shouldn't use your own personal account for archiving.
```
Once the extension is installed in your preferred browser, login to Twitter in this browser, and then activate the extension and export the cookies. You can choose to export all your cookies for your browser, or just cookies for this specific site. In the image below, we're only exporting cookies for Twitter/x.com:

132
docs/source/how_to/gsheets_setup.md
View File

@@ -6,20 +6,58 @@ This guide explains how to set up Google Sheets to process URLs automatically an
2. Setting up a service account so Auto Archiver can access the sheet
3. Setting the Auto Archiver settings
### 1. Setting up your Google Sheet
Any Google sheet must have at least *one* column, with the name 'link' (you can change this name afterwards). This is the column with the URLs that you want the Auto Archiver to archive. Your sheet can have many other columns that the Auto Archiver can use, and you can also include any other columns for your own personal use.
## 1. Setting up a Google Service Account
We recommend copying [this template Google Sheet](https://docs.google.com/spreadsheets/d/1NJZo_XZUBKTI1Ghlgi4nTPVvCfb0HXAs6j5tNGas72k/edit?usp=sharing) as a starting point for your project.
Once your Google Sheet is set up, you need to create what's called a 'service account' that will allow the Auto Archiver to access it.
To do this, you can either:
* a) follow the steps in [this guide](https://gspread.readthedocs.io/en/latest/oauth2.html) all the way up until step 8. You should have downloaded a file called `service_account.json` and should save it in the `secrets/` folder
* b) run the following script to automatically generate the file:
```{code} bash
https://raw.githubusercontent.com/bellingcat/auto-archiver/refs/heads/main/scripts/generate_google_services.sh | bash -s --
```
This uses gcloud to create a new project, a new user and downloads the service account automatically for you. The service account file will have the name `service_account-XXXXXXX.json` where XXXXXXX is a random 16 letter/digit string for the project created.
```{note}
To save the generated file to a different folder, pass an argument as follows:
```{code} bash
https://raw.githubusercontent.com/bellingcat/auto-archiver/refs/heads/main/scripts/generate_google_services.sh | bash -s -- /path/to/secrets
```
----------
Once you've downloaded the file, you can save it to `secrets/service_account.json` (the default name), or to another file and then change the location in the settings (see step 4).
Also make sure to **note down** the email address for this service account. You'll need that for step 3.
```{note}
The email address created in this step can be found either by opening the `service_account.json` file, or if you used b) the `generate_google_services.sh` script, then the script will have printed it out for you.
The email address will look something like `user@project-name.iam.gserviceaccount.com`
```
## 2. Setting up your Google Sheet
We recommend copying [this template Google Sheet](https://docs.google.com/spreadsheets/d/1NJZo_XZUBKTI1Ghlgi4nTPVvCfb0HXAs6j5tNGas72k/edit?usp=sharing) as a starting point for your project, as this matches all the columns required.
But if you like, you can also create your own custom sheet. The only columns required are 'link', 'archive status', and 'archive location'. 'link' is the column with the URLs that you want the Auto Archiver to archive, the other two record the archival status and result.
Here's an overview of all the columns, and what a complete sheet would look like.
Inputs:
**Inputs:**
* **Link** *(required)*: the URL of the post to archive
These are processed by the Gsheet Feeder and passed to the Auto Archiver.
* **Link** *(required)*: the URL of the post that is to be archived
* **Destination folder**: custom folder for archived file (regardless of storage)
Outputs:
**Outputs:**
These are updated by the Gsheet DB module during the archiving process.
Note the required columns are only required if you are using the Gsheet DB module as well as the feeder.
* **Archive status** *(required)*: Status of archive operation
* **Archive location**: URL of archived post
* **Archive date**: Date archived
@@ -33,61 +71,67 @@ Outputs:
* **WACZ**: Link to a WACZ web archive of post
* **ReplayWebpage**: Link to a ReplayWebpage viewer of the WACZ archive
For example, this is a spreadsheet configured with all of the columns for the auto archiver and a few URLs to archive. (Note that the column names are not case sensitive.)
For example, this is a spreadsheet configured with all of the columns for the auto archiver and a few URLs to archive.
In this example the Ghseet Feeder and Gsheet DB are being used, and the archive is in progress.
(Note that the column names are not case sensitive.)
![A screenshot of a Google Spreadsheet with column headers defined as above, and several Youtube and Twitter URLs in the "Link" column](../demo-before.png)
![A screenshot of a Google Spreadsheet with column headers defined as above, and several Youtube and Twitter URLs in the "Link" column](../../demo-before.png)
We'll change the name of the 'Destination Folder' column in step 3.
We'll change the name of the 'Destination Folder' column in the Step 4a.
## 2. Setting up your Service Account
## 3. Share your Google Sheet with your Service Account email address
Once your Google Sheet is set up, you need to create what's called a 'service account' that will allow the Auto Archiver to access it.
Remember that email address you copied in Step 1? Now that you've set up your Google sheet, click 'Share' in the top
right hand corner and enter the email address. Make sure to give the account **Editor** access. Here's how that looks:
To do this, follow the steps in [this guide](https://gspread.readthedocs.io/en/latest/oauth2.html) all the way up until step 8. You should have downloaded a file called `service_account.json` and shared the Google Sheet with the log 'client_email' email address in this file.
![Share sheet](share_sheet.png)
Once you've downloaded the file, save it to `secrets/service_account.json`
## 4. Setting up the configuration file
## 3. Setting up the configuration file
Now that you've set up your Google sheet, and you've set up the service account so Auto Archiver can access the sheet, the final step is to set your configuration.
First, make sure you have `gsheet_feeder` set in the `steps.feeders` section of your config. If you wish to store the results of the archiving process back in your Google sheet, make sure to also set the `ghseet_db` settig in the `steps.databases` section. Here's how this might look:
The final step is to set your configuration. First, make sure you have `gsheet_feeder_db` set in the `steps.feeders` section of your config. If you wish to store the results of the archiving process back in your Google sheet, make sure to also put `gsheet_feeder_db` setting in the `steps.databases` section. Here's how this might look:
```{code} yaml
steps:
feeders:
- gsheet_feeder
- gsheet_feeder_db
...
databases:
- gsheet_db # optional, if you also want to store the results in the Google sheet
- gsheet_feeder_db # optional, if you also want to store the results in the Google sheet and tract the status of active archivals.
...
```
Next, set up the `gsheet_feeder` configuration settings in the 'Configurations' part of the config `orchestration.yaml` file. Open up he file, and set the `gsheet_feeder.sheet` setting or the `gsheet_feeder.sheet_id` setting. The `sheet` should be the name of your sheet, as it shows in the top left of the sheet. For example, the sheet [here](https://docs.google.com/spreadsheets/d/1NJZo_XZUBKTI1Ghlgi4nTPVvCfb0HXAs6j5tNGas72k/edit?gid=0#gid=0) is called 'Public Auto Archiver template'.
Next, set up the `gsheet_feeder_db` configuration settings in the 'Configurations' part of the config `orchestration.yaml` file. Open up the file, and set the `gsheet_feeder_db.sheet` setting or the `gsheet_feeder_db.sheet_id` setting. The `sheet` should be the name of your sheet, as it shows in the top left of the sheet.
For example, the sheet [here](https://docs.google.com/spreadsheets/d/1NJZo_XZUBKTI1Ghlgi4nTPVvCfb0HXAs6j5tNGas72k/edit?gid=0#gid=0) is called 'Public Auto Archiver template'.
If you saved your `service_account.json` file to anywhere other than the default location (`secrets/service_account.json`), then also make sure to change that now:
Here's how this might look:
```{code} yaml
...
gsheet_feeder:
gsheet_feeder_db:
sheet: 'My Awesome Sheet'
service_account: secrets/service_account-XXXXX.json # or leave as secrets/service_account.json
...
```
You can also pass these settings directly on the command line without having to edit the file, here'a an example of how to do that (using docker):
`docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver:dockerize --gsheet_feeder.sheet "Auto archive test 2023-2"`.
`docker run -it --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver:dockerize --gsheet_feeder_db.sheet "My Awesome Sheet 2"`.
Here, the sheet name has been overridden/specified in the command line invocation.
### 3a. (Optional) Changing the column names
### 4a. (Optional) Changing the column names
In step 1, we said we would change the name of the 'Destination Folder'. Perhaps you don't like this name, or already have a sheet with a different name. In our example here, we want to name this column 'Save Folder'. To do this, we need to edit the `ghseet_feeder.column` setting in the configuration file. For more information on this setting, see the [Gsheet Feeder docs](../modules/autogen/feeder/gsheet_feeder.md#configuration-options). We will first copy the default settings from the Gsheet Feeder docs for the 'column' settings, and then edit the 'Destination Folder' section to rename it 'Save Folder'. Our final configuration section looks like:
In step 1, we said we would change the name of the 'Destination Folder'. Perhaps you don't like this name, or already have a sheet with a different name. In our example here, we want to name this column 'Save Folder'. To do this, we need to edit the `ghseet_feeder_db.column` setting in the configuration file.
For more information on this setting, see the [Gsheet Feeder Database docs](../modules/autogen/feeder/gsheet_feeder_db.md#configuration-options). We will first copy the default settings from the Gsheet Feeder docs for the 'column' settings, and then edit the 'Destination Folder' section to rename it 'Save Folder'. Our final configuration section looks like:
```{code} yaml
...
gsheet_feeder:
gsheet_feeder_db:
sheet: 'My Awesome Sheet'
header: 1
service_account: secrets/service_account.json
columns:
url: link
status: archive status
@@ -103,20 +147,44 @@ gsheet_feeder:
pdq_hash: perceptual hashes
wacz: wacz
replaywebpage: replaywebpage
```
## 4. Running the Auto Archiver
### Feeding the URLs to the Auto Archiver
## Viewing the Results after archiving
The URLs to be archived should be added to the Google Sheet, and optionally a folder value. Leave all the other configured columns empty (but you may add additional columns for your own use, as long as they don't conflict with the column names mapped in the configuration file).
The Auto Archiver will archive any URLs which have an empty 'status' column
With the `ghseet_db` installed, once you start running the Auto Archiver, it will updates the "Archive status" column.
### Viewing the Results after archiving
![A screenshot of a Google Spreadsheet with column headers defined as above, and several Youtube and Twitter URLs in the "Link" column. The auto archiver has added "archive in progress" to one of the status columns.](../demo-progress.png)
With the `ghseet_feeder_db` installed, once you start running the Auto Archiver, it will update the "Archive status" column.
The status will be set to "Archive in progress" once the archival starts. If the archival is stopped during a run, either manually or because an error is raised the status value should be cleared.
![A screenshot of a Google Spreadsheet with column headers defined as above, and several Youtube and Twitter URLs in the "Link" column. The auto archiver has added "archive in progress" to one of the status columns.](../../demo-progress.png)
The links are downloaded and archived, and the spreadsheet is updated to the following:
![A screenshot of a Google Spreadsheet with videos archived and metadata added per the description of the columns above.](../demo-after.png)
![A screenshot of a Google Spreadsheet with videos archived and metadata added per the description of the columns above.](../../demo-after.png)
Note that the first row is skipped, as it is assumed to be a header row (`--gsheet_feeder.header=1` and you can change it if you use more rows above). Rows with an empty URL column, or a non-empty archive column are also skipped. All sheets in the document will be checked.
Note that the first row is skipped, as it is assumed to be a header row (`--gsheet_feeder_db.header=1` and you can change it if you use more rows above). Rows with an empty URL column, or a non-empty archive column are also skipped. All sheets in the document will be checked.
The "archive location" link contains the path of the archived file, in local storage, S3, or in Google Drive.
![The archive result for a link in the demo sheet.](../demo-archive.png)
![The archive result for a link in the demo sheet.](../../demo-archive.png)
### Troubleshooting
**Hanging Archival in progress status**
Occasionally system crashes or other unexpected events can cause the Auto Archiver to exit without cleaning up the status value.
If you are sure that all archival processes have stopped but you still see "Archive in progress" in the status column, you can manually clear the status column to allow the Auto Archiver to retry that archival on the next run.
**Nothing archived status**
Sometimes this means the tool is genuinely unable to extract the content at this point in time, but sometimes it can be resolved with different configurations.
Try:
- Turning on additional 'extractor' types in the configuration file (this can appear as 'no archiver' in the status column).
- Changing credentials or refreshing session files for extractors which require them
- Check if the extractors can accept any additional configurations such as adding a cookie file.

61
docs/source/how_to/new_config_format.md
View File

@@ -1,11 +1,11 @@
# Upgrading to v0.13
# Upgrading from v0.12
```{note} This how-to is only relevant for people who used Auto Archiver before February 2025 (versions prior to 0.13).
If you are new to Auto Archiver, then you are already using the latest configuration format and this how-to is not relevant for you.
```
Version 0.13 of Auto Archiver has breaking changes in the configuration format, which means earlier configuration formats will not work without slight modifications.
Versions 0.13+ of Auto Archiver has breaking changes in the configuration format, which means earlier configuration formats will not work without slight modifications.
## How do I know if I need to update my configuration format?
@@ -22,15 +22,13 @@ your configuration file or on the command line (using --feeders)
```{code} yaml
steps:
feeder: gsheet_feeder
feeder: cli_feeder
...
```
## Updating your configuration file
The next two sections outline the two methods you have for updating your file.
To update your configuration file, you can either:
### 1. Manually edit the configuration file and change the values.
## 1. Manually edit the configuration file and change the values.
This is recommended if you want to keep all your old settings. Follow the steps below to change the relevant settings:
@@ -75,28 +73,49 @@ The names of the actual modules have also changed, so for any extractor modules
- `wayback_archiver_enricher` → `wayback_extractor_enricher`
- `vk_archiver` → `vk_extractor`
Additionally, the `youtube_archiver` has been renamed to `generic_extractor` as it is considered the default/fallback extractor. Read more about the [generic extractor](../modules/autogen/extractor/generic_extractor.md).
#### c) Module Renaming
The `youtube_archiver` has been renamed to `generic_extractor` as it is considered the default/fallback extractor. Read more about the [generic extractor](../modules/autogen/extractor/generic_extractor.md).
The `atlos` modules have been merged into one, as have the `gsheets` feeder and database.
- `atlos_feeder` → `atlos_feeder_db_storage`
- `atlos_storage` → `atlos_feeder_db_storage`
- `atlos_db` → `atlos_feeder_db_storage`
- `gsheet_feeder` → `gsheet_feeder_db`
- `gsheet_db` → `gsheet_feeder_db`
Example:
```{code} yaml
steps:
feeders:
- gsheet_feeder_db # formerly gsheet_feeder
...
archivers:
- telethon_archiver
- youtube_archiver
- vk_archiver
# renaming 'archiver' to 'extractor', and renaming the youtube_archiver the above config will become:
steps:
extractors: # formerly 'archivers'
- telethon_extractor # formerly telethon_archiver
- generic_extractor # formerly youtube_archiver
- vk_extractor # formerly vk_archiver
databases:
- gsheet_feeder_db # formerly gsheet_db
...
extractors:
- telethon_extractor
- vk_extractor
- generic_extractor
```
#### c) Redundant / Obsolete Modules
```{note}
Don't forget to also rename the configuration settings. For example:
```{code} yaml
gsheet_feeder_db: # formerly gsheet_feeder
service_account: secrets/service_account.json
sheet: My Google Sheet
...
```
#### d) Redundant / Obsolete Modules
With v0.13 of Auto Archiver, the following modules have been removed and their features have been built in to the generic_extractor. You should remove them from the 'steps' section of your configuration file:
@@ -104,7 +123,7 @@ With v0.13 of Auto Archiver, the following modules have been removed and their f
* `tiktok_archiver` - use the `generic_extractor` to extract TikTok videos.
### 2. Auto-generate a new config, then copy over your settings.
## 2. Auto-generate a new config, then copy over your settings.
Using this method, you can have Auto Archiver auto-generate a configuration file for you, then you can copy over the desired settings from your old config file. This is probably the easiest method and quickest to setup, but it may require some trial and error as you copy over your settings.

BIN
docs/source/how_to/share_sheet.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 60 KiB

5
docs/source/installation/config_editor.md Normal file
View File

@@ -0,0 +1,5 @@
# Configuration Editor
```{raw} html
:file: settings.html
```

60
docs/source/installation/faq.md Normal file
View File

@@ -0,0 +1,60 @@
# Frequently Asked Questions
### Q: What websites does the Auto Archiver support?
**A:** The Auto Archiver works for a large variety of sites. Firstly, the Auto Archiver can download
and archive any video website supported by YT-DLP, a powerful video-downloading tool ([full list of of
sites here](https://github.com/yt-dlp/yt-dlp/blob/master/supportedsites.md)). Aside from these sites,
there are various different 'Extractors' for specific websites. See the full list of extractors that
are available on the [extractors](../modules/extractor.md) page. Some sites supported include:
* Twitter
* Instagram
* Telegram
* VKontact
* Tiktok
* Bluesky
```{note} What websites the Auto Archiver can archie depends on what extractors you have enabled in
your configuration. See [configuration](./configurations.md) for more info.
```
### Q: Does the Auto Archiver only work for social media posts ?
**A:** No, the Auto Archiver can archive any web page on the internet, not just social media posts.
However, for social media posts Auto Archiver can extract more relevant/useful information (such as
post comments, likes, author etc.) which may not be available for a generic website. If you are looking
to more generally archive webpages, then you should make sure to enable the [](../modules/autogen/extractor/wacz_extractor_enricher.md)
and the [](../modules/autogen/extractor/wayback_extractor_enricher.md).
### Q: What kind of data is stored for each webpage that's archived?
**A:** This depends on the website archived, but more generally, for social media posts any videos and photos in
the post will be archived. For video sites, the video will be downloaded separately. For most of these sites, additional
metadata such as published date, uploader/author and ratings/comments will also be saved. Additionally, further data can be
saved depending on the enrichers that you have enabled. Some other types of data saved are timestamps if you have the
[](../modules/autogen/enricher/timestamping_enricher.md) or [](../modules/autogen/enricher/opentimestamps_enricher.md) enabled,
screenshots of the web page with the [](../modules/autogen/enricher/screenshot_enricher.md), and for videos, thumbnails of the
video with the [](../modules/autogen/enricher/thumbnail_enricher.md). You can also store things like hashes (SHA256, or pdq hashes)
with the various hash enrichers.
### Q: Where is my data stored?
**A:** With the default configuration, data is stored on your local computer in the `local_storage` folder. You can adjust these settings by
changing the [storage modules](../modules/storage.md) you have enabled. For example, you could choose to store your data in an S3 bucket or
on Google Drive.
```{note}
You can choose to store your data in multiple places, for example your local drive **and** an S3 bucket for redundancy.
```
### Q: What should I do is something doesn't work?
**A:** First, read through the log files to see if you can find a specific reason why something isn't working. Learn more about logging
and how to enable debug logging in the [Logging Howto](../how_to/logging.md).
If you cannot find an answer in the logs, then try searching this documentation or existing / closed issues on the [Github Issue Tracker](https://github.com/bellingcat/auto-archiver/issues?q=is%3Aissue%20). If you still cannot find an answer, then consider opening an issue on the Github Issue Tracker or asking in the Bellingcat Discord
'Auto Archiver' group.
#### Common reasons why an archiving might not work:
* The website may have temporarily adjusted its settings - sometimes sites like Telegram or Twitter adjust their scraping protection settings. Often,
waiting a day or two and then trying again can work.
* The site requires you to be logged in - you could try using cookies or authentication to bypass any blocks. See [](../installation/authentication.md) for more information.
* The website you're trying to archive has changed its settings/structure. Make sure you're using the latest version of Auto Archiver and try again.

6
docs/source/installation/installation.md
View File

@@ -1,5 +1,11 @@
# Installation
```{toctree}
:maxdepth: 1
upgrading.md
```
There are 3 main ways to use the auto-archiver. We recommend the 'docker' method for most uses. This installs all the requirements in one command.
1. Easiest (recommended): [via docker](#installing-with-docker)

395
docs/source/installation/settings.html Normal file
View File

File diff suppressed because one or more lines are too long

19
docs/source/installation/setup.md
View File

@@ -1,13 +1,14 @@
# Getting Started
```{toctree}
:maxdepth: 1
:hidden:
installation.md
configurations.md
config_editor.md
authentication.md
requirements.md
faq.md
config_cheatsheet.md
```
@@ -26,17 +27,18 @@ The way you run the Auto Archiver depends on how you installed it (docker instal
If you installed Auto Archiver using docker, open up your terminal, and copy-paste / type the following command:
```bash
docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver
docker run -it --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver
```
breaking this command down:
1. `docker run` tells docker to start a new container (an instance of the image)
2. `--rm` makes sure this container is removed after execution (less garbage locally)
3. `-v $PWD/secrets:/app/secrets` - your secrets folder with settings
2. `-it` tells docker to run in 'interactive mode' so that we get nice colour logs
3. `--rm` makes sure this container is removed after execution (less garbage locally)
4. `-v $PWD/secrets:/app/secrets` - your secrets folder with settings
1. `-v` is a volume flag which means a folder that you have on your computer will be connected to a folder inside the docker container
2. `$PWD/secrets` points to a `secrets/` folder in your current working directory (where your console points to), we use this folder as a best practice to hold all the secrets/tokens/passwords/... you use
3. `/app/secrets` points to the path the docker container where this image can be found
4. `-v $PWD/local_archive:/app/local_archive` - (optional) if you use local_storage
5. `-v $PWD/local_archive:/app/local_archive` - (optional) if you use local_storage
1. `-v` same as above, this is a volume instruction
2. `$PWD/local_archive` is a folder `local_archive/` in case you want to archive locally and have the files accessible outside docker
3. `/app/local_archive` is a folder inside docker that you can reference in your orchestration.yml file
@@ -47,13 +49,14 @@ The invocations below will run the auto-archiver Docker image using a configurat
```bash
# Have auto-archiver run with the default settings, generating a settings file in ./secrets/orchestration.yaml
docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver
docker run -it --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver
# uses the same configuration, but with the `gsheet_feeder`, a header on row 2 and with some different column names
# Note this expects you to have followed the [Google Sheets setup](how_to/google_sheets.md) and added your service_account.json to the `secrets/` folder
# notice that columns is a dictionary so you need to pass it as JSON and it will override only the values provided
docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver --feeders=gsheet_feeder --gsheet_feeder.sheet="use it on another sheets doc" --gsheet_feeder.header=2 --gsheet_feeder.columns='{"url": "link"}'
docker run -it --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver --feeders=gsheet_feeder --gsheet_feeder.sheet="use it on another sheets doc" --gsheet_feeder.header=2 --gsheet_feeder.columns='{"url": "link"}'
# Runs auto-archiver for the first time, but in 'full' mode, enabling all modules to get a full settings file
docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver --mode full
docker run -it --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver --mode full
```
------------

30
docs/source/installation/upgrading.md Normal file
View File

@@ -0,0 +1,30 @@
# Upgrading
If an update is available, then you will see a message in the logs when you
run Auto Archiver. Here's what those logs look like:
```{code} bash
********* IMPORTANT: UPDATE AVAILABLE ********
A new version of auto-archiver is available (v0.13.6, you have 0.13.4)
Make sure to update to the latest version using: `pip install --upgrade auto-archiver`
```
Upgrading Auto Archiver depends on the way you installed it.
## Docker
To upgrade using docker, update the docker image with:
```
docker pull bellingcat/auto-archiver:latest
```
## Pip
To upgrade the pip package, use:
```
pip install --upgrade auto-archiver
```

478
poetry.lock generated
View File

@@ -1,4 +1,4 @@
# This file is automatically @generated by Poetry 2.0.1 and should not be changed by hand.
# This file is automatically @generated by Poetry 2.1.1 and should not be changed by hand.
[[package]]
name = "accessible-pygments"
@@ -51,7 +51,7 @@ typing_extensions = {version = ">=4.5", markers = "python_version < \"3.13\""}
[package.extras]
doc = ["Sphinx (>=7.4,<8.0)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx_rtd_theme"]
test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "trustme", "truststore (>=0.9.1)", "uvloop (>=0.21)"]
test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "trustme", "truststore (>=0.9.1) ; python_version >= \"3.10\"", "uvloop (>=0.21) ; platform_python_implementation == \"CPython\" and platform_system != \"Windows\" and python_version < \"3.14\""]
trio = ["trio (>=0.26.1)"]
[[package]]
@@ -94,23 +94,23 @@ files = [
]
[package.extras]
benchmark = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins", "pytest-xdist[psutil]"]
cov = ["cloudpickle", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"]
dev = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pre-commit-uv", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"]
benchmark = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"]
cov = ["cloudpickle ; platform_python_implementation == \"CPython\"", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"]
dev = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pre-commit-uv", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"]
docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier (<24.7)"]
tests = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"]
tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"]
tests = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"]
tests-mypy = ["mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\""]
[[package]]
name = "authlib"
version = "1.4.1"
version = "1.5.1"
description = "The ultimate Python library in building OAuth and OpenID Connect servers and clients."
optional = false
python-versions = ">=3.9"
groups = ["main"]
files = [
{file = "Authlib-1.4.1-py2.py3-none-any.whl", hash = "sha256:edc29c3f6a3e72cd9e9f45fff67fc663a2c364022eb0371c003f22d5405915c1"},
{file = "authlib-1.4.1.tar.gz", hash = "sha256:30ead9ea4993cdbab821dc6e01e818362f92da290c04c7f6a1940f86507a790d"},
{file = "authlib-1.5.1-py2.py3-none-any.whl", hash = "sha256:8408861cbd9b4ea2ff759b00b6f02fd7d81ac5a56d0b2b22c08606c6049aae11"},
{file = "authlib-1.5.1.tar.gz", hash = "sha256:5cbc85ecb0667312c1cdc2f9095680bb735883b123fb509fde1e65b1c5df972e"},
]
[package.dependencies]
@@ -145,7 +145,7 @@ files = [
]
[package.extras]
dev = ["backports.zoneinfo", "freezegun (>=1.0,<2.0)", "jinja2 (>=3.0)", "pytest (>=6.0)", "pytest-cov", "pytz", "setuptools", "tzdata"]
dev = ["backports.zoneinfo ; python_version < \"3.9\"", "freezegun (>=1.0,<2.0)", "jinja2 (>=3.0)", "pytest (>=6.0)", "pytest-cov", "pytz", "setuptools", "tzdata ; sys_platform == \"win32\""]
[[package]]
name = "beautifulsoup4"
@@ -172,18 +172,18 @@ lxml = ["lxml"]
[[package]]
name = "boto3"
version = "1.36.22"
version = "1.37.8"
description = "The AWS SDK for Python"
optional = false
python-versions = ">=3.8"
groups = ["main"]
files = [
{file = "boto3-1.36.22-py3-none-any.whl", hash = "sha256:39957eabdce009353d72d131046489fbbfa15891865d5f069f1e8bfa414e6b81"},
{file = "boto3-1.36.22.tar.gz", hash = "sha256:768c8a4d4a6227fe2258105efa086f1424cba5ca915a5eb2305b2cd979306ad1"},
{file = "boto3-1.37.8-py3-none-any.whl", hash = "sha256:b9f506e08c9f54687d6c073ef1c550a24a62cc2d1e0bc7cda9f13112a38818bf"},
{file = "boto3-1.37.8.tar.gz", hash = "sha256:9448f4a079189e19c3253cfdc5b8ef6dc51a3b82431e8347a51f4c1b2d9dab42"},
]
[package.dependencies]
botocore = ">=1.36.22,<1.37.0"
botocore = ">=1.37.8,<1.38.0"
jmespath = ">=0.7.1,<2.0.0"
s3transfer = ">=0.11.0,<0.12.0"
@@ -192,14 +192,14 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"]
[[package]]
name = "botocore"
version = "1.36.22"
version = "1.37.8"
description = "Low-level, data-driven core of boto 3."
optional = false
python-versions = ">=3.8"
groups = ["main"]
files = [
{file = "botocore-1.36.22-py3-none-any.whl", hash = "sha256:75d6b34acb0686ee4d54ff6eb285e78ccfe318407428769d1e3e13351714d890"},
{file = "botocore-1.36.22.tar.gz", hash = "sha256:59520247d5a479731724f97c995d5a1c2aae3b303b324f39d99efcfad1d3019e"},
{file = "botocore-1.37.8-py3-none-any.whl", hash = "sha256:a6c94f33de12f4b10b10684019e554c980469b8394c6d82448a738cbd8452cef"},
{file = "botocore-1.37.8.tar.gz", hash = "sha256:b5825e08dd3e25642aa22a0d7d92bf81fef1ef857117e4155f923bbccf5aba63"},
]
[package.dependencies]
@@ -363,14 +363,14 @@ beautifulsoup4 = "*"
[[package]]
name = "cachetools"
version = "5.5.1"
version = "5.5.2"
description = "Extensible memoizing collections and decorators"
optional = false
python-versions = ">=3.7"
groups = ["main"]
files = [
{file = "cachetools-5.5.1-py3-none-any.whl", hash = "sha256:b76651fdc3b24ead3c648bbdeeb940c1b04d365b38b4af66788f9ec4a81d42bb"},
{file = "cachetools-5.5.1.tar.gz", hash = "sha256:70f238fbba50383ef62e55c6aff6d9673175fe59f7c6782c7a0b9e38f4a9df95"},
{file = "cachetools-5.5.2-py3-none-any.whl", hash = "sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a"},
{file = "cachetools-5.5.2.tar.gz", hash = "sha256:1a661caa9175d26759571b2e19580f9d6393969e5dfca11fdb1f947a23e640d4"},
]
[[package]]
@@ -482,6 +482,18 @@ files = [
[package.dependencies]
pycparser = "*"
[[package]]
name = "cfgv"
version = "3.4.0"
description = "Validate configuration and produce human readable error messages."
optional = false
python-versions = ">=3.8"
groups = ["dev"]
files = [
{file = "cfgv-3.4.0-py2.py3-none-any.whl", hash = "sha256:b7265b1f29fd3316bfcd2b330d63d024f2bfd8bcb8b0272f8e19a504856c48f9"},
{file = "cfgv-3.4.0.tar.gz", hash = "sha256:e52591d4c5f5dead8e0f673fb16db7949d2cfb3f7da4582893288f0ded8fe560"},
]
[[package]]
name = "charset-normalizer"
version = "3.4.1"
@@ -701,6 +713,18 @@ calendars = ["convertdate (>=2.2.1)", "hijridate"]
fasttext = ["fasttext (>=0.9.1)", "numpy (>=1.19.3,<2)"]
langdetect = ["langdetect (>=1.0.0)"]
[[package]]
name = "distlib"
version = "0.3.9"
description = "Distribution utilities"
optional = false
python-versions = "*"
groups = ["dev"]
files = [
{file = "distlib-0.3.9-py2.py3-none-any.whl", hash = "sha256:47f8c22fd27c27e25a65601af709b38e4f0a45ea4fc2e710f65755fa8caaaf87"},
{file = "distlib-0.3.9.tar.gz", hash = "sha256:a60f20dea646b8a33f3e7772f74dc0b2d0772d2837ee1342a00645c81edf9403"},
]
[[package]]
name = "docutils"
version = "0.21.2"
@@ -747,6 +771,23 @@ future = "*"
[package.extras]
dev = ["Sphinx (==2.1.0)", "future (==0.17.1)", "numpy (==1.16.4)", "pytest (==4.6.1)", "pytest-mock (==1.10.4)", "tox (==3.12.1)"]
[[package]]
name = "filelock"
version = "3.17.0"
description = "A platform independent file lock."
optional = false
python-versions = ">=3.9"
groups = ["dev"]
files = [
{file = "filelock-3.17.0-py3-none-any.whl", hash = "sha256:533dc2f7ba78dc2f0f531fc6c4940addf7b70a481e269a5a3b93be94ffbe8338"},
{file = "filelock-3.17.0.tar.gz", hash = "sha256:ee4e77401ef576ebb38cd7f13b9b28893194acc20a8e68e18730ba9c0e54660e"},
]
[package.extras]
docs = ["furo (>=2024.8.6)", "sphinx (>=8.1.3)", "sphinx-autodoc-typehints (>=3)"]
testing = ["covdefaults (>=2.3)", "coverage (>=7.6.10)", "diff-cover (>=9.2.1)", "pytest (>=8.3.4)", "pytest-asyncio (>=0.25.2)", "pytest-cov (>=6)", "pytest-mock (>=3.14)", "pytest-timeout (>=2.3.1)", "virtualenv (>=20.28.1)"]
typing = ["typing-extensions (>=4.12.2) ; python_version < \"3.11\""]
[[package]]
name = "future"
version = "1.0.0"
@@ -780,20 +821,20 @@ requests = ">=2.18.0,<3.0.0.dev0"
[package.extras]
async-rest = ["google-auth[aiohttp] (>=2.35.0,<3.0.dev0)"]
grpc = ["grpcio (>=1.33.2,<2.0dev)", "grpcio (>=1.49.1,<2.0dev)", "grpcio-status (>=1.33.2,<2.0.dev0)", "grpcio-status (>=1.49.1,<2.0.dev0)"]
grpc = ["grpcio (>=1.33.2,<2.0dev)", "grpcio (>=1.49.1,<2.0dev) ; python_version >= \"3.11\"", "grpcio-status (>=1.33.2,<2.0.dev0)", "grpcio-status (>=1.49.1,<2.0.dev0) ; python_version >= \"3.11\""]
grpcgcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"]
grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"]
[[package]]
name = "google-api-python-client"
version = "2.161.0"
version = "2.163.0"
description = "Google API Client Library for Python"
optional = false
python-versions = ">=3.7"
groups = ["main"]
files = [
{file = "google_api_python_client-2.161.0-py2.py3-none-any.whl", hash = "sha256:9476a5a4f200bae368140453df40f9cda36be53fa7d0e9a9aac4cdb859a26448"},
{file = "google_api_python_client-2.161.0.tar.gz", hash = "sha256:324c0cce73e9ea0a0d2afd5937e01b7c2d6a4d7e2579cdb6c384f9699d6c9f37"},
{file = "google_api_python_client-2.163.0-py2.py3-none-any.whl", hash = "sha256:080e8bc0669cb4c1fb8efb8da2f5b91a2625d8f0e7796cfad978f33f7016c6c4"},
{file = "google_api_python_client-2.163.0.tar.gz", hash = "sha256:88dee87553a2d82176e2224648bf89272d536c8f04dcdda37ef0a71473886dd7"},
]
[package.dependencies]
@@ -865,14 +906,14 @@ tool = ["click (>=6.0.0)"]
[[package]]
name = "googleapis-common-protos"
version = "1.67.0"
version = "1.69.1"
description = "Common protobufs used in Google APIs"
optional = false
python-versions = ">=3.7"
groups = ["main"]
files = [
{file = "googleapis_common_protos-1.67.0-py2.py3-none-any.whl", hash = "sha256:579de760800d13616f51cf8be00c876f00a9f146d3e6510e19d1f4111758b741"},
{file = "googleapis_common_protos-1.67.0.tar.gz", hash = "sha256:21398025365f138be356d5923e9168737d94d46a72aefee4a6110a1f23463c86"},
{file = "googleapis_common_protos-1.69.1-py2.py3-none-any.whl", hash = "sha256:4077f27a6900d5946ee5a369fab9c8ded4c0ef1c6e880458ea2f70c14f7b70d5"},
{file = "googleapis_common_protos-1.69.1.tar.gz", hash = "sha256:e20d2d8dda87da6fe7340afbbdf4f0bcb4c8fae7e6cadf55926c31f946b0b9b1"},
]
[package.dependencies]
@@ -883,14 +924,14 @@ grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"]
[[package]]
name = "gspread"
version = "6.1.4"
version = "6.2.0"
description = "Google Spreadsheets Python API"
optional = false
python-versions = ">=3.8"
groups = ["main"]
files = [
{file = "gspread-6.1.4-py3-none-any.whl", hash = "sha256:c34781c426031a243ad154952b16f21ac56a5af90687885fbee3d1fba5280dcd"},
{file = "gspread-6.1.4.tar.gz", hash = "sha256:b8eec27de7cadb338bb1b9f14a9be168372dee8965c0da32121816b5050ac1de"},
{file = "gspread-6.2.0-py3-none-any.whl", hash = "sha256:7fa1a11e1ecacc6c5946fa016be05941baca8540404314f59aec963dd8ae5db3"},
{file = "gspread-6.2.0.tar.gz", hash = "sha256:bc3d02d1c39e0b40bfc8035b4fec407aa71a17f343fc81cc7e3f75bfa6555de6"},
]
[package.dependencies]
@@ -924,6 +965,21 @@ files = [
[package.dependencies]
pyparsing = {version = ">=2.4.2,<3.0.0 || >3.0.0,<3.0.1 || >3.0.1,<3.0.2 || >3.0.2,<3.0.3 || >3.0.3,<4", markers = "python_version > \"3.0\""}
[[package]]
name = "identify"
version = "2.6.9"
description = "File identification library for Python"
optional = false
python-versions = ">=3.9"
groups = ["dev"]
files = [
{file = "identify-2.6.9-py2.py3-none-any.whl", hash = "sha256:c98b4322da415a8e5a70ff6e51fbc2d2932c015532d77e9f8537b4ba7813b150"},
{file = "identify-2.6.9.tar.gz", hash = "sha256:d40dfe3142a1421d8518e3d3985ef5ac42890683e32306ad614a29490abeb6bf"},
]
[package.extras]
license = ["ukkonen"]
[[package]]
name = "idna"
version = "3.10"
@@ -983,14 +1039,14 @@ browser-cookie3 = ["browser_cookie3 (>=0.19.1)"]
[[package]]
name = "jinja2"
version = "3.1.5"
version = "3.1.6"
description = "A very fast and expressive template engine."
optional = false
python-versions = ">=3.7"
groups = ["main", "docs"]
files = [
{file = "jinja2-3.1.5-py3-none-any.whl", hash = "sha256:aba0f4dc9ed8013c424088f68a5c226f7d6097ed89b246d7749c2ec4175c6adb"},
{file = "jinja2-3.1.5.tar.gz", hash = "sha256:8fefff8dc3034e27bb80d67c671eb8a9bc424c0ef4c0826edbff304cceff43bb"},
{file = "jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67"},
{file = "jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d"},
]
[package.dependencies]
@@ -1064,7 +1120,7 @@ colorama = {version = ">=0.3.4", markers = "sys_platform == \"win32\""}
win32-setctime = {version = ">=1.0.0", markers = "sys_platform == \"win32\""}
[package.extras]
dev = ["Sphinx (==8.1.3)", "build (==1.2.2)", "colorama (==0.4.5)", "colorama (==0.4.6)", "exceptiongroup (==1.1.3)", "freezegun (==1.1.0)", "freezegun (==1.5.0)", "mypy (==v0.910)", "mypy (==v0.971)", "mypy (==v1.13.0)", "mypy (==v1.4.1)", "myst-parser (==4.0.0)", "pre-commit (==4.0.1)", "pytest (==6.1.2)", "pytest (==8.3.2)", "pytest-cov (==2.12.1)", "pytest-cov (==5.0.0)", "pytest-cov (==6.0.0)", "pytest-mypy-plugins (==1.9.3)", "pytest-mypy-plugins (==3.1.0)", "sphinx-rtd-theme (==3.0.2)", "tox (==3.27.1)", "tox (==4.23.2)", "twine (==6.0.1)"]
dev = ["Sphinx (==8.1.3) ; python_version >= \"3.11\"", "build (==1.2.2) ; python_version >= \"3.11\"", "colorama (==0.4.5) ; python_version < \"3.8\"", "colorama (==0.4.6) ; python_version >= \"3.8\"", "exceptiongroup (==1.1.3) ; python_version >= \"3.7\" and python_version < \"3.11\"", "freezegun (==1.1.0) ; python_version < \"3.8\"", "freezegun (==1.5.0) ; python_version >= \"3.8\"", "mypy (==v0.910) ; python_version < \"3.6\"", "mypy (==v0.971) ; python_version == \"3.6\"", "mypy (==v1.13.0) ; python_version >= \"3.8\"", "mypy (==v1.4.1) ; python_version == \"3.7\"", "myst-parser (==4.0.0) ; python_version >= \"3.11\"", "pre-commit (==4.0.1) ; python_version >= \"3.9\"", "pytest (==6.1.2) ; python_version < \"3.8\"", "pytest (==8.3.2) ; python_version >= \"3.8\"", "pytest-cov (==2.12.1) ; python_version < \"3.8\"", "pytest-cov (==5.0.0) ; python_version == \"3.8\"", "pytest-cov (==6.0.0) ; python_version >= \"3.9\"", "pytest-mypy-plugins (==1.9.3) ; python_version >= \"3.6\" and python_version < \"3.8\"", "pytest-mypy-plugins (==3.1.0) ; python_version >= \"3.8\"", "sphinx-rtd-theme (==3.0.2) ; python_version >= \"3.11\"", "tox (==3.27.1) ; python_version < \"3.8\"", "tox (==4.23.2) ; python_version >= \"3.8\"", "twine (==6.0.1) ; python_version >= \"3.11\""]
[[package]]
name = "markdown-it-py"
@@ -1265,6 +1321,18 @@ rtd = ["ipython", "sphinx (>=7)", "sphinx-autodoc2 (>=0.5.0,<0.6.0)", "sphinx-bo
testing = ["beautifulsoup4", "coverage[toml]", "defusedxml", "pygments (<2.19)", "pytest (>=8,<9)", "pytest-cov", "pytest-param-files (>=0.6.0,<0.7.0)", "pytest-regressions", "sphinx-pytest"]
testing-docutils = ["pygments", "pytest (>=8,<9)", "pytest-param-files (>=0.6.0,<0.7.0)"]
[[package]]
name = "nodeenv"
version = "1.9.1"
description = "Node.js virtual environment builder"
optional = false
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7"
groups = ["dev"]
files = [
{file = "nodeenv-1.9.1-py2.py3-none-any.whl", hash = "sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9"},
{file = "nodeenv-1.9.1.tar.gz", hash = "sha256:6ec12890a2dab7946721edbfbcd91f3319c6ccc9aec47be7c7e6b7011ee6645f"},
]
[[package]]
name = "numpy"
version = "2.1.3"
@@ -1366,6 +1434,22 @@ rsa = ["cryptography (>=3.0.0)"]
signals = ["blinker (>=1.4.0)"]
signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"]
[[package]]
name = "opentimestamps"
version = "0.4.5"
description = "Create and verify OpenTimestamps proofs"
optional = false
python-versions = "*"
groups = ["main"]
files = [
{file = "opentimestamps-0.4.5-py3-none-any.whl", hash = "sha256:a4912b3bd1b612a3ef5fac925b9137889e6c5cb91cc9e76c8202a2bf8abe26b5"},
{file = "opentimestamps-0.4.5.tar.gz", hash = "sha256:56726ccde97fb67f336a7f237ce36808e5593c3089d68d900b1c83d0ebf9dcfa"},
]
[package.dependencies]
pycryptodomex = ">=3.3.1"
python-bitcoinlib = ">=0.9.0,<0.13.0"
[[package]]
name = "oscrypto"
version = "1.3.0"
@@ -1515,9 +1599,26 @@ docs = ["furo", "olefile", "sphinx (>=8.1)", "sphinx-copybutton", "sphinx-inline
fpx = ["olefile"]
mic = ["olefile"]
tests = ["check-manifest", "coverage (>=7.4.2)", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout", "trove-classifiers (>=2024.10.12)"]
typing = ["typing-extensions"]
typing = ["typing-extensions ; python_version < \"3.10\""]
xmp = ["defusedxml"]
[[package]]
name = "platformdirs"
version = "4.3.6"
description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`."
optional = false
python-versions = ">=3.8"
groups = ["dev"]
files = [
{file = "platformdirs-4.3.6-py3-none-any.whl", hash = "sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb"},
{file = "platformdirs-4.3.6.tar.gz", hash = "sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907"},
]
[package.extras]
docs = ["furo (>=2024.8.6)", "proselint (>=0.14)", "sphinx (>=8.0.2)", "sphinx-autodoc-typehints (>=2.4)"]
test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=8.3.2)", "pytest-cov (>=5)", "pytest-mock (>=3.14)"]
type = ["mypy (>=1.11.2)"]
[[package]]
name = "pluggy"
version = "1.5.0"
@@ -1534,6 +1635,25 @@ files = [
dev = ["pre-commit", "tox"]
testing = ["pytest", "pytest-benchmark"]
[[package]]
name = "pre-commit"
version = "4.1.0"
description = "A framework for managing and maintaining multi-language pre-commit hooks."
optional = false
python-versions = ">=3.9"
groups = ["dev"]
files = [
{file = "pre_commit-4.1.0-py2.py3-none-any.whl", hash = "sha256:d29e7cb346295bcc1cc75fc3e92e343495e3ea0196c9ec6ba53f49f10ab6ae7b"},
{file = "pre_commit-4.1.0.tar.gz", hash = "sha256:ae3f018575a588e30dfddfab9a05448bfbd6b73d78709617b5a2b853549716d4"},
]
[package.dependencies]
cfgv = ">=2.0.0"
identify = ">=1.0.0"
nodeenv = ">=0.11.1"
pyyaml = ">=5.1"
virtualenv = ">=20.10.0"
[[package]]
name = "proto-plus"
version = "1.26.0"
@@ -1680,14 +1800,14 @@ files = [
[[package]]
name = "pydata-sphinx-theme"
version = "0.16.1"
version = "0.15.4"
description = "Bootstrap-based Sphinx theme from the PyData community"
optional = false
python-versions = ">=3.9"
groups = ["docs"]
files = [
{file = "pydata_sphinx_theme-0.16.1-py3-none-any.whl", hash = "sha256:225331e8ac4b32682c18fcac5a57a6f717c4e632cea5dd0e247b55155faeccde"},
{file = "pydata_sphinx_theme-0.16.1.tar.gz", hash = "sha256:a08b7f0b7f70387219dc659bff0893a7554d5eb39b59d3b8ef37b8401b7642d7"},
{file = "pydata_sphinx_theme-0.15.4-py3-none-any.whl", hash = "sha256:2136ad0e9500d0949f96167e63f3e298620040aea8f9c74621959eda5d4cf8e6"},
{file = "pydata_sphinx_theme-0.15.4.tar.gz", hash = "sha256:7762ec0ac59df3acecf49fd2f889e1b4565dbce8b88b2e29ee06fdd90645a06d"},
]
[package.dependencies]
@@ -1695,8 +1815,9 @@ accessible-pygments = "*"
Babel = "*"
beautifulsoup4 = "*"
docutils = "!=0.17.0"
packaging = "*"
pygments = ">=2.7"
sphinx = ">=6.1"
sphinx = ">=5"
typing-extensions = "*"
[package.extras]
@@ -1782,14 +1903,14 @@ files = [
[[package]]
name = "pytest"
version = "8.3.4"
version = "8.3.5"
description = "pytest: simple powerful testing with Python"
optional = false
python-versions = ">=3.8"
groups = ["dev"]
files = [
{file = "pytest-8.3.4-py3-none-any.whl", hash = "sha256:50e16d954148559c9a74109af1eaf0c945ba2d8f30f0a3d3335edde19788b6f6"},
{file = "pytest-8.3.4.tar.gz", hash = "sha256:965370d062bce11e73868e0335abac31b4d3de0e82f4007408d242b4f8610761"},
{file = "pytest-8.3.5-py3-none-any.whl", hash = "sha256:c69214aa47deac29fad6c2a4f590b9c4a9fdb16a403176fe154b79c0b4d4d820"},
{file = "pytest-8.3.5.tar.gz", hash = "sha256:f4efe70cc14e511565ac476b57c279e12a855b11f48f212af1080ef2263d3845"},
]
[package.dependencies]
@@ -1839,6 +1960,18 @@ pytest = ">=6.2.5"
[package.extras]
dev = ["pre-commit", "pytest-asyncio", "tox"]
[[package]]
name = "python-bitcoinlib"
version = "0.12.2"
description = "The Swiss Army Knife of the Bitcoin protocol."
optional = false
python-versions = "*"
groups = ["main"]
files = [
{file = "python-bitcoinlib-0.12.2.tar.gz", hash = "sha256:c65ab61427c77c38d397bfc431f71d86fd355b453a536496ec3fcb41bd10087d"},
{file = "python_bitcoinlib-0.12.2-py3-none-any.whl", hash = "sha256:2f29a9f475f21c12169b3a6cc8820f34f11362d7ff1200a5703dce3e4e903a44"},
]
[[package]]
name = "python-dateutil"
version = "2.9.0.post0"
@@ -1907,7 +2040,7 @@ version = "6.0.2"
description = "YAML parser and emitter for Python"
optional = false
python-versions = ">=3.8"
groups = ["docs"]
groups = ["dev", "docs"]
files = [
{file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"},
{file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"},
@@ -2282,34 +2415,62 @@ files = [
{file = "ruamel.yaml.clib-0.2.12.tar.gz", hash = "sha256:6c8fbb13ec503f99a91901ab46e0b07ae7941cd527393187039aec586fdfd36f"},
]
[[package]]
name = "ruff"
version = "0.9.10"
description = "An extremely fast Python linter and code formatter, written in Rust."
optional = false
python-versions = ">=3.7"
groups = ["dev"]
files = [
{file = "ruff-0.9.10-py3-none-linux_armv6l.whl", hash = "sha256:eb4d25532cfd9fe461acc83498361ec2e2252795b4f40b17e80692814329e42d"},
{file = "ruff-0.9.10-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:188a6638dab1aa9bb6228a7302387b2c9954e455fb25d6b4470cb0641d16759d"},
{file = "ruff-0.9.10-py3-none-macosx_11_0_arm64.whl", hash = "sha256:5284dcac6b9dbc2fcb71fdfc26a217b2ca4ede6ccd57476f52a587451ebe450d"},
{file = "ruff-0.9.10-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:47678f39fa2a3da62724851107f438c8229a3470f533894b5568a39b40029c0c"},
{file = "ruff-0.9.10-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:99713a6e2766b7a17147b309e8c915b32b07a25c9efd12ada79f217c9c778b3e"},
{file = "ruff-0.9.10-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:524ee184d92f7c7304aa568e2db20f50c32d1d0caa235d8ddf10497566ea1a12"},
{file = "ruff-0.9.10-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:df92aeac30af821f9acf819fc01b4afc3dfb829d2782884f8739fb52a8119a16"},
{file = "ruff-0.9.10-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de42e4edc296f520bb84954eb992a07a0ec5a02fecb834498415908469854a52"},
{file = "ruff-0.9.10-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d257f95b65806104b6b1ffca0ea53f4ef98454036df65b1eda3693534813ecd1"},
{file = "ruff-0.9.10-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b60dec7201c0b10d6d11be00e8f2dbb6f40ef1828ee75ed739923799513db24c"},
{file = "ruff-0.9.10-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:d838b60007da7a39c046fcdd317293d10b845001f38bcb55ba766c3875b01e43"},
{file = "ruff-0.9.10-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:ccaf903108b899beb8e09a63ffae5869057ab649c1e9231c05ae354ebc62066c"},
{file = "ruff-0.9.10-py3-none-musllinux_1_2_i686.whl", hash = "sha256:f9567d135265d46e59d62dc60c0bfad10e9a6822e231f5b24032dba5a55be6b5"},
{file = "ruff-0.9.10-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:5f202f0d93738c28a89f8ed9eaba01b7be339e5d8d642c994347eaa81c6d75b8"},
{file = "ruff-0.9.10-py3-none-win32.whl", hash = "sha256:bfb834e87c916521ce46b1788fbb8484966e5113c02df216680102e9eb960029"},
{file = "ruff-0.9.10-py3-none-win_amd64.whl", hash = "sha256:f2160eeef3031bf4b17df74e307d4c5fb689a6f3a26a2de3f7ef4044e3c484f1"},
{file = "ruff-0.9.10-py3-none-win_arm64.whl", hash = "sha256:5fd804c0327a5e5ea26615550e706942f348b197d5475ff34c19733aee4b2e69"},
{file = "ruff-0.9.10.tar.gz", hash = "sha256:9bacb735d7bada9cfb0f2c227d3658fc443d90a727b47f206fb33f52f3c0eac7"},
]
[[package]]
name = "s3transfer"
version = "0.11.2"
version = "0.11.4"
description = "An Amazon S3 Transfer Manager"
optional = false
python-versions = ">=3.8"
groups = ["main"]
files = [
{file = "s3transfer-0.11.2-py3-none-any.whl", hash = "sha256:be6ecb39fadd986ef1701097771f87e4d2f821f27f6071c872143884d2950fbc"},
{file = "s3transfer-0.11.2.tar.gz", hash = "sha256:3b39185cb72f5acc77db1a58b6e25b977f28d20496b6e58d6813d75f464d632f"},
{file = "s3transfer-0.11.4-py3-none-any.whl", hash = "sha256:ac265fa68318763a03bf2dc4f39d5cbd6a9e178d81cc9483ad27da33637e320d"},
{file = "s3transfer-0.11.4.tar.gz", hash = "sha256:559f161658e1cf0a911f45940552c696735f5c74e64362e515f333ebed87d679"},
]
[package.dependencies]
botocore = ">=1.36.0,<2.0a.0"
botocore = ">=1.37.4,<2.0a.0"
[package.extras]
crt = ["botocore[crt] (>=1.36.0,<2.0a.0)"]
crt = ["botocore[crt] (>=1.37.4,<2.0a.0)"]
[[package]]
name = "selenium"
version = "4.28.1"
version = "4.29.0"
description = "Official Python bindings for Selenium WebDriver"
optional = false
python-versions = ">=3.9"
groups = ["main"]
files = [
{file = "selenium-4.28.1-py3-none-any.whl", hash = "sha256:4238847e45e24e4472cfcf3554427512c7aab9443396435b1623ef406fff1cc1"},
{file = "selenium-4.28.1.tar.gz", hash = "sha256:0072d08670d7ec32db901bd0107695a330cecac9f196e3afb3fa8163026e022a"},
{file = "selenium-4.29.0-py3-none-any.whl", hash = "sha256:ce5d26f1ddc1111641113653af33694c13947dd36c2df09cdd33f554351d372e"},
{file = "selenium-4.29.0.tar.gz", hash = "sha256:3a62f7ec33e669364a6c0562a701deb69745b569c50d55f1a912bf8eb33358ba"},
]
[package.dependencies]
@@ -2462,19 +2623,19 @@ test = ["httpx", "pytest (>=6)"]
[[package]]
name = "sphinx-book-theme"
version = "1.1.3"
version = "1.1.4"
description = "A clean book theme for scientific explanations and documentation with Sphinx"
optional = false
python-versions = ">=3.9"
groups = ["docs"]
files = [
{file = "sphinx_book_theme-1.1.3-py3-none-any.whl", hash = "sha256:a554a9a7ac3881979a87a2b10f633aa2a5706e72218a10f71be38b3c9e831ae9"},
{file = "sphinx_book_theme-1.1.3.tar.gz", hash = "sha256:1f25483b1846cb3d353a6bc61b3b45b031f4acf845665d7da90e01ae0aef5b4d"},
{file = "sphinx_book_theme-1.1.4-py3-none-any.whl", hash = "sha256:843b3f5c8684640f4a2d01abd298beb66452d1b2394cd9ef5be5ebd5640ea0e1"},
{file = "sphinx_book_theme-1.1.4.tar.gz", hash = "sha256:73efe28af871d0a89bd05856d300e61edce0d5b2fbb7984e84454be0fedfe9ed"},
]
[package.dependencies]
pydata-sphinx-theme = ">=0.15.2"
sphinx = ">=5"
pydata-sphinx-theme = "0.15.4"
sphinx = ">=6.1"
[package.extras]
code-style = ["pre-commit"]
@@ -2621,14 +2782,14 @@ test = ["pytest"]
[[package]]
name = "starlette"
version = "0.45.3"
version = "0.46.0"
description = "The little ASGI library that shines."
optional = false
python-versions = ">=3.9"
groups = ["docs"]
files = [
{file = "starlette-0.45.3-py3-none-any.whl", hash = "sha256:dfb6d332576f136ec740296c7e8bb8c8a7125044e7c6da30744718880cdd059d"},
{file = "starlette-0.45.3.tar.gz", hash = "sha256:2cbcba2a75806f8a41c722141486f37c28e30a0921c5f6fe4346cb0dcee1302f"},
{file = "starlette-0.46.0-py3-none-any.whl", hash = "sha256:913f0798bd90ba90a9156383bcf1350a17d6259451d0d8ee27fc0cf2db609038"},
{file = "starlette-0.46.0.tar.gz", hash = "sha256:b359e4567456b28d473d0193f34c0de0ed49710d75ef183a74a5ce0499324f50"},
]
[package.dependencies]
@@ -2639,14 +2800,14 @@ full = ["httpx (>=0.27.0,<0.29.0)", "itsdangerous", "jinja2", "python-multipart
[[package]]
name = "telethon"
version = "1.38.1"
version = "1.39.0"
description = "Full-featured Telegram client library for Python 3"
optional = false
python-versions = ">=3.5"
groups = ["main"]
files = [
{file = "Telethon-1.38.1-py3-none-any.whl", hash = "sha256:30c187017501bfb982b8af5659f864dda4108f77ea49cfce61e8f6fdb8a18d6e"},
{file = "Telethon-1.38.1.tar.gz", hash = "sha256:f9866c1e37197a0894e0c02aa56a6359bffb14a585e88e18e3e819df4fda399a"},
{file = "Telethon-1.39.0-py3-none-any.whl", hash = "sha256:aa9f394b94be144799a6f6a93ab463867bc7c63503ede9631751940a98f6c703"},
{file = "telethon-1.39.0.tar.gz", hash = "sha256:35d4795d8c91deac515fb0bcb3723866b924de1c724e1d5c230460e96f284a63"},
]
[package.dependencies]
@@ -2756,14 +2917,14 @@ sortedcontainers = "*"
[[package]]
name = "trio-websocket"
version = "0.12.1"
version = "0.12.2"
description = "WebSocket library for Trio"
optional = false
python-versions = ">=3.8"
groups = ["main"]
files = [
{file = "trio_websocket-0.12.1-py3-none-any.whl", hash = "sha256:608ec746bb287e5d5a66baf483e41194193c5cf05ffaad6240e7d1fcd80d1e6f"},
{file = "trio_websocket-0.12.1.tar.gz", hash = "sha256:d55ccd4d3eae27c494f3fdae14823317839bdcb8214d1173eacc4d42c69fc91b"},
{file = "trio_websocket-0.12.2-py3-none-any.whl", hash = "sha256:df605665f1db533f4a386c94525870851096a223adcb97f72a07e8b4beba45b6"},
{file = "trio_websocket-0.12.2.tar.gz", hash = "sha256:22c72c436f3d1e264d0910a3951934798dcc5b00ae56fc4ee079d46c7cf20fae"},
]
[package.dependencies]
@@ -2815,14 +2976,14 @@ files = [
[[package]]
name = "tzlocal"
version = "5.3"
version = "5.3.1"
description = "tzinfo object for the local timezone"
optional = false
python-versions = ">=3.9"
groups = ["main"]
files = [
{file = "tzlocal-5.3-py3-none-any.whl", hash = "sha256:3814135a1bb29763c6e4f08fd6e41dbb435c7a60bfbb03270211bcc537187d8c"},
{file = "tzlocal-5.3.tar.gz", hash = "sha256:2fafbfc07e9d8b49ade18f898d6bcd37ae88ce3ad6486842a2e4f03af68323d2"},
{file = "tzlocal-5.3.1-py3-none-any.whl", hash = "sha256:eb1a66c3ef5847adf7a834f1be0800581b683b5608e74f86ecbcef8ab91bb85d"},
{file = "tzlocal-5.3.1.tar.gz", hash = "sha256:cceffc7edecefea1f595541dbd6e990cb1ea3d19bf01b2809f362a03dd7921fd"},
]
[package.dependencies]
@@ -2874,7 +3035,7 @@ files = [
pysocks = {version = ">=1.5.6,<1.5.7 || >1.5.7,<2.0", optional = true, markers = "extra == \"socks\""}
[package.extras]
brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"]
brotli = ["brotli (>=1.0.9) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\""]
h2 = ["h2 (>=4,<5)"]
socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"]
zstd = ["zstandard (>=0.18.0)"]
@@ -2897,7 +3058,28 @@ h11 = ">=0.8"
typing-extensions = {version = ">=4.0", markers = "python_version < \"3.11\""}
[package.extras]
standard = ["colorama (>=0.4)", "httptools (>=0.6.3)", "python-dotenv (>=0.13)", "pyyaml (>=5.1)", "uvloop (>=0.14.0,!=0.15.0,!=0.15.1)", "watchfiles (>=0.13)", "websockets (>=10.4)"]
standard = ["colorama (>=0.4) ; sys_platform == \"win32\"", "httptools (>=0.6.3)", "python-dotenv (>=0.13)", "pyyaml (>=5.1)", "uvloop (>=0.14.0,!=0.15.0,!=0.15.1) ; sys_platform != \"win32\" and sys_platform != \"cygwin\" and platform_python_implementation != \"PyPy\"", "watchfiles (>=0.13)", "websockets (>=10.4)"]
[[package]]
name = "virtualenv"
version = "20.29.3"
description = "Virtual Python Environment builder"
optional = false
python-versions = ">=3.8"
groups = ["dev"]
files = [
{file = "virtualenv-20.29.3-py3-none-any.whl", hash = "sha256:3e3d00f5807e83b234dfb6122bf37cfadf4be216c53a49ac059d02414f819170"},
{file = "virtualenv-20.29.3.tar.gz", hash = "sha256:95e39403fcf3940ac45bc717597dba16110b74506131845d9b687d5e73d947ac"},
]
[package.dependencies]
distlib = ">=0.3.7,<1"
filelock = ">=3.12.2,<4"
platformdirs = ">=3.9.1,<5"
[package.extras]
docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.2,!=7.3)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=23.6)"]
test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23.1)", "pytest (>=7.4)", "pytest-env (>=0.8.2)", "pytest-freezer (>=0.4.8) ; platform_python_implementation == \"PyPy\" or platform_python_implementation == \"CPython\" and sys_platform == \"win32\" and python_version >= \"3.13\"", "pytest-mock (>=3.11.1)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)", "setuptools (>=68)", "time-machine (>=2.10) ; platform_python_implementation == \"CPython\""]
[[package]]
name = "vk-api"
@@ -3068,81 +3250,81 @@ test = ["websockets"]
[[package]]
name = "websockets"
version = "15.0"
version = "15.0.1"
description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)"
optional = false
python-versions = ">=3.9"
groups = ["main", "docs"]
files = [
{file = "websockets-15.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5e6ee18a53dd5743e6155b8ff7e8e477c25b29b440f87f65be8165275c87fef0"},
{file = "websockets-15.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ee06405ea2e67366a661ed313e14cf2a86e84142a3462852eb96348f7219cee3"},
{file = "websockets-15.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8711682a629bbcaf492f5e0af72d378e976ea1d127a2d47584fa1c2c080b436b"},
{file = "websockets-15.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94c4a9b01eede952442c088d415861b0cf2053cbd696b863f6d5022d4e4e2453"},
{file = "websockets-15.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:45535fead66e873f411c1d3cf0d3e175e66f4dd83c4f59d707d5b3e4c56541c4"},
{file = "websockets-15.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e389efe46ccb25a1f93d08c7a74e8123a2517f7b7458f043bd7529d1a63ffeb"},
{file = "websockets-15.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:67a04754d121ea5ca39ddedc3f77071651fb5b0bc6b973c71c515415b44ed9c5"},
{file = "websockets-15.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:bd66b4865c8b853b8cca7379afb692fc7f52cf898786537dfb5e5e2d64f0a47f"},
{file = "websockets-15.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:a4cc73a6ae0a6751b76e69cece9d0311f054da9b22df6a12f2c53111735657c8"},
{file = "websockets-15.0-cp310-cp310-win32.whl", hash = "sha256:89da58e4005e153b03fe8b8794330e3f6a9774ee9e1c3bd5bc52eb098c3b0c4f"},
{file = "websockets-15.0-cp310-cp310-win_amd64.whl", hash = "sha256:4ff380aabd7a74a42a760ee76c68826a8f417ceb6ea415bd574a035a111fd133"},
{file = "websockets-15.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:dd24c4d256558429aeeb8d6c24ebad4e982ac52c50bc3670ae8646c181263965"},
{file = "websockets-15.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f83eca8cbfd168e424dfa3b3b5c955d6c281e8fc09feb9d870886ff8d03683c7"},
{file = "websockets-15.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4095a1f2093002c2208becf6f9a178b336b7572512ee0a1179731acb7788e8ad"},
{file = "websockets-15.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb915101dfbf318486364ce85662bb7b020840f68138014972c08331458d41f3"},
{file = "websockets-15.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:45d464622314973d78f364689d5dbb9144e559f93dca11b11af3f2480b5034e1"},
{file = "websockets-15.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ace960769d60037ca9625b4c578a6f28a14301bd2a1ff13bb00e824ac9f73e55"},
{file = "websockets-15.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c7cd4b1015d2f60dfe539ee6c95bc968d5d5fad92ab01bb5501a77393da4f596"},
{file = "websockets-15.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:4f7290295794b5dec470867c7baa4a14182b9732603fd0caf2a5bf1dc3ccabf3"},
{file = "websockets-15.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3abd670ca7ce230d5a624fd3d55e055215d8d9b723adee0a348352f5d8d12ff4"},
{file = "websockets-15.0-cp311-cp311-win32.whl", hash = "sha256:110a847085246ab8d4d119632145224d6b49e406c64f1bbeed45c6f05097b680"},
{file = "websockets-15.0-cp311-cp311-win_amd64.whl", hash = "sha256:8d7bbbe2cd6ed80aceef2a14e9f1c1b61683194c216472ed5ff33b700e784e37"},
{file = "websockets-15.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:cccc18077acd34c8072578394ec79563664b1c205f7a86a62e94fafc7b59001f"},
{file = "websockets-15.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d4c22992e24f12de340ca5f824121a5b3e1a37ad4360b4e1aaf15e9d1c42582d"},
{file = "websockets-15.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1206432cc6c644f6fc03374b264c5ff805d980311563202ed7fef91a38906276"},
{file = "websockets-15.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d3cc75ef3e17490042c47e0523aee1bcc4eacd2482796107fd59dd1100a44bc"},
{file = "websockets-15.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b89504227a5311610e4be16071465885a0a3d6b0e82e305ef46d9b064ce5fb72"},
{file = "websockets-15.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:56e3efe356416bc67a8e093607315951d76910f03d2b3ad49c4ade9207bf710d"},
{file = "websockets-15.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0f2205cdb444a42a7919690238fb5979a05439b9dbb73dd47c863d39640d85ab"},
{file = "websockets-15.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:aea01f40995fa0945c020228ab919b8dfc93fc8a9f2d3d705ab5b793f32d9e99"},
{file = "websockets-15.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a9f8e33747b1332db11cf7fcf4a9512bef9748cb5eb4d3f7fbc8c30d75dc6ffc"},
{file = "websockets-15.0-cp312-cp312-win32.whl", hash = "sha256:32e02a2d83f4954aa8c17e03fe8ec6962432c39aca4be7e8ee346b05a3476904"},
{file = "websockets-15.0-cp312-cp312-win_amd64.whl", hash = "sha256:ffc02b159b65c05f2ed9ec176b715b66918a674bd4daed48a9a7a590dd4be1aa"},
{file = "websockets-15.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:d2244d8ab24374bed366f9ff206e2619345f9cd7fe79aad5225f53faac28b6b1"},
{file = "websockets-15.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:3a302241fbe825a3e4fe07666a2ab513edfdc6d43ce24b79691b45115273b5e7"},
{file = "websockets-15.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:10552fed076757a70ba2c18edcbc601c7637b30cdfe8c24b65171e824c7d6081"},
{file = "websockets-15.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c53f97032b87a406044a1c33d1e9290cc38b117a8062e8a8b285175d7e2f99c9"},
{file = "websockets-15.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1caf951110ca757b8ad9c4974f5cac7b8413004d2f29707e4d03a65d54cedf2b"},
{file = "websockets-15.0-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8bf1ab71f9f23b0a1d52ec1682a3907e0c208c12fef9c3e99d2b80166b17905f"},
{file = "websockets-15.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bfcd3acc1a81f106abac6afd42327d2cf1e77ec905ae11dc1d9142a006a496b6"},
{file = "websockets-15.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:c8c5c8e1bac05ef3c23722e591ef4f688f528235e2480f157a9cfe0a19081375"},
{file = "websockets-15.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:86bfb52a9cfbcc09aba2b71388b0a20ea5c52b6517c0b2e316222435a8cdab72"},
{file = "websockets-15.0-cp313-cp313-win32.whl", hash = "sha256:26ba70fed190708551c19a360f9d7eca8e8c0f615d19a574292b7229e0ae324c"},
{file = "websockets-15.0-cp313-cp313-win_amd64.whl", hash = "sha256:ae721bcc8e69846af00b7a77a220614d9b2ec57d25017a6bbde3a99473e41ce8"},
{file = "websockets-15.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c348abc5924caa02a62896300e32ea80a81521f91d6db2e853e6b1994017c9f6"},
{file = "websockets-15.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5294fcb410ed0a45d5d1cdedc4e51a60aab5b2b3193999028ea94afc2f554b05"},
{file = "websockets-15.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c24ba103ecf45861e2e1f933d40b2d93f5d52d8228870c3e7bf1299cd1cb8ff1"},
{file = "websockets-15.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc8821a03bcfb36e4e4705316f6b66af28450357af8a575dc8f4b09bf02a3dee"},
{file = "websockets-15.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffc5ae23ada6515f31604f700009e2df90b091b67d463a8401c1d8a37f76c1d7"},
{file = "websockets-15.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ac67b542505186b3bbdaffbc303292e1ee9c8729e5d5df243c1f20f4bb9057e"},
{file = "websockets-15.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:c86dc2068f1c5ca2065aca34f257bbf4f78caf566eb230f692ad347da191f0a1"},
{file = "websockets-15.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:30cff3ef329682b6182c01c568f551481774c476722020b8f7d0daacbed07a17"},
{file = "websockets-15.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:98dcf978d4c6048965d1762abd534c9d53bae981a035bfe486690ba11f49bbbb"},
{file = "websockets-15.0-cp39-cp39-win32.whl", hash = "sha256:37d66646f929ae7c22c79bc73ec4074d6db45e6384500ee3e0d476daf55482a9"},
{file = "websockets-15.0-cp39-cp39-win_amd64.whl", hash = "sha256:24d5333a9b2343330f0f4eb88546e2c32a7f5c280f8dd7d3cc079beb0901781b"},
{file = "websockets-15.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:b499caef4bca9cbd0bd23cd3386f5113ee7378094a3cb613a2fa543260fe9506"},
{file = "websockets-15.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:17f2854c6bd9ee008c4b270f7010fe2da6c16eac5724a175e75010aacd905b31"},
{file = "websockets-15.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:89f72524033abbfde880ad338fd3c2c16e31ae232323ebdfbc745cbb1b3dcc03"},
{file = "websockets-15.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1657a9eecb29d7838e3b415458cc494e6d1b194f7ac73a34aa55c6fb6c72d1f3"},
{file = "websockets-15.0-pp310-pypy310_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e413352a921f5ad5d66f9e2869b977e88d5103fc528b6deb8423028a2befd842"},
{file = "websockets-15.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:8561c48b0090993e3b2a54db480cab1d23eb2c5735067213bb90f402806339f5"},
{file = "websockets-15.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:190bc6ef8690cd88232a038d1b15714c258f79653abad62f7048249b09438af3"},
{file = "websockets-15.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:327adab7671f3726b0ba69be9e865bba23b37a605b585e65895c428f6e47e766"},
{file = "websockets-15.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2bd8ef197c87afe0a9009f7a28b5dc613bfc585d329f80b7af404e766aa9e8c7"},
{file = "websockets-15.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:789c43bf4a10cd067c24c321238e800b8b2716c863ddb2294d2fed886fa5a689"},
{file = "websockets-15.0-pp39-pypy39_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7394c0b7d460569c9285fa089a429f58465db930012566c03046f9e3ab0ed181"},
{file = "websockets-15.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:2ea4f210422b912ebe58ef0ad33088bc8e5c5ff9655a8822500690abc3b1232d"},
{file = "websockets-15.0-py3-none-any.whl", hash = "sha256:51ffd53c53c4442415b613497a34ba0aa7b99ac07f1e4a62db5dcd640ae6c3c3"},
{file = "websockets-15.0.tar.gz", hash = "sha256:ca36151289a15b39d8d683fd8b7abbe26fc50be311066c5f8dcf3cb8cee107ab"},
{file = "websockets-15.0.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d63efaa0cd96cf0c5fe4d581521d9fa87744540d4bc999ae6e08595a1014b45b"},
{file = "websockets-15.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ac60e3b188ec7574cb761b08d50fcedf9d77f1530352db4eef1707fe9dee7205"},
{file = "websockets-15.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5756779642579d902eed757b21b0164cd6fe338506a8083eb58af5c372e39d9a"},
{file = "websockets-15.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0fdfe3e2a29e4db3659dbd5bbf04560cea53dd9610273917799f1cde46aa725e"},
{file = "websockets-15.0.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c2529b320eb9e35af0fa3016c187dffb84a3ecc572bcee7c3ce302bfeba52bf"},
{file = "websockets-15.0.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac1e5c9054fe23226fb11e05a6e630837f074174c4c2f0fe442996112a6de4fb"},
{file = "websockets-15.0.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5df592cd503496351d6dc14f7cdad49f268d8e618f80dce0cd5a36b93c3fc08d"},
{file = "websockets-15.0.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:0a34631031a8f05657e8e90903e656959234f3a04552259458aac0b0f9ae6fd9"},
{file = "websockets-15.0.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3d00075aa65772e7ce9e990cab3ff1de702aa09be3940d1dc88d5abf1ab8a09c"},
{file = "websockets-15.0.1-cp310-cp310-win32.whl", hash = "sha256:1234d4ef35db82f5446dca8e35a7da7964d02c127b095e172e54397fb6a6c256"},
{file = "websockets-15.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:39c1fec2c11dc8d89bba6b2bf1556af381611a173ac2b511cf7231622058af41"},
{file = "websockets-15.0.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:823c248b690b2fd9303ba00c4f66cd5e2d8c3ba4aa968b2779be9532a4dad431"},
{file = "websockets-15.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678999709e68425ae2593acf2e3ebcbcf2e69885a5ee78f9eb80e6e371f1bf57"},
{file = "websockets-15.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d50fd1ee42388dcfb2b3676132c78116490976f1300da28eb629272d5d93e905"},
{file = "websockets-15.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d99e5546bf73dbad5bf3547174cd6cb8ba7273062a23808ffea025ecb1cf8562"},
{file = "websockets-15.0.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:66dd88c918e3287efc22409d426c8f729688d89a0c587c88971a0faa2c2f3792"},
{file = "websockets-15.0.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8dd8327c795b3e3f219760fa603dcae1dcc148172290a8ab15158cf85a953413"},
{file = "websockets-15.0.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8fdc51055e6ff4adeb88d58a11042ec9a5eae317a0a53d12c062c8a8865909e8"},
{file = "websockets-15.0.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:693f0192126df6c2327cce3baa7c06f2a117575e32ab2308f7f8216c29d9e2e3"},
{file = "websockets-15.0.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:54479983bd5fb469c38f2f5c7e3a24f9a4e70594cd68cd1fa6b9340dadaff7cf"},
{file = "websockets-15.0.1-cp311-cp311-win32.whl", hash = "sha256:16b6c1b3e57799b9d38427dda63edcbe4926352c47cf88588c0be4ace18dac85"},
{file = "websockets-15.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:27ccee0071a0e75d22cb35849b1db43f2ecd3e161041ac1ee9d2352ddf72f065"},
{file = "websockets-15.0.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:3e90baa811a5d73f3ca0bcbf32064d663ed81318ab225ee4f427ad4e26e5aff3"},
{file = "websockets-15.0.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:592f1a9fe869c778694f0aa806ba0374e97648ab57936f092fd9d87f8bc03665"},
{file = "websockets-15.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0701bc3cfcb9164d04a14b149fd74be7347a530ad3bbf15ab2c678a2cd3dd9a2"},
{file = "websockets-15.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8b56bdcdb4505c8078cb6c7157d9811a85790f2f2b3632c7d1462ab5783d215"},
{file = "websockets-15.0.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0af68c55afbd5f07986df82831c7bff04846928ea8d1fd7f30052638788bc9b5"},
{file = "websockets-15.0.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64dee438fed052b52e4f98f76c5790513235efaa1ef7f3f2192c392cd7c91b65"},
{file = "websockets-15.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d5f6b181bb38171a8ad1d6aa58a67a6aa9d4b38d0f8c5f496b9e42561dfc62fe"},
{file = "websockets-15.0.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:5d54b09eba2bada6011aea5375542a157637b91029687eb4fdb2dab11059c1b4"},
{file = "websockets-15.0.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3be571a8b5afed347da347bfcf27ba12b069d9d7f42cb8c7028b5e98bbb12597"},
{file = "websockets-15.0.1-cp312-cp312-win32.whl", hash = "sha256:c338ffa0520bdb12fbc527265235639fb76e7bc7faafbb93f6ba80d9c06578a9"},
{file = "websockets-15.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:fcd5cf9e305d7b8338754470cf69cf81f420459dbae8a3b40cee57417f4614a7"},
{file = "websockets-15.0.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ee443ef070bb3b6ed74514f5efaa37a252af57c90eb33b956d35c8e9c10a1931"},
{file = "websockets-15.0.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5a939de6b7b4e18ca683218320fc67ea886038265fd1ed30173f5ce3f8e85675"},
{file = "websockets-15.0.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:746ee8dba912cd6fc889a8147168991d50ed70447bf18bcda7039f7d2e3d9151"},
{file = "websockets-15.0.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:595b6c3969023ecf9041b2936ac3827e4623bfa3ccf007575f04c5a6aa318c22"},
{file = "websockets-15.0.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c714d2fc58b5ca3e285461a4cc0c9a66bd0e24c5da9911e30158286c9b5be7f"},
{file = "websockets-15.0.1-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f3c1e2ab208db911594ae5b4f79addeb3501604a165019dd221c0bdcabe4db8"},
{file = "websockets-15.0.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:229cf1d3ca6c1804400b0a9790dc66528e08a6a1feec0d5040e8b9eb14422375"},
{file = "websockets-15.0.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:756c56e867a90fb00177d530dca4b097dd753cde348448a1012ed6c5131f8b7d"},
{file = "websockets-15.0.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:558d023b3df0bffe50a04e710bc87742de35060580a293c2a984299ed83bc4e4"},
{file = "websockets-15.0.1-cp313-cp313-win32.whl", hash = "sha256:ba9e56e8ceeeedb2e080147ba85ffcd5cd0711b89576b83784d8605a7df455fa"},
{file = "websockets-15.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:e09473f095a819042ecb2ab9465aee615bd9c2028e4ef7d933600a8401c79561"},
{file = "websockets-15.0.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5f4c04ead5aed67c8a1a20491d54cdfba5884507a48dd798ecaf13c74c4489f5"},
{file = "websockets-15.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:abdc0c6c8c648b4805c5eacd131910d2a7f6455dfd3becab248ef108e89ab16a"},
{file = "websockets-15.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a625e06551975f4b7ea7102bc43895b90742746797e2e14b70ed61c43a90f09b"},
{file = "websockets-15.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d591f8de75824cbb7acad4e05d2d710484f15f29d4a915092675ad3456f11770"},
{file = "websockets-15.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:47819cea040f31d670cc8d324bb6435c6f133b8c7a19ec3d61634e62f8d8f9eb"},
{file = "websockets-15.0.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac017dd64572e5c3bd01939121e4d16cf30e5d7e110a119399cf3133b63ad054"},
{file = "websockets-15.0.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:4a9fac8e469d04ce6c25bb2610dc535235bd4aa14996b4e6dbebf5e007eba5ee"},
{file = "websockets-15.0.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:363c6f671b761efcb30608d24925a382497c12c506b51661883c3e22337265ed"},
{file = "websockets-15.0.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:2034693ad3097d5355bfdacfffcbd3ef5694f9718ab7f29c29689a9eae841880"},
{file = "websockets-15.0.1-cp39-cp39-win32.whl", hash = "sha256:3b1ac0d3e594bf121308112697cf4b32be538fb1444468fb0a6ae4feebc83411"},
{file = "websockets-15.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:b7643a03db5c95c799b89b31c036d5f27eeb4d259c798e878d6937d71832b1e4"},
{file = "websockets-15.0.1-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0c9e74d766f2818bb95f84c25be4dea09841ac0f734d1966f415e4edfc4ef1c3"},
{file = "websockets-15.0.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:1009ee0c7739c08a0cd59de430d6de452a55e42d6b522de7aa15e6f67db0b8e1"},
{file = "websockets-15.0.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76d1f20b1c7a2fa82367e04982e708723ba0e7b8d43aa643d3dcd404d74f1475"},
{file = "websockets-15.0.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f29d80eb9a9263b8d109135351caf568cc3f80b9928bccde535c235de55c22d9"},
{file = "websockets-15.0.1-pp310-pypy310_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b359ed09954d7c18bbc1680f380c7301f92c60bf924171629c5db97febb12f04"},
{file = "websockets-15.0.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:cad21560da69f4ce7658ca2cb83138fb4cf695a2ba3e475e0559e05991aa8122"},
{file = "websockets-15.0.1-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7f493881579c90fc262d9cdbaa05a6b54b3811c2f300766748db79f098db9940"},
{file = "websockets-15.0.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:47b099e1f4fbc95b701b6e85768e1fcdaf1630f3cbe4765fa216596f12310e2e"},
{file = "websockets-15.0.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:67f2b6de947f8c757db2db9c71527933ad0019737ec374a8a6be9a956786aaf9"},
{file = "websockets-15.0.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d08eb4c2b7d6c41da6ca0600c077e93f5adcfd979cd777d747e9ee624556da4b"},
{file = "websockets-15.0.1-pp39-pypy39_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b826973a4a2ae47ba357e4e82fa44a463b8f168e1ca775ac64521442b19e87f"},
{file = "websockets-15.0.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:21c1fa28a6a7e3cbdc171c694398b6df4744613ce9b36b1a498e816787e28123"},
{file = "websockets-15.0.1-py3-none-any.whl", hash = "sha256:f7a866fbc1e97b5c617ee4116daaa09b722101d4a3c170c787450ba409f9736f"},
{file = "websockets-15.0.1.tar.gz", hash = "sha256:82544de02076bafba038ce055ee6412d68da13ab47f0c60cab827346de828dee"},
]
[[package]]
@@ -3159,7 +3341,7 @@ files = [
]
[package.extras]
dev = ["black (>=19.3b0)", "pytest (>=4.6.2)"]
dev = ["black (>=19.3b0) ; python_version >= \"3.6\"", "pytest (>=4.6.2)"]
[[package]]
name = "wsproto"
@@ -3178,20 +3360,20 @@ h11 = ">=0.9.0,<1"
[[package]]
name = "yt-dlp"
version = "2025.1.26"
version = "2025.2.19"
description = "A feature-rich command-line audio/video downloader"
optional = false
python-versions = ">=3.9"
groups = ["main"]
files = [
{file = "yt_dlp-2025.1.26-py3-none-any.whl", hash = "sha256:3e76bd896b9f96601021ca192ca0fbdd195e3c3dcc28302a3a34c9bc4979da7b"},
{file = "yt_dlp-2025.1.26.tar.gz", hash = "sha256:1c9738266921ad43c568ad01ac3362fb7c7af549276fbec92bd72f140da16240"},
{file = "yt_dlp-2025.2.19-py3-none-any.whl", hash = "sha256:3ed218eaeece55e9d715afd41abc450dc406ee63bf79355169dfde312d38fdb8"},
{file = "yt_dlp-2025.2.19.tar.gz", hash = "sha256:f33ca76df2e4db31880f2fe408d44f5058d9f135015b13e50610dfbe78245bea"},
]
[package.extras]
build = ["build", "hatchling", "pip", "setuptools (>=71.0.2)", "wheel"]
curl-cffi = ["curl-cffi (==0.5.10)", "curl-cffi (>=0.5.10,!=0.6.*,<0.7.2)"]
default = ["brotli", "brotlicffi", "certifi", "mutagen", "pycryptodomex", "requests (>=2.32.2,<3)", "urllib3 (>=1.26.17,<3)", "websockets (>=13.0)"]
curl-cffi = ["curl-cffi (==0.5.10) ; os_name == \"nt\" and implementation_name == \"cpython\"", "curl-cffi (>=0.5.10,!=0.6.*,<0.7.2) ; os_name != \"nt\" and implementation_name == \"cpython\""]
default = ["brotli ; implementation_name == \"cpython\"", "brotlicffi ; implementation_name != \"cpython\"", "certifi", "mutagen", "pycryptodomex", "requests (>=2.32.2,<3)", "urllib3 (>=1.26.17,<3)", "websockets (>=13.0)"]
dev = ["autopep8 (>=2.0,<3.0)", "pre-commit", "pytest (>=8.1,<9.0)", "pytest-rerunfailures (>=14.0,<15.0)", "ruff (>=0.9.0,<0.10.0)"]
pyinstaller = ["pyinstaller (>=6.11.1)"]
secretstorage = ["cffi", "secretstorage"]
@@ -3201,4 +3383,4 @@ test = ["pytest (>=8.1,<9.0)", "pytest-rerunfailures (>=14.0,<15.0)"]
[metadata]
lock-version = "2.1"
python-versions = ">=3.10,<3.13"
content-hash = "1a548dd3eeeaf1fb408b8339bf84ba31c1362a6f4c3c0aa98cdcb82ece256acc"
content-hash = "feb24e7029ddb05300ccc057ce64299871443d0c82038152e0fe3d810ec6fc9f"

32
pyproject.toml
View File

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
[project]
name = "auto-archiver"
version = "0.13.4"
version = "0.13.8"
description = "Automatically archive links to videos, images, and social media content from Google Sheets (and more)."
requires-python = ">=3.10,<3.13"
@@ -56,6 +56,7 @@ dependencies = [
"rich-argparse (>=1.6.0,<2.0.0)",
"ruamel-yaml (>=0.18.10,<0.19.0)",
"rfc3161-client (>=1.0.0,<2.0.0)",
"opentimestamps (>=0.4.5,<0.5.0)",
]
[tool.poetry.group.dev.dependencies]
@@ -63,6 +64,8 @@ pytest = "^8.3.4"
autopep8 = "^2.3.1"
pytest-loguru = "^0.4.0"
pytest-mock = "^3.14.0"
ruff = "^0.9.10"
pre-commit = "^4.1.0"
[tool.poetry.group.docs.dependencies]
sphinx = "^8.1.3"
@@ -88,4 +91,29 @@ documentation = "https://github.com/bellingcat/auto-archiver"
markers = [
"download: marks tests that download content from the network",
"incremental: marks a class to run tests incrementally. If a test fails in the class, the remaining tests will be skipped",
]
]
[tool.ruff]
#exclude = ["docs"]
line-length = 120
# Remove this for a more detailed lint report
output-format = "concise"
# TODO: temp ignore rule for timestamping_enricher to allow for open PR
exclude = ["src/auto_archiver/modules/timestamping_enricher/*"]
[tool.ruff.lint]
# Extend the rules to check for by adding them to this option:
# See documentation for more details: https://docs.astral.sh/ruff/rules/
#extend-select = ["B"]
[tool.ruff.lint.per-file-ignores]
# Ignore import violations in __init__.py files
"__init__.py" = ["F401", "F403"]
# Ignore 'useless expression' in manifest files.
"__manifest__.py" = ["B018"]
[tool.ruff.format]
docstring-code-format = false

9
scripts/create_update_gdrive_oauth_token.py
View File

@@ -1,5 +1,6 @@
import os.path
import click, json
import click
import json
from google.auth.transport.requests import Request
from google.oauth2.credentials import Credentials
@@ -70,11 +71,7 @@ def main(credentials, token):
print(emailAddress)
# Call the Drive v3 API and return some files
results = (
service.files()
.list(pageSize=10, fields="nextPageToken, files(id, name)")
.execute()
)
results = service.files().list(pageSize=10, fields="nextPageToken, files(id, name)").execute()
items = results.get("files", [])
if not items:

135
scripts/generate_google_services.sh Normal file
View File

@@ -0,0 +1,135 @@
#!/usr/bin/env bash
set -e # Exit on error
UUID=$(LC_ALL=C tr -dc a-z0-9 </dev/urandom | head -c 16)
PROJECT_NAME="auto-archiver-$UUID"
ACCOUNT_NAME="autoarchiver"
KEY_FILE="service_account-$UUID.json"
DEST_DIR="$1"
echo "====================================================="
echo "🔧 Auto-Archiver Google Services Setup Script"
echo "====================================================="
echo "This script will:"
echo " 1. Install Google Cloud SDK if needed"
echo " 2. Create a Google Cloud project named $PROJECT_NAME"
echo " 3. Create a service account for Auto-Archiver"
echo " 4. Generate a key file for API access"
echo ""
echo " Tip: Pass a directory path as an argument to this script to move the key file there"
echo " e.g. ./generate_google_services.sh /path/to/secrets"
echo "====================================================="
# Check and install Google Cloud SDK based on platform
install_gcloud_sdk() {
if command -v gcloud &> /dev/null; then
echo "✅ Google Cloud SDK is already installed"
return 0
fi
echo "📦 Installing Google Cloud SDK..."
# Detect OS
case "$(uname -s)" in
Darwin*)
if command -v brew &> /dev/null; then
echo "🍺 Installing via Homebrew..."
brew install google-cloud-sdk --cask
else
echo "📥 Downloading Google Cloud SDK for macOS..."
curl -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-latest-darwin-x86_64.tar.gz
tar -xf google-cloud-cli-latest-darwin-x86_64.tar.gz
./google-cloud-sdk/install.sh --quiet
rm google-cloud-cli-latest-darwin-x86_64.tar.gz
echo "🔄 Please restart your terminal and run this script again"
exit 0
fi
;;
Linux*)
echo "📥 Downloading Google Cloud SDK for Linux..."
curl -O https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-latest-linux-x86_64.tar.gz
tar -xf google-cloud-cli-latest-linux-x86_64.tar.gz
./google-cloud-sdk/install.sh --quiet
rm google-cloud-cli-latest-linux-x86_64.tar.gz
echo "🔄 Please restart your terminal and run this script again"
exit 0
;;
CYGWIN*|MINGW*|MSYS*)
echo "⚠️ Windows detected. Please follow manual installation instructions at:"
echo "https://cloud.google.com/sdk/docs/install-sdk"
exit 1
;;
*)
echo "⚠️ Unknown operating system. Please follow manual installation instructions at:"
echo "https://cloud.google.com/sdk/docs/install-sdk"
exit 1
;;
esac
echo "✅ Google Cloud SDK installed"
}
# Install Google Cloud SDK if needed
install_gcloud_sdk
# Login to Google Cloud
if gcloud auth list --filter=status:ACTIVE --format="value(account)" | grep -q "@"; then
echo "✅ Already authenticated with Google Cloud"
else
echo "🔑 Authenticating with Google Cloud..."
gcloud auth login
fi
# Create project
echo "🌟 Creating Google Cloud project: $PROJECT_NAME"
gcloud projects create $PROJECT_NAME
# Create service account
echo "👤 Creating service account: $ACCOUNT_NAME"
gcloud iam service-accounts create $ACCOUNT_NAME --project $PROJECT_NAME
# Enable required APIs (uncomment and add APIs as needed)
echo "⬆️ Enabling required Google APIs..."
gcloud services enable sheets.googleapis.com --project $PROJECT_NAME
gcloud services enable drive.googleapis.com --project $PROJECT_NAME
# Get the service account email
echo "📧 Retrieving service account email..."
ACCOUNT_EMAIL=$(gcloud iam service-accounts list --project $PROJECT_NAME --format="value(email)")
# Create and download key
echo "🔑 Generating service account key file: $KEY_FILE"
gcloud iam service-accounts keys create $KEY_FILE --iam-account=$ACCOUNT_EMAIL
# move the file to TARGET_DIR if provided
if [[ -n "$DEST_DIR" ]]; then
# Expand `~` if used
DEST_DIR=$(eval echo "$DEST_DIR")
# Ensure the directory exists
if [[ ! -d "$DEST_DIR" ]]; then
mkdir -p "$DEST_DIR"
fi
DEST_PATH="$DEST_DIR/$KEY_FILE"
echo "🚚 Moving key file to: $DEST_PATH"
mv "$KEY_FILE" "$DEST_PATH"
KEY_FILE="$DEST_PATH"
fi
echo "====================================================="
echo "✅ SETUP COMPLETE!"
echo "====================================================="
echo "📝 Important Information:"
echo " • Project Name: $PROJECT_NAME"
echo " • Service Account: $ACCOUNT_EMAIL"
echo " • Key File: $KEY_FILE"
echo ""
echo "📋 Next Steps:"
echo " 1. Share any Google Sheets with this email address:"
echo " $ACCOUNT_EMAIL"
echo " 2. Move $KEY_FILE to your auto-archiver secrets directory"
echo " 3. Update your auto-archiver config to use this key file (if needed)"
echo "====================================================="

63
scripts/generate_settings_schema.py Normal file
View File

@@ -0,0 +1,63 @@
import json
import os
import io
from ruamel.yaml import YAML
from auto_archiver.core.module import ModuleFactory
from auto_archiver.core.consts import MODULE_TYPES
from auto_archiver.core.config import EMPTY_CONFIG
class SchemaEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, set):
return list(obj)
return json.JSONEncoder.default(self, obj)
# Get available modules
module_factory = ModuleFactory()
available_modules = module_factory.available_modules()
modules_by_type = {}
# Categorize modules by type
for module in available_modules:
for type in module.manifest.get("type", []):
modules_by_type.setdefault(type, []).append(module)
all_modules_ordered_by_type = sorted(
available_modules, key=lambda x: (MODULE_TYPES.index(x.type[0]), not x.requires_setup)
)
yaml: YAML = YAML()
config_string = io.BytesIO()
yaml.dump(EMPTY_CONFIG, config_string)
config_string = config_string.getvalue().decode("utf-8")
output_schema = {
"modules": dict(
(
module.name,
{
"name": module.name,
"display_name": module.display_name,
"manifest": module.manifest,
"configs": module.configs or None,
},
)
for module in all_modules_ordered_by_type
),
"steps": dict(
(f"{module_type}s", [module.name for module in modules_by_type[module_type]]) for module_type in MODULE_TYPES
),
"configs": [m.name for m in all_modules_ordered_by_type if m.configs],
"module_types": MODULE_TYPES,
"empty_config": config_string,
}
current_file_dir = os.path.dirname(os.path.abspath(__file__))
output_file = os.path.join(current_file_dir, "settings/src/schema.json")
with open(output_file, "w") as file:
print(f"Writing schema to {output_file}")
json.dump(output_schema, file, indent=4, cls=SchemaEncoder)

24
scripts/settings/.gitignore vendored Normal file
View File

@@ -0,0 +1,24 @@
# Logs
logs
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
pnpm-debug.log*
lerna-debug.log*
node_modules
dist
dist-ssr
*.local
# Editor directories and files
.vscode/*
!.vscode/extensions.json
.idea
.DS_Store
*.suo
*.ntvs*
*.njsproj
*.sln
*.sw?

3
scripts/settings/index.html Normal file
View File

@@ -0,0 +1,3 @@
<div id="root"></div>
<script type="module" src="/src/main.tsx"></script>

3743
scripts/settings/package-lock.json generated Normal file
View File

File diff suppressed because it is too large Load Diff

31
scripts/settings/package.json Normal file
View File

@@ -0,0 +1,31 @@
{
"name": "material-ui-vite-ts",
"private": true,
"version": "5.0.0",
"type": "module",
"scripts": {
"dev": "vite",
"build": "vite build",
"preview": "vite preview"
},
"dependencies": {
"@dnd-kit/core": "^6.3.1",
"@dnd-kit/sortable": "^10.0.0",
"@emotion/react": "latest",
"@emotion/styled": "latest",
"@mui/icons-material": "^6.4.7",
"@mui/material": "latest",
"react": "19.0.0",
"react-dom": "19.0.0",
"react-markdown": "^10.0.0",
"yaml": "^2.7.0"
},
"devDependencies": {
"@types/react": "latest",
"@types/react-dom": "latest",
"@vitejs/plugin-react": "latest",
"typescript": "latest",
"vite": "latest",
"vite-plugin-singlefile": "^2.1.0"
}
}

450
scripts/settings/src/App.tsx Normal file
View File

@@ -0,0 +1,450 @@
import * as React from 'react';
import { useEffect, useState, useRef } from 'react';
import Container from '@mui/material/Container';
import Typography from '@mui/material/Typography';
import Box from '@mui/material/Box';
import FileUploadIcon from '@mui/icons-material/FileUpload';
import {
DndContext,
closestCenter,
KeyboardSensor,
PointerSensor,
useSensor,
useSensors,
DragOverlay
} from "@dnd-kit/core";
import {
arrayMove,
SortableContext,
sortableKeyboardCoordinates,
rectSortingStrategy
} from "@dnd-kit/sortable";
import type { DragStartEvent, DragEndEvent, UniqueIdentifier } from "@dnd-kit/core";
import { Module } from './types';
import { modules, steps, module_types, empty_config } from './schema.json';
import {
Stack,
Button,
} from '@mui/material';
import Grid from '@mui/material/Grid2';
import { parseDocument, Document, YAMLSeq, YAMLMap, Scalar } from 'yaml'
import StepCard from './StepCard';
function FileDrop({ setYamlFile }: { setYamlFile: React.Dispatch<React.SetStateAction<Document>> }) {
const [showError, setShowError] = useState(false);
const [label, setLabel] = useState(<>Drag and drop your orchestration.yaml file here, or click to select a file.</>);
const wrapperRef = useRef(null);
function openYAMLFile(event: any) {
let file = event.target.files[0];
if (file.type.indexOf('yaml') === -1) {
setShowError(true);
setLabel(<>Invalid type, only YAML files are accepted.</>)
return;
}
let reader = new FileReader();
reader.onload = function (e) {
let contents = e.target ? e.target.result : '';
try {
let document = parseDocument(contents as string);
if (document.errors.length > 0) {
// not a valid yaml file
setShowError(true);
setLabel(<>Invalid file. Make sure your Orchestration is a valid YAML file with a 'steps' section in it.</>)
return;
} else {
setShowError(false);
setLabel(<>File loaded successfully.</>)
}
// do some basic validation of 'steps'
let steps = document.get('steps');
if (!steps) {
setShowError(true);
setLabel(<>Invalid file. Your orchestration file must have a 'steps' section in it.</>)
return;
}
const replacements = {
feeder: 'feeders',
formatter: 'formatters',
archivers: 'extractors',
};
let error = false;
for (let stepType of Object.keys(replacements)) {
if (steps.get(stepType) !== undefined) {
setShowError(true);
setLabel(<>Invalid file. Your orchestration file appears to be in the old (v0.12) format with a '{stepType}' section.<br/>You should manually update your orchestration file first (hint: {stepType} → {replacements[stepType]})</>);
error = true;
return;
}
};
setYamlFile(document);
} catch (e) {
console.error(e);
}
}
reader.readAsText(file);
}
return (
<>
<div
style={{
position: 'relative',
width: '100%',
border: 'dashed',
borderRadius:'5px',
textAlign: 'center',
borderWidth: '1px',
padding: '20px' }}
onDragEnter={(e) => {
e.currentTarget.style.backgroundColor = 'var(--mui-palette-LinearProgress-infoBg)';
}}
onDragLeave={(e) => {
e.currentTarget.style.backgroundColor = '';
}}
onDrop={(e) => {
e.currentTarget.style.backgroundColor = '';
}}
>
<FileUploadIcon style={{ fontSize: 50 }} />
<input style={{
opacity: 0,
position: 'absolute',
top: 0,
left: 0,
width: '100%',
height: '100%',
cursor: 'pointer',
}}
type="file" id="file"
accept=".yaml"
onChange={openYAMLFile} />
<Typography variant="body1" color={showError ? 'error' : ''} >
{label}
</Typography>
</div>
</>
);
}
function ModuleTypes({ stepType, setEnabledModules, enabledModules, configValues }: { stepType: string, setEnabledModules: any, enabledModules: any, configValues: any }) {
const [showError, setShowError] = useState<boolean>(false);
const [activeId, setActiveId] = useState<UniqueIdentifier>();
const [items, setItems] = useState<string[]>([]);
useEffect(() => {
setItems(enabledModules[stepType].map(([name, enabled]: [string, boolean]) => name));
}
, [enabledModules]);
const toggleModule = (event: any) => {
// make sure that 'feeder' and 'formatter' types only have one value
let name = event.target.id;
let checked = event.target.checked;
if (stepType === 'feeders' || stepType === 'formatters') {
// check how many modules of this type are enabled
const checkedModules = enabledModules[stepType].filter(([m, enabled]: [string, boolean]) => {
return (m !== name && enabled) || (checked && m === name)
});
if (checkedModules.length > 1) {
setShowError(true);
} else {
setShowError(false);
}
} else {
setShowError(false);
}
let newEnabledModules = { ...enabledModules };
newEnabledModules[stepType] = enabledModules[stepType].map(([m, enabled]: [string, boolean]) => {
return (m === name) ? [m, checked] : [m, enabled];
});
setEnabledModules(newEnabledModules);
}
const sensors = useSensors(
useSensor(PointerSensor),
useSensor(KeyboardSensor, {
coordinateGetter: sortableKeyboardCoordinates
})
);
const handleDragStart = (event: DragStartEvent) => {
setActiveId(event.active.id);
};
const handleDragEnd = (event: DragEndEvent) => {
setActiveId(undefined);
const { active, over } = event;
if (active.id !== over?.id) {
const oldIndex = items.indexOf(active.id as string);
const newIndex = items.indexOf(over?.id as string);
let newArray = arrayMove(items, oldIndex, newIndex);
// set it also on steps
let newEnabledModules = { ...enabledModules };
newEnabledModules[stepType] = enabledModules[stepType].sort((a, b) => {
return newArray.indexOf(a[0]) - newArray.indexOf(b[0]);
})
setEnabledModules(newEnabledModules);
}
};
return (
<>
<Box sx={{ my: 4 }}>
<Typography id={stepType} variant="h6" style={{ textTransform: 'capitalize' }} >
{stepType}
</Typography>
<Typography variant="body1" >
Select the <a href={`https://auto-archiver.readthedocs.io/en/latest/modules/${stepType.slice(0,-1)}.html`} target="_blank">{stepType}</a> you wish to enable. Drag to reorder.
</Typography>
</Box>
{showError ? <Typography variant="body1" color="error" >Only one {stepType.slice(0,-1)} can be enabled at a time.</Typography> : null}
<DndContext
sensors={sensors}
collisionDetection={closestCenter}
onDragEnd={handleDragEnd}
onDragStart={handleDragStart}
>
<Grid container spacing={1} key={stepType}>
<SortableContext items={items} strategy={rectSortingStrategy}>
{items.map((name: string) => {
let m: Module = modules[name];
return (
<StepCard key={name} type={stepType} module={m} toggleModule={toggleModule} enabledModules={enabledModules} configValues={configValues} />
);
})}
<DragOverlay>
{activeId ? (
<div
style={{
width: "100%",
height: "100%",
backgroundColor: "grey",
opacity: 0.1,
}}
></div>
) : null}
</DragOverlay>
</SortableContext>
</Grid>
</DndContext>
</>
);
}
export default function App() {
const [yamlFile, setYamlFile] = useState<Document>(new Document());
const [enabledModules, setEnabledModules] = useState<{}>(Object.fromEntries(Object.keys(steps).map(type => [type, steps[type].map((name: string) => [name, false])])));
const [configValues, setConfigValues] = useState<{
[key: string]: {
[key: string
]: any
}
}>(
Object.keys(modules).reduce((acc, module) => {
acc[module] = {};
return acc;
}, {})
);
const saveSettings = function (copy: boolean = false) {
// edit the yamlFile
// generate the steps config
let stepsConfig = enabledModules;
let finalYamlFile: Document = null;
if (!yamlFile || yamlFile.contents == null) {
// create the yaml file from
finalYamlFile = parseDocument(empty_config as string);
} else {
finalYamlFile = yamlFile;
}
// set the steps
module_types.forEach((type: string) => {
let stepType = type + 's';
let existingSteps = finalYamlFile.getIn(['steps', stepType]) as YAMLSeq;
stepsConfig[stepType].forEach(([name, enabled]: [string, boolean]) => {
let index = existingSteps.items.findIndex((item) => {
return (item.value || item) === name
});
let stepItem = finalYamlFile.getIn(['steps', stepType], true) as YAMLSeq;
if (enabled && index === -1) {
finalYamlFile.addIn(['steps', stepType], name);
stepItem.commentBefore = stepItem.commentBefore?.replace("\n - " + name, '');
stepItem.comment = stepItem.comment?.replace("\n - " + name, '');
} else if (!enabled && index !== -1) {
// set the value to empty and add a comment before with the commented value
finalYamlFile.deleteIn(['steps', stepType, index]);
stepItem.commentBefore += "\n - " + name;
finalYamlFile.setIn(['steps', stepType], stepItem);
}
});
// sort the items
existingSteps.items.sort((a: Scalar | string, b: Scalar | string) => {
return (stepsConfig[stepType].findIndex((val: [string, boolean]) => {return val[0] === (a.value || a)}) -
stepsConfig[stepType].findIndex((val: [string, boolean]) => {return val[0] === (b.value || b)}))
});
existingSteps.flow = existingSteps.items.length ? false : true;
});
// set all other settings
// loop through each item that isn't 'steps' in the finalYamlFile and check if it exists in configValues
Object.keys(configValues).forEach((module_name: string) => {
// get an existing key
let existingConfig = finalYamlFile.get(module_name, true) as YAMLMap;
if (existingConfig) {
Object.keys(configValues[module_name]).forEach((config_name: string) => {
let existingConfigYAML = existingConfig.get(config_name, true) as Scalar;
if (existingConfigYAML) {
existingConfigYAML.value = configValues[module_name][config_name];
existingConfig.set(config_name, existingConfigYAML);
} else {
existingConfig.set(config_name, configValues[module_name][config_name]);
}
});
finalYamlFile.set(module_name, existingConfig);
} else {
if (configValues[module_name] && Object.keys(configValues[module_name]).length > 0) {
finalYamlFile.set(module_name, configValues[module_name]);
}
}
});
if (copy) {
navigator.clipboard.writeText(String(finalYamlFile)).then(() => {
alert("Settings copied to clipboard.");
});
} else {
// offer the file for download
const blob = new Blob([String(finalYamlFile)], { type: 'application/x-yaml' });
const url = URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url;
a.download = 'orchestration.yaml';
a.click();
}
}
useEffect(() => {
// load the configs, and set the default values if they exist
let newConfigValues = {};
Object.keys(modules).map((module: string) => {
let m = modules[module];
let configs = m.configs;
if (!configs) {
return;
}
newConfigValues[module] = {};
Object.keys(configs).map((config: string) => {
let config_args = configs[config];
if (config_args.default !== undefined) {
newConfigValues[module][config] = config_args.default;
}
});
})
setConfigValues(newConfigValues);
}, []);
useEffect(() => {
if (!yamlFile || yamlFile.contents == null) {
return;
}
let settings = yamlFile.toJS();
// make a deep copy of settings
let stepSettings = settings['steps'];
let newEnabledModules = Object.fromEntries(Object.keys(steps).map((type: string) => {
return [type, steps[type].map((name: string) => {
return [name, stepSettings[type].indexOf(name) !== -1];
}).sort((a, b) => {
let aIndex = stepSettings[type].indexOf(a[0]);
let bIndex = stepSettings[type].indexOf(b[0]);
if (aIndex === -1 && bIndex === -1) {
return a - b;
}
if (bIndex === -1) {
return -1;
}
if (aIndex === -1) {
return 1;
}
return aIndex - bIndex;
})];
}).sort((a, b) => {
return module_types.indexOf(a[0]) - module_types.indexOf(b[0]);
}));
setEnabledModules(newEnabledModules);
// set the config values
let newConfigValues = settings;
delete newConfigValues['steps'];
setConfigValues(Object.keys(modules).reduce((acc, module) => {
acc[module] = newConfigValues[module] || {};
return acc;
}, {}));
}, [yamlFile]);
return (
<Container maxWidth="lg">
<Box sx={{ my: 4 }}>
<Box sx={{ my: 4 }}>
<Typography variant="h5" >
1. Select your orchestration.yaml settings file.
</Typography>
<Typography variant="body1">Or skip this step to start from scratch</Typography>
<FileDrop setYamlFile={setYamlFile} />
</Box>
<Box sx={{ my: 4 }}>
<Typography variant="h5" >
2. Choose the Modules you wish to enable/disable
</Typography>
{Object.keys(steps).map((stepType: string) => {
return (
<Box key={stepType} sx={{ my: 4 }}>
<ModuleTypes stepType={stepType} setEnabledModules={setEnabledModules} enabledModules={enabledModules} configValues={configValues} />
</Box>
);
})}
</Box>
<Box sx={{ my: 4 }}>
<Typography variant="h5" >
3. Configure your Enabled Modules
</Typography>
<Typography variant="body1" >
Next to each module you've enabled, you can click 'Configure' to set the module's settings.
</Typography>
</Box>
<Box sx={{ my: 4 }}>
<Typography variant="h5" >
4. Save your settings
</Typography>
<Stack direction="row" spacing={2} sx={{ my: 2 }}>
<Button variant="contained" color="primary" onClick={() => saveSettings(true)}>Copy Settings to Clipboard</Button>
<Button variant="contained" color="primary" onClick={() => saveSettings()}>Save Settings to File</Button>
</Stack>
</Box>
</Box>
</Container>
);
}

258
scripts/settings/src/StepCard.tsx Normal file
View File

@@ -0,0 +1,258 @@
import { useState } from "react";
import { useSortable } from "@dnd-kit/sortable";
import ReactMarkdown from 'react-markdown';
import { CSS } from "@dnd-kit/utilities";
import {
Card,
CardActions,
CardHeader,
Button,
Dialog,
DialogTitle,
DialogContent,
Box,
IconButton,
Checkbox,
Select,
MenuItem,
FormControl,
FormControlLabel,
FormHelperText,
TextField,
Stack,
Typography,
InputAdornment,
} from '@mui/material';
import Grid from '@mui/material/Grid2';
import DragIndicatorIcon from '@mui/icons-material/DragIndicator';
import Visibility from '@mui/icons-material/Visibility';
import VisibilityOff from '@mui/icons-material/VisibilityOff';
import HelpIconOutlined from '@mui/icons-material/HelpOutline';
import { Module, Config } from "./types";
// adds 'capitalize' method to String prototype
declare global {
interface String {
capitalize(): string;
}
}
String.prototype.capitalize = function (this: string) {
return this.charAt(0).toUpperCase() + this.slice(1);
};
const StepCard = ({
type,
module,
toggleModule,
enabledModules,
configValues
}: {
type: string,
module: Module,
toggleModule: any,
enabledModules: any,
configValues: any
}) => {
const {
attributes,
listeners,
setNodeRef,
transform,
transition,
isDragging
} = useSortable({ id: module.name });
const style = {
...Card.style,
transform: CSS.Transform.toString(transform),
transition,
zIndex: isDragging ? "100" : "auto",
opacity: isDragging ? 0.3 : 1
};
let name = module.name;
const [helpOpen, setHelpOpen] = useState(false);
const [configOpen, setConfigOpen] = useState(false);
const enabled = enabledModules[type].find((m: any) => m[0] === name)[1];
return (
<Grid ref={setNodeRef} size={{ xs: 6, sm: 4, md: 3 }} style={style}>
<Card >
<CardHeader
title={
<FormControlLabel
style={{paddingRight: '0 !important'}}
control={<Checkbox title="Check to enable this module" sx={{paddingTop:0, paddingBottom:0}} id={name} onClick={toggleModule} checked={enabled} />}
label={module.display_name} />
}
/>
<CardActions>
<Box sx={{ justifyContent: 'space-between', display: 'flex', width: '100%' }}>
<Box>
<IconButton title="Module information" size="small" onClick={() => setHelpOpen(true)}>
<HelpIconOutlined />
</IconButton>
{enabled && module.configs && name != 'cli_feeder' ? (
<Button size="small" onClick={() => setConfigOpen(true)}>Configure</Button>
) : null}
</Box>
<IconButton size="small" title="Drag to reorder" sx={{ cursor: 'grab' }} {...listeners} {...attributes}>
<DragIndicatorIcon/>
</IconButton>
</Box>
</CardActions>
</Card>
<Dialog
open={helpOpen}
onClose={() => setHelpOpen(false)}
maxWidth="lg"
>
<DialogTitle>
{module.display_name}
</DialogTitle>
<DialogContent>
<ReactMarkdown>
{module.manifest.description.split("\n").map((line: string) => line.trim()).join("\n")}
</ReactMarkdown>
</DialogContent>
</Dialog>
{module.configs && name != 'cli_feeder' && <ConfigPanel module={module} open={configOpen} setOpen={setConfigOpen} configValues={configValues} />}
</Grid>
)
}
function ConfigField({ config_value, module, configValues }: { config_value: any, module: Module, configValues: any }) {
const [showPassword, setShowPassword] = useState(false);
const handleClickShowPassword = () => setShowPassword((show) => !show);
const handleMouseDownPassword = (event: React.MouseEvent<HTMLButtonElement>) => {
event.preventDefault();
};
const handleMouseUpPassword = (event: React.MouseEvent<HTMLButtonElement>) => {
event.preventDefault();
};
function setConfigValue(config: any, value: any) {
configValues[module.name][config] = value;
}
const config_args: Config = module.configs[config_value];
const config_name: string = config_value.replace(/_/g, " ");
const config_display_name = config_name.capitalize();
const value = configValues[module.name][config_value] || config_args.default;
const config_value_lower = config_value.toLowerCase();
const is_password = config_value_lower.includes('password') ||
config_value_lower.includes('secret') ||
config_value_lower.includes('token') ||
config_value_lower.includes('key') ||
config_value_lower.includes('api_hash') ||
config_args.type === 'password';
const text_input_type = is_password ? 'password' : (config_args.type === 'int' ? 'number' : 'text');
return (
<Box>
<Typography variant='body1' style={{ fontWeight: 'bold' }}>{config_display_name} {config_args.required && (`(required)`)} </Typography>
<FormControl size="small">
{config_args.type === 'bool' ?
<FormControlLabel control={
<Checkbox defaultChecked={value} size="small" id={`${module}.${config_value}`}
onChange={(e) => {
setConfigValue(config_value, e.target.checked);
}}
/>} label={config_args.help.capitalize()}
/>
:
(
config_args.choices !== undefined ?
<Select size="small" id={`${module}.${config_value}`}
defaultValue={config_args.default}
value={value}
onChange={(e) => {
setConfigValue(config_value, e.target.value);
}}
>
{config_args.choices.map((choice: any) => {
return (
<MenuItem key={`${module}.${config_value}.${choice}`}
value={choice}>{choice}</MenuItem>
);
})}
</Select>
:
(config_args.type === 'json_loader' ?
<TextField multiline size="small" id={`${module}.${config_value}`} defaultValue={JSON.stringify(value, null, 2)} rows={6} onChange={
(e) => {
try {
let val = JSON.parse(e.target.value);
setConfigValue(config_value, val);
} catch (e) {
console.log(e);
}
}
} />
:
<TextField size="small" id={`${module}.${config_value}`} defaultValue={value} type={showPassword ? 'text' : text_input_type}
onChange={(e) => {
setConfigValue(config_value, e.target.value);
}}
required={config_args.required}
slotProps={ is_password ? {
input: { endAdornment: (
<InputAdornment position="end">
<IconButton
aria-label="toggle password visibility"
onClick={handleClickShowPassword}
onMouseDown={handleMouseDownPassword}
onMouseUp={handleMouseUpPassword}
>
{showPassword ? <VisibilityOff /> : <Visibility />}
</IconButton>
</InputAdornment>
)}
} : {}}
/>
)
)
}
{config_args.type !== 'bool' && (
<FormHelperText >{config_args.help.capitalize()}</FormHelperText>
)}
</FormControl>
</Box>
)
}
function ConfigPanel({ module, open, setOpen, configValues }: { module: Module, open: boolean, setOpen: any, configValues: any }) {
return (
<>
<Dialog
open={open}
onClose={() => setOpen(false)}
maxWidth="lg"
>
<DialogTitle>
{module.display_name}
</DialogTitle>
<DialogContent>
<Stack direction="column" spacing={1}>
{Object.keys(module.configs).map((config_value: any) => {
return (
<ConfigField key={config_value} config_value={config_value} module={module} configValues={configValues} />
);
})}
</Stack>
</DialogContent>
</Dialog>
</>
);
}
export default StepCard;

44
scripts/settings/src/main.tsx Normal file
View File

@@ -0,0 +1,44 @@
import * as React from 'react';
import * as ReactDOM from 'react-dom/client';
import { ThemeProvider } from '@mui/material/styles';
import { CssBaseline } from '@mui/material';
import App from './App';
import { createTheme } from '@mui/material/styles';
import { red } from '@mui/material/colors';
import { useState, useEffect } from 'react';
function RootApp() {
const [mode, setMode] = useState('light');
useEffect(() => {
setMode(window.localStorage.getItem('theme') || 'light');
}, []);
var observer = new MutationObserver(function(mutations) {
setMode(window.localStorage.getItem('theme') || 'light');
})
observer.observe(document.documentElement, {attributes: true, attributeFilter: ['data-theme']});
// A custom theme for this app
const theme = createTheme({
palette: {
mode: mode == 'light' ? 'light' : 'dark',
},
cssVariables: true
});
return (
<ThemeProvider theme={theme}>
<CssBaseline />
<App />
</ThemeProvider>
);
}
ReactDOM.createRoot(document.getElementById('root')!).render(
<React.StrictMode>
<RootApp />
</React.StrictMode>,
);

21
scripts/settings/src/types.d.ts vendored Normal file
View File

@@ -0,0 +1,21 @@
export interface Config {
name: string;
description: string;
type: string?;
default: any;
help: string;
choices: string[];
required: boolean;
}
interface Manifest {
description: string;
}
export interface Module {
name: string;
description: string;
configs: { [key: string]: Config };
manifest: Manifest;
display_name: string;
}

21
scripts/settings/tsconfig.json Normal file
View File

@@ -0,0 +1,21 @@
{
"compilerOptions": {
"target": "ESNext",
"useDefineForClassFields": true,
"lib": ["DOM", "DOM.Iterable", "ESNext"],
"allowJs": false,
"skipLibCheck": true,
"esModuleInterop": false,
"allowSyntheticDefaultImports": true,
"strict": true,
"forceConsistentCasingInFileNames": true,
"module": "ESNext",
"moduleResolution": "Node",
"resolveJsonModule": true,
"isolatedModules": true,
"noEmit": true,
"jsx": "react-jsx"
},
"include": ["src"],
"references": [{ "path": "./tsconfig.node.json" }]
}

9
scripts/settings/tsconfig.node.json Normal file
View File

@@ -0,0 +1,9 @@
{
"compilerOptions": {
"composite": true,
"module": "ESNext",
"moduleResolution": "Node",
"allowSyntheticDefaultImports": true
},
"include": ["vite.config.ts"]
}

12
scripts/settings/vite.config.ts Normal file
View File

@@ -0,0 +1,12 @@
import { defineConfig } from 'vite';
import react from '@vitejs/plugin-react';
import { viteSingleFile } from "vite-plugin-singlefile"
// https://vite.dev/config/
export default defineConfig({
plugins: [react(), viteSingleFile()],
build: {
// minify: false,
// sourcemap: true,
}
});

2
scripts/telegram_setup.py
View File

@@ -12,7 +12,6 @@ Then run this script to create a new session file.
You will need to provide your phone number and a 2FA code the first time you run this script.
"""
import os
from telethon.sync import TelegramClient
from loguru import logger
@@ -26,4 +25,3 @@ SESSION_FILE = "secrets/anon-insta"
os.makedirs("secrets", exist_ok=True)
with TelegramClient(SESSION_FILE, API_ID, API_HASH) as client:
logger.success(f"New session file created: {SESSION_FILE}.session")

8
src/auto_archiver/__main__.py
View File

@@ -1,9 +1,13 @@
""" Entry point for the auto_archiver package. """
"""Entry point for the auto_archiver package."""
from auto_archiver.core.orchestrator import ArchivingOrchestrator
import sys
def main():
for _ in ArchivingOrchestrator()._command_line_run(sys.argv[1:]): pass
for _ in ArchivingOrchestrator()._command_line_run(sys.argv[1:]):
pass
if __name__ == "__main__":
main()

5
src/auto_archiver/core/__init__.py
View File

@@ -1,6 +1,5 @@
""" Core modules to handle things such as orchestration, metadata and configs..
"""Core modules to handle things such as orchestration, metadata and configs.."""
"""
from .metadata import Metadata
from .media import Media
from .base_module import BaseModule
@@ -14,4 +13,4 @@ from .enricher import Enricher
from .feeder import Feeder
from .storage import Storage
from .extractor import Extractor
from .formatter import Formatter
from .formatter import Formatter

73
src/auto_archiver/core/base_module.py
View File

@@ -1,9 +1,8 @@
from __future__ import annotations
from typing import Mapping, Any, Type, TYPE_CHECKING
from typing import Mapping, Any, TYPE_CHECKING
from abc import ABC
from copy import deepcopy, copy
from copy import deepcopy
from tempfile import TemporaryDirectory
from auto_archiver.utils import url as UrlUtil
from auto_archiver.core.consts import MODULE_TYPES as CONF_MODULE_TYPES
@@ -13,8 +12,8 @@ from loguru import logger
if TYPE_CHECKING:
from .module import ModuleFactory
class BaseModule(ABC):
class BaseModule(ABC):
"""
Base module class. All modules should inherit from this class.
@@ -46,15 +45,13 @@ class BaseModule(ABC):
@property
def storages(self) -> list:
return self.config.get('storages', [])
return self.config.get("storages", [])
def config_setup(self, config: dict):
authentication = config.get('authentication', {})
# this is important. Each instance is given its own deepcopied config, so modules cannot
# change values to affect other modules
config = deepcopy(config)
authentication = deepcopy(config.pop('authentication', {}))
authentication = deepcopy(config.pop("authentication", {}))
self.authentication = authentication
self.config = config
@@ -62,18 +59,28 @@ class BaseModule(ABC):
setattr(self, key, val)
def setup(self):
# For any additional setup required by modules, e.g. autehntication
# For any additional setup required by modules outside of the configs in the manifesst,
# e.g. authentication
pass
def auth_for_site(self, site: str, extract_cookies=True) -> Mapping[str, Any]:
"""
Returns the authentication information for a given site. This is used to authenticate
with a site before extracting data. The site should be the domain of the site, e.g. 'twitter.com'
:param site: the domain of the site to get authentication information for
:param extract_cookies: whether or not to extract cookies from the given browser/file and return the cookie jar (disabling can speed up processing if you don't actually need the cookies jar).
:returns: authdict dict of login information for the given site
:returns: authdict dict -> {
"username": str,
"password": str,
"api_key": str,
"api_secret": str,
"cookie": str,
"cookies_file": str,
"cookies_from_browser": str,
"cookies_jar": CookieJar
}
**Global options:**\n
* cookies_from_browser: str - the name of the browser to extract cookies from (e.g. 'chrome', 'firefox' - uses ytdlp under the hood to extract\n
@@ -87,15 +94,15 @@ class BaseModule(ABC):
* cookie: str - a cookie string to use for login (specific to this site)\n
* cookies_file: str - the path to a cookies file to use for login (specific to this site)\n
* cookies_from_browser: str - the name of the browser to extract cookies from (specitic for this site)\n
"""
# TODO: think about if/how we can deal with sites that have multiple domains (main one is x.com/twitter.com)
# for now the user must enter them both, like "x.com,twitter.com" in their config. Maybe we just hard-code?
site = UrlUtil.domain_for_url(site).lstrip("www.")
site = UrlUtil.domain_for_url(site).removeprefix("www.")
# add the 'www' version of the site to the list of sites to check
authdict = {}
for to_try in [site, f"www.{site}"]:
if to_try in self.authentication:
authdict.update(self.authentication[to_try])
@@ -105,19 +112,22 @@ class BaseModule(ABC):
if not authdict:
for key in self.authentication.keys():
if key in site or site in key:
logger.debug(f"Could not find exact authentication information for site '{site}'. \
did find information for '{key}' which is close, is this what you meant? \
If so, edit your authentication settings to make sure it exactly matches.")
logger.debug(
f"Could not find exact authentication information for site '{site}'. \
did find information for '{key}' which is close, is this what you meant? \
If so, edit your authentication settings to make sure it exactly matches."
)
def get_ytdlp_cookiejar(args):
import yt_dlp
from yt_dlp import parse_options
logger.debug(f"Extracting cookies from settings: {args[1]}")
# parse_options returns a named tuple as follows, we only need the ydl_options part
# collections.namedtuple('ParsedOptions', ('parser', 'options', 'urls', 'ydl_opts'))
ytdlp_opts = getattr(parse_options(args), 'ydl_opts')
ytdlp_opts = getattr(parse_options(args), "ydl_opts")
return yt_dlp.YoutubeDL(ytdlp_opts).cookiejar
get_cookiejar_options = None
# order of priority:
@@ -126,22 +136,21 @@ class BaseModule(ABC):
# 3. cookies_from_browser setting in global config
# 4. cookies_file setting in global config
if 'cookies_from_browser' in authdict:
get_cookiejar_options = ['--cookies-from-browser', authdict['cookies_from_browser']]
elif 'cookies_file' in authdict:
get_cookiejar_options = ['--cookies', authdict['cookies_file']]
elif 'cookies_from_browser' in self.authentication:
authdict['cookies_from_browser'] = self.authentication['cookies_from_browser']
get_cookiejar_options = ['--cookies-from-browser', self.authentication['cookies_from_browser']]
elif 'cookies_file' in self.authentication:
authdict['cookies_file'] = self.authentication['cookies_file']
get_cookiejar_options = ['--cookies', self.authentication['cookies_file']]
if "cookies_from_browser" in authdict:
get_cookiejar_options = ["--cookies-from-browser", authdict["cookies_from_browser"]]
elif "cookies_file" in authdict:
get_cookiejar_options = ["--cookies", authdict["cookies_file"]]
elif "cookies_from_browser" in self.authentication:
authdict["cookies_from_browser"] = self.authentication["cookies_from_browser"]
get_cookiejar_options = ["--cookies-from-browser", self.authentication["cookies_from_browser"]]
elif "cookies_file" in self.authentication:
authdict["cookies_file"] = self.authentication["cookies_file"]
get_cookiejar_options = ["--cookies", self.authentication["cookies_file"]]
if get_cookiejar_options:
authdict['cookies_jar'] = get_ytdlp_cookiejar(get_cookiejar_options)
authdict["cookies_jar"] = get_ytdlp_cookiejar(get_cookiejar_options)
return authdict
def repr(self):
return f"Module<'{self.display_name}' (config: {self.config[self.name]})>"
return f"Module<'{self.display_name}' (config: {self.config[self.name]})>"

66
src/auto_archiver/core/config.py
View File

@@ -6,26 +6,28 @@ flexible setup in various environments.
"""
import argparse
from ruamel.yaml import YAML, CommentedMap, add_representer
from ruamel.yaml import YAML, CommentedMap
import json
import os
from loguru import logger
from copy import deepcopy
from auto_archiver.core.consts import MODULE_TYPES
from typing import Any, List, Type, Tuple
_yaml: YAML = YAML()
DEFAULT_CONFIG_FILE = "secrets/orchestration.yaml"
EMPTY_CONFIG = _yaml.load("""
EMPTY_CONFIG = _yaml.load(
"""
# Auto Archiver Configuration
# Steps are the modules that will be run in the order they are defined
steps:""" + "".join([f"\n {module}s: []" for module in MODULE_TYPES]) + \
"""
steps:"""
+ "".join([f"\n {module}s: []" for module in MODULE_TYPES])
+ """
# Global configuration
@@ -52,50 +54,54 @@ authentication: {}
logging:
level: INFO
""")
"""
)
# note: 'logging' is explicitly added above in order to better format the config file
# Arg Parse Actions/Classes
class AuthenticationJsonParseAction(argparse.Action):
def __call__(self, parser, namespace, values, option_string=None):
try:
auth_dict = json.loads(values)
setattr(namespace, self.dest, auth_dict)
except json.JSONDecodeError as e:
raise argparse.ArgumentTypeError(f"Invalid JSON input for argument '{self.dest}': {e}")
raise argparse.ArgumentTypeError(f"Invalid JSON input for argument '{self.dest}': {e}") from e
def load_from_file(path):
try:
with open(path, 'r') as f:
with open(path, "r") as f:
try:
auth_dict = json.load(f)
except json.JSONDecodeError:
f.seek(0)
# maybe it's yaml, try that
auth_dict = _yaml.load(f)
if auth_dict.get('authentication'):
auth_dict = auth_dict['authentication']
auth_dict['load_from_file'] = path
if auth_dict.get("authentication"):
auth_dict = auth_dict["authentication"]
auth_dict["load_from_file"] = path
return auth_dict
except:
except Exception:
return None
if isinstance(auth_dict, dict) and auth_dict.get('from_file'):
auth_dict = load_from_file(auth_dict['from_file'])
if isinstance(auth_dict, dict) and auth_dict.get("from_file"):
auth_dict = load_from_file(auth_dict["from_file"])
elif isinstance(auth_dict, str):
# if it's a string
auth_dict = load_from_file(auth_dict)
if not isinstance(auth_dict, dict):
raise argparse.ArgumentTypeError("Authentication must be a dictionary of site names and their authentication methods")
global_options = ['cookies_from_browser', 'cookies_file', 'load_from_file']
raise argparse.ArgumentTypeError(
"Authentication must be a dictionary of site names and their authentication methods"
)
global_options = ["cookies_from_browser", "cookies_file", "load_from_file"]
for key, auth in auth_dict.items():
if key in global_options:
continue
if not isinstance(key, str) or not isinstance(auth, dict):
raise argparse.ArgumentTypeError(f"Authentication must be a dictionary of site names and their authentication methods. Valid global configs are {global_options}")
raise argparse.ArgumentTypeError(
f"Authentication must be a dictionary of site names and their authentication methods. Valid global configs are {global_options}"
)
setattr(namespace, self.dest, auth_dict)
@@ -106,8 +112,8 @@ class UniqueAppendAction(argparse.Action):
if value not in getattr(namespace, self.dest):
getattr(namespace, self.dest).append(value)
class DefaultValidatingParser(argparse.ArgumentParser):
class DefaultValidatingParser(argparse.ArgumentParser):
def error(self, message):
"""
Override of error to format a nicer looking error message using logger
@@ -136,8 +142,10 @@ class DefaultValidatingParser(argparse.ArgumentParser):
return super().parse_known_args(args, namespace)
# Config Utils
def to_dot_notation(yaml_conf: CommentedMap | dict) -> dict:
dotdict = {}
@@ -151,6 +159,7 @@ def to_dot_notation(yaml_conf: CommentedMap | dict) -> dict:
process_subdict(yaml_conf)
return dotdict
def from_dot_notation(dotdict: dict) -> dict:
normal_dict = {}
@@ -171,9 +180,11 @@ def from_dot_notation(dotdict: dict) -> dict:
def is_list_type(value):
return isinstance(value, list) or isinstance(value, tuple) or isinstance(value, set)
def is_dict_type(value):
return isinstance(value, dict) or isinstance(value, CommentedMap)
def merge_dicts(dotdict: dict, yaml_dict: CommentedMap) -> CommentedMap:
yaml_dict: CommentedMap = deepcopy(yaml_dict)
@@ -184,7 +195,7 @@ def merge_dicts(dotdict: dict, yaml_dict: CommentedMap) -> CommentedMap:
yaml_subdict[key] = value
continue
if key == 'steps':
if key == "steps":
for module_type, modules in value.items():
# overwrite the 'steps' from the config file with the ones from the CLI
yaml_subdict[key][module_type] = modules
@@ -199,6 +210,7 @@ def merge_dicts(dotdict: dict, yaml_dict: CommentedMap) -> CommentedMap:
update_dict(from_dot_notation(dotdict), yaml_dict)
return yaml_dict
def read_yaml(yaml_filename: str) -> CommentedMap:
config = None
try:
@@ -212,20 +224,26 @@ def read_yaml(yaml_filename: str) -> CommentedMap:
return config
# TODO: make this tidier/find a way to notify of which keys should not be stored
def store_yaml(config: CommentedMap, yaml_filename: str) -> None:
config_to_save = deepcopy(config)
## if the save path is the default location (secrets) then create the 'secrets' folder
if os.path.dirname(yaml_filename) == "secrets":
os.makedirs("secrets", exist_ok=True)
auth_dict = config_to_save.get("authentication", {})
if auth_dict and auth_dict.get('load_from_file'):
if auth_dict and auth_dict.get("load_from_file"):
# remove all other values from the config, don't want to store it in the config file
auth_dict = {"load_from_file": auth_dict["load_from_file"]}
config_to_save.pop('urls', None)
config_to_save.pop("urls", None)
with open(yaml_filename, "w", encoding="utf-8") as outf:
_yaml.dump(config_to_save, outf)
def is_valid_config(config: CommentedMap) -> bool:
return config and config != EMPTY_CONFIG
return config and config != EMPTY_CONFIG

32
src/auto_archiver/core/consts.py
View File

@@ -1,23 +1,19 @@
class SetupError(ValueError):
pass
MODULE_TYPES = [
'feeder',
'extractor',
'enricher',
'database',
'storage',
'formatter'
]
MODULE_TYPES = ["feeder", "extractor", "enricher", "database", "storage", "formatter"]
MANIFEST_FILE = "__manifest__.py"
DEFAULT_MANIFEST = {
'name': '', # the display name of the module
'author': 'Bellingcat', # creator of the module, leave this as Bellingcat or set your own name!
'type': [], # the type of the module, can be one or more of MODULE_TYPES
'requires_setup': True, # whether or not this module requires additional setup such as setting API Keys or installing additional softare
'description': '', # a description of the module
'dependencies': {}, # external dependencies, e.g. python packages or binaries, in dictionary format
'entry_point': '', # the entry point for the module, in the format 'module_name::ClassName'. This can be left blank to use the default entry point of module_name::ModuleName
'version': '1.0', # the version of the module
'configs': {} # any configuration options this module has, these will be exposed to the user in the config file or via the command line
}
"name": "", # the display name of the module
"author": "Bellingcat", # creator of the module, leave this as Bellingcat or set your own name!
"type": [], # the type of the module, can be one or more of MODULE_TYPES
"requires_setup": True, # whether or not this module requires additional setup such as setting API Keys or installing additional software
"description": "", # a description of the module
"dependencies": {}, # external dependencies, e.g. python packages or binaries, in dictionary format
"entry_point": "", # the entry point for the module, in the format 'module_name::ClassName'. This can be left blank to use the default entry point of module_name::ModuleName
"version": "1.0", # the version of the module
"configs": {}, # any configuration options this module has, these will be exposed to the user in the config file or via the command line
}

7
src/auto_archiver/core/database.py
View File

@@ -1,6 +1,6 @@
"""
Database module for the auto-archiver that defines the interface for implementing database modules
in the media archiving framework.
in the media archiving framework.
"""
from __future__ import annotations
@@ -9,6 +9,7 @@ from typing import Union
from auto_archiver.core import Metadata, BaseModule
class Database(BaseModule):
"""
Base class for implementing database modules in the media archiving framework.
@@ -20,7 +21,7 @@ class Database(BaseModule):
"""signals the DB that the given item archival has started"""
pass
def failed(self, item: Metadata, reason:str) -> None:
def failed(self, item: Metadata, reason: str) -> None:
"""update DB accordingly for failure"""
pass
@@ -34,6 +35,6 @@ class Database(BaseModule):
return False
@abstractmethod
def done(self, item: Metadata, cached: bool=False) -> None:
def done(self, item: Metadata, cached: bool = False) -> None:
"""archival result ready - should be saved to DB"""
pass

4
src/auto_archiver/core/enricher.py
View File

@@ -8,13 +8,15 @@ the archiving step and before storage or formatting.
Enrichers are optional but highly useful for making the archived data more powerful.
"""
from __future__ import annotations
from abc import abstractmethod
from auto_archiver.core import Metadata, BaseModule
class Enricher(BaseModule):
"""Base classes and utilities for enrichers in the Auto Archiver system.
Enricher modules must implement the `enrich` method to define their behavior.
"""

31
src/auto_archiver/core/extractor.py
View File

@@ -1,17 +1,15 @@
""" The `extractor` module defines the base functionality for implementing extractors in the media archiving framework.
This class provides common utility methods and a standard interface for extractors.
"""The `extractor` module defines the base functionality for implementing extractors in the media archiving framework.
This class provides common utility methods and a standard interface for extractors.
Factory method to initialize an extractor instance based on its name.
Factory method to initialize an extractor instance based on its name.
"""
from __future__ import annotations
from pathlib import Path
from abc import abstractmethod
from dataclasses import dataclass
import mimetypes
import os
import mimetypes
import requests
from loguru import logger
from retrying import retry
@@ -39,7 +37,7 @@ class Extractor(BaseModule):
Used to clean unnecessary URL parameters OR unfurl redirect links
"""
return url
def match_link(self, url: str) -> re.Match:
"""
Returns a match object if the given URL matches the valid_url pattern or False/None if not.
@@ -58,7 +56,7 @@ class Extractor(BaseModule):
"""
if self.valid_url:
return self.match_link(url) is not None
return True
def _guess_file_type(self, path: str) -> str:
@@ -74,16 +72,17 @@ class Extractor(BaseModule):
@retry(wait_random_min=500, wait_random_max=3500, stop_max_attempt_number=5)
def download_from_url(self, url: str, to_filename: str = None, verbose=True) -> str:
"""
downloads a URL to provided filename, or inferred from URL, returns local filename
downloads a URL to provided filename, or inferred from URL, returns local filename
"""
if not to_filename:
to_filename = url.split('/')[-1].split('?')[0]
to_filename = url.split("/")[-1].split("?")[0]
if len(to_filename) > 64:
to_filename = to_filename[-64:]
to_filename = os.path.join(self.tmp_dir, to_filename)
if verbose: logger.debug(f"downloading {url[0:50]=} {to_filename=}")
if verbose:
logger.debug(f"downloading {url[0:50]=} {to_filename=}")
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36'
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36"
}
try:
d = requests.get(url, stream=True, headers=headers, timeout=30)
@@ -91,12 +90,12 @@ class Extractor(BaseModule):
# get mimetype from the response headers
if not mimetypes.guess_type(to_filename)[0]:
content_type = d.headers.get('Content-Type') or self._guess_file_type(url)
content_type = d.headers.get("Content-Type") or self._guess_file_type(url)
extension = mimetypes.guess_extension(content_type)
if extension:
to_filename += extension
with open(to_filename, 'wb') as f:
with open(to_filename, "wb") as f:
for chunk in d.iter_content(chunk_size=8192):
f.write(chunk)
return to_filename
@@ -108,8 +107,8 @@ class Extractor(BaseModule):
def download(self, item: Metadata) -> Metadata | False:
"""
Downloads the media from the given URL and returns a Metadata object with the downloaded media.
If the URL is not supported or the download fails, this method should return False.
"""
pass
pass

8
src/auto_archiver/core/feeder.py
View File

@@ -1,5 +1,5 @@
"""
The feeder base module defines the interface for implementing feeders in the media archiving framework.
The feeder base module defines the interface for implementing feeders in the media archiving framework.
"""
from __future__ import annotations
@@ -7,8 +7,8 @@ from abc import abstractmethod
from auto_archiver.core import Metadata
from auto_archiver.core import BaseModule
class Feeder(BaseModule):
class Feeder(BaseModule):
"""
Base class for implementing feeders in the media archiving framework.
@@ -19,7 +19,7 @@ class Feeder(BaseModule):
def __iter__(self) -> Metadata:
"""
Returns an iterator (use `yield`) over the items to be archived.
These should be instances of Metadata, typically created with Metadata().set_url(url).
"""
return None
return None

4
src/auto_archiver/core/formatter.py
View File

@@ -12,7 +12,7 @@ from auto_archiver.core import Metadata, Media, BaseModule
class Formatter(BaseModule):
"""
Base class for implementing formatters in the media archiving framework.
Subclasses must implement the `format` method to define their behavior.
"""
@@ -21,4 +21,4 @@ class Formatter(BaseModule):
"""
Formats a Metadata object into a user-viewable format (e.g. HTML) and stores it if needed.
"""
return None
return None

29
src/auto_archiver/core/media.py
View File

@@ -6,7 +6,7 @@ nested media retrieval, and type validation.
from __future__ import annotations
import os
import traceback
from typing import Any, List
from typing import Any, List, Iterator
from dataclasses import dataclass, field
from dataclasses_json import dataclass_json, config
import mimetypes
@@ -21,14 +21,14 @@ class Media:
Represents a media file with associated properties and storage details.
Attributes:
- filename: The file path of the media.
- key: An optional identifier for the media.
- filename: The file path of the media as saved locally (temporarily, before uploading to the storage).
- urls: A list of URLs where the media is stored or accessible.
- properties: Additional metadata or transformations for the media.
- _mimetype: The media's mimetype (e.g., image/jpeg, video/mp4).
"""
filename: str
key: str = None
_key: str = None
urls: List[str] = field(default_factory=list)
properties: dict = field(default_factory=dict)
_mimetype: str = None # eg: image/jpeg
@@ -47,19 +47,20 @@ class Media:
for any_media in self.all_inner_media(include_self=True):
s.store(any_media, url, metadata=metadata)
def all_inner_media(self, include_self=False):
def all_inner_media(self, include_self=False) -> Iterator[Media]:
"""Retrieves all media, including nested media within properties or transformations on original media.
This function returns a generator for all the inner media.
"""
if include_self: yield self
if include_self:
yield self
for prop in self.properties.values():
if isinstance(prop, Media):
if isinstance(prop, Media):
for inner_media in prop.all_inner_media(include_self=True):
yield inner_media
if isinstance(prop, list):
for prop_media in prop:
if isinstance(prop_media, Media):
if isinstance(prop_media, Media):
for inner_media in prop_media.all_inner_media(include_self=True):
yield inner_media
@@ -67,6 +68,10 @@ class Media:
# checks if the media is already stored in the given storage
return len(self.urls) > 0 and len(self.urls) == len(in_storage.config["steps"]["storages"])
@property
def key(self) -> str:
return self._key
def set(self, key: str, value: Any) -> Media:
self.properties[key] = value
return self
@@ -110,15 +115,17 @@ class Media:
# checks for video streams with ffmpeg, or min file size for a video
# self.is_video() should be used together with this method
try:
streams = ffmpeg.probe(self.filename, select_streams='v')['streams']
streams = ffmpeg.probe(self.filename, select_streams="v")["streams"]
logger.warning(f"STREAMS FOR {self.filename} {streams}")
return any(s.get("duration_ts", 0) > 0 for s in streams)
except Error: return False # ffmpeg errors when reading bad files
except Error:
return False # ffmpeg errors when reading bad files
except Exception as e:
logger.error(e)
logger.error(traceback.format_exc())
try:
fsize = os.path.getsize(self.filename)
return fsize > 20_000
except: pass
except Exception as e:
pass
return True

95
src/auto_archiver/core/metadata.py
View File

@@ -13,7 +13,7 @@ from __future__ import annotations
import hashlib
from typing import Any, List, Union, Dict
from dataclasses import dataclass, field
from dataclasses_json import dataclass_json, config
from dataclasses_json import dataclass_json
import datetime
from urllib.parse import urlparse
from dateutil.parser import parse as parse_dt
@@ -21,6 +21,7 @@ from loguru import logger
from .media import Media
@dataclass_json # annotation order matters
@dataclass
class Metadata:
@@ -40,19 +41,23 @@ class Metadata:
- If `True`, this instance's values are overwritten by `right`.
- If `False`, the inverse applies.
"""
if not right: return self
if not right:
return self
if overwrite_left:
if right.status and len(right.status):
self.status = right.status
self._context.update(right._context)
for k, v in right.metadata.items():
assert k not in self.metadata or type(v) == type(self.get(k))
if type(v) not in [dict, list, set] or k not in self.metadata:
assert k not in self.metadata or type(v) is type(self.get(k))
if not isinstance(v, (dict, list, set)) or k not in self.metadata:
self.set(k, v)
else: # key conflict
if type(v) in [dict, set]: self.set(k, self.get(k) | v)
elif type(v) == list: self.set(k, self.get(k) + v)
if isinstance(v, (dict, set)):
self.set(k, self.get(k) | v)
elif type(v) is list:
self.set(k, self.get(k) + v)
self.media.extend(right.media)
else: # invert and do same logic
return right.merge(self)
return self
@@ -69,7 +74,7 @@ class Metadata:
def append(self, key: str, val: Any) -> Metadata:
if key not in self.metadata:
self.metadata[key] = []
self.metadata[key] = []
self.metadata[key] = val
return self
@@ -80,24 +85,26 @@ class Metadata:
return self.metadata.get(key, default)
def success(self, context: str = None) -> Metadata:
if context: self.status = f"{context}: success"
else: self.status = "success"
if context:
self.status = f"{context}: success"
else:
self.status = "success"
return self
def is_success(self) -> bool:
return "success" in self.status
def is_empty(self) -> bool:
meaningfull_ids = set(self.metadata.keys()) - set(["_processed_at", "url", "total_bytes", "total_size", "archive_duration_seconds"])
meaningfull_ids = set(self.metadata.keys()) - set(
["_processed_at", "url", "total_bytes", "total_size", "archive_duration_seconds"]
)
return not self.is_success() and len(self.media) == 0 and len(meaningfull_ids) == 0
@property # getter .netloc
def netloc(self) -> str:
return urlparse(self.get_url()).netloc
# custom getter/setters
# custom getter/setters
def set_url(self, url: str) -> Metadata:
assert type(url) is str and len(url) > 0, "invalid URL"
@@ -120,36 +127,43 @@ class Metadata:
return self.get("title")
def set_timestamp(self, timestamp: datetime.datetime) -> Metadata:
if type(timestamp) == str:
if isinstance(timestamp, str):
timestamp = parse_dt(timestamp)
assert type(timestamp) == datetime.datetime, "set_timestamp expects a datetime instance"
assert isinstance(timestamp, datetime.datetime), "set_timestamp expects a datetime instance"
return self.set("timestamp", timestamp)
def get_timestamp(self, utc=True, iso=True) -> datetime.datetime:
def get_timestamp(self, utc=True, iso=True) -> datetime.datetime | str | None:
ts = self.get("timestamp")
if not ts: return
if not ts:
return None
try:
if type(ts) == str: ts = datetime.datetime.fromisoformat(ts)
if type(ts) == float: ts = datetime.datetime.fromtimestamp(ts)
if utc: ts = ts.replace(tzinfo=datetime.timezone.utc)
if iso: return ts.isoformat()
return ts
if isinstance(ts, str):
ts = datetime.datetime.fromisoformat(ts)
elif isinstance(ts, float):
ts = datetime.datetime.fromtimestamp(ts)
if utc:
ts = ts.replace(tzinfo=datetime.timezone.utc)
return ts.isoformat() if iso else ts
except Exception as e:
logger.error(f"Unable to parse timestamp {ts}: {e}")
return
return None
def add_media(self, media: Media, id: str = None) -> Metadata:
# adds a new media, optionally including an id
if media is None: return
if media is None:
return
if id is not None:
assert not len([1 for m in self.media if m.get("id") == id]), f"cannot add 2 pieces of media with the same id {id}"
assert not len([1 for m in self.media if m.get("id") == id]), (
f"cannot add 2 pieces of media with the same id {id}"
)
media.set("id", id)
self.media.append(media)
return media
def get_media_by_id(self, id: str, default=None) -> Media:
for m in self.media:
if m.get("id") == id: return m
if m.get("id") == id:
return m
return default
def remove_duplicate_media_by_hash(self) -> None:
@@ -159,7 +173,8 @@ class Metadata:
with open(filename, "rb") as f:
while True:
buf = f.read(chunksize)
if not buf: break
if not buf:
break
hash_algo.update(buf)
return hash_algo.hexdigest()
@@ -167,15 +182,18 @@ class Metadata:
new_media = []
for m in self.media:
h = m.get("hash")
if not h: h = calculate_hash_in_chunks(hashlib.sha256(), int(1.6e7), m.filename)
if len(h) and h in media_hashes: continue
if not h:
h = calculate_hash_in_chunks(hashlib.sha256(), int(1.6e7), m.filename)
if len(h) and h in media_hashes:
continue
media_hashes.add(h)
new_media.append(m)
self.media = new_media
def get_first_image(self, default=None) -> Media:
for m in self.media:
if "image" in m.mimetype: return m
if "image" in m.mimetype:
return m
return default
def set_final_media(self, final: Media) -> Metadata:
@@ -193,22 +211,25 @@ class Metadata:
def __str__(self) -> str:
return self.__repr__()
@staticmethod
def choose_most_complete(results: List[Metadata]) -> Metadata:
# returns the most complete result from a list of results
# prioritizes results with more media, then more metadata
if len(results) == 0: return None
if len(results) == 1: return results[0]
if len(results) == 0:
return None
if len(results) == 1:
return results[0]
most_complete = results[0]
for r in results[1:]:
if len(r.media) > len(most_complete.media): most_complete = r
elif len(r.media) == len(most_complete.media) and len(r.metadata) > len(most_complete.metadata): most_complete = r
if len(r.media) > len(most_complete.media):
most_complete = r
elif len(r.media) == len(most_complete.media) and len(r.metadata) > len(most_complete.metadata):
most_complete = r
return most_complete
def set_context(self, key: str, val: Any) -> Metadata:
self._context[key] = val
return self
def get_context(self, key: str, default: Any = None) -> Any:
return self._context.get(key, default)
return self._context.get(key, default)

129
src/auto_archiver/core/module.py
View File

@@ -3,10 +3,12 @@ Defines the Step abstract base class, which acts as a blueprint for steps in the
by handling user configuration, validating the steps properties, and implementing dynamic instantiation.
"""
from __future__ import annotations
import subprocess
from dataclasses import dataclass
from typing import List, TYPE_CHECKING
from typing import List, TYPE_CHECKING, Type
import shutil
import ast
import copy
@@ -16,7 +18,7 @@ import os
from os.path import join
from loguru import logger
import auto_archiver
from auto_archiver.core.consts import DEFAULT_MANIFEST, MANIFEST_FILE
from auto_archiver.core.consts import DEFAULT_MANIFEST, MANIFEST_FILE, SetupError
if TYPE_CHECKING:
from .base_module import BaseModule
@@ -24,17 +26,17 @@ if TYPE_CHECKING:
HAS_SETUP_PATHS = False
class ModuleFactory:
class ModuleFactory:
def __init__(self):
self._lazy_modules = {}
def setup_paths(self, paths: list[str]) -> None:
"""
Sets up the paths for the modules to be loaded from
This is necessary for the modules to be imported correctly
"""
global HAS_SETUP_PATHS
@@ -46,45 +48,55 @@ class ModuleFactory:
# see odoo/module/module.py -> initialize_sys_path
if path not in auto_archiver.modules.__path__:
if HAS_SETUP_PATHS == True:
logger.warning(f"You are attempting to re-initialise the module paths with: '{path}' for a 2nd time. \
if HAS_SETUP_PATHS:
logger.warning(
f"You are attempting to re-initialise the module paths with: '{path}' for a 2nd time. \
This could lead to unexpected behaviour. It is recommended to only use a single modules path. \
If you wish to load modules from different paths then load a 2nd python interpreter (e.g. using multiprocessing).")
auto_archiver.modules.__path__.append(path)
If you wish to load modules from different paths then load a 2nd python interpreter (e.g. using multiprocessing)."
)
auto_archiver.modules.__path__.append(path)
# sort based on the length of the path, so that the longest path is last in the list
auto_archiver.modules.__path__ = sorted(auto_archiver.modules.__path__, key=len, reverse=True)
HAS_SETUP_PATHS = True
def get_module(self, module_name: str, config: dict) -> BaseModule:
def get_module(self, module_name: str, config: dict) -> Type[BaseModule]:
"""
Gets and sets up a module using the provided config
This will actually load and instantiate the module, and load all its dependencies (i.e. not lazy)
"""
return self.get_module_lazy(module_name).load(config)
def get_module_lazy(self, module_name: str, suppress_warnings: bool = False) -> LazyBaseModule:
"""
Lazily loads a module, returning a LazyBaseModule
This has all the information about the module, but does not load the module itself or its dependencies
To load an actual module, call .setup() on a lazy module
"""
if module_name in self._lazy_modules:
return self._lazy_modules[module_name]
available = self.available_modules(limit_to_modules=[module_name], suppress_warnings=suppress_warnings)
if not available:
raise IndexError(f"Module '{module_name}' not found. Are you sure it's installed/exists?")
message = f"Module '{module_name}' not found. Are you sure it's installed/exists?"
if "archiver" in module_name:
message += f" Did you mean '{module_name.replace('archiver', 'extractor')}'?"
elif "gsheet" in module_name:
message += " Did you mean 'gsheet_feeder_db'?"
elif "atlos" in module_name:
message += " Did you mean 'atlos_feeder_db_storage'?"
raise IndexError(message)
return available[0]
def available_modules(self, limit_to_modules: List[str]= [], suppress_warnings: bool = False) -> List[LazyBaseModule]:
def available_modules(
self, limit_to_modules: List[str] = [], suppress_warnings: bool = False
) -> List[LazyBaseModule]:
# search through all valid 'modules' paths. Default is 'modules' in the current directory
# see odoo/modules/module.py -> get_modules
@@ -116,7 +128,7 @@ class ModuleFactory:
self._lazy_modules[possible_module] = lazy_module
all_modules.append(lazy_module)
if not suppress_warnings:
for module in limit_to_modules:
if not any(module == m.name for m in all_modules):
@@ -124,15 +136,16 @@ class ModuleFactory:
return all_modules
@dataclass
class LazyBaseModule:
"""
A lazy module class, which only loads the manifest and does not load the module itself.
This is useful for getting information about a module without actually loading it.
"""
name: str
description: str
path: str
@@ -149,30 +162,30 @@ class LazyBaseModule:
@property
def type(self):
return self.manifest['type']
return self.manifest["type"]
@property
def entry_point(self):
if not self._entry_point and not self.manifest['entry_point']:
if not self._entry_point and not self.manifest["entry_point"]:
# try to create the entry point from the module name
self._entry_point = f"{self.name}::{self.name.replace('_', ' ').title().replace(' ', '')}"
return self._entry_point
@property
def dependencies(self) -> dict:
return self.manifest['dependencies']
return self.manifest["dependencies"]
@property
def configs(self) -> dict:
return self.manifest['configs']
return self.manifest["configs"]
@property
def requires_setup(self) -> bool:
return self.manifest['requires_setup']
return self.manifest["requires_setup"]
@property
def display_name(self) -> str:
return self.manifest['name']
return self.manifest["name"]
@property
def manifest(self) -> dict:
@@ -186,17 +199,16 @@ class LazyBaseModule:
try:
manifest.update(ast.literal_eval(f.read()))
except (ValueError, TypeError, SyntaxError, MemoryError, RecursionError) as e:
raise ValueError(f"Error loading manifest from file {self.path}/{MANIFEST_FILE}: {e}")
raise ValueError(f"Error loading manifest from file {self.path}/{MANIFEST_FILE}: {e}") from e
self._manifest = manifest
self._entry_point = manifest['entry_point']
self.description = manifest['description']
self.version = manifest['version']
self._entry_point = manifest["entry_point"]
self.description = manifest["description"]
self.version = manifest["version"]
return manifest
def load(self, config) -> BaseModule:
if self._instance:
return self._instance
@@ -207,19 +219,21 @@ class LazyBaseModule:
# clear out any empty strings that a user may have erroneously added
continue
if not check(dep):
logger.error(f"Module '{self.name}' requires external dependency '{dep}' which is not available/setup. \
Have you installed the required dependencies for the '{self.name}' module? See the README for more information.")
exit(1)
logger.error(
f"Module '{self.name}' requires external dependency '{dep}' which is not available/setup. \
Have you installed the required dependencies for the '{self.name}' module? See the documentation for more information."
)
raise SetupError()
def check_python_dep(dep):
# first check if it's a module:
try:
m = self.module_factory.get_module_lazy(dep, suppress_warnings=True)
try:
# we must now load this module and set it up with the config
# we must now load this module and set it up with the config
m.load(config)
return True
except:
except Exception:
logger.error(f"Unable to setup module '{dep}' for use in module '{self.name}'")
return False
except IndexError:
@@ -228,13 +242,26 @@ class LazyBaseModule:
return find_spec(dep)
check_deps(self.dependencies.get('python', []), check_python_dep)
check_deps(self.dependencies.get('bin', []), lambda dep: shutil.which(dep))
def check_bin_dep(dep):
dep_exists = shutil.which(dep)
if dep == "docker":
if os.environ.get("RUNNING_IN_DOCKER"):
# this is only for the WACZ enricher, which requires docker
# if we're already running in docker then we don't need docker
return True
# check if docker daemon is running
return dep_exists and subprocess.run(["docker", "ps", "-q"]).returncode == 0
return dep_exists
check_deps(self.dependencies.get("python", []), check_python_dep)
check_deps(self.dependencies.get("bin", []), check_bin_dep)
logger.debug(f"Loading module '{self.display_name}'...")
for qualname in [self.name, f'auto_archiver.modules.{self.name}']:
for qualname in [self.name, f"auto_archiver.modules.{self.name}"]:
try:
# first import the whole module, to make sure it's working properly
__import__(qualname)
@@ -243,10 +270,10 @@ class LazyBaseModule:
pass
# then import the file for the entry point
file_name, class_name = self.entry_point.split('::')
sub_qualname = f'{qualname}.{file_name}'
file_name, class_name = self.entry_point.split("::")
sub_qualname = f"{qualname}.{file_name}"
__import__(f'{qualname}.{file_name}', fromlist=[self.entry_point])
__import__(f"{qualname}.{file_name}", fromlist=[self.entry_point])
# finally, get the class instance
instance: BaseModule = getattr(sys.modules[sub_qualname], class_name)()
@@ -254,11 +281,11 @@ class LazyBaseModule:
instance.name = self.name
instance.display_name = self.display_name
instance.module_factory = self.module_factory
# merge the default config with the user config
default_config = dict((k, v['default']) for k, v in self.configs.items() if 'default' in v)
config[self.name] = default_config | config.get(self.name, {})
# merge the default config with the user config
default_config = dict((k, v["default"]) for k, v in self.configs.items() if "default" in v)
config[self.name] = default_config | config.get(self.name, {})
instance.config_setup(config)
instance.setup()
@@ -267,4 +294,4 @@ class LazyBaseModule:
return instance
def __repr__(self):
return f"Module<'{self.display_name}' ({self.name})>"
return f"Module<'{self.display_name}' ({self.name})>"

353
src/auto_archiver/core/orchestrator.py
View File

@@ -1,6 +1,6 @@
""" Orchestrates all archiving steps, including feeding items,
archiving them with specific archivers, enrichment, storage,
formatting, database operations and clean up.
"""Orchestrates all archiving steps, including feeding items,
archiving them with specific archivers, enrichment, storage,
formatting, database operations and clean up.
"""
@@ -15,24 +15,32 @@ from copy import copy
from rich_argparse import RichHelpFormatter
from loguru import logger
import requests
from .metadata import Metadata, Media
from auto_archiver.version import __version__
from .config import read_yaml, store_yaml, to_dot_notation, merge_dicts, is_valid_config, \
DefaultValidatingParser, UniqueAppendAction, AuthenticationJsonParseAction, DEFAULT_CONFIG_FILE
from .config import (
read_yaml,
store_yaml,
to_dot_notation,
merge_dicts,
is_valid_config,
DefaultValidatingParser,
UniqueAppendAction,
AuthenticationJsonParseAction,
DEFAULT_CONFIG_FILE,
)
from .module import ModuleFactory, LazyBaseModule
from . import validators, Feeder, Extractor, Database, Storage, Formatter, Enricher
from .consts import MODULE_TYPES
from .consts import MODULE_TYPES, SetupError
from auto_archiver.utils.url import check_url_or_raise
if TYPE_CHECKING:
from .base_module import BaseModule
from .module import LazyBaseModule
class SetupError(ValueError):
pass
class ArchivingOrchestrator:
class ArchivingOrchestrator:
# instance variables
module_factory: ModuleFactory
setup_finished: bool
@@ -62,20 +70,63 @@ class ArchivingOrchestrator:
epilog="Check the code at https://github.com/bellingcat/auto-archiver",
formatter_class=RichHelpFormatter,
)
parser.add_argument('--help', '-h', action='store_true', dest='help', help='show a full help message and exit')
parser.add_argument('--version', action='version', version=__version__)
parser.add_argument('--config', action='store', dest="config_file", help='the filename of the YAML configuration file (defaults to \'config.yaml\')', default=DEFAULT_CONFIG_FILE)
parser.add_argument('--mode', action='store', dest='mode', type=str, choices=['simple', 'full'], help='the mode to run the archiver in', default='simple')
parser.add_argument("--help", "-h", action="store_true", dest="help", help="show a full help message and exit")
parser.add_argument("--version", action="version", version=__version__)
parser.add_argument(
"--config",
action="store",
dest="config_file",
help="the filename of the YAML configuration file (defaults to 'config.yaml')",
default=DEFAULT_CONFIG_FILE,
)
parser.add_argument(
"--mode",
action="store",
dest="mode",
type=str,
choices=["simple", "full"],
help="the mode to run the archiver in",
default="simple",
)
# override the default 'help' so we can inject all the configs and show those
parser.add_argument('-s', '--store', dest='store', default=False, help='Store the created config in the config file', action=argparse.BooleanOptionalAction)
parser.add_argument('--module_paths', dest='module_paths', nargs='+', default=[], help='additional paths to search for modules', action=UniqueAppendAction)
parser.add_argument(
"-s",
"--store",
dest="store",
default=False,
help="Store the created config in the config file",
action=argparse.BooleanOptionalAction,
)
parser.add_argument(
"--module_paths",
dest="module_paths",
nargs="+",
default=[],
help="additional paths to search for modules",
action=UniqueAppendAction,
)
self.basic_parser = parser
return parser
def check_steps(self, config):
for module_type in MODULE_TYPES:
if not config["steps"].get(f"{module_type}s", []):
if (module_type == "feeder" or module_type == "formatter") and config["steps"].get(f"{module_type}"):
raise SetupError(
f"It appears you have '{module_type}' set under 'steps' in your configuration file, but as of version 0.13.0 of Auto Archiver, you must use '{module_type}s'. Change this in your configuration file and try again. \
Here's how that would look: \n\nsteps:\n {module_type}s:\n - [your_{module_type}_name_here]\n {'extractors:...' if module_type == 'feeder' else '...'}\n"
)
if module_type == "extractor" and config["steps"].get("archivers"):
raise SetupError(
"As of version 0.13.0 of Auto Archiver, the 'archivers' step name has been changed to 'extractors'. Change this in your configuration file and try again. \
Here's how that would look: \n\nsteps:\n extractors:\n - [your_extractor_name_here]\n enrichers:...\n"
)
raise SetupError(
f"No {module_type}s were configured. Make sure to set at least one {module_type} in your configuration file or on the command line (using --{module_type}s)"
)
def setup_complete_parser(self, basic_config: dict, yaml_config: dict, unused_args: list[str]) -> None:
# modules parser to get the overridden 'steps' values
modules_parser = argparse.ArgumentParser(
add_help=False,
@@ -83,7 +134,9 @@ class ArchivingOrchestrator:
self.add_modules_args(modules_parser)
cli_modules, unused_args = modules_parser.parse_known_args(unused_args)
for module_type in MODULE_TYPES:
yaml_config['steps'][f"{module_type}s"] = getattr(cli_modules, f"{module_type}s", []) or yaml_config['steps'].get(f"{module_type}s", [])
yaml_config["steps"][f"{module_type}s"] = getattr(cli_modules, f"{module_type}s", []) or yaml_config[
"steps"
].get(f"{module_type}s", [])
parser = DefaultValidatingParser(
add_help=False,
@@ -100,39 +153,38 @@ class ArchivingOrchestrator:
# but should we add them? Or should we just add them to the 'complete' parser?
if is_valid_config(yaml_config):
self.check_steps(yaml_config)
# only load the modules enabled in config
# TODO: if some steps are empty (e.g. 'feeders' is empty), should we default to the 'simple' ones? Or only if they are ALL empty?
enabled_modules = []
# first loads the modules from the config file, then from the command line
for module_type in MODULE_TYPES:
enabled_modules.extend(yaml_config['steps'].get(f"{module_type}s", []))
enabled_modules.extend(yaml_config["steps"].get(f"{module_type}s", []))
# clear out duplicates, but keep the order
enabled_modules = list(dict.fromkeys(enabled_modules))
avail_modules = self.module_factory.available_modules(limit_to_modules=enabled_modules, suppress_warnings=True)
avail_modules = self.module_factory.available_modules(
limit_to_modules=enabled_modules, suppress_warnings=True
)
self.add_individual_module_args(avail_modules, parser)
elif basic_config.mode == 'simple':
elif basic_config.mode == "simple":
simple_modules = [module for module in self.module_factory.available_modules() if not module.requires_setup]
self.add_individual_module_args(simple_modules, parser)
# for simple mode, we use the cli_feeder and any modules that don't require setup
if not yaml_config['steps']['feeders']:
yaml_config['steps']['feeders'] = ['cli_feeder']
# add them to the config
for module in simple_modules:
for module_type in module.type:
yaml_config['steps'].setdefault(f"{module_type}s", []).append(module.name)
yaml_config["steps"].setdefault(f"{module_type}s", []).append(module.name)
else:
# load all modules, they're not using the 'simple' mode
all_modules = self.module_factory.available_modules()
# add all the modules to the steps
for module in all_modules:
for module_type in module.type:
yaml_config['steps'].setdefault(f"{module_type}s", []).append(module.name)
yaml_config["steps"].setdefault(f"{module_type}s", []).append(module.name)
self.add_individual_module_args(all_modules, parser)
parser.set_defaults(**to_dot_notation(yaml_config))
# reload the parser with the new arguments, now that we have them
@@ -158,41 +210,75 @@ class ArchivingOrchestrator:
store_yaml(config, basic_config.config_file)
return config
def add_modules_args(self, parser: argparse.ArgumentParser = None):
if not parser:
parser = self.parser
# Module loading from the command line
for module_type in MODULE_TYPES:
parser.add_argument(f'--{module_type}s', dest=f'{module_type}s', nargs='+', help=f'the {module_type}s to use', default=[], action=UniqueAppendAction)
parser.add_argument(
f"--{module_type}s",
dest=f"{module_type}s",
nargs="+",
help=f"the {module_type}s to use",
default=[],
action=UniqueAppendAction,
)
def add_additional_args(self, parser: argparse.ArgumentParser = None):
if not parser:
parser = self.parser
# allow passing URLs directly on the command line
parser.add_argument('urls', nargs='*', default=[], help='URL(s) to archive, either a single URL or a list of urls, should not come from config.yaml')
parser.add_argument('--authentication', dest='authentication', help='A dictionary of sites and their authentication methods \
parser.add_argument(
"--authentication",
dest="authentication",
help="A dictionary of sites and their authentication methods \
(token, username etc.) that extractors can use to log into \
a website. If passing this on the command line, use a JSON string. \
You may also pass a path to a valid JSON/YAML file which will be parsed.',
default={},
nargs="?",
action=AuthenticationJsonParseAction)
You may also pass a path to a valid JSON/YAML file which will be parsed.",
default={},
nargs="?",
action=AuthenticationJsonParseAction,
)
# logging arguments
parser.add_argument('--logging.level', action='store', dest='logging.level', choices=['INFO', 'DEBUG', 'ERROR', 'WARNING'], help='the logging level to use', default='INFO', type=str.upper)
parser.add_argument('--logging.file', action='store', dest='logging.file', help='the logging file to write to', default=None)
parser.add_argument('--logging.rotation', action='store', dest='logging.rotation', help='the logging rotation to use', default=None)
def add_individual_module_args(self, modules: list[LazyBaseModule] = None, parser: argparse.ArgumentParser = None) -> None:
parser.add_argument(
"--logging.level",
action="store",
dest="logging.level",
choices=["INFO", "DEBUG", "ERROR", "WARNING"],
help="the logging level to use",
default="INFO",
type=str.upper,
)
parser.add_argument(
"--logging.file", action="store", dest="logging.file", help="the logging file to write to", default=None
)
parser.add_argument(
"--logging.rotation",
action="store",
dest="logging.rotation",
help="the logging rotation to use",
default=None,
)
def add_individual_module_args(
self, modules: list[LazyBaseModule] = None, parser: argparse.ArgumentParser = None
) -> None:
if not modules:
modules = self.module_factory.available_modules()
for module in modules:
if module.name == "cli_feeder":
# special case. For the CLI feeder, allow passing URLs directly on the command line without setting --cli_feeder.urls=
parser.add_argument(
"urls",
nargs="*",
default=[],
help="URL(s) to archive, either a single URL or a list of urls, should not come from config.yaml",
)
continue
if not module.configs:
# this module has no configs, don't show anything in the help
@@ -202,21 +288,21 @@ class ArchivingOrchestrator:
group = parser.add_argument_group(module.display_name or module.name, f"{module.description[:100]}...")
for name, kwargs in module.configs.items():
if not kwargs.get('metavar', None):
if not kwargs.get("metavar", None):
# make a nicer metavar, metavar is what's used in the help, e.g. --cli_feeder.urls [METAVAR]
kwargs['metavar'] = name.upper()
kwargs["metavar"] = name.upper()
if kwargs.get('required', False):
if kwargs.get("required", False):
# required args shouldn't have a 'default' value, remove it
kwargs.pop('default', None)
kwargs.pop("default", None)
kwargs.pop('cli_set', None)
should_store = kwargs.pop('should_store', False)
kwargs['dest'] = f"{module.name}.{kwargs.pop('dest', name)}"
kwargs.pop("cli_set", None)
should_store = kwargs.pop("should_store", False)
kwargs["dest"] = f"{module.name}.{kwargs.pop('dest', name)}"
try:
kwargs['type'] = getattr(validators, kwargs.get('type', '__invalid__'))
kwargs["type"] = getattr(validators, kwargs.get("type", "__invalid__"))
except AttributeError:
kwargs['type'] = __builtins__.get(kwargs.get('type'), str)
kwargs["type"] = __builtins__.get(kwargs.get("type"), str)
arg = group.add_argument(f"--{module.name}.{name}", **kwargs)
arg.should_store = should_store
@@ -231,12 +317,11 @@ class ArchivingOrchestrator:
self.basic_parser.exit()
def setup_logging(self, config):
logging_config = config["logging"]
logging_config = config['logging']
if logging_config.get('enabled', True) is False:
if logging_config.get("enabled", True) is False:
# disabled logging settings, they're set on a higher level
logger.disable('auto_archiver')
logger.disable("auto_archiver")
return
# setup loguru logging
@@ -246,67 +331,63 @@ class ArchivingOrchestrator:
pass
# add other logging info
if self.logger_id is None: # note - need direct comparison to None since need to consider falsy value 0
self.logger_id = logger.add(sys.stderr, level=logging_config['level'])
if log_file := logging_config['file']:
logger.add(log_file) if not logging_config['rotation'] else logger.add(log_file, rotation=logging_config['rotation'])
if self.logger_id is None: # note - need direct comparison to None since need to consider falsy value 0
self.logger_id = logger.add(sys.stderr, level=logging_config["level"])
if log_file := logging_config["file"]:
logger.add(log_file) if not logging_config["rotation"] else logger.add(
log_file, rotation=logging_config["rotation"]
)
def install_modules(self, modules_by_type):
"""
Traverses all modules in 'steps' and loads them into the orchestrator, storing them in the
Traverses all modules in 'steps' and loads them into the orchestrator, storing them in the
orchestrator's attributes (self.feeders, self.extractors etc.). If no modules of a certain type
are loaded, the program will exit with an error message.
"""
invalid_modules = []
for module_type in MODULE_TYPES:
step_items = []
modules_to_load = modules_by_type[f"{module_type}s"]
if not modules_to_load:
raise SetupError(f"No {module_type}s were configured. Make sure to set at least one {module_type} in your configuration file or on the command line (using --{module_type}s)")
raise SetupError(
f"No {module_type}s were configured. Make sure to set at least one {module_type} in your configuration file or on the command line (using --{module_type}s)"
)
def check_steps_ok():
if not len(step_items):
if len(modules_to_load):
logger.error(f"Unable to load any {module_type}s. Tried the following, but none were available: {modules_to_load}")
raise SetupError(f"NO {module_type.upper()}S LOADED. Please check your configuration and try again.")
logger.error(
f"Unable to load any {module_type}s. Tried the following, but none were available: {modules_to_load}"
)
raise SetupError(
f"NO {module_type.upper()}S LOADED. Please check your configuration and try again."
)
if (module_type == 'feeder' or module_type == 'formatter') and len(step_items) > 1:
raise SetupError(f"Only one {module_type} is allowed, found {len(step_items)} {module_type}s. Please remove one of the following from your configuration file: {modules_to_load}")
if (module_type == "feeder" or module_type == "formatter") and len(step_items) > 1:
raise SetupError(
f"Only one {module_type} is allowed, found {len(step_items)} {module_type}s. Please remove one of the following from your configuration file: {modules_to_load}"
)
for module in modules_to_load:
if module == 'cli_feeder':
# cli_feeder is a pseudo module, it just takes the command line args for [URLS]
urls = self.config['urls']
if not urls:
raise SetupError("No URLs provided. Please provide at least one URL via the command line, or set up an alternative feeder. Use --help for more information.")
def feed(self) -> Generator[Metadata]:
for url in urls:
logger.debug(f"Processing URL: '{url}'")
yield Metadata().set_url(url)
pseudo_module = type('CLIFeeder', (Feeder,), {
'name': 'cli_feeder',
'display_name': 'CLI Feeder',
'__iter__': feed
})()
pseudo_module.__iter__ = feed
step_items.append(pseudo_module)
continue
if module in invalid_modules:
continue
# check to make sure that we're trying to load it as the correct type - i.e. make sure the user hasn't put it under the wrong 'step'
lazy_module: LazyBaseModule = self.module_factory.get_module_lazy(module)
if module_type not in lazy_module.type:
types = ",".join(f"'{t}'" for t in lazy_module.type)
raise SetupError(
f"Configuration Error: Module '{module}' is not a {module_type}, but has the types: {types}. Please check you set this module up under the right step in your orchestration file."
)
loaded_module = None
try:
loaded_module: BaseModule = self.module_factory.get_module(module, self.config)
loaded_module: BaseModule = lazy_module.load(self.config)
except (KeyboardInterrupt, Exception) as e:
logger.error(f"Error during setup of modules: {e}\n{traceback.format_exc()}")
if module_type == 'extractor' and loaded_module.name == module:
if not isinstance(e, KeyboardInterrupt) and not isinstance(e, SetupError):
logger.error(f"Error during setup of modules: {e}\n{traceback.format_exc()}")
if loaded_module and module_type == "extractor":
loaded_module.cleanup()
raise e
@@ -321,11 +402,13 @@ class ArchivingOrchestrator:
def load_config(self, config_file: str) -> dict:
if not os.path.exists(config_file) and config_file != DEFAULT_CONFIG_FILE:
logger.error(f"The configuration file {config_file} was not found. Make sure the file exists and try again, or run without the --config file to use the default settings.")
logger.error(
f"The configuration file {config_file} was not found. Make sure the file exists and try again, or run without the --config file to use the default settings."
)
raise FileNotFoundError(f"Configuration file {config_file} not found")
return read_yaml(config_file)
def setup_config(self, args: list) -> dict:
"""
Sets up the configuration file, merging the default config with the user's config
@@ -349,30 +432,51 @@ class ArchivingOrchestrator:
return self.setup_complete_parser(basic_config, yaml_config, unused_args)
def check_for_updates(self):
response = requests.get("https://pypi.org/pypi/auto-archiver/json").json()
latest_version = response["info"]["version"]
# check version compared to current version
if latest_version != __version__:
if os.environ.get("RUNNING_IN_DOCKER"):
update_cmd = "`docker pull bellingcat/auto-archiver:latest`"
else:
update_cmd = "`pip install --upgrade auto-archiver`"
logger.warning("")
logger.warning("********* IMPORTANT: UPDATE AVAILABLE ********")
logger.warning(f"A new version of auto-archiver is available (v{latest_version}, you have {__version__})")
logger.warning(f"Make sure to update to the latest version using: {update_cmd}")
logger.warning("")
def setup(self, args: list):
"""
Function to configure all setup of the orchestrator: setup configs and load modules.
This method should only ever be called once
"""
self.check_for_updates()
if self.setup_finished:
logger.warning("The `setup_config()` function should only ever be run once. \
logger.warning(
"The `setup_config()` function should only ever be run once. \
If you need to re-run the setup, please re-instantiate a new instance of the orchestrator. \
For code implementatations, you should call .setup_config() once then you may call .feed() \
multiple times to archive multiple URLs.")
multiple times to archive multiple URLs."
)
return
self.setup_basic_parser()
self.config = self.setup_config(args)
logger.info(f"======== Welcome to the AUTO ARCHIVER ({__version__}) ==========")
self.install_modules(self.config['steps'])
self.install_modules(self.config["steps"])
# log out the modules that were loaded
for module_type in MODULE_TYPES:
logger.info(f"{module_type.upper()}S: " + ", ".join(m.display_name for m in getattr(self, f"{module_type}s")))
logger.info(
f"{module_type.upper()}S: " + ", ".join(m.display_name for m in getattr(self, f"{module_type}s"))
)
self.setup_finished = True
def _command_line_run(self, args: list) -> Generator[Metadata]:
@@ -380,9 +484,9 @@ class ArchivingOrchestrator:
This is the main entry point for the orchestrator, when run from the command line.
:param args: list of arguments to pass to the orchestrator - these are the command line args
You should not call this method from code implementations.
This method sets up the configuration, loads the modules, and runs the feed.
If you wish to make code invocations yourself, you should use the 'setup' and 'feed' methods separately.
To test configurations, without loading any modules you can also first call 'setup_configs'
@@ -400,7 +504,6 @@ class ArchivingOrchestrator:
e.cleanup()
def feed(self) -> Generator[Metadata]:
url_count = 0
for feeder in self.feeders:
for item in feeder:
@@ -431,9 +534,9 @@ class ArchivingOrchestrator:
self.cleanup()
exit()
except Exception as e:
logger.error(f'Got unexpected error on item {item}: {e}\n{traceback.format_exc()}')
logger.error(f"Got unexpected error on item {item}: {e}\n{traceback.format_exc()}")
for d in self.databases:
if type(e) == AssertionError:
if isinstance(e, AssertionError):
d.failed(item, str(e))
else:
d.failed(item, reason="unexpected error")
@@ -446,13 +549,13 @@ class ArchivingOrchestrator:
def archive(self, result: Metadata) -> Union[Metadata, None]:
"""
Runs the archiving process for a single URL
1. Each archiver can sanitize its own URLs
2. Check for cached results in Databases, and signal start to the databases
3. Call Archivers until one succeeds
4. Call Enrichers
5. Store all downloaded/generated media
6. Call selected Formatter and store formatted if needed
Runs the archiving process for a single URL
1. Each archiver can sanitize its own URLs
2. Check for cached results in Databases, and signal start to the databases
3. Call Archivers until one succeeds
4. Call Enrichers
5. Store all downloaded/generated media
6. Call selected Formatter and store formatted if needed
"""
original_url = result.get_url().strip()
@@ -468,7 +571,8 @@ class ArchivingOrchestrator:
url = a.sanitize_url(url)
result.set_url(url)
if original_url != url: result.set("original_url", original_url)
if original_url != url:
result.set("original_url", original_url)
# 2 - notify start to DBs, propagate already archived if feature enabled in DBs
cached_result = None
@@ -479,7 +583,8 @@ class ArchivingOrchestrator:
if cached_result:
logger.debug("Found previously archived entry")
for d in self.databases:
try: d.done(cached_result, cached=True)
try:
d.done(cached_result, cached=True)
except Exception as e:
logger.error(f"ERROR database {d.name}: {e}: {traceback.format_exc()}")
return cached_result
@@ -489,13 +594,15 @@ class ArchivingOrchestrator:
logger.info(f"Trying extractor {a.name} for {url}")
try:
result.merge(a.download(result))
if result.is_success(): break
if result.is_success():
break
except Exception as e:
logger.error(f"ERROR archiver {a.name}: {e}: {traceback.format_exc()}")
# 4 - call enrichers to work with archived content
for e in self.enrichers:
try: e.enrich(result)
try:
e.enrich(result)
except Exception as exc:
logger.error(f"ERROR enricher {e.name}: {exc}: {traceback.format_exc()}")
@@ -513,12 +620,12 @@ class ArchivingOrchestrator:
# signal completion to databases and archivers
for d in self.databases:
try: d.done(result)
try:
d.done(result)
except Exception as e:
logger.error(f"ERROR database {d.name}: {e}: {traceback.format_exc()}")
return result
def setup_authentication(self, config: dict) -> dict:
"""
@@ -527,7 +634,7 @@ class ArchivingOrchestrator:
Split up strings into multiple sites if they are comma separated
"""
authentication = config.get('authentication', {})
authentication = config.get("authentication", {})
# extract out concatenated sites
for key, val in copy(authentication).items():
@@ -536,8 +643,8 @@ class ArchivingOrchestrator:
site = site.strip()
authentication[site] = val
del authentication[key]
config['authentication'] = authentication
config["authentication"] = authentication
return config
# Helper Properties

61
src/auto_archiver/core/storage.py
View File

@@ -1,5 +1,22 @@
"""
Base module for Storage modules – modular components that store media objects in various locations.
If you are looking to implement a new storage module, you should subclass the `Storage` class and
implement the `get_cdn_url` and `uploadf` methods.
Your module **must** also have two config variables 'path_generator' and 'filename_generator' which
determine how the key is generated for the media object. The 'path_generator' and 'filename_generator'
variables can be set to one of the following values:
- 'flat': A flat structure with no subfolders
- 'url': A structure based on the URL of the media object
- 'random': A random structure
The 'filename_generator' variable can be set to one of the following values:
- 'random': A random string
- 'static': A replicable strategy such as a hash
If you don't want to use this naming convention, you can override the `set_key` method in your subclass.
"""
from __future__ import annotations
@@ -15,18 +32,19 @@ from auto_archiver.utils.misc import random_str
from auto_archiver.core import Media, BaseModule, Metadata
from auto_archiver.modules.hash_enricher.hash_enricher import HashEnricher
class Storage(BaseModule):
"""
Base class for implementing storage modules in the media archiving framework.
Subclasses must implement the `get_cdn_url` and `uploadf` methods to define their behavior.
"""
def store(self, media: Media, url: str, metadata: Metadata=None) -> None:
if media.is_stored(in_storage=self):
def store(self, media: Media, url: str, metadata: Metadata = None) -> None:
if media.is_stored(in_storage=self):
logger.debug(f"{media.key} already stored, skipping")
return
self.set_key(media, url, metadata)
self.upload(media, metadata=metadata)
media.add_url(self.get_cdn_url(media))
@@ -42,42 +60,55 @@ class Storage(BaseModule):
def uploadf(self, file: IO[bytes], key: str, **kwargs: dict) -> bool:
"""
Uploads (or saves) a file to the storage service/location.
This method should not be called directly, but instead through the 'store' method,
which sets up the media for storage.
"""
pass
def upload(self, media: Media, **kwargs) -> bool:
logger.debug(f'[{self.__class__.__name__}] storing file {media.filename} with key {media.key}')
with open(media.filename, 'rb') as f:
"""
Uploads a media object to the storage service.
This method should not be called directly, but instead be called through the 'store' method,
which sets up the media for storage.
"""
logger.debug(f"[{self.__class__.__name__}] storing file {media.filename} with key {media.key}")
with open(media.filename, "rb") as f:
return self.uploadf(f, media, **kwargs)
def set_key(self, media: Media, url, metadata: Metadata) -> None:
def set_key(self, media: Media, url: str, metadata: Metadata) -> None:
"""takes the media and optionally item info and generates a key"""
if media.key is not None and len(media.key) > 0: return
folder = metadata.get_context('folder', '')
if media.key is not None and len(media.key) > 0:
# media key is already set
return
folder = metadata.get_context("folder", "")
filename, ext = os.path.splitext(media.filename)
# Handle path_generator logic
path_generator = self.config.get("path_generator", "url")
path_generator = self.path_generator
if path_generator == "flat":
path = ""
filename = slugify(filename) # Ensure filename is slugified
elif path_generator == "url":
path = slugify(url)
path = slugify(url)[:70]
elif path_generator == "random":
path = self.config.get("random_path", random_str(24), True)
path = random_str(24)
else:
raise ValueError(f"Invalid path_generator: {path_generator}")
# Handle filename_generator logic
filename_generator = self.config.get("filename_generator", "random")
filename_generator = self.filename_generator
if filename_generator == "random":
filename = random_str(24)
elif filename_generator == "static":
# load the hash_enricher module
he = self.module_factory.get_module(HashEnricher, self.config)
he: HashEnricher = self.module_factory.get_module("hash_enricher", self.config)
hd = he.calculate_hash(media.filename)
filename = hd[:24]
else:
raise ValueError(f"Invalid filename_generator: {filename_generator}")
media.key = os.path.join(folder, path, f"{filename}{ext}")
key = os.path.join(folder, path, f"{filename}{ext}")
media._key = key

5
src/auto_archiver/core/validators.py
View File

@@ -3,11 +3,13 @@ from pathlib import Path
import argparse
import json
def example_validator(value):
if "example" not in value:
raise argparse.ArgumentTypeError(f"{value} is not a valid value for this argument")
return value
def positive_number(value):
if value < 0:
raise argparse.ArgumentTypeError(f"{value} is not a positive number")
@@ -19,5 +21,6 @@ def valid_file(value):
raise argparse.ArgumentTypeError(f"File '{value}' does not exist.")
return value
def json_loader(cli_val):
return json.loads(cli_val)
return json.loads(cli_val)

2
src/auto_archiver/modules/api_db/__init__.py
View File

@@ -1 +1 @@
from .api_db import AAApiDb
from .api_db import AAApiDb

7
src/auto_archiver/modules/api_db/__manifest__.py
View File

@@ -11,8 +11,7 @@
"required": True,
"help": "API endpoint where calls are made to",
},
"api_token": {"default": None,
"help": "API Bearer token."},
"api_token": {"default": None, "help": "API Bearer token."},
"public": {
"default": False,
"type": "bool",
@@ -24,9 +23,9 @@
"help": "which group of users have access to the archive in case public=false as author",
},
"use_api_cache": {
"default": True,
"default": False,
"type": "bool",
"help": "if False then the API database will be queried prior to any archiving operations and stop if the link has already been archived",
"help": "if True then the API database will be queried prior to any archiving operations and stop if the link has already been archived",
},
"store_results": {
"default": True,

26
src/auto_archiver/modules/api_db/api_db.py
View File

@@ -12,10 +12,11 @@ class AAApiDb(Database):
"""Connects to auto-archiver-api instance"""
def fetch(self, item: Metadata) -> Union[Metadata, bool]:
""" query the database for the existence of this item.
Helps avoid re-archiving the same URL multiple times.
"""query the database for the existence of this item.
Helps avoid re-archiving the same URL multiple times.
"""
if not self.use_api_cache: return
if not self.use_api_cache:
return
params = {"url": item.get_url(), "limit": 15}
headers = {"Authorization": f"Bearer {self.api_token}", "accept": "application/json"}
@@ -32,22 +33,25 @@ class AAApiDb(Database):
def done(self, item: Metadata, cached: bool = False) -> None:
"""archival result ready - should be saved to DB"""
if not self.store_results: return
if not self.store_results:
return
if cached:
logger.debug(f"skipping saving archive of {item.get_url()} to the AA API because it was cached")
return
logger.debug(f"saving archive of {item.get_url()} to the AA API.")
payload = {
'author_id': self.author_id,
'url': item.get_url(),
'public': self.public,
'group_id': self.group_id,
'tags': list(self.tags),
'result': item.to_json(),
"author_id": self.author_id,
"url": item.get_url(),
"public": self.public,
"group_id": self.group_id,
"tags": list(self.tags),
"result": item.to_json(),
}
headers = {"Authorization": f"Bearer {self.api_token}"}
response = requests.post(os.path.join(self.api_endpoint, "interop/submit-archive"), json=payload, headers=headers)
response = requests.post(
os.path.join(self.api_endpoint, "interop/submit-archive"), json=payload, headers=headers
)
if response.status_code == 201:
logger.success(f"AA API: {response.json()}")

1
src/auto_archiver/modules/atlos_db/__init__.py
View File

@@ -1 +0,0 @@
from .atlos_db import AtlosDb

38
src/auto_archiver/modules/atlos_db/__manifest__.py
View File

@@ -1,38 +0,0 @@
{
"name": "Atlos Database",
"type": ["database"],
"entry_point": "atlos_db::AtlosDb",
"requires_setup": True,
"dependencies":
{"python": ["loguru",
""],
"bin": [""]},
"configs": {
"api_token": {
"default": None,
"help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/",
"required": True,
"type": "str",
},
"atlos_url": {
"default": "https://platform.atlos.org",
"help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.",
"type": "str"
},
},
"description": """
Handles integration with the Atlos platform for managing archival results.
### Features
- Outputs archival results to the Atlos API for storage and tracking.
- Updates failure status with error details when archiving fails.
- Processes and formats metadata, including ISO formatting for datetime fields.
- Skips processing for items without an Atlos ID.
### Setup
Required configs:
- atlos_url: Base URL for the Atlos API.
- api_token: Authentication token for API access.
"""
,
}

66
src/auto_archiver/modules/atlos_db/atlos_db.py
View File

@@ -1,66 +0,0 @@
from typing import Union
import requests
from loguru import logger
from auto_archiver.core import Database
from auto_archiver.core import Metadata
class AtlosDb(Database):
"""
Outputs results to Atlos
"""
def failed(self, item: Metadata, reason: str) -> None:
"""Update DB accordingly for failure"""
# If the item has no Atlos ID, there's nothing for us to do
if not item.metadata.get("atlos_id"):
logger.info(f"Item {item.get_url()} has no Atlos ID, skipping")
return
requests.post(
f"{self.atlos_url}/api/v2/source_material/metadata/{item.metadata['atlos_id']}/auto_archiver",
headers={"Authorization": f"Bearer {self.api_token}"},
json={"metadata": {"processed": True, "status": "error", "error": reason}},
).raise_for_status()
logger.info(
f"Stored failure for {item.get_url()} (ID {item.metadata['atlos_id']}) on Atlos: {reason}"
)
def fetch(self, item: Metadata) -> Union[Metadata, bool]:
"""check and fetch if the given item has been archived already, each
database should handle its own caching, and configuration mechanisms"""
return False
def _process_metadata(self, item: Metadata) -> dict:
"""Process metadata for storage on Atlos. Will convert any datetime
objects to ISO format."""
return {
k: v.isoformat() if hasattr(v, "isoformat") else v
for k, v in item.metadata.items()
}
def done(self, item: Metadata, cached: bool = False) -> None:
"""archival result ready - should be saved to DB"""
if not item.metadata.get("atlos_id"):
logger.info(f"Item {item.get_url()} has no Atlos ID, skipping")
return
requests.post(
f"{self.atlos_url}/api/v2/source_material/metadata/{item.metadata['atlos_id']}/auto_archiver",
headers={"Authorization": f"Bearer {self.api_token}"},
json={
"metadata": dict(
processed=True,
status="success",
results=self._process_metadata(item),
)
},
).raise_for_status()
logger.info(
f"Stored success for {item.get_url()} (ID {item.metadata['atlos_id']}) on Atlos"
)

1
src/auto_archiver/modules/atlos_feeder/__init__.py
View File

@@ -1 +0,0 @@
from .atlos_feeder import AtlosFeeder

34
src/auto_archiver/modules/atlos_feeder/__manifest__.py
View File

@@ -1,34 +0,0 @@
{
"name": "Atlos Feeder",
"type": ["feeder"],
"requires_setup": True,
"dependencies": {
"python": ["loguru", "requests"],
},
"configs": {
"api_token": {
"type": "str",
"required": True,
"help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/",
},
"atlos_url": {
"default": "https://platform.atlos.org",
"help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.",
"type": "str"
},
},
"description": """
AtlosFeeder: A feeder module that integrates with the Atlos API to fetch source material URLs for archival.
### Features
- Connects to the Atlos API to retrieve a list of source material URLs.
- Filters source materials based on visibility, processing status, and metadata.
- Converts filtered source materials into `Metadata` objects with the relevant `atlos_id` and URL.
- Iterates through paginated results using a cursor for efficient API interaction.
### Notes
- Requires an Atlos API endpoint and a valid API token for authentication.
- Ensures only unprocessed, visible, and ready-to-archive URLs are returned.
- Handles pagination transparently when retrieving data from the Atlos API.
"""
}

42
src/auto_archiver/modules/atlos_feeder/atlos_feeder.py
View File

@@ -1,42 +0,0 @@
import requests
from loguru import logger
from auto_archiver.core import Feeder
from auto_archiver.core import Metadata
class AtlosFeeder(Feeder):
def __iter__(self) -> Metadata:
# Get all the urls from the Atlos API
count = 0
cursor = None
while True:
response = requests.get(
f"{self.atlos_url}/api/v2/source_material",
headers={"Authorization": f"Bearer {self.api_token}"},
params={"cursor": cursor},
)
data = response.json()
response.raise_for_status()
cursor = data["next"]
for item in data["results"]:
if (
item["source_url"] not in [None, ""]
and (
item["metadata"]
.get("auto_archiver", {})
.get("processed", False)
!= True
)
and item["visibility"] == "visible"
and item["status"] not in ["processing", "pending"]
):
yield Metadata().set_url(item["source_url"]).set(
"atlos_id", item["id"]
)
count += 1
if len(data["results"]) == 0 or cursor is None:
break

1
src/auto_archiver/modules/atlos_feeder_db_storage/__init__.py Normal file
View File

@@ -0,0 +1 @@
from .atlos_feeder_db_storage import AtlosFeederDbStorage

46
src/auto_archiver/modules/atlos_feeder_db_storage/__manifest__.py Normal file
View File

@@ -0,0 +1,46 @@
{
"name": "Atlos Feeder Database Storage",
"type": ["feeder", "database", "storage"],
"entry_point": "atlos_feeder_db_storage::AtlosFeederDbStorage",
"requires_setup": True,
"dependencies": {
"python": ["loguru", "requests"],
},
"configs": {
"api_token": {
"type": "str",
"required": True,
"help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/",
},
"atlos_url": {
"default": "https://platform.atlos.org",
"help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.",
"type": "str",
},
},
"description": """
A module that integrates with the Atlos API to fetch source material URLs for archival, uplaod extracted media,
[Atlos](https://www.atlos.org/) is a visual investigation and archiving platform designed for investigative research, journalism, and open-source intelligence (OSINT).
It helps users organize, analyze, and store media from various sources, making it easier to track and investigate digital evidence.
To get started create a new project and obtain an API token from the settings page. You can group event's into Atlos's 'incidents'.
Here you can add 'source material' by URLn and the Atlos feeder will fetch these URLs for archival.
You can use Atlos only as a 'feeder', however you can also implement the 'database' and 'storage' features to store the media files in Atlos which is recommended.
The Auto Archiver will retain the Atlos ID for each item, ensuring that the media and database outputs are uplaoded back into the relevant media item.
### Features
- Connects to the Atlos API to retrieve a list of source material URLs.
- Iterates through the URLs from all source material items which are unprocessed, visible, and ready to archive.
- If the storage option is selected, it will store the media files alongside the original source material item in Atlos.
- Is the database option is selected it will output the results to the media item, as well as updating failure status with error details when archiving fails.
- Skips Storege/ database upload for items without an Atlos ID - restricting that you must use the Atlos feeder so that it has the Atlos ID to store the results with.
### Notes
- Requires an Atlos account with a project and a valid API token for authentication.
- Ensures only unprocessed, visible, and ready-to-archive URLs are returned.
- Feches any media items within an Atlos project, regardless of separation into incidents.
""",
}

143
src/auto_archiver/modules/atlos_feeder_db_storage/atlos_feeder_db_storage.py Normal file
View File

@@ -0,0 +1,143 @@
import hashlib
import os
from typing import IO, Iterator, Optional, Union
import requests
from loguru import logger
from auto_archiver.core import Database, Feeder, Media, Metadata, Storage
from auto_archiver.utils import calculate_file_hash
class AtlosFeederDbStorage(Feeder, Database, Storage):
def setup(self) -> requests.Session:
"""create and return a persistent session."""
self.session = requests.Session()
def _get(self, endpoint: str, params: Optional[dict] = None) -> dict:
"""Wrapper for GET requests to the Atlos API."""
url = f"{self.atlos_url}{endpoint}"
response = self.session.get(url, headers={"Authorization": f"Bearer {self.api_token}"}, params=params)
response.raise_for_status()
return response.json()
def _post(
self,
endpoint: str,
json: Optional[dict] = None,
params: Optional[dict] = None,
files: Optional[dict] = None,
) -> dict:
"""Wrapper for POST requests to the Atlos API."""
url = f"{self.atlos_url}{endpoint}"
response = self.session.post(
url,
headers={"Authorization": f"Bearer {self.api_token}"},
json=json,
params=params,
files=files,
)
response.raise_for_status()
return response.json()
# ! Atlos Module - Feeder Methods
def __iter__(self) -> Iterator[Metadata]:
"""Iterate over unprocessed, visible source materials from Atlos."""
cursor = None
while True:
data = self._get("/api/v2/source_material", params={"cursor": cursor})
cursor = data.get("next")
results = data.get("results", [])
for item in results:
if (
item.get("source_url") not in [None, ""]
and not item.get("metadata", {}).get("auto_archiver", {}).get("processed", False)
and item.get("visibility") == "visible"
and item.get("status") not in ["processing", "pending"]
):
yield Metadata().set_url(item["source_url"]).set("atlos_id", item["id"])
if not results or cursor is None:
break
# ! Atlos Module - Database Methods
def failed(self, item: Metadata, reason: str) -> None:
"""Mark an item as failed in Atlos, if the ID exists."""
atlos_id = item.metadata.get("atlos_id")
if not atlos_id:
logger.info(f"Item {item.get_url()} has no Atlos ID, skipping")
return
self._post(
f"/api/v2/source_material/metadata/{atlos_id}/auto_archiver",
json={"metadata": {"processed": True, "status": "error", "error": reason}},
)
logger.info(f"Stored failure for {item.get_url()} (ID {atlos_id}) on Atlos: {reason}")
def fetch(self, item: Metadata) -> Union[Metadata, bool]:
"""check and fetch if the given item has been archived already, each
database should handle its own caching, and configuration mechanisms"""
return False
def _process_metadata(self, item: Metadata) -> dict:
"""Process metadata for storage on Atlos. Will convert any datetime
objects to ISO format."""
return {k: v.isoformat() if hasattr(v, "isoformat") else v for k, v in item.metadata.items()}
def done(self, item: Metadata, cached: bool = False) -> None:
"""Mark an item as successfully archived in Atlos."""
atlos_id = item.metadata.get("atlos_id")
if not atlos_id:
logger.info(f"Item {item.get_url()} has no Atlos ID, skipping")
return
self._post(
f"/api/v2/source_material/metadata/{atlos_id}/auto_archiver",
json={
"metadata": {
"processed": True,
"status": "success",
"results": self._process_metadata(item),
}
},
)
logger.info(f"Stored success for {item.get_url()} (ID {atlos_id}) on Atlos")
# ! Atlos Module - Storage Methods
def get_cdn_url(self, _media: Media) -> str:
"""Return the base Atlos URL as the CDN URL."""
return self.atlos_url
def upload(self, media: Media, metadata: Optional[Metadata] = None, **_kwargs) -> bool:
"""Upload a media file to Atlos if it has not been uploaded already."""
if metadata is None:
logger.error(f"No metadata provided for {media.filename}")
return False
atlos_id = metadata.get("atlos_id")
if not atlos_id:
logger.error(f"No Atlos ID found in metadata; can't store {media.filename} in Atlos.")
return False
media_hash = calculate_file_hash(media.filename, hash_algo=hashlib.sha256, chunksize=4096)
# Check whether the media has already been uploaded
source_material = self._get(f"/api/v2/source_material/{atlos_id}")["result"]
existing_media = [artifact.get("file_hash_sha256") for artifact in source_material.get("artifacts", [])]
if media_hash in existing_media:
logger.info(f"{media.filename} with SHA256 {media_hash} already uploaded to Atlos")
return True
# Upload the media to the Atlos API
with open(media.filename, "rb") as file_obj:
self._post(
f"/api/v2/source_material/upload/{atlos_id}",
params={"title": media.properties},
files={"file": (os.path.basename(media.filename), file_obj)},
)
logger.info(f"Uploaded {media.filename} to Atlos with ID {atlos_id} and title {media.key}")
return True
def uploadf(self, file: IO[bytes], key: str, **kwargs: dict) -> bool:
"""Upload a file-like object; not implemented."""
pass

1
src/auto_archiver/modules/atlos_storage/__init__.py
View File

@@ -1 +0,0 @@
from .atlos_storage import AtlosStorage

32
src/auto_archiver/modules/atlos_storage/__manifest__.py
View File

@@ -1,32 +0,0 @@
{
"name": "Atlos Storage",
"type": ["storage"],
"requires_setup": True,
"dependencies": {
"python": ["loguru", "boto3"],
"bin": []
},
"description": """
Stores media files in a [Atlos](https://www.atlos.org/).
### Features
- Saves media files to Atlos, organizing them into folders based on the provided path structure.
### Notes
- Requires setup with Atlos credentials.
- Files are uploaded to the specified `root_folder_id` and organized by the `media.key` structure.
""",
"configs": {
"api_token": {
"default": None,
"help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/",
"required": True,
"type": "str"
},
"atlos_url": {
"default": "https://platform.atlos.org",
"help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.",
"type": "str"
},
}
}

66
src/auto_archiver/modules/atlos_storage/atlos_storage.py
View File

@@ -1,66 +0,0 @@
import hashlib
import os
from typing import IO, Optional
import requests
from loguru import logger
from auto_archiver.core import Media, Metadata
from auto_archiver.core import Storage
class AtlosStorage(Storage):
def get_cdn_url(self, _media: Media) -> str:
# It's not always possible to provide an exact URL, because it's
# possible that the media once uploaded could have been copied to
# another project.
return self.atlos_url
def _hash(self, media: Media) -> str:
# Hash the media file using sha-256. We don't use the existing auto archiver
# hash because there's no guarantee that the configuerer is using sha-256, which
# is how Atlos hashes files.
sha256 = hashlib.sha256()
with open(media.filename, "rb") as f:
while True:
buf = f.read(4096)
if not buf: break
sha256.update(buf)
return sha256.hexdigest()
def upload(self, media: Media, metadata: Optional[Metadata]=None, **_kwargs) -> bool:
atlos_id = metadata.get("atlos_id")
if atlos_id is None:
logger.error(f"No Atlos ID found in metadata; can't store {media.filename} on Atlos")
return False
media_hash = self._hash(media)
# Check whether the media has already been uploaded
source_material = requests.get(
f"{self.atlos_url}/api/v2/source_material/{atlos_id}",
headers={"Authorization": f"Bearer {self.api_token}"},
).json()["result"]
existing_media = [x["file_hash_sha256"] for x in source_material.get("artifacts", [])]
if media_hash in existing_media:
logger.info(f"{media.filename} with SHA256 {media_hash} already uploaded to Atlos")
return True
# Upload the media to the Atlos API
requests.post(
f"{self.atlos_url}/api/v2/source_material/upload/{atlos_id}",
headers={"Authorization": f"Bearer {self.api_token}"},
params={
"title": media.properties
},
files={"file": (os.path.basename(media.filename), open(media.filename, "rb"))},
).raise_for_status()
logger.info(f"Uploaded {media.filename} to Atlos with ID {atlos_id} and title {media.key}")
return True
# must be implemented even if unused
def uploadf(self, file: IO[bytes], key: str, **kwargs: dict) -> bool: pass

22
src/auto_archiver/modules/cli_feeder/__manifest__.py Normal file
View File

@@ -0,0 +1,22 @@
{
"name": "Command Line Feeder",
"type": ["feeder"],
"entry_point": "cli_feeder::CLIFeeder",
"requires_setup": False,
"configs": {
"urls": {
"default": None,
"help": "URL(s) to archive, either a single URL or a list of urls, should not come from config.yaml",
},
},
"description": """
The Command Line Feeder is the default enabled feeder for the Auto Archiver. It allows you to pass URLs directly to the orchestrator from the command line
without the need to specify any additional configuration or command line arguments:
`auto-archiver --feeder cli_feeder -- "https://example.com/1/,https://example.com/2/"`
You can pass multiple URLs by separating them with a space. The URLs will be processed in the order they are provided.
`auto-archiver --feeder cli_feeder -- https://example.com/1/ https://example.com/2/`
""",
}

23
src/auto_archiver/modules/cli_feeder/cli_feeder.py Normal file
View File

@@ -0,0 +1,23 @@
from loguru import logger
from auto_archiver.core.feeder import Feeder
from auto_archiver.core.metadata import Metadata
from auto_archiver.core.consts import SetupError
class CLIFeeder(Feeder):
def setup(self) -> None:
self.urls = self.config["urls"]
if not self.urls:
raise SetupError(
"No URLs provided. Please provide at least one URL via the command line, or set up an alternative feeder. Use --help for more information."
)
def __iter__(self) -> Metadata:
urls = self.config["urls"]
for url in urls:
logger.debug(f"Processing {url}")
m = Metadata().set_url(url)
yield m
logger.success(f"Processed {len(urls)} URL(s)")

2
src/auto_archiver/modules/console_db/__init__.py
View File

@@ -1 +1 @@
from .console_db import ConsoleDb
from .console_db import ConsoleDb

10
src/auto_archiver/modules/console_db/console_db.py
View File

@@ -6,18 +6,18 @@ from auto_archiver.core import Metadata
class ConsoleDb(Database):
"""
Outputs results to the console
Outputs results to the console
"""
def started(self, item: Metadata) -> None:
logger.warning(f"STARTED {item}")
logger.info(f"STARTED {item}")
def failed(self, item: Metadata, reason:str) -> None:
def failed(self, item: Metadata, reason: str) -> None:
logger.error(f"FAILED {item}: {reason}")
def aborted(self, item: Metadata) -> None:
logger.warning(f"ABORTED {item}")
def done(self, item: Metadata, cached: bool=False) -> None:
def done(self, item: Metadata, cached: bool = False) -> None:
"""archival result ready - should be saved to DB"""
logger.success(f"DONE {item}")
logger.success(f"DONE {item}")

2
src/auto_archiver/modules/csv_db/__init__.py
View File

@@ -1 +1 @@
from .csv_db import CSVDb
from .csv_db import CSVDb

9
src/auto_archiver/modules/csv_db/__manifest__.py
View File

@@ -2,12 +2,11 @@
"name": "CSV Database",
"type": ["database"],
"requires_setup": False,
"dependencies": {"python": ["loguru"]
},
'entry_point': 'csv_db::CSVDb',
"dependencies": {"python": ["loguru"]},
"entry_point": "csv_db::CSVDb",
"configs": {
"csv_file": {"default": "db.csv", "help": "CSV file name"}
},
"csv_file": {"default": "db.csv", "help": "CSV file name to save metadata to"},
},
"description": """
Handles exporting archival results to a CSV file.

7
src/auto_archiver/modules/csv_db/csv_db.py
View File

@@ -9,14 +9,15 @@ from auto_archiver.core import Metadata
class CSVDb(Database):
"""
Outputs results to a CSV file
Outputs results to a CSV file
"""
def done(self, item: Metadata, cached: bool=False) -> None:
def done(self, item: Metadata, cached: bool = False) -> None:
"""archival result ready - should be saved to DB"""
logger.success(f"DONE {item}")
is_empty = not os.path.isfile(self.csv_file) or os.path.getsize(self.csv_file) == 0
with open(self.csv_file, "a", encoding="utf-8") as outf:
writer = DictWriter(outf, fieldnames=asdict(Metadata()))
if is_empty: writer.writeheader()
if is_empty:
writer.writeheader()
writer.writerow(asdict(item))

2
src/auto_archiver/modules/csv_feeder/__init__.py
View File

@@ -1 +1 @@
from .csv_feeder import CSVFeeder
from .csv_feeder import CSVFeeder

34
src/auto_archiver/modules/csv_feeder/__manifest__.py
View File

@@ -1,27 +1,23 @@
{
"name": "CSV Feeder",
"type": ["feeder"],
"requires_setup": False,
"dependencies": {
"python": ["loguru"],
"bin": [""]
},
'requires_setup': True,
'entry_point': "csv_feeder::CSVFeeder",
"dependencies": {"python": ["loguru"], "bin": [""]},
"requires_setup": True,
"entry_point": "csv_feeder::CSVFeeder",
"configs": {
"files": {
"default": None,
"help": "Path to the input file(s) to read the URLs from, comma separated. \
"files": {
"default": None,
"help": "Path to the input file(s) to read the URLs from, comma separated. \
Input files should be formatted with one URL per line",
"required": True,
"type": "valid_file",
"nargs": "+",
},
"column": {
"default": None,
"help": "Column number or name to read the URLs from, 0-indexed",
}
"required": True,
"type": "valid_file",
"nargs": "+",
},
"column": {
"default": None,
"help": "Column number or name to read the URLs from, 0-indexed",
},
},
"description": """
Reads URLs from CSV files and feeds them into the archiving process.
@@ -33,5 +29,5 @@
### Setup
- Input files should be formatted with one URL per line, with or without a header row.
- If you have a header row, you can specify the column number or name to read URLs from using the 'column' config option.
"""
""",
}

11
src/auto_archiver/modules/csv_feeder/csv_feeder.py
View File

@@ -5,11 +5,10 @@ from auto_archiver.core import Feeder
from auto_archiver.core import Metadata
from auto_archiver.utils import url_or_none
class CSVFeeder(Feeder):
column = None
def __iter__(self) -> Metadata:
for file in self.files:
with open(file, "r") as f:
@@ -20,9 +19,11 @@ class CSVFeeder(Feeder):
try:
url_column = first_row.index(url_column)
except ValueError:
logger.error(f"Column {url_column} not found in header row: {first_row}. Did you set the 'column' config correctly?")
logger.error(
f"Column {url_column} not found in header row: {first_row}. Did you set the 'column' config correctly?"
)
return
elif not(url_or_none(first_row[url_column])):
elif not (url_or_none(first_row[url_column])):
# it's a header row, but we've been given a column number already
logger.debug(f"Skipping header row: {first_row}")
else:
@@ -35,4 +36,4 @@ class CSVFeeder(Feeder):
continue
url = row[url_column]
logger.debug(f"Processing {url}")
yield Metadata().set_url(url)
yield Metadata().set_url(url)

2
src/auto_archiver/modules/gdrive_storage/__init__.py
View File

@@ -1 +1 @@
from .gdrive_storage import GDriveStorage
from .gdrive_storage import GDriveStorage

21
src/auto_archiver/modules/gdrive_storage/__manifest__.py
View File

@@ -19,14 +19,21 @@
},
"filename_generator": {
"default": "static",
"help": "how to name stored files: 'random' creates a random string; 'static' uses a replicable strategy such as a hash.",
"help": "how to name stored files: 'random' creates a random string; 'static' uses a hash, with the settings of the 'hash_enricher' module (defaults to SHA256 if not enabled).",
"choices": ["random", "static"],
},
"root_folder_id": {"required": True,
"help": "root google drive folder ID to use as storage, found in URL: 'https://drive.google.com/drive/folders/FOLDER_ID'"},
"oauth_token": {"default": None,
"help": "JSON filename with Google Drive OAuth token: check auto-archiver repository scripts folder for create_update_gdrive_oauth_token.py. NOTE: storage used will count towards owner of GDrive folder, therefore it is best to use oauth_token_filename over service_account."},
"service_account": {"default": "secrets/service_account.json", "help": "service account JSON file path, same as used for Google Sheets. NOTE: storage used will count towards the developer account."},
"root_folder_id": {
"required": True,
"help": "root google drive folder ID to use as storage, found in URL: 'https://drive.google.com/drive/folders/FOLDER_ID'",
},
"oauth_token": {
"default": None,
"help": "JSON filename with Google Drive OAuth token: check auto-archiver repository scripts folder for create_update_gdrive_oauth_token.py. NOTE: storage used will count towards owner of GDrive folder, therefore it is best to use oauth_token_filename over service_account.",
},
"service_account": {
"default": "secrets/service_account.json",
"help": "service account JSON file path, same as used for Google Sheets. NOTE: storage used will count towards the developer account.",
},
},
"description": """
@@ -94,5 +101,5 @@ This module integrates Google Drive as a storage backend, enabling automatic fol
https://davemateer.com/2022/04/28/google-drive-with-python#tokens
"""
""",
}

105
src/auto_archiver/modules/gdrive_storage/gdrive_storage.py
View File

@@ -1,4 +1,3 @@
import json
import os
import time
@@ -15,12 +14,9 @@ from auto_archiver.core import Media
from auto_archiver.core import Storage
class GDriveStorage(Storage):
def setup(self) -> None:
self.scopes = ['https://www.googleapis.com/auth/drive']
self.scopes = ["https://www.googleapis.com/auth/drive"]
# Initialize Google Drive service
self._setup_google_drive_service()
@@ -37,25 +33,25 @@ class GDriveStorage(Storage):
def _initialize_with_oauth_token(self):
"""Initialize Google Drive service with OAuth token."""
with open(self.oauth_token, 'r') as stream:
with open(self.oauth_token, "r") as stream:
creds_json = json.load(stream)
creds_json['refresh_token'] = creds_json.get("refresh_token", "")
creds_json["refresh_token"] = creds_json.get("refresh_token", "")
creds = Credentials.from_authorized_user_info(creds_json, self.scopes)
if not creds.valid and creds.expired and creds.refresh_token:
creds.refresh(Request())
with open(self.oauth_token, 'w') as token_file:
with open(self.oauth_token, "w") as token_file:
logger.debug("Saving refreshed OAuth token.")
token_file.write(creds.to_json())
elif not creds.valid:
raise ValueError("Invalid OAuth token. Please regenerate the token.")
return build('drive', 'v3', credentials=creds)
return build("drive", "v3", credentials=creds)
def _initialize_with_service_account(self):
"""Initialize Google Drive service with service account."""
creds = service_account.Credentials.from_service_account_file(self.service_account, scopes=self.scopes)
return build('drive', 'v3', credentials=creds)
return build("drive", "v3", credentials=creds)
def get_cdn_url(self, media: Media) -> str:
"""
@@ -79,7 +75,7 @@ class GDriveStorage(Storage):
return f"https://drive.google.com/file/d/{file_id}/view?usp=sharing"
def upload(self, media: Media, **kwargs) -> bool:
logger.debug(f'[{self.__class__.__name__}] storing file {media.filename} with key {media.key}')
logger.debug(f"[{self.__class__.__name__}] storing file {media.filename} with key {media.key}")
"""
1. for each sub-folder in the path check if exists or create
2. upload file to root_id/other_paths.../filename
@@ -95,25 +91,30 @@ class GDriveStorage(Storage):
parent_id = upload_to
# upload file to gd
logger.debug(f'uploading {filename=} to folder id {upload_to}')
file_metadata = {
'name': [filename],
'parents': [upload_to]
}
logger.debug(f"uploading {filename=} to folder id {upload_to}")
file_metadata = {"name": [filename], "parents": [upload_to]}
media = MediaFileUpload(media.filename, resumable=True)
gd_file = self.service.files().create(supportsAllDrives=True, body=file_metadata, media_body=media, fields='id').execute()
logger.debug(f'uploadf: uploaded file {gd_file["id"]} successfully in folder={upload_to}')
gd_file = (
self.service.files()
.create(supportsAllDrives=True, body=file_metadata, media_body=media, fields="id")
.execute()
)
logger.debug(f"uploadf: uploaded file {gd_file['id']} successfully in folder={upload_to}")
# must be implemented even if unused
def uploadf(self, file: IO[bytes], key: str, **kwargs: dict) -> bool: pass
def uploadf(self, file: IO[bytes], key: str, **kwargs: dict) -> bool:
pass
def _get_id_from_parent_and_name(self, parent_id: str,
name: str,
retries: int = 1,
sleep_seconds: int = 10,
use_mime_type: bool = False,
raise_on_missing: bool = True,
use_cache=False):
def _get_id_from_parent_and_name(
self,
parent_id: str,
name: str,
retries: int = 1,
sleep_seconds: int = 10,
use_mime_type: bool = False,
raise_on_missing: bool = True,
use_cache=False,
):
"""
Retrieves the id of a folder or file from its @name and the @parent_id folder
Optionally does multiple @retries and sleeps @sleep_seconds between them
@@ -134,32 +135,39 @@ class GDriveStorage(Storage):
debug_header: str = f"[searching {name=} in {parent_id=}]"
query_string = f"'{parent_id}' in parents and name = '{name}' and trashed = false "
if use_mime_type:
query_string += f" and mimeType='application/vnd.google-apps.folder' "
query_string += " and mimeType='application/vnd.google-apps.folder' "
for attempt in range(retries):
results = self.service.files().list(
# both below for Google Shared Drives
supportsAllDrives=True,
includeItemsFromAllDrives=True,
q=query_string,
spaces='drive', # ie not appDataFolder or photos
fields='files(id, name)'
).execute()
items = results.get('files', [])
results = (
self.service.files()
.list(
# both below for Google Shared Drives
supportsAllDrives=True,
includeItemsFromAllDrives=True,
q=query_string,
spaces="drive", # ie not appDataFolder or photos
fields="files(id, name)",
)
.execute()
)
items = results.get("files", [])
if len(items) > 0:
logger.debug(f"{debug_header} found {len(items)} matches, returning last of {','.join([i['id'] for i in items])}")
_id = items[-1]['id']
if use_cache: self.api_cache[cache_key] = _id
logger.debug(
f"{debug_header} found {len(items)} matches, returning last of {','.join([i['id'] for i in items])}"
)
_id = items[-1]["id"]
if use_cache:
self.api_cache[cache_key] = _id
return _id
else:
logger.debug(f'{debug_header} not found, attempt {attempt+1}/{retries}.')
logger.debug(f"{debug_header} not found, attempt {attempt + 1}/{retries}.")
if attempt < retries - 1:
logger.debug(f'sleeping for {sleep_seconds} second(s)')
logger.debug(f"sleeping for {sleep_seconds} second(s)")
time.sleep(sleep_seconds)
if raise_on_missing:
raise ValueError(f'{debug_header} not found after {retries} attempt(s)')
raise ValueError(f"{debug_header} not found after {retries} attempt(s)")
return None
def _mkdir(self, name: str, parent_id: str):
@@ -167,12 +175,7 @@ class GDriveStorage(Storage):
Creates a new GDrive folder @name inside folder @parent_id
Returns id of the created folder
"""
logger.debug(f'Creating new folder with {name=} inside {parent_id=}')
file_metadata = {
'name': [name],
'mimeType': 'application/vnd.google-apps.folder',
'parents': [parent_id]
}
gd_folder = self.service.files().create(supportsAllDrives=True, body=file_metadata, fields='id').execute()
return gd_folder.get('id')
logger.debug(f"Creating new folder with {name=} inside {parent_id=}")
file_metadata = {"name": [name], "mimeType": "application/vnd.google-apps.folder", "parents": [parent_id]}
gd_folder = self.service.files().create(supportsAllDrives=True, body=file_metadata, fields="id").execute()
return gd_folder.get("id")

2
src/auto_archiver/modules/generic_extractor/__init__.py
View File

@@ -1 +1 @@
from .generic_extractor import GenericExtractor
from .generic_extractor import GenericExtractor

27
src/auto_archiver/modules/generic_extractor/__manifest__.py
View File

@@ -15,6 +15,9 @@ supported by `yt-dlp`, such as YouTube, Facebook, and others. It provides functi
for retrieving videos, subtitles, comments, and other metadata, and it integrates with
the broader archiving framework.
For a full list of video platforms supported by `yt-dlp`, see the
[official documentation](https://github.com/yt-dlp/yt-dlp/blob/master/supportedsites.md)
### Features
- Supports downloading videos and playlists.
- Retrieves metadata like titles, descriptions, upload dates, and durations.
@@ -28,6 +31,13 @@ the broader archiving framework.
metadata objects. Some dropins are included in this generic_archiver by default, but
custom dropins can be created to handle additional websites and passed to the archiver
via the command line using the `--dropins` option (TODO!).
### Auto-Updates
The Generic Extractor will also automatically check for updates to `yt-dlp` (every 5 days by default).
This can be configured using the `ytdlp_update_interval` setting (or disabled by setting it to -1).
If you are having issues with the extractor, you can review the version of `yt-dlp` being used with `yt-dlp --version`.
""",
"configs": {
"subtitles": {"default": True, "help": "download subtitles if available", "type": "bool"},
@@ -64,5 +74,22 @@ via the command line using the `--dropins` option (TODO!).
"default": "inf",
"help": "Use to limit the number of videos to download when a channel or long page is being extracted. 'inf' means no limit.",
},
"extractor_args": {
"default": {},
"help": "Additional arguments to pass to the yt-dlp extractor. See https://github.com/yt-dlp/yt-dlp/blob/master/README.md#extractor-arguments.",
"type": "json_loader",
},
"ytdlp_update_interval": {
"default": 5,
"help": "How often to check for yt-dlp updates (days). If positive, will check and update yt-dlp every [num] days. Set it to -1 to disable, or 0 to always update on every run.",
"type": "int",
},
"ytdlp_args": {
"default": "",
"help": "Additional arguments to pass to yt-dlp, e.g. --no-check-certificate or --plugin-dirs.\
See yt-dlp documentation here for more information: https://github.com/yt-dlp/yt-dlp?tab=readme-ov-file#general-options\
Note: this is not to be confused with 'extractor_args' which are specific to the extractor itself.",
"type": "str",
},
},
}

16
src/auto_archiver/modules/generic_extractor/bluesky.py
View File

@@ -4,15 +4,16 @@ from auto_archiver.core.extractor import Extractor
from auto_archiver.core.metadata import Metadata, Media
from .dropin import GenericDropin, InfoExtractor
class Bluesky(GenericDropin):
class Bluesky(GenericDropin):
def create_metadata(self, post: dict, ie_instance: InfoExtractor, archiver: Extractor, url: str) -> Metadata:
result = Metadata()
result.set_url(url)
result.set_title(post["record"]["text"])
result.set_timestamp(post["record"]["createdAt"])
for k, v in self._get_post_data(post).items():
if v: result.set(k, v)
if v:
result.set(k, v)
# download if embeds present (1 video XOR >=1 images)
for media in self._download_bsky_embeds(post, archiver):
@@ -23,12 +24,12 @@ class Bluesky(GenericDropin):
def extract_post(self, url: str, ie_instance: InfoExtractor) -> dict:
# TODO: If/when this PR (https://github.com/yt-dlp/yt-dlp/pull/12098) is merged on ytdlp, remove the comments and delete the code below
handle, video_id = ie_instance._match_valid_url(url).group('handle', 'id')
handle, video_id = ie_instance._match_valid_url(url).group("handle", "id")
return ie_instance._extract_post(handle=handle, post_id=video_id)
def _download_bsky_embeds(self, post: dict, archiver: Extractor) -> list[Media]:
"""
Iterates over image(s) or video in a Bluesky post and downloads them
Iterates over image(s) or video in a Bluesky post and downloads them
"""
media = []
embed = post.get("record", {}).get("embed", {})
@@ -37,16 +38,15 @@ class Bluesky(GenericDropin):
media_url = "https://bsky.social/xrpc/com.atproto.sync.getBlob?cid={}&did={}"
for image_media in image_medias:
url = media_url.format(image_media['image']['ref']['$link'], post['author']['did'])
url = media_url.format(image_media["image"]["ref"]["$link"], post["author"]["did"])
image_media = archiver.download_from_url(url)
media.append(Media(image_media))
for video_media in video_medias:
url = media_url.format(video_media['ref']['$link'], post['author']['did'])
url = media_url.format(video_media["ref"]["$link"], post["author"]["did"])
video_media = archiver.download_from_url(url)
media.append(Media(video_media))
return media
def _get_post_data(self, post: dict) -> dict:
"""
Extracts relevant information returned by the .getPostThread api call (excluding text/created_at): author, mentions, tags, links.
@@ -74,4 +74,4 @@ class Bluesky(GenericDropin):
res["tags"] = tags
if links:
res["links"] = links
return res
return res

32
src/auto_archiver/modules/generic_extractor/dropin.py
View File

@@ -1,12 +1,14 @@
from typing import Type
from yt_dlp.extractor.common import InfoExtractor
from auto_archiver.core.metadata import Metadata
from auto_archiver.core.extractor import Extractor
class GenericDropin:
"""Base class for dropins for the generic extractor.
In many instances, an extractor will exist in ytdlp, but it will only process videos.
Dropins can be created and used to make use of the already-written private code of a
Dropins can be created and used to make use of the already-written private code of a
specific extractor from ytdlp.
The dropin should be able to handle the following methods:
@@ -23,26 +25,26 @@ class GenericDropin:
"""
extractor: Type[Extractor] = None
def extract_post(self, url: str, ie_instance: InfoExtractor):
"""
This method should return the post data from the url.
"""
raise NotImplementedError("This method should be implemented in the subclass")
def create_metadata(self, post: dict, ie_instance: InfoExtractor, archiver: Extractor, url: str) -> Metadata:
"""
This method should create a Metadata object from the post data.
"""
raise NotImplementedError("This method should be implemented in the subclass")
def skip_ytdlp_download(self, url: str, ie_instance: InfoExtractor):
"""
This method should return True if you want to skip the ytdlp download method.
"""
return False
def keys_to_clean(self, video_data: dict, info_extractor: InfoExtractor):
"""
This method should return a list of strings (keys) to clean from the video_data dict.
@@ -50,9 +52,25 @@ class GenericDropin:
E.g. ["uploader", "uploader_id", "tiktok_specific_field"]
"""
return []
def download_additional_media(self, video_data: dict, info_extractor: InfoExtractor, metadata: Metadata):
"""
This method should download any additional media from the post.
"""
return metadata
return metadata
def suitable(self, url, info_extractor: InfoExtractor):
"""
A method to allow dropins to override their InfoExtractor's 'suitable' method.
Dropins should override this method and return True if the url is suitable for the extractor
(based on being able to parse other URLs). See the `suitable_extractors` method in the
`GenericExtractor` class for how this is implemented.
The default behaviour of this method is to return the result of the InfoExtractor's 'suitable' method.
### Example: An example of where this is useful is for the FacebookIE extractor in yt-dlp. By default,
it's 'suitable' method only returns True for video URLs. However, we can override this method in the
Facebook dropin to return True for all Facebook URLs (photo/post types). This way, the Facebook dropin
can be used for all Facebook URLs.
"""
return info_extractor.suitable(url)

160
src/auto_archiver/modules/generic_extractor/facebook.py
View File

@@ -1,18 +1,154 @@
import re
from .dropin import GenericDropin
from auto_archiver.core.metadata import Metadata
from yt_dlp.extractor.facebook import FacebookIE
# TODO: Remove if / when https://github.com/yt-dlp/yt-dlp/pull/12275 is merged
from yt_dlp.utils import (
clean_html,
get_element_by_id,
traverse_obj,
get_first,
merge_dicts,
int_or_none,
parse_count,
)
def _extract_metadata(self, webpage, video_id):
post_data = [
self._parse_json(j, video_id, fatal=False)
for j in re.findall(r"data-sjs>({.*?ScheduledServerJS.*?})</script>", webpage)
]
post = (
traverse_obj(
post_data,
(..., "require", ..., ..., ..., "__bbox", "require", ..., ..., ..., "__bbox", "result", "data"),
expected_type=dict,
)
or []
)
media = traverse_obj(
post,
(
...,
"attachments",
...,
lambda k, v: (k == "media" and str(v["id"]) == video_id and v["__typename"] == "Video"),
),
expected_type=dict,
)
title = get_first(media, ("title", "text"))
description = get_first(media, ("creation_story", "comet_sections", "message", "story", "message", "text"))
page_title = title or self._html_search_regex(
(
r'<h2\s+[^>]*class="uiHeaderTitle"[^>]*>(?P<content>[^<]*)</h2>',
r'(?s)<span class="fbPhotosPhotoCaption".*?id="fbPhotoPageCaption"><span class="hasCaption">(?P<content>.*?)</span>',
self._meta_regex("og:title"),
self._meta_regex("twitter:title"),
r"<title>(?P<content>.+?)</title>",
),
webpage,
"title",
default=None,
group="content",
)
description = description or self._html_search_meta(
["description", "og:description", "twitter:description"], webpage, "description", default=None
)
uploader_data = (
get_first(media, ("owner", {dict}))
or get_first(
post, ("video", "creation_story", "attachments", ..., "media", lambda k, v: k == "owner" and v["name"])
)
or get_first(post, (..., "video", lambda k, v: k == "owner" and v["name"]))
or get_first(post, ("node", "actors", ..., {dict}))
or get_first(post, ("event", "event_creator", {dict}))
or get_first(post, ("video", "creation_story", "short_form_video_context", "video_owner", {dict}))
or {}
)
uploader = uploader_data.get("name") or (
clean_html(get_element_by_id("fbPhotoPageAuthorName", webpage))
or self._search_regex(
(r'ownerName\s*:\s*"([^"]+)"', *self._og_regexes("title")), webpage, "uploader", fatal=False
)
)
timestamp = int_or_none(self._search_regex(r'<abbr[^>]+data-utime=["\'](\d+)', webpage, "timestamp", default=None))
thumbnail = self._html_search_meta(["og:image", "twitter:image"], webpage, "thumbnail", default=None)
# some webpages contain unretrievable thumbnail urls
# like https://lookaside.fbsbx.com/lookaside/crawler/media/?media_id=10155168902769113&get_thumbnail=1
# in https://www.facebook.com/yaroslav.korpan/videos/1417995061575415/
if thumbnail and not re.search(r"\.(?:jpg|png)", thumbnail):
thumbnail = None
info_dict = {
"description": description,
"uploader": uploader,
"uploader_id": uploader_data.get("id"),
"timestamp": timestamp,
"thumbnail": thumbnail,
"view_count": parse_count(
self._search_regex(
(r'\bviewCount\s*:\s*["\']([\d,.]+)', r'video_view_count["\']\s*:\s*(\d+)'),
webpage,
"view count",
default=None,
)
),
"concurrent_view_count": get_first(
post, (("video", (..., ..., "attachments", ..., "media")), "liveViewerCount", {int_or_none})
),
**traverse_obj(
post,
(
lambda _, v: video_id in v["url"],
"feedback",
{
"like_count": ("likers", "count", {int}),
"comment_count": ("total_comment_count", {int}),
"repost_count": ("share_count_reduced", {parse_count}),
},
),
get_all=False,
),
}
info_json_ld = self._search_json_ld(webpage, video_id, default={})
info_json_ld["title"] = (
re.sub(r"\s*\|\s*Facebook$", "", title or info_json_ld.get("title") or page_title or "")
or (description or "").replace("\n", " ")
or f"Facebook video #{video_id}"
)
return merge_dicts(info_json_ld, info_dict)
class Facebook(GenericDropin):
def extract_post(self, url: str, ie_instance):
video_id = ie_instance._match_valid_url(url).group('id')
ie_instance._download_webpage(
url.replace('://m.facebook.com/', '://www.facebook.com/'), video_id)
webpage = ie_instance._download_webpage(url, ie_instance._match_valid_url(url).group('id'))
def extract_post(self, url: str, ie_instance: FacebookIE):
post_id_regex = r"(?P<id>pfbid[A-Za-z0-9]+|\d+|t\.(\d+\/\d+))"
post_id = re.search(post_id_regex, url).group("id")
webpage = ie_instance._download_webpage(url.replace("://m.facebook.com/", "://www.facebook.com/"), post_id)
# TODO: fix once https://github.com/yt-dlp/yt-dlp/pull/12275 is merged
post_data = ie_instance._extract_metadata(webpage)
# TODO: For long posts, this _extract_metadata only seems to return the first 100 or so characters, followed by ...
# TODO: If/when https://github.com/yt-dlp/yt-dlp/pull/12275 is merged, uncomment next line and delete the one after
# post_data = ie_instance._extract_metadata(webpage, post_id)
post_data = _extract_metadata(ie_instance, webpage, post_id)
return post_data
def create_metadata(self, post: dict, ie_instance, archiver, url):
metadata = archiver.create_metadata(url)
metadata.set_title(post.get('title')).set_content(post.get('description')).set_post_data(post)
return metadata
def create_metadata(self, post: dict, ie_instance: FacebookIE, archiver, url):
result = Metadata()
result.set_content(post.get("description", ""))
result.set_title(post.get("title", ""))
result.set("author", post.get("uploader", ""))
result.set_url(url)
return result
def suitable(self, url, info_extractor: FacebookIE):
regex = r"(?:https?://(?:[\w-]+\.)?(?:facebook\.com||facebookwkhpilnemxj7asaniu7vnjjbiltxjqhye3mhbshg7kx5tfyd\.onion)/)"
return re.match(regex, url)
def skip_ytdlp_download(self, url: str, is_instance: FacebookIE):
"""
Skip using the ytdlp download method for Facebook *photo* posts, they have a URL with an id of t.XXXXX/XXXXX
"""
if re.search(r"/t.\d+/\d+", url):
return True

332
src/auto_archiver/modules/generic_extractor/generic_extractor.py
View File

@@ -1,31 +1,94 @@
import datetime, os, yt_dlp, pysubs2
import datetime
import os
import importlib
import subprocess
from typing import Generator, Type
import yt_dlp
from yt_dlp.extractor.common import InfoExtractor
import pysubs2
from loguru import logger
from auto_archiver.core.extractor import Extractor
from auto_archiver.core import Metadata, Media
from auto_archiver.utils import get_datetime_from_str
from .dropin import GenericDropin
class SkipYtdlp(Exception):
pass
class GenericExtractor(Extractor):
_dropins = {}
def setup(self):
# check for file .ytdlp-update in the secrets folder
if self.ytdlp_update_interval < 0:
return
use_secrets = os.path.exists("secrets")
path = os.path.join("secrets" if use_secrets else "", ".ytdlp-update")
next_update_check = None
if os.path.exists(path):
with open(path, "r") as f:
next_update_check = datetime.datetime.fromisoformat(f.read())
if not next_update_check or next_update_check < datetime.datetime.now():
self.update_ytdlp()
next_update_check = datetime.datetime.now() + datetime.timedelta(days=self.ytdlp_update_interval)
with open(path, "w") as f:
f.write(next_update_check.isoformat())
def update_ytdlp(self):
logger.info("Checking and updating yt-dlp...")
logger.info(
f"Tip: change the 'ytdlp_update_interval' setting to control how often yt-dlp is updated. Set to -1 to disable or 0 to enable on every run. Current setting: {self.ytdlp_update_interval}"
)
from importlib.metadata import version as get_version
old_version = get_version("yt-dlp")
try:
# try and update with pip (this works inside poetry environment and in a normal virtualenv)
result = subprocess.run(["pip", "install", "--upgrade", "yt-dlp"], check=True, capture_output=True)
if "Successfully installed yt-dlp" in result.stdout.decode():
new_version = importlib.metadata.version("yt-dlp")
logger.info(f"yt-dlp successfully (from {old_version} to {new_version})")
importlib.reload(yt_dlp)
else:
logger.info("yt-dlp already up to date")
except Exception as e:
logger.error(f"Error updating yt-dlp: {e}")
def suitable_extractors(self, url: str) -> Generator[str, None, None]:
"""
Returns a list of valid extractors for the given URL"""
for info_extractor in yt_dlp.YoutubeDL()._ies.values():
if info_extractor.suitable(url) and info_extractor.working():
if not info_extractor.working():
continue
# check if there's a dropin and see if that declares whether it's suitable
dropin: GenericDropin = self.dropin_for_name(info_extractor.ie_key())
if dropin and dropin.suitable(url, info_extractor):
yield info_extractor
elif info_extractor.suitable(url):
yield info_extractor
def suitable(self, url: str) -> bool:
"""
Checks for valid URLs out of all ytdlp extractors.
Returns False for the GenericIE, which as labelled by yt-dlp: 'Generic downloader that works on some sites'
"""
return any(self.suitable_extractors(url))
def download_additional_media(self, video_data: dict, info_extractor: InfoExtractor, metadata: Metadata) -> Metadata:
def download_additional_media(
self, video_data: dict, info_extractor: InfoExtractor, metadata: Metadata
) -> Metadata:
"""
Downloads additional media like images, comments, subtitles, etc.
@@ -34,7 +97,7 @@ class GenericExtractor(Extractor):
# Just get the main thumbnail. More thumbnails are available in
# video_data['thumbnails'] should they be required
thumbnail_url = video_data.get('thumbnail')
thumbnail_url = video_data.get("thumbnail")
if thumbnail_url:
try:
cover_image_path = self.download_from_url(thumbnail_url)
@@ -57,15 +120,65 @@ class GenericExtractor(Extractor):
Clean up the ytdlp generic video data to make it more readable and remove unnecessary keys that ytdlp adds
"""
base_keys = ['formats', 'thumbnail', 'display_id', 'epoch', 'requested_downloads',
'duration_string', 'thumbnails', 'http_headers', 'webpage_url_basename', 'webpage_url_domain',
'extractor', 'extractor_key', 'playlist', 'playlist_index', 'duration_string', 'protocol', 'requested_subtitles',
'format_id', 'acodec', 'vcodec', 'ext', 'epoch', '_has_drm', 'filesize', 'audio_ext', 'video_ext', 'vbr', 'abr',
'resolution', 'dynamic_range', 'aspect_ratio', 'cookies', 'format', 'quality', 'preference', 'artists',
'channel_id', 'subtitles', 'tbr', 'url', 'original_url', 'automatic_captions', 'playable_in_embed', 'live_status',
'_format_sort_fields', 'chapters', 'requested_formats', 'format_note',
'audio_channels', 'asr', 'fps', 'was_live', 'is_live', 'heatmap', 'age_limit', 'stretched_ratio']
base_keys = [
"formats",
"thumbnail",
"display_id",
"epoch",
"requested_downloads",
"duration_string",
"thumbnails",
"http_headers",
"webpage_url_basename",
"webpage_url_domain",
"extractor",
"extractor_key",
"playlist",
"playlist_index",
"duration_string",
"protocol",
"requested_subtitles",
"format_id",
"acodec",
"vcodec",
"ext",
"epoch",
"_has_drm",
"filesize",
"audio_ext",
"video_ext",
"vbr",
"abr",
"resolution",
"dynamic_range",
"aspect_ratio",
"cookies",
"format",
"quality",
"preference",
"artists",
"channel_id",
"subtitles",
"tbr",
"url",
"original_url",
"automatic_captions",
"playable_in_embed",
"live_status",
"_format_sort_fields",
"chapters",
"requested_formats",
"format_note",
"audio_channels",
"asr",
"fps",
"was_live",
"is_live",
"heatmap",
"age_limit",
"stretched_ratio",
]
dropin = self.dropin_for_name(info_extractor.ie_key())
if dropin:
try:
@@ -74,8 +187,8 @@ class GenericExtractor(Extractor):
pass
return base_keys
def add_metadata(self, video_data: dict, info_extractor: InfoExtractor, url:str, result: Metadata) -> Metadata:
def add_metadata(self, video_data: dict, info_extractor: InfoExtractor, url: str, result: Metadata) -> Metadata:
"""
Creates a Metadata object from the given video_data
"""
@@ -84,29 +197,43 @@ class GenericExtractor(Extractor):
result = self.download_additional_media(video_data, info_extractor, result)
# keep both 'title' and 'fulltitle', but prefer 'title', falling back to 'fulltitle' if it doesn't exist
result.set_title(video_data.pop('title', video_data.pop('fulltitle', "")))
result.set_url(url)
if not result.get_title():
result.set_title(video_data.pop("title", video_data.pop("fulltitle", "")))
if not result.get("url"):
result.set_url(url)
if "description" in video_data and not result.get("content"):
result.set_content(video_data["description"])
# extract comments if enabled
if self.comments:
result.set("comments", [{
"text": c["text"],
"author": c["author"],
"timestamp": datetime.datetime.fromtimestamp(c.get("timestamp"), tz = datetime.timezone.utc)
} for c in video_data.get("comments", [])])
result.set(
"comments",
[
{
"text": c["text"],
"author": c["author"],
"timestamp": datetime.datetime.fromtimestamp(c.get("timestamp"), tz=datetime.timezone.utc),
}
for c in video_data.get("comments", [])
],
)
# then add the common metadata
if timestamp := video_data.pop("timestamp", None):
timestamp = datetime.datetime.fromtimestamp(timestamp, tz = datetime.timezone.utc).isoformat()
timestamp = video_data.pop("timestamp", None)
if timestamp and not result.get("timestamp"):
timestamp = datetime.datetime.fromtimestamp(timestamp, tz=datetime.timezone.utc).isoformat()
result.set_timestamp(timestamp)
if upload_date := video_data.pop("upload_date", None):
upload_date = datetime.datetime.strptime(upload_date, '%Y%m%d').replace(tzinfo=datetime.timezone.utc)
upload_date = video_data.pop("upload_date", None)
if upload_date and not result.get("upload_date"):
upload_date = get_datetime_from_str(upload_date, "%Y%m%d").replace(tzinfo=datetime.timezone.utc)
result.set("upload_date", upload_date)
# then clean away any keys we don't want
for clean_key in self.keys_to_clean(info_extractor, video_data):
video_data.pop(clean_key, None)
# then add the rest of the video data
for k, v in video_data.items():
if v:
@@ -124,25 +251,28 @@ class GenericExtractor(Extractor):
if not dropin:
# TODO: add a proper link to 'how to create your own dropin'
logger.debug(f"""Could not find valid dropin for {info_extractor.IE_NAME}.
logger.debug(f"""Could not find valid dropin for {info_extractor.ie_key()}.
Why not try creating your own, and make sure it has a valid function called 'create_metadata'. Learn more: https://auto-archiver.readthedocs.io/en/latest/user_guidelines.html#""")
return False
post_data = dropin.extract_post(url, ie_instance)
return dropin.create_metadata(post_data, ie_instance, self, url)
def get_metadata_for_video(self, data: dict, info_extractor: Type[InfoExtractor], url: str, ydl: yt_dlp.YoutubeDL) -> Metadata:
result = dropin.create_metadata(post_data, ie_instance, self, url)
return self.add_metadata(post_data, info_extractor, url, result)
def get_metadata_for_video(
self, data: dict, info_extractor: Type[InfoExtractor], url: str, ydl: yt_dlp.YoutubeDL
) -> Metadata:
# this time download
ydl.params['getcomments'] = self.comments
#TODO: for playlist or long lists of videos, how to download one at a time so they can be stored before the next one is downloaded?
ydl.params["getcomments"] = self.comments
# TODO: for playlist or long lists of videos, how to download one at a time so they can be stored before the next one is downloaded?
data = ydl.extract_info(url, ie_key=info_extractor.ie_key(), download=True)
if "entries" in data:
entries = data.get("entries", [])
if not len(entries):
logger.warning('YoutubeDLArchiver could not find any video')
logger.warning("YoutubeDLArchiver could not find any video")
return False
else: entries = [data]
else:
entries = [data]
result = Metadata()
@@ -150,17 +280,18 @@ class GenericExtractor(Extractor):
try:
filename = ydl.prepare_filename(entry)
if not os.path.exists(filename):
filename = filename.split('.')[0] + '.mkv'
filename = filename.split(".")[0] + ".mkv"
new_media = Media(filename)
for x in ["duration", "original_url", "fulltitle", "description", "upload_date"]:
if x in entry: new_media.set(x, entry[x])
if x in entry:
new_media.set(x, entry[x])
# read text from subtitles if enabled
if self.subtitles:
for lang, val in (data.get('requested_subtitles') or {}).items():
try:
subs = pysubs2.load(val.get('filepath'), encoding="utf-8")
for lang, val in (data.get("requested_subtitles") or {}).items():
try:
subs = pysubs2.load(val.get("filepath"), encoding="utf-8")
text = " ".join([line.text for line in subs])
new_media.set(f"subtitles_{lang}", text)
except Exception as e:
@@ -170,8 +301,8 @@ class GenericExtractor(Extractor):
logger.error(f"Error processing entry {entry}: {e}")
return self.add_metadata(data, info_extractor, url, result)
def dropin_for_name(self, dropin_name: str, additional_paths = [], package=__package__) -> Type[InfoExtractor]:
def dropin_for_name(self, dropin_name: str, additional_paths=[], package=__package__) -> GenericDropin:
dropin_name = dropin_name.lower()
if dropin_name == "generic":
@@ -179,8 +310,10 @@ class GenericExtractor(Extractor):
return None
dropin_class_name = dropin_name.title()
def _load_dropin(dropin):
dropin_class = getattr(dropin, dropin_class_name)()
dropin.extractor = self
return self._dropins.setdefault(dropin_name, dropin_class)
try:
@@ -202,7 +335,7 @@ class GenericExtractor(Extractor):
return _load_dropin(dropin)
except (FileNotFoundError, ModuleNotFoundError):
pass
# fallback to loading the dropins within auto-archiver
try:
return _load_dropin(importlib.import_module(f".{dropin_name}", package=package))
@@ -214,43 +347,53 @@ class GenericExtractor(Extractor):
def download_for_extractor(self, info_extractor: InfoExtractor, url: str, ydl: yt_dlp.YoutubeDL) -> Metadata:
"""
Tries to download the given url using the specified extractor
It first tries to use ytdlp directly to download the video. If the post is not a video, it will then try to
use the extractor's _extract_post method to get the post metadata if possible.
"""
# when getting info without download, we also don't need the comments
ydl.params['getcomments'] = False
ydl.params["getcomments"] = False
result = False
dropin_submodule = self.dropin_for_name(info_extractor.ie_key())
try:
if dropin_submodule and dropin_submodule.skip_ytdlp_download(info_extractor, url):
raise Exception(f"Skipping using ytdlp to download files for {info_extractor.ie_key()}")
if dropin_submodule and dropin_submodule.skip_ytdlp_download(url, info_extractor):
logger.debug(f"Skipping using ytdlp to download files for {info_extractor.ie_key()}")
raise SkipYtdlp()
# don't download since it can be a live stream
data = ydl.extract_info(url, ie_key=info_extractor.ie_key(), download=False)
if data.get('is_live', False) and not self.livestreams:
if data.get("is_live", False) and not self.livestreams:
logger.warning("Livestream detected, skipping due to 'livestreams' configuration setting")
return False
# it's a valid video, that the youtubdedl can download out of the box
result = self.get_metadata_for_video(data, info_extractor, url, ydl)
except Exception as e:
if info_extractor.ie_key() == "generic":
if info_extractor.IE_NAME == "generic":
# don't clutter the logs with issues about the 'generic' extractor not having a dropin
return False
logger.debug(f'Issue using "{info_extractor.IE_NAME}" extractor to download video (error: {repr(e)}), attempting to use extractor to get post data instead')
if not isinstance(e, SkipYtdlp):
logger.debug(
f'Issue using "{info_extractor.IE_NAME}" extractor to download video (error: {repr(e)}), attempting to use dropin to get post data instead'
)
try:
result = self.get_metadata_for_post(info_extractor, url, ydl)
except (yt_dlp.utils.DownloadError, yt_dlp.utils.ExtractorError) as post_e:
logger.error(f'Error downloading metadata for post: {post_e}')
logger.error("Error downloading metadata for post: {error}", error=str(post_e))
return False
except Exception as generic_e:
logger.debug(f'Attempt to extract using ytdlp extractor "{info_extractor.IE_NAME}" failed: \n {repr(generic_e)}', exc_info=True)
logger.debug(
'Attempt to extract using ytdlp extractor "{name}" failed: \n {error}',
name=info_extractor.IE_NAME,
error=str(generic_e),
exc_info=True,
)
return False
if result:
extractor_name = "yt-dlp"
if info_extractor:
@@ -266,43 +409,70 @@ class GenericExtractor(Extractor):
def download(self, item: Metadata) -> Metadata:
url = item.get_url()
#TODO: this is a temporary hack until this issue is closed: https://github.com/yt-dlp/yt-dlp/issues/11025
# TODO: this is a temporary hack until this issue is closed: https://github.com/yt-dlp/yt-dlp/issues/11025
if url.startswith("https://ya.ru"):
url = url.replace("https://ya.ru", "https://yandex.ru")
item.set("replaced_url", url)
ydl_options = [
"-o",
os.path.join(self.tmp_dir, "%(id)s.%(ext)s"),
"--quiet",
"--no-playlist" if not self.allow_playlist else "--yes-playlist",
"--write-subs" if self.subtitles else "--no-write-subs",
"--write-auto-subs" if self.subtitles else "--no-write-auto-subs",
"--live-from-start" if self.live_from_start else "--no-live-from-start",
]
# proxy handling
if self.proxy:
ydl_options.extend(["--proxy", self.proxy])
# max_downloads handling
if self.max_downloads != "inf":
ydl_options.extend(["--max-downloads", str(self.max_downloads)])
ydl_options.extend(["--playlist-end", str(self.max_downloads)])
ydl_options = {'outtmpl': os.path.join(self.tmp_dir, f'%(id)s.%(ext)s'),
'quiet': False, 'noplaylist': not self.allow_playlist ,
'writesubtitles': self.subtitles,'writeautomaticsub': self.subtitles,
"live_from_start": self.live_from_start, "proxy": self.proxy,
"max_downloads": self.max_downloads, "playlistend": self.max_downloads}
# set up auth
auth = self.auth_for_site(url, extract_cookies=False)
# order of importance: username/pasword -> api_key -> cookie -> cookies_from_browser -> cookies_file
# order of importance: username/password -> api_key -> cookie -> cookies_from_browser -> cookies_file
if auth:
if 'username' in auth and 'password' in auth:
logger.debug(f'Using provided auth username and password for {url}')
ydl_options['username'] = auth['username']
ydl_options['password'] = auth['password']
elif 'cookie' in auth:
logger.debug(f'Using provided auth cookie for {url}')
yt_dlp.utils.std_headers['cookie'] = auth['cookie']
elif 'cookies_from_browser' in auth:
logger.debug(f'Using extracted cookies from browser {auth["cookies_from_browser"]} for {url}')
ydl_options['cookiesfrombrowser'] = auth['cookies_from_browser']
elif 'cookies_file' in auth:
logger.debug(f'Using cookies from file {auth["cookies_file"]} for {url}')
ydl_options['cookiefile'] = auth['cookies_file']
if "username" in auth and "password" in auth:
logger.debug(f"Using provided auth username and password for {url}")
ydl_options.extend(("--username", auth["username"]))
ydl_options.extend(("--password", auth["password"]))
elif "cookie" in auth:
logger.debug(f"Using provided auth cookie for {url}")
yt_dlp.utils.std_headers["cookie"] = auth["cookie"]
elif "cookies_from_browser" in auth:
logger.debug(f"Using extracted cookies from browser {auth['cookies_from_browser']} for {url}")
ydl_options.extend(("--cookies-from-browser", auth["cookies_from_browser"]))
elif "cookies_file" in auth:
logger.debug(f"Using cookies from file {auth['cookies_file']} for {url}")
ydl_options.extend(("--cookies", auth["cookies_file"]))
ydl = yt_dlp.YoutubeDL(ydl_options) # allsubtitles and subtitleslangs not working as expected, so default lang is always "en"
# Applying user-defined extractor_args
if self.extractor_args:
for key, args in self.extractor_args.items():
logger.debug(f"Setting extractor_args: {key}")
if isinstance(args, dict):
arg_str = ";".join(f"{k}={v}" for k, v in args.items())
else:
arg_str = str(args)
ydl_options.extend(["--extractor-args", f"{key}:{arg_str}"])
if self.ytdlp_args:
logger.debug("Adding additional ytdlp arguments: {self.ytdlp_args}")
ydl_options += self.ytdlp_args.split(" ")
*_, validated_options = yt_dlp.parse_options(ydl_options)
ydl = yt_dlp.YoutubeDL(
validated_options
) # allsubtitles and subtitleslangs not working as expected, so default lang is always "en"
for info_extractor in self.suitable_extractors(url):
result = self.download_for_extractor(info_extractor, url, ydl)
if result:
return result
return False

83
src/auto_archiver/modules/generic_extractor/tiktok.py Normal file
View File

@@ -0,0 +1,83 @@
import requests
from loguru import logger
from yt_dlp.extractor.tiktok import TikTokIE, TikTokLiveIE, TikTokVMIE, TikTokUserIE
from auto_archiver.core import Metadata, Media
from datetime import datetime, timezone
from .dropin import GenericDropin
class Tiktok(GenericDropin):
"""
TikTok droping for the Generic Extractor that uses an unofficial API if/when ytdlp fails.
It's useful for capturing content that requires a login, like sensitive content.
"""
TIKWM_ENDPOINT = "https://www.tikwm.com/api/?url={url}"
def suitable(self, url, info_extractor) -> bool:
"""This dropin (which uses Tikvm) is suitable for *all* Tiktok type URLs - videos, lives, VMs, and users.
Return the 'suitable' method from the TikTokIE class."""
return any(extractor().suitable(url) for extractor in (TikTokIE, TikTokLiveIE, TikTokVMIE, TikTokUserIE))
def extract_post(self, url: str, ie_instance):
logger.debug(f"Using Tikwm API to attempt to download tiktok video from {url=}")
endpoint = self.TIKWM_ENDPOINT.format(url=url)
r = requests.get(endpoint)
if r.status_code != 200:
raise ValueError(f"unexpected status code '{r.status_code}' from tikwm.com for {url=}:")
try:
json_response = r.json()
except ValueError:
raise ValueError(f"failed to parse JSON response from tikwm.com for {url=}")
if not json_response.get("msg") == "success" or not (api_data := json_response.get("data", {})):
raise ValueError(f"failed to get a valid response from tikwm.com for {url=}: {repr(json_response)}")
# tries to get the non-watermarked version first
video_url = api_data.pop("play", api_data.pop("wmplay", None))
if not video_url:
raise ValueError(f"no valid video URL found in response from tikwm.com for {url=}")
api_data["video_url"] = video_url
return api_data
def keys_to_clean(self, video_data: dict, info_extractor):
return ["video_url", "title", "create_time", "author", "cover", "origin_cover", "ai_dynamic_cover", "duration"]
def create_metadata(self, post: dict, ie_instance, archiver, url):
# prepare result, start by downloading video
result = Metadata()
video_url = post.pop("video_url")
# get the cover if possible
cover_url = post.pop("origin_cover", post.pop("cover", post.pop("ai_dynamic_cover", None)))
if cover_url and (cover_downloaded := archiver.download_from_url(cover_url)):
result.add_media(Media(cover_downloaded))
# get the video or fail
video_downloaded = archiver.download_from_url(video_url, f"vid_{post.get('id', '')}")
if not video_downloaded:
logger.error(f"failed to download video from {video_url}")
return False
video_media = Media(video_downloaded)
if duration := post.get("duration", None):
video_media.set("duration", duration)
result.add_media(video_media)
# add remaining metadata
result.set_title(post.get("title", ""))
if created_at := post.get("create_time", None):
result.set_timestamp(datetime.fromtimestamp(created_at, tz=timezone.utc))
if author := post.get("author", None):
result.set("author", author)
result.set("api_data", post)
return result

41
src/auto_archiver/modules/generic_extractor/truth.py
View File

@@ -9,11 +9,11 @@ from dateutil.parser import parse as parse_dt
from .dropin import GenericDropin
class Truth(GenericDropin):
class Truth(GenericDropin):
def extract_post(self, url, ie_instance: InfoExtractor) -> dict:
video_id = ie_instance._match_id(url)
truthsocial_url = f'https://truthsocial.com/api/v1/statuses/{video_id}'
truthsocial_url = f"https://truthsocial.com/api/v1/statuses/{video_id}"
return ie_instance._download_json(truthsocial_url, video_id)
def skip_ytdlp_download(self, url, ie_instance: Type[InfoExtractor]) -> bool:
@@ -22,31 +22,42 @@ class Truth(GenericDropin):
def create_metadata(self, post: dict, ie_instance: InfoExtractor, archiver: Extractor, url: str) -> Metadata:
"""
Creates metadata from a truth social post
Only used for posts that contain no media. ytdlp.TruthIE extractor can handle posts with media
Format is:
{'id': '109598702184774628', 'created_at': '2022-12-29T19:51:18.161Z', 'in_reply_to_id': None, 'quote_id': None, 'in_reply_to_account_id': None, 'sensitive': False, 'spoiler_text': '', 'visibility': 'public', 'language': 'en', 'uri': 'https://truthsocial.com/@bbcnewa/109598702184774628', 'url': 'https://truthsocial.com/@bbcnewa/109598702184774628', 'content': '<p>Pele, regarded by many as football\'s greatest ever player, has died in Brazil at the age of 82. <a href="https://www.bbc.com/sport/football/42751517" rel="nofollow noopener noreferrer" target="_blank"><span class="invisible">https://www.</span><span class="ellipsis">bbc.com/sport/football/4275151</span><span class="invisible">7</span></a></p>', 'account': {'id': '107905163010312793', 'username': 'bbcnewa', 'acct': 'bbcnewa', 'display_name': 'BBC News', 'locked': False, 'bot': False, 'discoverable': True, 'group': False, 'created_at': '2022-03-05T17:42:01.159Z', 'note': '<p>News, features and analysis by the BBC</p>', 'url': 'https://truthsocial.com/@bbcnewa', 'avatar': 'https://static-assets-1.truthsocial.com/tmtg:prime-ts-assets/accounts/avatars/107/905/163/010/312/793/original/e7c07550dc22c23a.jpeg', 'avatar_static': 'https://static-assets-1.truthsocial.com/tmtg:prime-ts-assets/accounts/avatars/107/905/163/010/312/793/original/e7c07550dc22c23a.jpeg', 'header': 'https://static-assets-1.truthsocial.com/tmtg:prime-ts-assets/accounts/headers/107/905/163/010/312/793/original/a00eeec2b57206c7.jpeg', 'header_static': 'https://static-assets-1.truthsocial.com/tmtg:prime-ts-assets/accounts/headers/107/905/163/010/312/793/original/a00eeec2b57206c7.jpeg', 'followers_count': 1131, 'following_count': 3, 'statuses_count': 9, 'last_status_at': '2024-11-12', 'verified': False, 'location': '', 'website': 'https://www.bbc.com/news', 'unauth_visibility': True, 'chats_onboarded': True, 'feeds_onboarded': True, 'accepting_messages': False, 'show_nonmember_group_statuses': None, 'emojis': [], 'fields': [], 'tv_onboarded': True, 'tv_account': False}, 'media_attachments': [], 'mentions': [], 'tags': [], 'card': None, 'group': None, 'quote': None, 'in_reply_to': None, 'reblog': None, 'sponsored': False, 'replies_count': 1, 'reblogs_count': 0, 'favourites_count': 2, 'favourited': False, 'reblogged': False, 'muted': False, 'pinned': False, 'bookmarked': False, 'poll': None, 'emojis': []}
"""
result = Metadata()
result.set_url(url)
timestamp = post['created_at'] # format is 2022-12-29T19:51:18.161Z
timestamp = post["created_at"] # format is 2022-12-29T19:51:18.161Z
result.set_timestamp(parse_dt(timestamp))
result.set('description', post['content'])
result.set('author', post['account']['username'])
result.set("description", post["content"])
result.set("author", post["account"]["username"])
for key in ['replies_count', 'reblogs_count', 'favourites_count', ('account', 'followers_count'), ('account', 'following_count'), ('account', 'statuses_count'), ('account', 'display_name'), 'language', 'in_reply_to_account', 'replies_count']:
for key in [
"replies_count",
"reblogs_count",
"favourites_count",
("account", "followers_count"),
("account", "following_count"),
("account", "statuses_count"),
("account", "display_name"),
"language",
"in_reply_to_account",
"replies_count",
]:
if isinstance(key, tuple):
store_key = " ".join(key)
else:
store_key = key
result.set(store_key, traverse_obj(post, key))
# add the media
for media in post.get('media_attachments', []):
filename = archiver.download_from_url(media['url'])
result.add_media(Media(filename), id=media.get('id'))
return result
# add the media
for media in post.get("media_attachments", []):
filename = archiver.download_from_url(media["url"])
result.add_media(Media(filename), id=media.get("id"))
return result

51
src/auto_archiver/modules/generic_extractor/twitter.py
View File

@@ -1,18 +1,17 @@
import re, mimetypes, json
from datetime import datetime
import re
import mimetypes
from loguru import logger
from slugify import slugify
from auto_archiver.core.metadata import Metadata, Media
from auto_archiver.utils import url as UrlUtil
from auto_archiver.utils import url as UrlUtil, get_datetime_from_str
from auto_archiver.core.extractor import Extractor
from .dropin import GenericDropin, InfoExtractor
class Twitter(GenericDropin):
def choose_variant(self, variants):
# choosing the highest quality possible
variant, width, height = None, 0, 0
@@ -27,44 +26,48 @@ class Twitter(GenericDropin):
else:
variant = var if not variant else variant
return variant
def extract_post(self, url: str, ie_instance: InfoExtractor):
twid = ie_instance._match_valid_url(url).group('id')
twid = ie_instance._match_valid_url(url).group("id")
return ie_instance._extract_status(twid=twid)
def keys_to_clean(self, video_data, info_extractor):
return ["user", "created_at", "entities", "favorited", "translator_type"]
def create_metadata(self, tweet: dict, ie_instance: InfoExtractor, archiver: Extractor, url: str) -> Metadata:
result = Metadata()
try:
if not tweet.get("user") or not tweet.get("created_at"):
raise ValueError(f"Error retreiving post. Are you sure it exists?")
timestamp = datetime.strptime(tweet["created_at"], "%a %b %d %H:%M:%S %z %Y")
raise ValueError("Error retreiving post. Are you sure it exists?")
timestamp = get_datetime_from_str(tweet["created_at"], "%a %b %d %H:%M:%S %z %Y")
except (ValueError, KeyError) as ex:
logger.warning(f"Unable to parse tweet: {str(ex)}\nRetreived tweet data: {tweet}")
return False
result\
.set_title(tweet.get('full_text', ''))\
.set_content(json.dumps(tweet, ensure_ascii=False))\
.set_timestamp(timestamp)
full_text = tweet.pop("full_text", "")
author = tweet["user"].get("name", "")
result.set("author", author).set_url(url)
result.set_title(f"{author} - {full_text}").set_content(full_text).set_timestamp(timestamp)
if not tweet.get("entities", {}).get("media"):
logger.debug('No media found, archiving tweet text only')
logger.debug("No media found, archiving tweet text only")
result.status = "twitter-ytdl"
return result
for i, tw_media in enumerate(tweet["entities"]["media"]):
media = Media(filename="")
mimetype = ""
if tw_media["type"] == "photo":
media.set("src", UrlUtil.twitter_best_quality_url(tw_media['media_url_https']))
media.set("src", UrlUtil.twitter_best_quality_url(tw_media["media_url_https"]))
mimetype = "image/jpeg"
elif tw_media["type"] == "video":
variant = self.choose_variant(tw_media['video_info']['variants'])
media.set("src", variant['url'])
mimetype = variant['content_type']
variant = self.choose_variant(tw_media["video_info"]["variants"])
media.set("src", variant["url"])
mimetype = variant["content_type"]
elif tw_media["type"] == "animated_gif":
variant = tw_media['video_info']['variants'][0]
media.set("src", variant['url'])
mimetype = variant['content_type']
variant = tw_media["video_info"]["variants"][0]
media.set("src", variant["url"])
mimetype = variant["content_type"]
ext = mimetypes.guess_extension(mimetype)
media.filename = archiver.download_from_url(media.get("src"), f'{slugify(url)}_{i}{ext}')
media.filename = archiver.download_from_url(media.get("src"), f"{slugify(url)}_{i}{ext}")
result.add_media(media)
return result
return result

Some files were not shown because too many files have changed in this diff Show More