Compare commits

..

1 Commits

Author SHA1 Message Date
Patrick Robertson
c4bf30d302 Dockerfile for scoop 2025-03-23 13:23:54 +04:00
46 changed files with 742 additions and 1938 deletions

View File

@@ -1,40 +0,0 @@
# To get started with Dependabot version updates, you'll need to specify which
# package ecosystems to update and where the package manifests are located.
# Please see the documentation for all configuration options:
# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
version: 2
updates:
- package-ecosystem: "pip"
directory: "/"
groups:
python:
patterns:
- "*"
schedule:
interval: "weekly"
- package-ecosystem: "github-actions"
directory: "/"
groups:
actions:
patterns:
- "*"
schedule:
interval: "weekly"
- package-ecosystem: "npm"
directory: "/scripts/settings/"
groups:
actions:
patterns:
- "*"
schedule:
interval: "weekly"
- package-ecosystem: "docker"
# Look for a `Dockerfile` in the `root` directory
directory: "/"
# Check for updates once a week
schedule:
interval: "weekly"

View File

@@ -22,7 +22,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Check out the repo
uses: actions/checkout@v4
uses: actions/checkout@v3
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
@@ -33,14 +33,14 @@ jobs:
uses: docker/setup-buildx-action@v3
- name: Log in to Docker Hub
uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772
uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Extract metadata (tags, labels) for Docker
id: meta
uses: docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804
uses: docker/metadata-action@369eb591f429131d6889c46b94e711f089e6ca96
with:
images: bellingcat/auto-archiver

View File

@@ -3,18 +3,8 @@ name: Ruff Formatting & Linting
on:
push:
branches: [ main ]
paths-ignore:
- "README.md"
- ".github"
- "poetry.lock"
- "scripts/settings"
pull_request:
branches: [ main ]
paths-ignore:
- "README.md"
- ".github"
- "poetry.lock"
- "scripts/settings"
jobs:
build:

View File

@@ -20,7 +20,8 @@ jobs:
fail-fast: false
matrix:
python-version: ["3.10", "3.11", "3.12"]
os: [ubuntu-22.04, ubuntu-latest]
os: [ubuntu-22.04]
#TODO: re-enable ubuntu-latest, this is disabled as oscrypto cannot be pinned to github commit and pushed to pypi
defaults:
run:
working-directory: ./
@@ -28,23 +29,16 @@ jobs:
steps:
- uses: actions/checkout@v4
- name: Install Poetry
run: pipx install poetry
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
cache: 'poetry'
- name: Install latest Poetry
run: pipx install poetry
- name: Cache Poetry and pip artifacts
uses: actions/cache@v4
with:
path: |
~/.cache/pypoetry
~/.cache/pip
key: poetry-${{ matrix.os }}-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }}
- name: Install dependencies from source only
- name: Install dependencies
run: poetry install --no-interaction --with dev
- name: Run Core Tests

View File

@@ -22,23 +22,16 @@ jobs:
steps:
- uses: actions/checkout@v4
- name: Install poetry
run: pipx install poetry
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
cache: 'poetry'
- name: Install latest Poetry
run: pipx install poetry
- name: Cache Poetry and pip artifacts
uses: actions/cache@v4
with:
path: |
~/.cache/pypoetry
~/.cache/pip
key: poetry-${{ matrix.os }}-${{ matrix.python-version }}-${{ hashFiles('**/poetry.lock') }}
- name: Install dependencies from source only
- name: Install dependencies
run: poetry install --no-interaction --with dev
- name: Run Download Tests

View File

@@ -1,4 +1,4 @@
FROM webrecorder/browsertrix-crawler:1.5.8 AS base
FROM ubuntu:noble AS base
ENV RUNNING_IN_DOCKER=1 \
LANG=C.UTF-8 \
@@ -11,9 +11,14 @@ ENV RUNNING_IN_DOCKER=1 \
ARG TARGETARCH
# Installing system dependencies
RUN add-apt-repository ppa:mozillateam/ppa && \
RUN apt-get update && \
apt install -y --no-install-recommends software-properties-common && \
add-apt-repository ppa:mozillateam/ppa && \
apt-get update && \
apt-get install -y --no-install-recommends gcc ffmpeg fonts-noto exiftool && \
apt-get install -y --no-install-recommends ca-certificates build-essential python3-pip python3-dev python3-venv gcc wget ffmpeg fonts-noto exiftool \
fonts-arphic-ukai fonts-arphic-uming fonts-freefont-ttf fonts-gfs-neohellenic fonts-indic fonts-ipafont-mincho fonts-ipafont-gothic fonts-kacst \
fonts-liberation fonts-noto-cjk fonts-noto-color-emoji fonts-roboto fonts-stix fonts-thai-tlwg fonts-sil-padauk fonts-ubuntu fonts-unfonts-core fonts-wqy-zenhei \
&& \
apt-get install -y --no-install-recommends firefox-esr && \
ln -s /usr/bin/firefox-esr /usr/bin/firefox
@@ -35,6 +40,18 @@ RUN if [ $(uname -m) = "aarch64" ]; then \
# Poetry and runtime
FROM base AS runtime
# Download and install node + Scoop
RUN wget -q -O - https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.2/install.sh | bash \
&& . "$HOME/.nvm/nvm.sh" \
&& nvm install 23 && nvm use 23
# Install Scoop - global install so `scoop` is available in the PATH
RUN . "$HOME/.nvm/nvm.sh" && npm install -g @harvard-lil/scoop && \
npx playwright install-deps chromium && \
corepack enable yarn
ENV POETRY_NO_INTERACTION=1 \
POETRY_VIRTUALENVS_IN_PROJECT=1 \
POETRY_VIRTUALENVS_CREATE=1

View File

@@ -1,13 +1,12 @@
<h1 align="center">Auto Archiver</h1>
[![Documentation Status](https://readthedocs.org/projects/auto-archiver/badge/?version=latest)](https://auto-archiver.readthedocs.io/en/latest/?badge=latest)
[![PyPI version](https://badge.fury.io/py/auto-archiver.svg)](https://badge.fury.io/py/auto-archiver)
[![Docker Image Version (latest by date)](https://img.shields.io/docker/v/bellingcat/auto-archiver?sort=semver&logo=docker&color=#69F0AE)](https://hub.docker.com/r/bellingcat/auto-archiver)
[![Docker Image Version (latest by date)](https://img.shields.io/docker/v/bellingcat/auto-archiver?label=version&logo=docker)](https://hub.docker.com/r/bellingcat/auto-archiver)
[![Core Test Status](https://github.com/bellingcat/auto-archiver/workflows/Core%20Tests/badge.svg)](https://github.com/bellingcat/auto-archiver/actions/workflows/tests-core.yaml)
<!-- [![Download Test Status](https://github.com/bellingcat/auto-archiver/workflows/Download%20Tests/badge.svg)](https://github.com/bellingcat/auto-archiver/actions/workflows/tests-download.yaml) -->
[![Download Test Status](https://github.com/bellingcat/auto-archiver/workflows/Download%20Tests/badge.svg)](https://github.com/bellingcat/auto-archiver/actions/workflows/tests-download.yaml)
<!-- ![Docker Pulls](https://img.shields.io/docker/pulls/bellingcat/auto-archiver) -->
<!-- [![PyPI download month](https://img.shields.io/pypi/dm/auto-archiver.svg)](https://pypi.python.org/pypi/auto-archiver/) -->
<!-- [![Documentation Status](https://readthedocs.org/projects/vk-url-scraper/badge/?version=latest)](https://vk-url-scraper.readthedocs.io/en/latest/?badge=latest) -->

View File

@@ -106,117 +106,5 @@ Finally,Some important things to remember:
## Authenticating on XXXX site with username/password
```{note}
This section is still under construction 🚧
```{note} This section is still under construction 🚧
```
# Proof of Origin Tokens
YouTube uses **Proof of Origin Tokens (POT)** as part of its bot detection system to verify that requests originate from valid clients. If a token is missing or invalid, some videos may return errors like "Sign in to confirm you're not a bot."
yt-dlp provides [a detailed guide to POTs](https://github.com/yt-dlp/yt-dlp/wiki/PO-Token-Guide).
### How Auto Archiver Uses POT
This feature is enabled for the Generic Archiver via two yt-dlp plugins:
- **Client-side plugin**: [yt-dlp-get-pot](https://github.com/coletdjnz/yt-dlp-get-pot)
Detects when a token is required and requests one from a provider.
- **Provider plugin**: [bgutil-ytdlp-pot-provider](https://github.com/Brainicism/bgutil-ytdlp-pot-provider)
Includes both a Python plugin and a **Node.js server or script** to generate the token.
These are installed in our Poetry environment.
### Integration Methods
**Docker (Recommended)**:
When running the Auto Archiver using the Docker image, we use the [Node.js token generation script](https://github.com/Brainicism/bgutil-ytdlp-pot-provider/tree/master/server).
This is to avoid managing a separate server process, and is handled automatically inside the Docker container when needed.
This is already included in the Docker image, however if you need to disable this you can set the config option `bguils_po_token_method` under the `generic_extractor` section of your `orchestration.yaml` config file to "disabled".
```yaml
generic_extractor:
bguils_po_token_method: "disabled"
```
**PyPi/ Local**:
When using the Auto Archiver PyPI package, or running locally, you will need additional system requirements to run the token generation script, namely either Docker, or Node.js and Yarn.
See the [bgutil-ytdlp-pot-provider](https://github.com/Brainicism/bgutil-ytdlp-pot-provider?tab=readme-ov-file#a-http-server-option) documentation for more details.
WARNING⚠: This will add the server scripts to the home directory of wherever this is running.
- You can set the config option `bguils_po_token_method` under the `generic_extractor` section of your `orchestration.yaml` config file to "script" to enable the token generation script process locally.
- Alternatively you can run the bgutil-ytdlp-pot-provider server separately using their Docker image or Node.js server.
### Notes
- The token generation script is only triggered when needed by yt-dlp, so it should have no effect unless YouTube requests a POT.
- If you're running the Auto Archiver in Docker, this is set up automatically.
- If you're running locally, you'll need to run the setup script manually or enable the feature in your config.
- You can set up both the server and the script, and the plugin will fallback on each other if needed. This is recommended for robustness!
### Configurations:
## Configurations Summary
| Option | Behavior | Docker Default? |
|------------| ------------------------------------------------------------------------------------------------------------------------------------------ | --------------- |
| `auto` | Docker: Automatically downloads and uses the token generation script. Local: Does nothing; assumes a separate server is running externally. | ✅ Yes |
| `script` | Explicitly downloads and uses the token generation script, even locally. | ❌ No |
| `disabled` | Disables token generation completely. | ❌ No |
Example configuration:
```yaml
generic_extractor:
# ...
bguils_po_token_method: "script"
# For debugging add the verbose flag here:
ytdlp_args: "--no-abort-on-error --abort-on-error --verbose"
```
**Advanced Configuration:**
If you change the default port of the bgutil-ytdlp-pot-provider server, you can pass the updated values using our `extractor_args` option for the gereric extractor.
```yaml
generic_extractor:
ytdlp_args: "--no-abort-on-error --abort-on-error --verbose"
ytdlp_update_interval: 5
bguils_po_token_method: "script"
extractor_args:
youtube:
getpot_bgutil_baseurl: "http://127.0.0.1:8080"
player_client: web,tv
```
For more details on this for bgutils see [here](https://github.com/Brainicism/bgutil-ytdlp-pot-provider?tab=readme-ov-file#usage)
### Checking the logs
To verify that the POT process working, look for the following lines in your log after adding the config option:
```shell
[GetPOT] BgUtilScript: Generating POT via script: /Users/you/bgutil-ytdlp-pot-provider/server/build/generate_once.js
[debug] [GetPOT] BgUtilScript: Executing command to get POT via script: /Users/you/.nvm/versions/node/v20.18.0/bin/node /Users/you/bgutil-ytdlp-pot-provider/server/build/generate_once.js -v ymCMy8OflKM
[debug] [GetPOT] BgUtilScript: stdout:
{"poToken":"MlMxojNFhEJvUzGeHEkVRSK_luXtwcDnwSNIOgaUutqB7t99nmlNvtWgYayboopG6ZopZgmQ-6PJCWEMHv89MIiFGGlJRY25Fkwzxmia_8uYgf5AWf==","generatedAt":"2025-03-26T10:45:26.156Z","visitIdentifier":"ymCMy8OflKM"}
[debug] [GetPOT] Fetching gvs PO Token for tv client
```
If it can't find the script or something, you'll see something like this:
```shell
[debug] [GetPOT] Fetching player PO Token for tv client
WARNING: [GetPOT] BgUtilScript: Script path doesn't exist: /Users/you/bgutil-ytdlp-pot-provider/server/build/generate_once.js. Please make sure the script has been transpiled correctly.
WARNING: [GetPOT] BgUtilHTTP: Error reaching GET http://127.0.0.1:4416/ping (caused by TransportError). Please make sure that the server is reachable at http://127.0.0.1:4416.
[debug] [GetPOT] No player PO Token provider available for tv client
```
In this case check that the script has been transpiled correctly and is available at the path specified in the log,
or that the server is running and reachable.

View File

@@ -1,169 +0,0 @@
# InstagrAPI Server
The instagram API Extractor requires access to a running instance of the InstagrAPI server.
We have a lightweight script with the endpoints required for our Instagram API Extractor module which you can run locally, or via Docker.
⚠️ Warning: Remember that it's best not to use your own personal account for archiving. [Here's why](../installation/authentication.md#recommendations-for-authentication).
## Quick Start: Using Docker
We've provided a convenient shell script (`run_instagrapi_server.sh`) that simplifies the process of setting up and running the Instagrapi server in Docker. This script handles building the Docker image, setting up credentials, and starting the container.
### 🔧 Running the script:
Run this script either from the repository root or from within the `scripts/instagrapi_server` directory:
```bash
./scripts/instagrapi_server/run_instagrapi_server.sh
```
This script will:
- Prompt for your Instagram username and password.
- Create the necessary `.env` file.
- Build the Docker image.
- Start the Docker container and authenticate with Instagram, creating a session automatically.
### ⏱ To run the server again later:
```bash
docker start ig-instasrv
```
### 🐛 Debugging:
View logs:
```bash
docker logs ig-instasrv
```
### Overview: How the Setup Works
1. You enter your Instagram credentials in a local `.env` file
2. You run the server **once locally** to generate a session file
3. After that, you can choose to run the server again locally or inside Docker without needing to log in again
---
## Optional: Manual / Local Setup
If you'd prefer to run the server manually (without Docker), you can follow these steps:
1. **Navigate to the server folder (and stay there for the rest of this guide)**:
```bash
cd scripts/instagrapi_server
```
2. **Create a `secrets/` folder** (if it doesn't already exist in `scripts/instagrapi_server`):
```bash
mkdir -p secrets
```
3. **Create a `.env` file** inside `secrets/` with your Instagram credentials:
```dotenv
INSTAGRAM_USERNAME="your_username"
INSTAGRAM_PASSWORD="your_password"
```
4. **Install dependencies** using the pyproject.toml file:
```bash
poetry install --no-root
```
5. **Run the server locally**:
```bash
poetry run uvicorn src.instaserver:app --port 8000
```
6. **Watch for the message**:
```
Login successful, session saved.
```
✅ Your session is now saved to `secrets/instagrapi_session.json`.
### To run it again locally:
```bash
poetry run uvicorn src.instaserver:app --port 8000
```
---
## Adding the API Endpoint to Auto Archiver
The server should now be running within that session, and accessible at http://127.0.0.1:8000
You can set this in the Auto Archiver orchestration.yaml file like this:
```yaml
instagram_api_extractor:
api_endpoint: http://127.0.0.1:8000
```
---
## 2. Running the Server Again
Once the session file is created, you should be able to run the server without logging in again.
### To run it locally (from scripts/instagrapi_server):
```bash
poetry run uvicorn src.instgrapinstance.instaserver:app --port 8000
```
---
## 3. Running via Docker (After Setup is Complete, either locally or via the script)
Once the `instagrapi_session.json` and `.env` files are set up, you can pass them Docker and it should authenticate successfully.
### 🔨 Build the Docker image manually:
```bash
docker build -t instagrapi-server .
```
### ▶️ Run the container:
```bash
docker run -d \
--env-file secrets/.env \
-v "$(pwd)/secrets:/app/secrets" \
-p 8000:8000 \
--name ig-instasrv \
instagrapi-server
```
This passes the /secrets/ directory to docker as well as the environment variables from the `.env` file.
---
## 4. Optional Cleanup
- **Stop the Docker container**:
```bash
docker stop ig-instasrv
```
- **Remove the container**:
```bash
docker rm ig-instasrv
```
- **Remove the Docker image**:
```bash
docker rmi instagrapi-server
```
### ⏱ To run again later:
```bash
docker start ig-instasrv
```
---
## Notes
- Never share your `.env` or `instagrapi_session.json` — these contain sensitive login data.
- If you want to reset your session, simply delete the `secrets/instagrapi_session.json` file and re-run the local server.

View File

@@ -6,15 +6,6 @@ There are two main use cases for authentication:
* Some websites require some kind of authentication in order to view the content. Examples include Facebook, Telegram etc.
* Some websites use anti-bot systems to block bot-like tools from accessing the website. Adding real login information to auto-archiver can sometimes bypass this.
```{note}
The Authentication framework currently only works with the following modules:
* Generic Extractor
* Screenshot Enricher
To authenticate for WACZ archiving, see the instructions on the [](../modules/autogen/enricher/wacz_extractor_enricher.md) page.
```
## The Authentication Config
You can save your authentication information directly inside your orchestration config file, or as a separate file (for security/multi-deploy purposes). Whether storing your settings inside the orchestration file, or as a separate file, the configuration format is the same. Currently, auto-archiver supports the following authentication types:
@@ -36,7 +27,7 @@ You can save your authentication information directly inside your orchestration
The Username & Password, and API settings only work with the Generic Extractor. Other modules (like the screenshot enricher) can only use the `cookies` options. Furthermore, many sites can still detect bots and block username/password logins. Twitter/X and YouTube are two prominent ones that block username/password logging.
One of the 'Cookies' options is recommended for the most robust archiving, but it still isn't guaranteed to work.
One of the 'Cookies' options is recommended for the most robust archiving.
```
```{code} yaml

461
poetry.lock generated
View File

@@ -33,14 +33,14 @@ files = [
[[package]]
name = "anyio"
version = "4.9.0"
version = "4.8.0"
description = "High level compatibility layer for multiple asynchronous event loop implementations"
optional = false
python-versions = ">=3.9"
groups = ["docs"]
files = [
{file = "anyio-4.9.0-py3-none-any.whl", hash = "sha256:9f76d541cad6e36af7beb62e978876f3b41e3e04f2c1fbf0884604c0a9c4d93c"},
{file = "anyio-4.9.0.tar.gz", hash = "sha256:673c0c244e15788651a4ff38710fea9675823028a6f08a5eda409e0c9840a028"},
{file = "anyio-4.8.0-py3-none-any.whl", hash = "sha256:b5011f270ab5eb0abf13385f851315585cc37ef330dd88e27ec3d34d651fd47a"},
{file = "anyio-4.8.0.tar.gz", hash = "sha256:1d9fe889df5212298c0c0723fa20479d1b94883a2df44bd3897aa91083316f7a"},
]
[package.dependencies]
@@ -50,20 +50,32 @@ sniffio = ">=1.1"
typing_extensions = {version = ">=4.5", markers = "python_version < \"3.13\""}
[package.extras]
doc = ["Sphinx (>=8.2,<9.0)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx_rtd_theme"]
test = ["anyio[trio]", "blockbuster (>=1.5.23)", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "trustme", "truststore (>=0.9.1) ; python_version >= \"3.10\"", "uvloop (>=0.21) ; platform_python_implementation == \"CPython\" and platform_system != \"Windows\" and python_version < \"3.14\""]
doc = ["Sphinx (>=7.4,<8.0)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx_rtd_theme"]
test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "trustme", "truststore (>=0.9.1) ; python_version >= \"3.10\"", "uvloop (>=0.21) ; platform_python_implementation == \"CPython\" and platform_system != \"Windows\" and python_version < \"3.14\""]
trio = ["trio (>=0.26.1)"]
[[package]]
name = "asn1crypto"
version = "1.5.1"
description = "Fast ASN.1 parser and serializer with definitions for private keys, public keys, certificates, CRL, OCSP, CMS, PKCS#3, PKCS#7, PKCS#8, PKCS#12, PKCS#5, X.509 and TSP"
optional = false
python-versions = "*"
groups = ["main"]
files = [
{file = "asn1crypto-1.5.1-py2.py3-none-any.whl", hash = "sha256:db4e40728b728508912cbb3d44f19ce188f218e9eba635821bb4b68564f8fd67"},
{file = "asn1crypto-1.5.1.tar.gz", hash = "sha256:13ae38502be632115abf8a24cbe5f4da52e3b5231990aff31123c805306ccb9c"},
]
[[package]]
name = "astroid"
version = "3.3.9"
version = "3.3.8"
description = "An abstract syntax tree for Python with inference support."
optional = false
python-versions = ">=3.9.0"
groups = ["docs"]
files = [
{file = "astroid-3.3.9-py3-none-any.whl", hash = "sha256:d05bfd0acba96a7bd43e222828b7d9bc1e138aaeb0649707908d3702a9831248"},
{file = "astroid-3.3.9.tar.gz", hash = "sha256:622cc8e3048684aa42c820d9d218978021c3c3d174fb03a9f0d615921744f550"},
{file = "astroid-3.3.8-py3-none-any.whl", hash = "sha256:187ccc0c248bfbba564826c26f070494f7bc964fd286b6d9fff4420e55de828c"},
{file = "astroid-3.3.8.tar.gz", hash = "sha256:a88c7994f914a4ea8572fac479459f4955eeccc877be3f2d959a33273b0cf40b"},
]
[package.dependencies]
@@ -71,21 +83,21 @@ typing-extensions = {version = ">=4.0.0", markers = "python_version < \"3.11\""}
[[package]]
name = "attrs"
version = "25.3.0"
version = "25.1.0"
description = "Classes Without Boilerplate"
optional = false
python-versions = ">=3.8"
groups = ["main"]
files = [
{file = "attrs-25.3.0-py3-none-any.whl", hash = "sha256:427318ce031701fea540783410126f03899a97ffc6f61596ad581ac2e40e3bc3"},
{file = "attrs-25.3.0.tar.gz", hash = "sha256:75d7cefc7fb576747b2c81b4442d4d4a1ce0900973527c011d1030fd3bf4af1b"},
{file = "attrs-25.1.0-py3-none-any.whl", hash = "sha256:c75a69e28a550a7e93789579c22aa26b0f5b83b75dc4e08fe092980051e1090a"},
{file = "attrs-25.1.0.tar.gz", hash = "sha256:1c97078a80c814273a76b2a298a932eb681c87415c11dee0a6921de7f1b02c3e"},
]
[package.extras]
benchmark = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"]
cov = ["cloudpickle ; platform_python_implementation == \"CPython\"", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"]
dev = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pre-commit-uv", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"]
docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier"]
docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier (<24.7)"]
tests = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"]
tests-mypy = ["mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\""]
@@ -158,35 +170,20 @@ charset-normalizer = ["charset-normalizer"]
html5lib = ["html5lib"]
lxml = ["lxml"]
[[package]]
name = "bgutil-ytdlp-pot-provider"
version = "0.7.4"
description = ""
optional = false
python-versions = ">=3.8"
groups = ["main"]
files = [
{file = "bgutil_ytdlp_pot_provider-0.7.4-py3-none-any.whl", hash = "sha256:5f0b1d884fec66dff703c421ea06f5fc9b11022d9c0babdaa0cab13ed99b9d77"},
{file = "bgutil_ytdlp_pot_provider-0.7.4.tar.gz", hash = "sha256:b6c1462b8f979540078085cd82462ef967b8b70cd0810d469243a31f5081e5c6"},
]
[package.dependencies]
yt-dlp-get-pot = ">=0.1.1"
[[package]]
name = "boto3"
version = "1.37.18"
version = "1.37.8"
description = "The AWS SDK for Python"
optional = false
python-versions = ">=3.8"
groups = ["main"]
files = [
{file = "boto3-1.37.18-py3-none-any.whl", hash = "sha256:1545c943f36db41853cdfdb6ff09c4eda9220dd95bd2fae76fc73091603525d1"},
{file = "boto3-1.37.18.tar.gz", hash = "sha256:9b272268794172b0b8bb9fb1f3c470c3b6c0ffb92fbd4882465cc740e40fbdcd"},
{file = "boto3-1.37.8-py3-none-any.whl", hash = "sha256:b9f506e08c9f54687d6c073ef1c550a24a62cc2d1e0bc7cda9f13112a38818bf"},
{file = "boto3-1.37.8.tar.gz", hash = "sha256:9448f4a079189e19c3253cfdc5b8ef6dc51a3b82431e8347a51f4c1b2d9dab42"},
]
[package.dependencies]
botocore = ">=1.37.18,<1.38.0"
botocore = ">=1.37.8,<1.38.0"
jmespath = ">=0.7.1,<2.0.0"
s3transfer = ">=0.11.0,<0.12.0"
@@ -195,14 +192,14 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"]
[[package]]
name = "botocore"
version = "1.37.18"
version = "1.37.8"
description = "Low-level, data-driven core of boto 3."
optional = false
python-versions = ">=3.8"
groups = ["main"]
files = [
{file = "botocore-1.37.18-py3-none-any.whl", hash = "sha256:a8b97d217d82b3c4f6bcc906e264df7ebb51e2c6a62b3548a97cd173fb8759a1"},
{file = "botocore-1.37.18.tar.gz", hash = "sha256:99e8eefd5df6347ead15df07ce55f4e62a51ea7b54de1127522a08597923b726"},
{file = "botocore-1.37.8-py3-none-any.whl", hash = "sha256:a6c94f33de12f4b10b10684019e554c980469b8394c6d82448a738cbd8452cef"},
{file = "botocore-1.37.8.tar.gz", hash = "sha256:b5825e08dd3e25642aa22a0d7d92bf81fef1ef857117e4155f923bbccf5aba63"},
]
[package.dependencies]
@@ -388,6 +385,22 @@ files = [
{file = "certifi-2025.1.31.tar.gz", hash = "sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651"},
]
[[package]]
name = "certvalidator"
version = "0.11.1"
description = "Validates X.509 certificates and paths"
optional = false
python-versions = "*"
groups = ["main"]
files = [
{file = "certvalidator-0.11.1-py2.py3-none-any.whl", hash = "sha256:77520b269f516d4fb0902998d5bd0eb3727fe153b659aa1cb828dcf12ea6b8de"},
{file = "certvalidator-0.11.1.tar.gz", hash = "sha256:922d141c94393ab285ca34338e18dd4093e3ae330b1f278e96c837cb62cffaad"},
]
[package.dependencies]
asn1crypto = ">=0.18.1"
oscrypto = ">=0.16.1"
[[package]]
name = "cffi"
version = "1.17.1"
@@ -395,7 +408,6 @@ description = "Foreign Function Interface for Python calling C code."
optional = false
python-versions = ">=3.8"
groups = ["main"]
markers = "os_name == \"nt\" and implementation_name != \"pypy\" or platform_python_implementation != \"PyPy\""
files = [
{file = "cffi-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:df8b1c11f177bc2313ec4b2d46baec87a5f3e71fc8b45dab2ee7cae86d9aba14"},
{file = "cffi-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f2cdc858323644ab277e9bb925ad72ae0e67f69e804f4898c070998d50b1a67"},
@@ -613,60 +625,48 @@ markers = {main = "sys_platform == \"win32\" or platform_system == \"Windows\"",
[[package]]
name = "cryptography"
version = "44.0.2"
version = "41.0.7"
description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers."
optional = false
python-versions = "!=3.9.0,!=3.9.1,>=3.7"
python-versions = ">=3.7"
groups = ["main"]
files = [
{file = "cryptography-44.0.2-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:efcfe97d1b3c79e486554efddeb8f6f53a4cdd4cf6086642784fa31fc384e1d7"},
{file = "cryptography-44.0.2-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29ecec49f3ba3f3849362854b7253a9f59799e3763b0c9d0826259a88efa02f1"},
{file = "cryptography-44.0.2-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc821e161ae88bfe8088d11bb39caf2916562e0a2dc7b6d56714a48b784ef0bb"},
{file = "cryptography-44.0.2-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:3c00b6b757b32ce0f62c574b78b939afab9eecaf597c4d624caca4f9e71e7843"},
{file = "cryptography-44.0.2-cp37-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7bdcd82189759aba3816d1f729ce42ffded1ac304c151d0a8e89b9996ab863d5"},
{file = "cryptography-44.0.2-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:4973da6ca3db4405c54cd0b26d328be54c7747e89e284fcff166132eb7bccc9c"},
{file = "cryptography-44.0.2-cp37-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:4e389622b6927d8133f314949a9812972711a111d577a5d1f4bee5e58736b80a"},
{file = "cryptography-44.0.2-cp37-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:f514ef4cd14bb6fb484b4a60203e912cfcb64f2ab139e88c2274511514bf7308"},
{file = "cryptography-44.0.2-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1bc312dfb7a6e5d66082c87c34c8a62176e684b6fe3d90fcfe1568de675e6688"},
{file = "cryptography-44.0.2-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:3b721b8b4d948b218c88cb8c45a01793483821e709afe5f622861fc6182b20a7"},
{file = "cryptography-44.0.2-cp37-abi3-win32.whl", hash = "sha256:51e4de3af4ec3899d6d178a8c005226491c27c4ba84101bfb59c901e10ca9f79"},
{file = "cryptography-44.0.2-cp37-abi3-win_amd64.whl", hash = "sha256:c505d61b6176aaf982c5717ce04e87da5abc9a36a5b39ac03905c4aafe8de7aa"},
{file = "cryptography-44.0.2-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:8e0ddd63e6bf1161800592c71ac794d3fb8001f2caebe0966e77c5234fa9efc3"},
{file = "cryptography-44.0.2-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:81276f0ea79a208d961c433a947029e1a15948966658cf6710bbabb60fcc2639"},
{file = "cryptography-44.0.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9a1e657c0f4ea2a23304ee3f964db058c9e9e635cc7019c4aa21c330755ef6fd"},
{file = "cryptography-44.0.2-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:6210c05941994290f3f7f175a4a57dbbb2afd9273657614c506d5976db061181"},
{file = "cryptography-44.0.2-cp39-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d1c3572526997b36f245a96a2b1713bf79ce99b271bbcf084beb6b9b075f29ea"},
{file = "cryptography-44.0.2-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:b042d2a275c8cee83a4b7ae30c45a15e6a4baa65a179a0ec2d78ebb90e4f6699"},
{file = "cryptography-44.0.2-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:d03806036b4f89e3b13b6218fefea8d5312e450935b1a2d55f0524e2ed7c59d9"},
{file = "cryptography-44.0.2-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:c7362add18b416b69d58c910caa217f980c5ef39b23a38a0880dfd87bdf8cd23"},
{file = "cryptography-44.0.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:8cadc6e3b5a1f144a039ea08a0bdb03a2a92e19c46be3285123d32029f40a922"},
{file = "cryptography-44.0.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:6f101b1f780f7fc613d040ca4bdf835c6ef3b00e9bd7125a4255ec574c7916e4"},
{file = "cryptography-44.0.2-cp39-abi3-win32.whl", hash = "sha256:3dc62975e31617badc19a906481deacdeb80b4bb454394b4098e3f2525a488c5"},
{file = "cryptography-44.0.2-cp39-abi3-win_amd64.whl", hash = "sha256:5f6f90b72d8ccadb9c6e311c775c8305381db88374c65fa1a68250aa8a9cb3a6"},
{file = "cryptography-44.0.2-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:af4ff3e388f2fa7bff9f7f2b31b87d5651c45731d3e8cfa0944be43dff5cfbdb"},
{file = "cryptography-44.0.2-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:0529b1d5a0105dd3731fa65680b45ce49da4d8115ea76e9da77a875396727b41"},
{file = "cryptography-44.0.2-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:7ca25849404be2f8e4b3c59483d9d3c51298a22c1c61a0e84415104dacaf5562"},
{file = "cryptography-44.0.2-pp310-pypy310_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:268e4e9b177c76d569e8a145a6939eca9a5fec658c932348598818acf31ae9a5"},
{file = "cryptography-44.0.2-pp310-pypy310_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:9eb9d22b0a5d8fd9925a7764a054dca914000607dff201a24c791ff5c799e1fa"},
{file = "cryptography-44.0.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:2bf7bf75f7df9715f810d1b038870309342bff3069c5bd8c6b96128cb158668d"},
{file = "cryptography-44.0.2-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:909c97ab43a9c0c0b0ada7a1281430e4e5ec0458e6d9244c0e821bbf152f061d"},
{file = "cryptography-44.0.2-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:96e7a5e9d6e71f9f4fca8eebfd603f8e86c5225bb18eb621b2c1e50b290a9471"},
{file = "cryptography-44.0.2-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:d1b3031093a366ac767b3feb8bcddb596671b3aaff82d4050f984da0c248b615"},
{file = "cryptography-44.0.2-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:04abd71114848aa25edb28e225ab5f268096f44cf0127f3d36975bdf1bdf3390"},
{file = "cryptography-44.0.2.tar.gz", hash = "sha256:c63454aa261a0cf0c5b4718349629793e9e634993538db841165b3df74f37ec0"},
{file = "cryptography-41.0.7-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:3c78451b78313fa81607fa1b3f1ae0a5ddd8014c38a02d9db0616133987b9cdf"},
{file = "cryptography-41.0.7-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:928258ba5d6f8ae644e764d0f996d61a8777559f72dfeb2eea7e2fe0ad6e782d"},
{file = "cryptography-41.0.7-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5a1b41bc97f1ad230a41657d9155113c7521953869ae57ac39ac7f1bb471469a"},
{file = "cryptography-41.0.7-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:841df4caa01008bad253bce2a6f7b47f86dc9f08df4b433c404def869f590a15"},
{file = "cryptography-41.0.7-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:5429ec739a29df2e29e15d082f1d9ad683701f0ec7709ca479b3ff2708dae65a"},
{file = "cryptography-41.0.7-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:43f2552a2378b44869fe8827aa19e69512e3245a219104438692385b0ee119d1"},
{file = "cryptography-41.0.7-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:af03b32695b24d85a75d40e1ba39ffe7db7ffcb099fe507b39fd41a565f1b157"},
{file = "cryptography-41.0.7-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:49f0805fc0b2ac8d4882dd52f4a3b935b210935d500b6b805f321addc8177406"},
{file = "cryptography-41.0.7-cp37-abi3-win32.whl", hash = "sha256:f983596065a18a2183e7f79ab3fd4c475205b839e02cbc0efbbf9666c4b3083d"},
{file = "cryptography-41.0.7-cp37-abi3-win_amd64.whl", hash = "sha256:90452ba79b8788fa380dfb587cca692976ef4e757b194b093d845e8d99f612f2"},
{file = "cryptography-41.0.7-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:079b85658ea2f59c4f43b70f8119a52414cdb7be34da5d019a77bf96d473b960"},
{file = "cryptography-41.0.7-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:b640981bf64a3e978a56167594a0e97db71c89a479da8e175d8bb5be5178c003"},
{file = "cryptography-41.0.7-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:e3114da6d7f95d2dee7d3f4eec16dacff819740bbab931aff8648cb13c5ff5e7"},
{file = "cryptography-41.0.7-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:d5ec85080cce7b0513cfd233914eb8b7bbd0633f1d1703aa28d1dd5a72f678ec"},
{file = "cryptography-41.0.7-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:7a698cb1dac82c35fcf8fe3417a3aaba97de16a01ac914b89a0889d364d2f6be"},
{file = "cryptography-41.0.7-pp38-pypy38_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:37a138589b12069efb424220bf78eac59ca68b95696fc622b6ccc1c0a197204a"},
{file = "cryptography-41.0.7-pp38-pypy38_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:68a2dec79deebc5d26d617bfdf6e8aab065a4f34934b22d3b5010df3ba36612c"},
{file = "cryptography-41.0.7-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:09616eeaef406f99046553b8a40fbf8b1e70795a91885ba4c96a70793de5504a"},
{file = "cryptography-41.0.7-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:48a0476626da912a44cc078f9893f292f0b3e4c739caf289268168d8f4702a39"},
{file = "cryptography-41.0.7-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:c7f3201ec47d5207841402594f1d7950879ef890c0c495052fa62f58283fde1a"},
{file = "cryptography-41.0.7-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c5ca78485a255e03c32b513f8c2bc39fedb7f5c5f8535545bdc223a03b24f248"},
{file = "cryptography-41.0.7-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:d6c391c021ab1f7a82da5d8d0b3cee2f4b2c455ec86c8aebbc84837a631ff309"},
{file = "cryptography-41.0.7.tar.gz", hash = "sha256:13f93ce9bea8016c253b34afc6bd6a75993e5c40672ed5405a9c832f0d4a00bc"},
]
[package.dependencies]
cffi = {version = ">=1.12", markers = "platform_python_implementation != \"PyPy\""}
cffi = ">=1.12"
[package.extras]
docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=3.0.0) ; python_version >= \"3.8\""]
docstest = ["pyenchant (>=3)", "readme-renderer (>=30.0)", "sphinxcontrib-spelling (>=7.3.1)"]
nox = ["nox (>=2024.4.15)", "nox[uv] (>=2024.3.2) ; python_version >= \"3.8\""]
pep8test = ["check-sdist ; python_version >= \"3.8\"", "click (>=8.0.1)", "mypy (>=1.4)", "ruff (>=0.3.6)"]
sdist = ["build (>=1.0.0)"]
docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=1.1.1)"]
docstest = ["pyenchant (>=1.6.11)", "sphinxcontrib-spelling (>=4.0.1)", "twine (>=1.12.0)"]
nox = ["nox"]
pep8test = ["black", "check-sdist", "mypy", "ruff"]
sdist = ["build"]
ssh = ["bcrypt (>=3.1.5)"]
test = ["certifi (>=2024)", "cryptography-vectors (==44.0.2)", "pretend (>=0.7)", "pytest (>=7.4.0)", "pytest-benchmark (>=4.0)", "pytest-cov (>=2.10.1)", "pytest-xdist (>=3.5.0)"]
test = ["pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"]
test-randomorder = ["pytest-randomly"]
[[package]]
@@ -768,14 +768,14 @@ dev = ["Sphinx (==2.1.0)", "future (==0.17.1)", "numpy (==1.16.4)", "pytest (==4
[[package]]
name = "filelock"
version = "3.18.0"
version = "3.17.0"
description = "A platform independent file lock."
optional = false
python-versions = ">=3.9"
groups = ["dev"]
files = [
{file = "filelock-3.18.0-py3-none-any.whl", hash = "sha256:c401f4f8377c4464e6db25fff06205fd89bdd83b65eb0488ed1b160f780e21de"},
{file = "filelock-3.18.0.tar.gz", hash = "sha256:adbc88eabb99d2fec8c9c1b229b171f18afa655400173ddc653d5d01501fb9f2"},
{file = "filelock-3.17.0-py3-none-any.whl", hash = "sha256:533dc2f7ba78dc2f0f531fc6c4940addf7b70a481e269a5a3b93be94ffbe8338"},
{file = "filelock-3.17.0.tar.gz", hash = "sha256:ee4e77401ef576ebb38cd7f13b9b28893194acc20a8e68e18730ba9c0e54660e"},
]
[package.extras]
@@ -797,22 +797,22 @@ files = [
[[package]]
name = "google-api-core"
version = "2.24.2"
version = "2.24.1"
description = "Google API client core library"
optional = false
python-versions = ">=3.7"
groups = ["main"]
files = [
{file = "google_api_core-2.24.2-py3-none-any.whl", hash = "sha256:810a63ac95f3c441b7c0e43d344e372887f62ce9071ba972eacf32672e072de9"},
{file = "google_api_core-2.24.2.tar.gz", hash = "sha256:81718493daf06d96d6bc76a91c23874dbf2fac0adbbf542831b805ee6e974696"},
{file = "google_api_core-2.24.1-py3-none-any.whl", hash = "sha256:bc78d608f5a5bf853b80bd70a795f703294de656c096c0968320830a4bc280f1"},
{file = "google_api_core-2.24.1.tar.gz", hash = "sha256:f8b36f5456ab0dd99a1b693a40a31d1e7757beea380ad1b38faaf8941eae9d8a"},
]
[package.dependencies]
google-auth = ">=2.14.1,<3.0.0"
googleapis-common-protos = ">=1.56.2,<2.0.0"
proto-plus = ">=1.22.3,<2.0.0"
protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<7.0.0"
requests = ">=2.18.0,<3.0.0"
google-auth = ">=2.14.1,<3.0.dev0"
googleapis-common-protos = ">=1.56.2,<2.0.dev0"
proto-plus = ">=1.22.3,<2.0.0dev"
protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<6.0.0.dev0"
requests = ">=2.18.0,<3.0.0.dev0"
[package.extras]
async-rest = ["google-auth[aiohttp] (>=2.35.0,<3.0.dev0)"]
@@ -822,21 +822,21 @@ grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"]
[[package]]
name = "google-api-python-client"
version = "2.165.0"
version = "2.163.0"
description = "Google API Client Library for Python"
optional = false
python-versions = ">=3.7"
groups = ["main"]
files = [
{file = "google_api_python_client-2.165.0-py2.py3-none-any.whl", hash = "sha256:4eaab7d4a20be0d3d1dde462fa95e9e0ccc2a3e177a656701bf73fe738ddef7d"},
{file = "google_api_python_client-2.165.0.tar.gz", hash = "sha256:0d2aee76727a104705630bebbc43669c864b766924e9329051ef7b7e2468eb72"},
{file = "google_api_python_client-2.163.0-py2.py3-none-any.whl", hash = "sha256:080e8bc0669cb4c1fb8efb8da2f5b91a2625d8f0e7796cfad978f33f7016c6c4"},
{file = "google_api_python_client-2.163.0.tar.gz", hash = "sha256:88dee87553a2d82176e2224648bf89272d536c8f04dcdda37ef0a71473886dd7"},
]
[package.dependencies]
google-api-core = ">=1.31.5,<2.0.dev0 || >2.3.0,<3.0.0"
google-auth = ">=1.32.0,<2.24.0 || >2.24.0,<2.25.0 || >2.25.0,<3.0.0"
google-api-core = ">=1.31.5,<2.0.dev0 || >2.3.0,<3.0.0.dev0"
google-auth = ">=1.32.0,<2.24.0 || >2.24.0,<2.25.0 || >2.25.0,<3.0.0.dev0"
google-auth-httplib2 = ">=0.2.0,<1.0.0"
httplib2 = ">=0.19.0,<1.0.0"
httplib2 = ">=0.19.0,<1.dev0"
uritemplate = ">=3.0.1,<5"
[[package]]
@@ -901,21 +901,21 @@ tool = ["click (>=6.0.0)"]
[[package]]
name = "googleapis-common-protos"
version = "1.69.2"
version = "1.69.1"
description = "Common protobufs used in Google APIs"
optional = false
python-versions = ">=3.7"
groups = ["main"]
files = [
{file = "googleapis_common_protos-1.69.2-py3-none-any.whl", hash = "sha256:0b30452ff9c7a27d80bfc5718954063e8ab53dd3697093d3bc99581f5fd24212"},
{file = "googleapis_common_protos-1.69.2.tar.gz", hash = "sha256:3e1b904a27a33c821b4b749fd31d334c0c9c30e6113023d495e48979a3dc9c5f"},
{file = "googleapis_common_protos-1.69.1-py2.py3-none-any.whl", hash = "sha256:4077f27a6900d5946ee5a369fab9c8ded4c0ef1c6e880458ea2f70c14f7b70d5"},
{file = "googleapis_common_protos-1.69.1.tar.gz", hash = "sha256:e20d2d8dda87da6fe7340afbbdf4f0bcb4c8fae7e6cadf55926c31f946b0b9b1"},
]
[package.dependencies]
protobuf = ">=3.20.2,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<7.0.0"
protobuf = ">=3.20.2,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<6.0.0.dev0"
[package.extras]
grpc = ["grpcio (>=1.44.0,<2.0.0)"]
grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"]
[[package]]
name = "gspread"
@@ -1004,14 +1004,14 @@ files = [
[[package]]
name = "iniconfig"
version = "2.1.0"
version = "2.0.0"
description = "brain-dead simple config-ini parsing"
optional = false
python-versions = ">=3.8"
python-versions = ">=3.7"
groups = ["dev"]
files = [
{file = "iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760"},
{file = "iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7"},
{file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"},
{file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"},
]
[[package]]
@@ -1445,6 +1445,21 @@ files = [
pycryptodomex = ">=3.3.1"
python-bitcoinlib = ">=0.9.0,<0.13.0"
[[package]]
name = "oscrypto"
version = "1.3.0"
description = "TLS (SSL) sockets, key generation, encryption, decryption, signing, verification and KDFs using the OS crypto libraries. Does not require a compiler, and relies on the OS for patching. Works on Windows, OS X and Linux/BSD."
optional = false
python-versions = "*"
groups = ["main"]
files = [
{file = "oscrypto-1.3.0-py2.py3-none-any.whl", hash = "sha256:2b2f1d2d42ec152ca90ccb5682f3e051fb55986e1b170ebde472b133713e7085"},
{file = "oscrypto-1.3.0.tar.gz", hash = "sha256:6f5fef59cb5b3708321db7cca56aed8ad7e662853351e7991fcf60ec606d47a4"},
]
[package.dependencies]
asn1crypto = ">=1.5.1"
[[package]]
name = "outcome"
version = "1.3.0.post0"
@@ -1584,20 +1599,20 @@ xmp = ["defusedxml"]
[[package]]
name = "platformdirs"
version = "4.3.7"
version = "4.3.6"
description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`."
optional = false
python-versions = ">=3.9"
python-versions = ">=3.8"
groups = ["dev"]
files = [
{file = "platformdirs-4.3.7-py3-none-any.whl", hash = "sha256:a03875334331946f13c549dbd8f4bac7a13a50a895a0eb1e8c6a8ace80d40a94"},
{file = "platformdirs-4.3.7.tar.gz", hash = "sha256:eb437d586b6a0986388f0d6f74aa0cde27b48d0e3d66843640bfb6bdcdb6e351"},
{file = "platformdirs-4.3.6-py3-none-any.whl", hash = "sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb"},
{file = "platformdirs-4.3.6.tar.gz", hash = "sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907"},
]
[package.extras]
docs = ["furo (>=2024.8.6)", "proselint (>=0.14)", "sphinx (>=8.1.3)", "sphinx-autodoc-typehints (>=3)"]
test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=8.3.4)", "pytest-cov (>=6)", "pytest-mock (>=3.14)"]
type = ["mypy (>=1.14.1)"]
docs = ["furo (>=2024.8.6)", "proselint (>=0.14)", "sphinx (>=8.0.2)", "sphinx-autodoc-typehints (>=2.4)"]
test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=8.3.2)", "pytest-cov (>=5)", "pytest-mock (>=3.14)"]
type = ["mypy (>=1.11.2)"]
[[package]]
name = "pluggy"
@@ -1617,14 +1632,14 @@ testing = ["pytest", "pytest-benchmark"]
[[package]]
name = "pre-commit"
version = "4.2.0"
version = "4.1.0"
description = "A framework for managing and maintaining multi-language pre-commit hooks."
optional = false
python-versions = ">=3.9"
groups = ["dev"]
files = [
{file = "pre_commit-4.2.0-py2.py3-none-any.whl", hash = "sha256:a009ca7205f1eb497d10b845e52c838a98b6cdd2102a6c8e4540e94ee75c58bd"},
{file = "pre_commit-4.2.0.tar.gz", hash = "sha256:601283b9757afd87d40c4c4a9b2b5de9637a8ea02eaff7adc2d0fb4e04841146"},
{file = "pre_commit-4.1.0-py2.py3-none-any.whl", hash = "sha256:d29e7cb346295bcc1cc75fc3e92e343495e3ea0196c9ec6ba53f49f10ab6ae7b"},
{file = "pre_commit-4.1.0.tar.gz", hash = "sha256:ae3f018575a588e30dfddfab9a05448bfbd6b73d78709617b5a2b853549716d4"},
]
[package.dependencies]
@@ -1636,39 +1651,41 @@ virtualenv = ">=20.10.0"
[[package]]
name = "proto-plus"
version = "1.26.1"
version = "1.26.0"
description = "Beautiful, Pythonic protocol buffers"
optional = false
python-versions = ">=3.7"
groups = ["main"]
files = [
{file = "proto_plus-1.26.1-py3-none-any.whl", hash = "sha256:13285478c2dcf2abb829db158e1047e2f1e8d63a077d94263c2b88b043c75a66"},
{file = "proto_plus-1.26.1.tar.gz", hash = "sha256:21a515a4c4c0088a773899e23c7bbade3d18f9c66c73edd4c7ee3816bc96a012"},
{file = "proto_plus-1.26.0-py3-none-any.whl", hash = "sha256:bf2dfaa3da281fc3187d12d224c707cb57214fb2c22ba854eb0c105a3fb2d4d7"},
{file = "proto_plus-1.26.0.tar.gz", hash = "sha256:6e93d5f5ca267b54300880fff156b6a3386b3fa3f43b1da62e680fc0c586ef22"},
]
[package.dependencies]
protobuf = ">=3.19.0,<7.0.0"
protobuf = ">=3.19.0,<6.0.0dev"
[package.extras]
testing = ["google-api-core (>=1.31.5)"]
[[package]]
name = "protobuf"
version = "6.30.1"
version = "5.29.3"
description = ""
optional = false
python-versions = ">=3.9"
python-versions = ">=3.8"
groups = ["main"]
files = [
{file = "protobuf-6.30.1-cp310-abi3-win32.whl", hash = "sha256:ba0706f948d0195f5cac504da156d88174e03218d9364ab40d903788c1903d7e"},
{file = "protobuf-6.30.1-cp310-abi3-win_amd64.whl", hash = "sha256:ed484f9ddd47f0f1bf0648806cccdb4fe2fb6b19820f9b79a5adf5dcfd1b8c5f"},
{file = "protobuf-6.30.1-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:aa4f7dfaed0d840b03d08d14bfdb41348feaee06a828a8c455698234135b4075"},
{file = "protobuf-6.30.1-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:47cd320b7db63e8c9ac35f5596ea1c1e61491d8a8eb6d8b45edc44760b53a4f6"},
{file = "protobuf-6.30.1-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:e3083660225fa94748ac2e407f09a899e6a28bf9c0e70c75def8d15706bf85fc"},
{file = "protobuf-6.30.1-cp39-cp39-win32.whl", hash = "sha256:554d7e61cce2aa4c63ca27328f757a9f3867bce8ec213bf09096a8d16bcdcb6a"},
{file = "protobuf-6.30.1-cp39-cp39-win_amd64.whl", hash = "sha256:b510f55ce60f84dc7febc619b47215b900466e3555ab8cb1ba42deb4496d6cc0"},
{file = "protobuf-6.30.1-py3-none-any.whl", hash = "sha256:3c25e51e1359f1f5fa3b298faa6016e650d148f214db2e47671131b9063c53be"},
{file = "protobuf-6.30.1.tar.gz", hash = "sha256:535fb4e44d0236893d5cf1263a0f706f1160b689a7ab962e9da8a9ce4050b780"},
{file = "protobuf-5.29.3-cp310-abi3-win32.whl", hash = "sha256:3ea51771449e1035f26069c4c7fd51fba990d07bc55ba80701c78f886bf9c888"},
{file = "protobuf-5.29.3-cp310-abi3-win_amd64.whl", hash = "sha256:a4fa6f80816a9a0678429e84973f2f98cbc218cca434abe8db2ad0bffc98503a"},
{file = "protobuf-5.29.3-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:a8434404bbf139aa9e1300dbf989667a83d42ddda9153d8ab76e0d5dcaca484e"},
{file = "protobuf-5.29.3-cp38-abi3-manylinux2014_aarch64.whl", hash = "sha256:daaf63f70f25e8689c072cfad4334ca0ac1d1e05a92fc15c54eb9cf23c3efd84"},
{file = "protobuf-5.29.3-cp38-abi3-manylinux2014_x86_64.whl", hash = "sha256:c027e08a08be10b67c06bf2370b99c811c466398c357e615ca88c91c07f0910f"},
{file = "protobuf-5.29.3-cp38-cp38-win32.whl", hash = "sha256:84a57163a0ccef3f96e4b6a20516cedcf5bb3a95a657131c5c3ac62200d23252"},
{file = "protobuf-5.29.3-cp38-cp38-win_amd64.whl", hash = "sha256:b89c115d877892a512f79a8114564fb435943b59067615894c3b13cd3e1fa107"},
{file = "protobuf-5.29.3-cp39-cp39-win32.whl", hash = "sha256:0eb32bfa5219fc8d4111803e9a690658aa2e6366384fd0851064b963b6d1f2a7"},
{file = "protobuf-5.29.3-cp39-cp39-win_amd64.whl", hash = "sha256:6ce8cc3389a20693bfde6c6562e03474c40851b44975c9b2bf6df7d8c4f864da"},
{file = "protobuf-5.29.3-py3-none-any.whl", hash = "sha256:0a18ed4a24198528f2333802eb075e59dea9d679ab7a6c5efb017a59004d849f"},
{file = "protobuf-5.29.3.tar.gz", hash = "sha256:5da0f41edaf117bde316404bad1a486cb4ededf8e4a54891296f648e8e076620"},
]
[[package]]
@@ -1728,7 +1745,6 @@ description = "C parser in Python"
optional = false
python-versions = ">=3.8"
groups = ["main"]
markers = "os_name == \"nt\" and implementation_name != \"pypy\" or platform_python_implementation != \"PyPy\""
files = [
{file = "pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc"},
{file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"},
@@ -1736,41 +1752,44 @@ files = [
[[package]]
name = "pycryptodomex"
version = "3.22.0"
version = "3.21.0"
description = "Cryptographic library for Python"
optional = false
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7"
python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7"
groups = ["main"]
files = [
{file = "pycryptodomex-3.22.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:41673e5cc39a8524557a0472077635d981172182c9fe39ce0b5f5c19381ffaff"},
{file = "pycryptodomex-3.22.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:276be1ed006e8fd01bba00d9bd9b60a0151e478033e86ea1cb37447bbc057edc"},
{file = "pycryptodomex-3.22.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:813e57da5ceb4b549bab96fa548781d9a63f49f1d68fdb148eeac846238056b7"},
{file = "pycryptodomex-3.22.0-cp27-cp27m-win32.whl", hash = "sha256:d7beeacb5394765aa8dabed135389a11ee322d3ee16160d178adc7f8ee3e1f65"},
{file = "pycryptodomex-3.22.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:b3746dedf74787da43e4a2f85bd78f5ec14d2469eb299ddce22518b3891f16ea"},
{file = "pycryptodomex-3.22.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:5ebc09b7d8964654aaf8a4f5ac325f2b0cc038af9bea12efff0cd4a5bb19aa42"},
{file = "pycryptodomex-3.22.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:aef4590263b9f2f6283469e998574d0bd45c14fb262241c27055b82727426157"},
{file = "pycryptodomex-3.22.0-cp37-abi3-macosx_10_9_x86_64.whl", hash = "sha256:5ac608a6dce9418d4f300fab7ba2f7d499a96b462f2b9b5c90d8d994cd36dcad"},
{file = "pycryptodomex-3.22.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a24f681365ec9757ccd69b85868bbd7216ba451d0f86f6ea0eed75eeb6975db"},
{file = "pycryptodomex-3.22.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:259664c4803a1fa260d5afb322972813c5fe30ea8b43e54b03b7e3a27b30856b"},
{file = "pycryptodomex-3.22.0-cp37-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7127d9de3c7ce20339e06bcd4f16f1a1a77f1471bcf04e3b704306dde101b719"},
{file = "pycryptodomex-3.22.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:ee75067b35c93cc18b38af47b7c0664998d8815174cfc66dd00ea1e244eb27e6"},
{file = "pycryptodomex-3.22.0-cp37-abi3-musllinux_1_2_i686.whl", hash = "sha256:1a8b0c5ba061ace4bcd03496d42702c3927003db805b8ec619ea6506080b381d"},
{file = "pycryptodomex-3.22.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:bfe4fe3233ef3e58028a3ad8f28473653b78c6d56e088ea04fe7550c63d4d16b"},
{file = "pycryptodomex-3.22.0-cp37-abi3-win32.whl", hash = "sha256:2cac9ed5c343bb3d0075db6e797e6112514764d08d667c74cb89b931aac9dddd"},
{file = "pycryptodomex-3.22.0-cp37-abi3-win_amd64.whl", hash = "sha256:ff46212fda7ee86ec2f4a64016c994e8ad80f11ef748131753adb67e9b722ebd"},
{file = "pycryptodomex-3.22.0-pp27-pypy_73-manylinux2010_x86_64.whl", hash = "sha256:5bf3ce9211d2a9877b00b8e524593e2209e370a287b3d5e61a8c45f5198487e2"},
{file = "pycryptodomex-3.22.0-pp27-pypy_73-win32.whl", hash = "sha256:684cb57812cd243217c3d1e01a720c5844b30f0b7b64bb1a49679f7e1e8a54ac"},
{file = "pycryptodomex-3.22.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:c8cffb03f5dee1026e3f892f7cffd79926a538c67c34f8b07c90c0bd5c834e27"},
{file = "pycryptodomex-3.22.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:140b27caa68a36d0501b05eb247bd33afa5f854c1ee04140e38af63c750d4e39"},
{file = "pycryptodomex-3.22.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:644834b1836bb8e1d304afaf794d5ae98a1d637bd6e140c9be7dd192b5374811"},
{file = "pycryptodomex-3.22.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:72c506aba3318505dbeecf821ed7b9a9f86f422ed085e2d79c4fba0ae669920a"},
{file = "pycryptodomex-3.22.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:7cd39f7a110c1ab97ce9ee3459b8bc615920344dc00e56d1b709628965fba3f2"},
{file = "pycryptodomex-3.22.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:e4eaaf6163ff13788c1f8f615ad60cdc69efac6d3bf7b310b21e8cfe5f46c801"},
{file = "pycryptodomex-3.22.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eac39e237d65981554c2d4c6668192dc7051ad61ab5fc383ed0ba049e4007ca2"},
{file = "pycryptodomex-3.22.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ab0d89d1761959b608952c7b347b0e76a32d1a5bb278afbaa10a7f3eaef9a0a"},
{file = "pycryptodomex-3.22.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5e64164f816f5e43fd69f8ed98eb28f98157faf68208cd19c44ed9d8e72d33e8"},
{file = "pycryptodomex-3.22.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:f005de31efad6f9acefc417296c641f13b720be7dbfec90edeaca601c0fab048"},
{file = "pycryptodomex-3.22.0.tar.gz", hash = "sha256:a1da61bacc22f93a91cbe690e3eb2022a03ab4123690ab16c46abb693a9df63d"},
{file = "pycryptodomex-3.21.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:dbeb84a399373df84a69e0919c1d733b89e049752426041deeb30d68e9867822"},
{file = "pycryptodomex-3.21.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:a192fb46c95489beba9c3f002ed7d93979423d1b2a53eab8771dbb1339eb3ddd"},
{file = "pycryptodomex-3.21.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:1233443f19d278c72c4daae749872a4af3787a813e05c3561c73ab0c153c7b0f"},
{file = "pycryptodomex-3.21.0-cp27-cp27m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bbb07f88e277162b8bfca7134b34f18b400d84eac7375ce73117f865e3c80d4c"},
{file = "pycryptodomex-3.21.0-cp27-cp27m-musllinux_1_1_aarch64.whl", hash = "sha256:e859e53d983b7fe18cb8f1b0e29d991a5c93be2c8dd25db7db1fe3bd3617f6f9"},
{file = "pycryptodomex-3.21.0-cp27-cp27m-win32.whl", hash = "sha256:ef046b2e6c425647971b51424f0f88d8a2e0a2a63d3531817968c42078895c00"},
{file = "pycryptodomex-3.21.0-cp27-cp27m-win_amd64.whl", hash = "sha256:da76ebf6650323eae7236b54b1b1f0e57c16483be6e3c1ebf901d4ada47563b6"},
{file = "pycryptodomex-3.21.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:c07e64867a54f7e93186a55bec08a18b7302e7bee1b02fd84c6089ec215e723a"},
{file = "pycryptodomex-3.21.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:56435c7124dd0ce0c8bdd99c52e5d183a0ca7fdcd06c5d5509423843f487dd0b"},
{file = "pycryptodomex-3.21.0-cp27-cp27mu-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:65d275e3f866cf6fe891411be9c1454fb58809ccc5de6d3770654c47197acd65"},
{file = "pycryptodomex-3.21.0-cp27-cp27mu-musllinux_1_1_aarch64.whl", hash = "sha256:5241bdb53bcf32a9568770a6584774b1b8109342bd033398e4ff2da052123832"},
{file = "pycryptodomex-3.21.0-cp36-abi3-macosx_10_9_universal2.whl", hash = "sha256:34325b84c8b380675fd2320d0649cdcbc9cf1e0d1526edbe8fce43ed858cdc7e"},
{file = "pycryptodomex-3.21.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:103c133d6cd832ae7266feb0a65b69e3a5e4dbbd6f3a3ae3211a557fd653f516"},
{file = "pycryptodomex-3.21.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77ac2ea80bcb4b4e1c6a596734c775a1615d23e31794967416afc14852a639d3"},
{file = "pycryptodomex-3.21.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9aa0cf13a1a1128b3e964dc667e5fe5c6235f7d7cfb0277213f0e2a783837cc2"},
{file = "pycryptodomex-3.21.0-cp36-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:46eb1f0c8d309da63a2064c28de54e5e614ad17b7e2f88df0faef58ce192fc7b"},
{file = "pycryptodomex-3.21.0-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:cc7e111e66c274b0df5f4efa679eb31e23c7545d702333dfd2df10ab02c2a2ce"},
{file = "pycryptodomex-3.21.0-cp36-abi3-musllinux_1_2_i686.whl", hash = "sha256:770d630a5c46605ec83393feaa73a9635a60e55b112e1fb0c3cea84c2897aa0a"},
{file = "pycryptodomex-3.21.0-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:52e23a0a6e61691134aa8c8beba89de420602541afaae70f66e16060fdcd677e"},
{file = "pycryptodomex-3.21.0-cp36-abi3-win32.whl", hash = "sha256:a3d77919e6ff56d89aada1bd009b727b874d464cb0e2e3f00a49f7d2e709d76e"},
{file = "pycryptodomex-3.21.0-cp36-abi3-win_amd64.whl", hash = "sha256:b0e9765f93fe4890f39875e6c90c96cb341767833cfa767f41b490b506fa9ec0"},
{file = "pycryptodomex-3.21.0-pp27-pypy_73-manylinux2010_x86_64.whl", hash = "sha256:feaecdce4e5c0045e7a287de0c4351284391fe170729aa9182f6bd967631b3a8"},
{file = "pycryptodomex-3.21.0-pp27-pypy_73-win32.whl", hash = "sha256:365aa5a66d52fd1f9e0530ea97f392c48c409c2f01ff8b9a39c73ed6f527d36c"},
{file = "pycryptodomex-3.21.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:3efddfc50ac0ca143364042324046800c126a1d63816d532f2e19e6f2d8c0c31"},
{file = "pycryptodomex-3.21.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0df2608682db8279a9ebbaf05a72f62a321433522ed0e499bc486a6889b96bf3"},
{file = "pycryptodomex-3.21.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5823d03e904ea3e53aebd6799d6b8ec63b7675b5d2f4a4bd5e3adcb512d03b37"},
{file = "pycryptodomex-3.21.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:27e84eeff24250ffec32722334749ac2a57a5fd60332cd6a0680090e7c42877e"},
{file = "pycryptodomex-3.21.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:8ef436cdeea794015263853311f84c1ff0341b98fc7908e8a70595a68cefd971"},
{file = "pycryptodomex-3.21.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a1058e6dfe827f4209c5cae466e67610bcd0d66f2f037465daa2a29d92d952b"},
{file = "pycryptodomex-3.21.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9ba09a5b407cbb3bcb325221e346a140605714b5e880741dc9a1e9ecf1688d42"},
{file = "pycryptodomex-3.21.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:8a9d8342cf22b74a746e3c6c9453cb0cfbb55943410e3a2619bd9164b48dc9d9"},
{file = "pycryptodomex-3.21.0.tar.gz", hash = "sha256:222d0bd05381dd25c32dd6065c071ebf084212ab79bab4599ba9e6a3e0009e6c"},
]
[[package]]
@@ -1817,16 +1836,35 @@ files = [
[package.extras]
windows-terminal = ["colorama (>=0.4.6)"]
[[package]]
name = "pyopenssl"
version = "24.2.1"
description = "Python wrapper module around the OpenSSL library"
optional = false
python-versions = ">=3.7"
groups = ["main"]
files = [
{file = "pyOpenSSL-24.2.1-py3-none-any.whl", hash = "sha256:967d5719b12b243588573f39b0c677637145c7a1ffedcd495a487e58177fbb8d"},
{file = "pyopenssl-24.2.1.tar.gz", hash = "sha256:4247f0dbe3748d560dcbb2ff3ea01af0f9a1a001ef5f7c4c647956ed8cbf0e95"},
]
[package.dependencies]
cryptography = ">=41.0.5,<44"
[package.extras]
docs = ["sphinx (!=5.2.0,!=5.2.0.post0,!=7.2.5)", "sphinx-rtd-theme"]
test = ["pretend", "pytest (>=3.0.1)", "pytest-rerunfailures"]
[[package]]
name = "pyparsing"
version = "3.2.2"
version = "3.2.1"
description = "pyparsing module - Classes and methods to define and execute parsing grammars"
optional = false
python-versions = ">=3.9"
groups = ["main"]
files = [
{file = "pyparsing-3.2.2-py3-none-any.whl", hash = "sha256:6ab05e1cb111cc72acc8ed811a3ca4c2be2af8d7b6df324347f04fd057d8d793"},
{file = "pyparsing-3.2.2.tar.gz", hash = "sha256:2a857aee851f113c2de9d4bfd9061baea478cb0f1c7ca6cbf594942d6d111575"},
{file = "pyparsing-3.2.1-py3-none-any.whl", hash = "sha256:506ff4f4386c4cec0590ec19e6302d3aedb992fdc02c761e90416f158dacf8e1"},
{file = "pyparsing-3.2.1.tar.gz", hash = "sha256:61980854fd66de3a90028d679a954d5f2623e83144b5afe5ee86f43d762e5f0a"},
]
[package.extras]
@@ -2214,37 +2252,6 @@ files = [
[package.dependencies]
six = ">=1.7.0"
[[package]]
name = "rfc3161-client"
version = "1.0.1"
description = ""
optional = false
python-versions = ">=3.9"
groups = ["main"]
files = [
{file = "rfc3161_client-1.0.1-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:75d8c9d255fa79b9ae4aa27cee519893599efd79f9e6c24a1194dd296ce1c210"},
{file = "rfc3161_client-1.0.1-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:0d3db059fe08d8b6b06aff89e133fcc352ffea1a1dafadb116dda9dae59d0689"},
{file = "rfc3161_client-1.0.1-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fdef0c9d3213ca5b79d7f76ada48ae10c5011cb25abed2f6df07b344d16d1c28"},
{file = "rfc3161_client-1.0.1-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7c34ce4d7d2bf5207c54de3a771e757f1f8bb04a8469d3cef6aefe074841064d"},
{file = "rfc3161_client-1.0.1-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e4809f2fcfb5f8b42261a7b831929f62a297b584c8d1f4d242eae5e9447674b6"},
{file = "rfc3161_client-1.0.1-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a644b220b7f0f0be7856f49b043651982bd76e7aa9eb17b3e4e303fde36ed5a1"},
{file = "rfc3161_client-1.0.1-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:bb03a5a77b07adf766b7daac6cb8b7a8337ffc8f6d6046af74469973f52df8e1"},
{file = "rfc3161_client-1.0.1-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:d6c6e4626780b1c531d32d6a126d6c27865b1eb59c65e8b0f1f8f94aa3205285"},
{file = "rfc3161_client-1.0.1-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:912c2f049ce23d0f1c173b6fbd8673f964a27ad97907064dbc74f86dd0d95d15"},
{file = "rfc3161_client-1.0.1-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:081211a1b602b6dff7feb314d39ca2229c8db4e8cf55eef0c35b460470f4b2bb"},
{file = "rfc3161_client-1.0.1-cp39-abi3-win32.whl", hash = "sha256:59efa8fddf72a15e397276fe512dbfb99c0dc95032b495815bfc4f8f16302f2c"},
{file = "rfc3161_client-1.0.1-cp39-abi3-win_amd64.whl", hash = "sha256:5381a63d5ed5b3c257cb18aacf3f737b1a1ad6df634290fe689b6d601c61cd24"},
{file = "rfc3161_client-1.0.1.tar.gz", hash = "sha256:1c951f3912b90c6d3f3505e644b74ee08543387253647b86459addbffb16f63f"},
]
[package.dependencies]
cryptography = ">=43,<45"
[package.extras]
dev = ["maturin (>=1.7,<2.0)", "rfc3161-client[doc,lint,test]"]
lint = ["interrogate", "ruff (>=0.7,<0.12)"]
test = ["coverage[toml]", "pretend", "pytest", "pytest-cov"]
[[package]]
name = "rich"
version = "13.9.4"
@@ -2419,14 +2426,14 @@ crt = ["botocore[crt] (>=1.37.4,<2.0a.0)"]
[[package]]
name = "selenium"
version = "4.30.0"
version = "4.29.0"
description = "Official Python bindings for Selenium WebDriver"
optional = false
python-versions = ">=3.9"
groups = ["main"]
files = [
{file = "selenium-4.30.0-py3-none-any.whl", hash = "sha256:90bcd3be86a1762100a093b33e5e4530b328226da94208caadb15ce13243dffd"},
{file = "selenium-4.30.0.tar.gz", hash = "sha256:16ab890fc7cb21a01e1b1e9a0fbaa9445fe30837eabc66e90b3bacf12138126a"},
{file = "selenium-4.29.0-py3-none-any.whl", hash = "sha256:ce5d26f1ddc1111641113653af33694c13947dd36c2df09cdd33f554351d372e"},
{file = "selenium-4.29.0.tar.gz", hash = "sha256:3a62f7ec33e669364a6c0562a701deb69745b569c50d55f1a912bf8eb33358ba"},
]
[package.dependencies]
@@ -2738,14 +2745,14 @@ test = ["pytest"]
[[package]]
name = "starlette"
version = "0.46.1"
version = "0.46.0"
description = "The little ASGI library that shines."
optional = false
python-versions = ">=3.9"
groups = ["docs"]
files = [
{file = "starlette-0.46.1-py3-none-any.whl", hash = "sha256:77c74ed9d2720138b25875133f3a2dae6d854af2ec37dceb56aef370c1d8a227"},
{file = "starlette-0.46.1.tar.gz", hash = "sha256:3c88d58ee4bd1bb807c0d1acb381838afc7752f9ddaec81bbe4383611d833230"},
{file = "starlette-0.46.0-py3-none-any.whl", hash = "sha256:913f0798bd90ba90a9156383bcf1350a17d6259451d0d8ee27fc0cf2db609038"},
{file = "starlette-0.46.0.tar.gz", hash = "sha256:b359e4567456b28d473d0193f34c0de0ed49710d75ef183a74a5ce0499324f50"},
]
[package.dependencies]
@@ -2889,6 +2896,26 @@ outcome = ">=1.2.0"
trio = ">=0.11"
wsproto = ">=0.14"
[[package]]
name = "tsp-client"
version = "0.2.0"
description = "An IETF Time-Stamp Protocol (TSP) (RFC 3161) client"
optional = false
python-versions = "*"
groups = ["main"]
files = [
{file = "tsp-client-0.2.0.tar.gz", hash = "sha256:6e66148dd116322eb44a7484e5ad33bbe640b997343c443de9cc70fc5eb19987"},
{file = "tsp_client-0.2.0-py3-none-any.whl", hash = "sha256:0b790d10a68d66782c13f1d7cc7f5206df26b49826c1da80944b7c05b1731784"},
]
[package.dependencies]
asn1crypto = ">=0.24.0"
pyOpenSSL = ">=20.0.0"
requests = ">=2.18.4"
[package.extras]
tests = ["build", "coverage", "mypy", "ruff", "wheel"]
[[package]]
name = "typing-extensions"
version = "4.12.2"
@@ -2919,15 +2946,15 @@ typing-extensions = ">=3.7.4"
[[package]]
name = "tzdata"
version = "2025.2"
version = "2025.1"
description = "Provider of IANA time zone data"
optional = false
python-versions = ">=2"
groups = ["main"]
markers = "platform_system == \"Windows\""
files = [
{file = "tzdata-2025.2-py2.py3-none-any.whl", hash = "sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8"},
{file = "tzdata-2025.2.tar.gz", hash = "sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9"},
{file = "tzdata-2025.1-py2.py3-none-any.whl", hash = "sha256:7e127113816800496f027041c570f50bcd464a020098a3b6b199517772303639"},
{file = "tzdata-2025.1.tar.gz", hash = "sha256:24894909e88cdb28bd1636c6887801df64cb485bd593f2fd83ef29075a81d694"},
]
[[package]]
@@ -3316,39 +3343,27 @@ h11 = ">=0.9.0,<1"
[[package]]
name = "yt-dlp"
version = "2025.3.21"
version = "2025.2.19"
description = "A feature-rich command-line audio/video downloader"
optional = false
python-versions = ">=3.9"
groups = ["main"]
files = [
{file = "yt_dlp-2025.3.21-py3-none-any.whl", hash = "sha256:80d5ce15f9223e0c27020b861a4c5b72c6ba5d6c957c1b8fd2a022a69783f482"},
{file = "yt_dlp-2025.3.21.tar.gz", hash = "sha256:5bcf47b2897254ea3816935a8dde47d243bff556782cced6b16a2b85e6b682ba"},
{file = "yt_dlp-2025.2.19-py3-none-any.whl", hash = "sha256:3ed218eaeece55e9d715afd41abc450dc406ee63bf79355169dfde312d38fdb8"},
{file = "yt_dlp-2025.2.19.tar.gz", hash = "sha256:f33ca76df2e4db31880f2fe408d44f5058d9f135015b13e50610dfbe78245bea"},
]
[package.extras]
build = ["build", "hatchling", "pip", "setuptools (>=71.0.2)", "wheel"]
curl-cffi = ["curl-cffi (==0.5.10) ; os_name == \"nt\" and implementation_name == \"cpython\"", "curl-cffi (>=0.5.10,!=0.6.*,<0.7.2) ; os_name != \"nt\" and implementation_name == \"cpython\""]
default = ["brotli ; implementation_name == \"cpython\"", "brotlicffi ; implementation_name != \"cpython\"", "certifi", "mutagen", "pycryptodomex", "requests (>=2.32.2,<3)", "urllib3 (>=1.26.17,<3)", "websockets (>=13.0)"]
dev = ["autopep8 (>=2.0,<3.0)", "pre-commit", "pytest (>=8.1,<9.0)", "pytest-rerunfailures (>=14.0,<15.0)", "ruff (>=0.11.0,<0.12.0)"]
dev = ["autopep8 (>=2.0,<3.0)", "pre-commit", "pytest (>=8.1,<9.0)", "pytest-rerunfailures (>=14.0,<15.0)", "ruff (>=0.9.0,<0.10.0)"]
pyinstaller = ["pyinstaller (>=6.11.1)"]
secretstorage = ["cffi", "secretstorage"]
static-analysis = ["autopep8 (>=2.0,<3.0)", "ruff (>=0.11.0,<0.12.0)"]
static-analysis = ["autopep8 (>=2.0,<3.0)", "ruff (>=0.9.0,<0.10.0)"]
test = ["pytest (>=8.1,<9.0)", "pytest-rerunfailures (>=14.0,<15.0)"]
[[package]]
name = "yt-dlp-get-pot"
version = "0.3.0"
description = ""
optional = false
python-versions = ">=3.9"
groups = ["main"]
files = [
{file = "yt_dlp_get_pot-0.3.0-py3-none-any.whl", hash = "sha256:a49a596a3e3c02cd9ce051192ea3fe8168cf24ece8954bed6aa331a87d86954f"},
{file = "yt_dlp_get_pot-0.3.0.tar.gz", hash = "sha256:ac9530b9e7b3d667235b9119da475f595d2dc7e6f6bbf98b965011be454e8833"},
]
[metadata]
lock-version = "2.1"
python-versions = ">=3.10,<3.13"
content-hash = "c612e9f98ca5199092141bb04a0de4cd5314a8fdc8cb12c1d63eafe26bbf16aa"
content-hash = "beb354960b8d8af491a13e09cb565c7e3099a2b150167c16147aa0438e970018"

View File

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
[project]
name = "auto-archiver"
version = "1.0.0"
version = "0.13.8"
description = "Automatically archive links to videos, images, and social media content from Google Sheets (and more)."
requires-python = ">=3.10,<3.13"
@@ -41,9 +41,11 @@ dependencies = [
"instaloader (>=0.0.0)",
"tqdm (>=0.0.0)",
"jinja2 (>=0.0.0)",
"pyOpenSSL (==24.2.1)",
"cryptography (>=41.0.0,<42.0.0)",
"boto3 (>=1.28.0,<2.0.0)",
"dataclasses-json (>=0.0.0)",
"yt-dlp (>=2025.3.21,<2026.0.0)",
"yt-dlp (>=2025.1.26,<2026.0.0)",
"numpy (==2.1.3)",
"vk-url-scraper (>=0.0.0)",
"requests[socks] (>=0.0.0)",
@@ -51,12 +53,11 @@ dependencies = [
"jsonlines (>=0.0.0)",
"pysubs2 (>=0.0.0)",
"retrying (>=0.0.0)",
"tsp-client (>=0.0.0)",
"certvalidator (>=0.0.0)",
"rich-argparse (>=1.6.0,<2.0.0)",
"ruamel-yaml (>=0.18.10,<0.19.0)",
"rfc3161-client (>=1.0.1,<2.0.0)",
"cryptography (>44.0.1,<45.0.0)",
"opentimestamps (>=0.4.5,<0.5.0)",
"bgutil-ytdlp-pot-provider (>=0.7.3,<0.8.0)",
]
[tool.poetry.group.dev.dependencies]

View File

@@ -1,2 +0,0 @@
secrets*
*instagrapi_session.json

View File

@@ -1,19 +0,0 @@
FROM python:3.12-slim
WORKDIR /app
# Install Poetry
RUN pip install --upgrade pip
RUN pip install poetry
# Copy all source code
COPY . .
# Prevent Poetry from creating a virtual environment
RUN poetry config virtualenvs.create false
# Install dependencies
RUN poetry install --no-root
# Use uvicorn to run the FastAPI app
CMD ["poetry", "run", "uvicorn", "src.instaserver:app", "--host", "0.0.0.0", "--port", "8000"]

View File

@@ -1,18 +0,0 @@
[project]
name = "instaserver"
version = "0.1.0"
description = "A FastAPI InstagrAPI server"
package-mode = false
requires-python = ">=3.10"
dependencies = [
"fastapi (>=0.115.12,<0.116.0)",
"instagrapi (>=2.1.3,<3.0.0)",
"uvicorn (>=0.34.0,<0.35.0)",
"pillow (>=11.1.0,<12.0.0)",
"python-dotenv (>=1.1.0,<2.0.0)"
]
[build-system]
requires = ["poetry-core>=2.0.0,<3.0.0"]
build-backend = "poetry.core.masonry.api"

View File

@@ -1,48 +0,0 @@
#!/usr/bin/env bash
#
# run_instagrapi_server.sh
# Usage:
# From repo root: ./scripts/instagrapi_server/run_instagrapi_server.sh
# Or from script dir: ./run_instagrapi_server.sh
#
set -e
# Step 1: cd to the script's directory (contains Dockerfile and secrets/)
cd "$(dirname "$0")" || exit 1
# Create secrets/ if it doesn't exist
if [[ ! -d "secrets" ]]; then
echo "Creating secrets/ directory..."
mkdir secrets
fi
echo "Enter your Instagram credentials to store in secrets/.env"
read -rp "Instagram Username: " IGUSER
read -rsp "Instagram Password: " IGPASS
echo ""
cat <<EOF > secrets/.env
INSTAGRAM_USERNAME=$IGUSER
INSTAGRAM_PASSWORD=$IGPASS
EOF
echo "Created secrets/.env with your credentials."
# Build Docker image
IMAGE_NAME="instagrapi-server"
echo "Building Docker image '$IMAGE_NAME'..."
docker build -t "$IMAGE_NAME" .
# Run container
CONTAINER_NAME="ig-instasrv"
echo "Running container '$CONTAINER_NAME'..."
docker run -d \
--env-file secrets/.env \
-v "$(pwd)/secrets:/app/secrets" \
-p 8000:8000 \
--name "$CONTAINER_NAME" \
"$IMAGE_NAME"
echo "Done! Instagrapi server is running on port 8000."
echo "Use 'docker logs $CONTAINER_NAME' to view logs."
echo "Use 'docker stop $CONTAINER_NAME' and 'docker rm $CONTAINER_NAME' to stop/remove the container."

View File

@@ -1,157 +0,0 @@
"""https://subzeroid.github.io/instagrapi/
Run using the following command:
uvicorn src.instgrapinstance.instaserver:app --host 0.0.0.0 --port 8000 --reload
"""
import logging
import os
import sys
from dotenv import load_dotenv
from fastapi import FastAPI, HTTPException
from instagrapi import Client
from instagrapi.exceptions import LoginRequired, BadCredentials
load_dotenv(dotenv_path="secrets/.env")
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s")
INSTAGRAM_USERNAME = os.getenv("INSTAGRAM_USERNAME")
INSTAGRAM_PASSWORD = os.getenv("INSTAGRAM_PASSWORD")
SESSION_FILE = "secrets/instagrapi_session.json"
app = FastAPI()
cl = Client()
@app.on_event("startup")
def startup_event():
"""Login automatically when server starts"""
try:
login_instagram()
except RuntimeError as e:
logging.error(f"API failed to start: {e}")
sys.exit(1)
def login_instagram():
"""Ensures Instagrapi is logged in and session is persistent"""
if not INSTAGRAM_USERNAME or not INSTAGRAM_PASSWORD:
raise RuntimeError("Instagram credentials are missing.")
if os.path.exists(SESSION_FILE):
try:
cl.load_settings(SESSION_FILE)
cl.get_timeline_feed()
logging.info("Using saved session.")
return
except LoginRequired:
logging.info("Session expired. Logging in again...")
try:
cl.login(INSTAGRAM_USERNAME, INSTAGRAM_PASSWORD)
cl.dump_settings(SESSION_FILE)
logging.info("Login successful, session saved.")
except BadCredentials as bc:
raise RuntimeError("Incorrect Instagram username or password.") from bc
except Exception as e:
raise RuntimeError(f"Login failed: {e}") from e
@app.get("/v1/media/by/id")
def get_media_by_id(id: str):
"""Fetch post details by media ID"""
logging.info(f"Fetching media by ID: {id}")
try:
media = cl.media_info(id)
return media.model_dump()
except Exception as e:
logging.warning(f"Media not found for ID {id}: {e}")
raise HTTPException(status_code=404, detail="Post not found") from e
@app.get("/v1/media/by/code")
def get_media_by_code(code: str):
"""Fetch post details by shortcode"""
logging.info(f"Fetching media by shortcode: {code}")
try:
media_id = cl.media_pk_from_code(code)
media = cl.media_info(media_id)
return media.model_dump()
except Exception as e:
logging.warning(f"Media not found for code {code}: {e}")
raise HTTPException(status_code=404, detail="Post not found") from e
@app.get("/v2/user/tag/medias")
def get_user_tagged_medias(user_id: str, page_id: str = None):
logging.info(f"Fetching tagged medias for user_id={user_id} page_id={page_id}")
try:
# Placeholder for now
items, next_page_id = [], None
return {"response": {"items": items}, "next_page_id": next_page_id}
except Exception as e:
logging.warning(f"Tagged media not found for {user_id}: {e}")
raise HTTPException(status_code=404, detail="Tagged media not found") from e
@app.get("/v1/user/highlights")
def get_user_highlights(user_id: str):
logging.info(f"Fetching highlights list for user_id={user_id}")
try:
highlights = cl.user_highlights(user_id)
return [h.model_dump() for h in highlights]
except Exception as e:
logging.warning(f"Highlights not found for {user_id}: {e}")
raise HTTPException(status_code=404, detail="No highlights found") from e
@app.get("/v2/highlight/by/id")
def get_highlight_by_id(id: str):
logging.info(f"Fetching highlight details for id={id}")
try:
highlight = cl.highlight_info(id)
return {"response": {"reels": {f"highlight:{id}": highlight.model_dump()}}}
except Exception as e:
logging.warning(f"Highlight not found for id {id}: {e}")
raise HTTPException(status_code=404, detail="Highlight not found") from e
@app.get("/v1/user/stories/by/username")
def get_stories(username: str):
logging.info(f"Fetching stories for username={username}")
try:
user_id = cl.user_id_from_username(username)
stories = cl.user_stories(user_id)
return [story.model_dump() for story in stories]
except Exception as e:
logging.warning(f"Stories not found for {username}: {e}")
raise HTTPException(status_code=404, detail="Stories not found") from e
@app.get("/v2/user/by/username")
def get_user_by_username(username: str):
logging.info(f"Fetching user profile for username={username}")
try:
user = cl.user_info_by_username(username)
return {"user": user.model_dump()}
except Exception as e:
logging.warning(f"User not found: {username}: {e}")
raise HTTPException(status_code=404, detail="User not found") from e
@app.get("/v1/user/medias/chunk")
def get_user_medias(user_id: str, end_cursor: str = None):
logging.info(f"Fetching paginated medias for user_id={user_id}, end_cursor={end_cursor}")
try:
posts, next_cursor = cl.user_medias_paginated(user_id, end_cursor=end_cursor)
return [[post.model_dump() for post in posts], next_cursor]
except Exception as e:
logging.warning(f"No posts found for user_id={user_id}: {e}")
raise HTTPException(status_code=404, detail="No posts found") from e
if __name__ == "__main__":
import uvicorn
uvicorn.run(app, host="0.0.0.0", port=8000)

File diff suppressed because it is too large Load Diff

View File

@@ -214,8 +214,11 @@ class LazyBaseModule:
# check external dependencies are installed
def check_deps(deps, check):
for dep in filter(lambda d: len(d.strip()) > 0, deps):
if not check(dep.strip()):
for dep in deps:
if not len(dep):
# clear out any empty strings that a user may have erroneously added
continue
if not check(dep):
logger.error(
f"Module '{self.name}' requires external dependency '{dep}' which is not available/setup. \
Have you installed the required dependencies for the '{self.name}' module? See the documentation for more information."
@@ -274,9 +277,6 @@ class LazyBaseModule:
# finally, get the class instance
instance: BaseModule = getattr(sys.modules[sub_qualname], class_name)()
# save the instance for future easy loading
self._instance = instance
# set the name, display name and module factory
instance.name = self.name
instance.display_name = self.display_name
@@ -289,6 +289,8 @@ class LazyBaseModule:
instance.config_setup(config)
instance.setup()
# save the instance for future easy loading
self._instance = instance
return instance
def __repr__(self):

View File

@@ -5,7 +5,6 @@ formatting, database operations and clean up.
"""
from __future__ import annotations
from packaging import version
from typing import Generator, Union, List, Type, TYPE_CHECKING
import argparse
import os
@@ -388,10 +387,8 @@ Here's how that would look: \n\nsteps:\n extractors:\n - [your_extractor_name_
except (KeyboardInterrupt, Exception) as e:
if not isinstance(e, KeyboardInterrupt) and not isinstance(e, SetupError):
logger.error(f"Error during setup of modules: {e}\n{traceback.format_exc()}")
# access the _instance here because loaded_module may not return if there's an error
if lazy_module._instance and module_type == "extractor":
lazy_module._instance.cleanup()
if loaded_module and module_type == "extractor":
loaded_module.cleanup()
raise e
if not loaded_module:
@@ -437,19 +434,16 @@ Here's how that would look: \n\nsteps:\n extractors:\n - [your_extractor_name_
def check_for_updates(self):
response = requests.get("https://pypi.org/pypi/auto-archiver/json").json()
latest_version = version.parse(response["info"]["version"])
current_version = version.parse(__version__)
latest_version = response["info"]["version"]
# check version compared to current version
if latest_version > current_version:
if latest_version != __version__:
if os.environ.get("RUNNING_IN_DOCKER"):
update_cmd = "`docker pull bellingcat/auto-archiver:latest`"
else:
update_cmd = "`pip install --upgrade auto-archiver`"
logger.warning("")
logger.warning("********* IMPORTANT: UPDATE AVAILABLE ********")
logger.warning(
f"A new version of auto-archiver is available (v{latest_version}, you have v{current_version})"
)
logger.warning(f"A new version of auto-archiver is available (v{latest_version}, you have {__version__})")
logger.warning(f"Make sure to update to the latest version using: {update_cmd}")
logger.warning("")

View File

@@ -4,6 +4,12 @@ import argparse
import json
def example_validator(value):
if "example" not in value:
raise argparse.ArgumentTypeError(f"{value} is not a valid value for this argument")
return value
def positive_number(value):
if value < 0:
raise argparse.ArgumentTypeError(f"{value} is not a positive number")

View File

@@ -74,11 +74,6 @@ If you are having issues with the extractor, you can review the version of `yt-d
"default": "inf",
"help": "Use to limit the number of videos to download when a channel or long page is being extracted. 'inf' means no limit.",
},
"bguils_po_token_method": {
"default": "auto",
"help": "Set up a Proof of origin token provider. This process has additional requirements. See [authentication](https://auto-archiver.readthedocs.io/en/latest/how_to/authentication_how_to.html) for more information.",
"choices": ["auto", "script", "disabled"],
},
"extractor_args": {
"default": {},
"help": "Additional arguments to pass to the yt-dlp extractor. See https://github.com/yt-dlp/yt-dlp/blob/master/README.md#extractor-arguments.",

View File

@@ -1,13 +1,9 @@
import shutil
import sys
import datetime
import os
import importlib
import subprocess
import zipfile
from typing import Generator, Type
from urllib.request import urlretrieve
import yt_dlp
from yt_dlp.extractor.common import InfoExtractor
@@ -29,138 +25,45 @@ class GenericExtractor(Extractor):
_dropins = {}
def setup(self):
self.check_for_extractor_updates()
self.setup_po_tokens()
def check_for_extractor_updates(self):
"""Checks whether yt-dlp or its plugins need updating and triggers a restart if so."""
# check for file .ytdlp-update in the secrets folder
if self.ytdlp_update_interval < 0:
return
update_file = os.path.join("secrets" if os.path.exists("secrets") else "", ".ytdlp-update")
next_check = None
if os.path.exists(update_file):
with open(update_file, "r") as f:
next_check = datetime.datetime.fromisoformat(f.read())
use_secrets = os.path.exists("secrets")
path = os.path.join("secrets" if use_secrets else "", ".ytdlp-update")
next_update_check = None
if os.path.exists(path):
with open(path, "r") as f:
next_update_check = datetime.datetime.fromisoformat(f.read())
if next_check and next_check > datetime.datetime.now():
return
if not next_update_check or next_update_check < datetime.datetime.now():
self.update_ytdlp()
yt_dlp_updated = self.update_package("yt-dlp")
bgutil_updated = self.update_package("bgutil-ytdlp-pot-provider")
next_update_check = datetime.datetime.now() + datetime.timedelta(days=self.ytdlp_update_interval)
with open(path, "w") as f:
f.write(next_update_check.isoformat())
# Write the new timestamp
with open(update_file, "w") as f:
next_check = datetime.datetime.now() + datetime.timedelta(days=self.ytdlp_update_interval)
f.write(next_check.isoformat())
if yt_dlp_updated or bgutil_updated:
if os.environ.get("AUTO_ARCHIVER_ALLOW_RESTART", "1") != "1":
logger.warning("yt-dlp or plugin was updated — please restart auto-archiver manually")
else:
logger.warning("yt-dlp or plugin was updated — restarting auto-archiver")
logger.warning(" ======= RESTARTING ======= ")
os.execv(sys.executable, [sys.executable] + sys.argv)
def update_package(self, package_name: str) -> bool:
logger.info(f"Checking and updating {package_name}...")
def update_ytdlp(self):
logger.info("Checking and updating yt-dlp...")
logger.info(
f"Tip: change the 'ytdlp_update_interval' setting to control how often yt-dlp is updated. Set to -1 to disable or 0 to enable on every run. Current setting: {self.ytdlp_update_interval}"
)
from importlib.metadata import version as get_version
old_version = get_version(package_name)
old_version = get_version("yt-dlp")
try:
result = subprocess.run(["pip", "install", "--upgrade", package_name], check=True, capture_output=True)
if f"Successfully installed {package_name}" in result.stdout.decode():
new_version = importlib.metadata.version(package_name)
logger.info(f"{package_name} updated from {old_version} to {new_version}")
return True
logger.info(f"{package_name} already up to date")
except Exception as e:
logger.error(f"Error updating {package_name}: {e}")
return False
# try and update with pip (this works inside poetry environment and in a normal virtualenv)
result = subprocess.run(["pip", "install", "--upgrade", "yt-dlp"], check=True, capture_output=True)
def setup_po_tokens(self) -> None:
"""Setup Proof of Origin Token method conditionally.
Uses provider: https://github.com/Brainicism/bgutil-ytdlp-pot-provider.
"""
in_docker = os.environ.get("RUNNING_IN_DOCKER")
if self.bguils_po_token_method == "disabled":
# This allows disabling of the PO Token generation script in the Docker implementation.
logger.warning("Proof of Origin Token generation is disabled.")
return
if self.bguils_po_token_method == "auto" and not in_docker:
logger.info(
"Proof of Origin Token method not explicitly set. "
"If you're running an external HTTP server separately, you can safely ignore this message. "
"To reduce the likelihood of bot detection, enable one of the methods described in the documentation: "
"https://auto-archiver.readthedocs.io/en/settings_page/installation/authentication.html#proof-of-origin-tokens"
)
return
# Either running in Docker, or "script" method is set beyond this point
self.setup_token_generation_script()
def setup_token_generation_script(self) -> None:
"""This function sets up the Proof of Origin Token generation script method for
bgutil-ytdlp-pot-provider if enabled or in Docker."""
missing_tools = [tool for tool in ("node", "yarn", "npx") if shutil.which(tool) is None]
if missing_tools:
logger.error(
f"Cannot set up PO Token script; missing required tools: {', '.join(missing_tools)}. "
"Install these tools or run bgutils via Docker. "
"See: https://github.com/Brainicism/bgutil-ytdlp-pot-provider"
)
return
try:
from importlib.metadata import version as get_version
plugin_version = get_version("bgutil-ytdlp-pot-provider")
base_dir = os.path.expanduser("~/bgutil-ytdlp-pot-provider")
server_dir = os.path.join(base_dir, "server")
version_file = os.path.join(server_dir, ".VERSION")
transpiled_script = os.path.join(server_dir, "build", "generate_once.js")
# Skip setup if version is correct and transpiled script exists
if os.path.isfile(transpiled_script) and os.path.isfile(version_file):
with open(version_file) as vf:
if vf.read().strip() == plugin_version:
logger.info("PO Token script already set up and up to date.")
if "Successfully installed yt-dlp" in result.stdout.decode():
new_version = importlib.metadata.version("yt-dlp")
logger.info(f"yt-dlp successfully (from {old_version} to {new_version})")
importlib.reload(yt_dlp)
else:
# Remove an outdated directory and pull a new version
if os.path.exists(base_dir):
shutil.rmtree(base_dir)
os.makedirs(base_dir, exist_ok=True)
zip_url = (
f"https://github.com/Brainicism/bgutil-ytdlp-pot-provider/archive/refs/tags/{plugin_version}.zip"
)
zip_path = os.path.join(base_dir, f"{plugin_version}.zip")
logger.info(f"Downloading bgutils release zip for version {plugin_version}...")
urlretrieve(zip_url, zip_path)
with zipfile.ZipFile(zip_path, "r") as z:
z.extractall(base_dir)
os.remove(zip_path)
extracted_root = os.path.join(base_dir, f"bgutil-ytdlp-pot-provider-{plugin_version}")
shutil.move(os.path.join(extracted_root, "server"), server_dir)
shutil.rmtree(extracted_root)
logger.info("Installing dependencies and transpiling PoT Generator script...")
subprocess.run(["yarn", "install", "--frozen-lockfile"], cwd=server_dir, check=True)
subprocess.run(["npx", "tsc"], cwd=server_dir, check=True)
with open(version_file, "w") as vf:
vf.write(plugin_version)
script_path = os.path.join(server_dir, "build", "generate_once.js")
if not os.path.exists(script_path):
logger.error("generate_once.js not found after transpilation.")
return
self.extractor_args.setdefault("youtube", {})["getpot_bgutil_script"] = script_path
logger.info(f"PO Token script configured at: {script_path}")
logger.info("yt-dlp already up to date")
except Exception as e:
logger.error(f"Failed to set up PO Token script: {e}")
logger.error(f"Error updating yt-dlp: {e}")
def suitable_extractors(self, url: str) -> Generator[str, None, None]:
"""

View File

@@ -88,7 +88,10 @@ class GsheetsFeederDB(Feeder, Database):
if len(self.allow_worksheets) and sheet_name not in self.allow_worksheets:
# ALLOW rules exist AND sheet name not explicitly allowed
return False
return not (self.block_worksheets and sheet_name in self.block_worksheets)
if len(self.block_worksheets) and sheet_name in self.block_worksheets:
# BLOCK rules exist AND sheet name is blocked
return False
return True
def missing_required_columns(self, gw: GWorksheet) -> list:
missing = []
@@ -158,8 +161,9 @@ class GsheetsFeederDB(Feeder, Database):
if (screenshot := item.get_media_by_id("screenshot")) and hasattr(screenshot, "urls"):
batch_if_valid("screenshot", "\n".join(screenshot.urls))
if (thumbnail := item.get_first_image("thumbnail")) and hasattr(thumbnail, "urls"):
batch_if_valid("thumbnail", f'=IMAGE("{thumbnail.urls[0]}")')
if thumbnail := item.get_first_image("thumbnail"):
if hasattr(thumbnail, "urls"):
batch_if_valid("thumbnail", f'=IMAGE("{thumbnail.urls[0]}")')
if browsertrix := item.get_media_by_id("browsertrix"):
batch_if_valid("wacz", "\n".join(browsertrix.urls))

View File

@@ -31,11 +31,9 @@
},
},
"description": """
Archives Instagram content using a deployment of the [Instagrapi API](https://subzeroid.github.io/instagrapi/).
Archives various types of Instagram content using the Instagrapi API.
Requires either getting a token from using a hosted [(paid) service](https://api.instagrapi.com/docs) and setting this in the configuration file.
Alternatively you can run your own server. We have a basic script which you can use for this which can be ran locally or using Docker.
For more information, read the [how to guide](https://auto-archiver.readthedocs.io/en/latest/how_to/run_instagrapi_server.html) on this.
Requires setting up an Instagrapi API deployment and providing an access token and API endpoint.
### Features
- Connects to an Instagrapi API deployment to fetch Instagram profiles, posts, stories, highlights, reels, and tagged content.

View File

@@ -88,9 +88,6 @@ class InstagramTbotExtractor(Extractor):
if message:
result.set_content(message).set_title(message[:128])
elif result.is_empty():
logger.debug(f"No media found for link {url=} for {self.name}: {message}")
return False
return result.success("insta-via-bot")
def _send_url_to_bot(self, url: str):
@@ -107,13 +104,13 @@ class InstagramTbotExtractor(Extractor):
message = ""
time.sleep(3)
# media is added before text by the bot so it can be used as a stop-logic mechanism
while attempts < max(self.timeout - 3, 15) and (not message or not len(seen_media)):
while attempts < max(self.timeout - 3, 3) and (not message or not len(seen_media)):
attempts += 1
time.sleep(1)
for post in self.client.iter_messages(chat, min_id=since_id):
since_id = max(since_id, post.id)
# Skip known filler message:
if "The bot receives information through https://hikerapi.com/" in post.message:
if post.message == "The bot receives information through https://hikerapi.com/p/hJqpppqi":
continue
if post.media and post.id not in seen_media:
filename_dest = os.path.join(tmp_dir, f"{chat.id}_{post.id}")

View File

@@ -19,7 +19,7 @@
},
"session_file": {
"default": "secrets/anon",
"help": "Path of the file to save the telegram login session for future usage, '.session' will be appended to the provided path.",
"help": "optional, records the telegram login session for future usage, '.session' will be appended to the provided value.",
},
"join_channels": {
"default": True,

View File

@@ -1,10 +1,4 @@
import os
import shutil
import re
import time
from pathlib import Path
from datetime import date
from telethon.sync import TelegramClient
from telethon.errors import ChannelInvalidError
from telethon.tl.functions.messages import ImportChatInviteRequest
@@ -14,9 +8,11 @@ from telethon.errors.rpcerrorlist import (
InviteRequestSentError,
InviteHashExpiredError,
)
from tqdm import tqdm
from loguru import logger
from tqdm import tqdm
import re
import time
import os
from auto_archiver.core import Extractor
from auto_archiver.core import Metadata, Media
@@ -35,22 +31,10 @@ class TelethonExtractor(Extractor):
"""
logger.info(f"SETUP {self.name} checking login...")
# in case the user already added '.session' to the session_file
base_session_name = self.session_file.removesuffix(".session")
base_session_filepath = f"{base_session_name}.session"
if self.session_file and not os.path.exists(base_session_filepath):
logger.warning(
f"SETUP - Session file {base_session_filepath} does not exist for {self.name}, creating an empty one."
)
Path(base_session_filepath).touch()
# make a copy of the session that is used exclusively with this archiver instance
self.session_file = os.path.join(
os.path.dirname(base_session_filepath), f"telethon-{date.today().strftime('%Y-%m-%d')}{random_str(8)}"
)
logger.debug(f"Making a copy of the session file {base_session_filepath} to {self.session_file}.session")
shutil.copy(base_session_filepath, f"{self.session_file}.session")
new_session_file = os.path.join("secrets/", f"telethon-{time.strftime('%Y-%m-%d')}{random_str(8)}.session")
shutil.copy(self.session_file + ".session", new_session_file)
self.session_file = new_session_file.replace(".session", "")
# initiate the client
self.client = TelegramClient(self.session_file, self.api_id, self.api_hash)
@@ -103,8 +87,8 @@ class TelethonExtractor(Extractor):
pbar.update()
def cleanup(self) -> None:
logger.info(f"CLEANUP {self.name} - removing session file {self.session_file}.session")
session_file_name = f"{self.session_file}.session"
logger.info(f"CLEANUP {self.name}.")
session_file_name = self.session_file + ".session"
if os.path.exists(session_file_name):
os.remove(session_file_name)
@@ -190,7 +174,7 @@ class TelethonExtractor(Extractor):
if getattr(original_post, "grouped_id", None) is None:
return [original_post] if getattr(original_post, "media", False) else []
search_ids = list(range(original_post.id - max_amp, original_post.id + max_amp + 1))
search_ids = [i for i in range(original_post.id - max_amp, original_post.id + max_amp + 1)]
posts = self.client.get_messages(chat, ids=search_ids)
media = []
for post in posts:

View File

@@ -3,38 +3,30 @@
"type": ["enricher"],
"requires_setup": True,
"dependencies": {
"python": ["loguru", "slugify", "cryptography", "rfc3161_client", "certifi"],
"python": ["loguru", "slugify", "tsp_client", "asn1crypto", "certvalidator", "certifi"],
},
"configs": {
"tsa_urls": {
"default": [
# See https://github.com/trailofbits/rfc3161-client/issues/46 for a list of valid TSAs
# Full list of TSAs: https://gist.github.com/Manouchehri/fd754e402d98430243455713efada710
"http://timestamp.identrust.com",
"http://timestamp.ssl.trustwave.com",
"http://zeitstempel.dfn.de",
"http://ts.ssl.com",
# "http://tsa.izenpe.com", # self-signed
"http://tsa.lex-persona.com/tsa",
# "http://ca.signfiles.com/TSAServer.aspx", # self-signed
# "http://tsa.sinpe.fi.cr/tsaHttp/", # self-signed
# "http://tsa.cra.ge/signserver/tsa?workerName=qtsa", # self-signed
"http://tss.cnbs.gob.hn/TSS/HttpTspServer",
"http://dss.nowina.lu/pki-factory/tsa/good-tsa",
# "https://freetsa.org/tsr", # self-signed
],
# [Adobe Approved Trust List] and [Windows Cert Store]
"http://timestamp.digicert.com",
"http://timestamp.identrust.com",
# "https://timestamp.entrust.net/TSS/RFC3161sha2TS", # not valid for timestamping
# "https://timestamp.sectigo.com", # wait 15 seconds between each request.
# [Adobe: European Union Trusted Lists].
# "https://timestamp.sectigo.com/qualified", # wait 15 seconds between each request.
# [Windows Cert Store]
"http://timestamp.globalsign.com/tsa/r6advanced1",
# [Adobe: European Union Trusted Lists] and [Windows Cert Store]
# "http://ts.quovadisglobal.com/eu", # not valid for timestamping
# "http://tsa.belgium.be/connect", # self-signed certificate in certificate chain
# "https://timestamp.aped.gov.gr/qtss", # self-signed certificate in certificate chain
# "http://tsa.sep.bg", # self-signed certificate in certificate chain
# "http://tsa.izenpe.com", #unable to get local issuer certificate
# "http://kstamp.keynectis.com/KSign", # unable to get local issuer certificate
"http://tss.accv.es:8318/tsa",
],
"help": "List of RFC3161 Time Stamp Authorities to use, separate with commas if passed via the command line.",
},
"cert_authorities": {
"default": None,
"help": "Path to a file containing trusted Certificate Authorities (CAs) in PEM format. If empty, the default system authorities are used.",
"type": "str",
},
"allow_selfsigned": {
"default": False,
"help": "Whether or not to allow and save self-signed Timestamping certificates. This allows for a greater range of timestamping servers to be used, \
but they are not trusted authorities",
"type": "bool"
}
},
"description": """

View File

@@ -1,23 +1,15 @@
import os
from importlib.metadata import version
import hashlib
from slugify import slugify
import requests
from loguru import logger
from rfc3161_client import (decode_timestamp_response,TimestampRequestBuilder,TimeStampResponse, VerifierBuilder)
from rfc3161_client import VerificationError as Rfc3161VerificationError
from rfc3161_client.base import HashAlgorithm
from rfc3161_client.tsp import SignedData
from cryptography import x509
from cryptography.hazmat.primitives import serialization
from tsp_client import TSPSigner, SigningSettings, TSPVerifier
from tsp_client.algorithms import DigestAlgorithm
from importlib.metadata import version
from asn1crypto.cms import ContentInfo
from certvalidator import CertificateValidator, ValidationContext
from asn1crypto import pem
import certifi
from auto_archiver.core import Enricher
from auto_archiver.core import Metadata, Media
from auto_archiver.version import __version__
class TimestampingEnricher(Enricher):
@@ -29,25 +21,6 @@ class TimestampingEnricher(Enricher):
See https://gist.github.com/Manouchehri/fd754e402d98430243455713efada710 for list of timestamp authorities.
"""
session = None
def setup(self):
self.session = requests.Session()
self.session.headers.update(
{
"Content-Type": "application/timestamp-query",
"User-Agent": f"Auto-Archiver {__version__}",
"Accept": "application/timestamp-reply",
}
)
def cleaup(self) -> None:
"""
Terminates the underlying network session.
"""
if self.session:
self.session.close()
def enrich(self, to_enrich: Metadata) -> None:
url = to_enrich.get_url()
logger.debug(f"RFC3161 timestamping existing files for {url=}")
@@ -61,8 +34,8 @@ class TimestampingEnricher(Enricher):
logger.warning(f"No hashes found in {url=}")
return
hashes_fn = os.path.join(self.tmp_dir, "hashes.txt")
tmp_dir = self.tmp_dir
hashes_fn = os.path.join(tmp_dir, "hashes.txt")
data_to_sign = "\n".join(hashes)
with open(hashes_fn, "w") as f:
@@ -70,160 +43,62 @@ class TimestampingEnricher(Enricher):
hashes_media = Media(filename=hashes_fn)
timestamp_tokens = []
from slugify import slugify
for tsa_url in self.tsa_urls:
try:
message = bytes(data_to_sign, encoding='utf8')
logger.debug(f"Timestamping {url=} with {tsa_url=}")
signed: TimeStampResponse = self.sign_data(tsa_url, message)
# fail if there's any issue with the certificates, uses certifi list of trusted CAs or the user-defined `cert_authorities`
root_cert = self.verify_signed(signed, message)
if not root_cert:
if self.allow_selfsigned:
logger.warning(f"Allowing self-signed certificat from TSA {tsa_url=}")
else:
raise ValueError(f"No valid root certificate found for {tsa_url=}. Are you sure it's a trusted TSA? Or define an alternative trusted root with `cert_authorities`. (tried: {self.cert_authorities or certifi.where()})")
# save the timestamping certificate
cert_chain = self.save_certificate(signed, root_cert)
timestamp_token_path = self.save_timestamp_token(signed.time_stamp_token(), tsa_url)
timestamp_tokens.append(Media(filename=timestamp_token_path).set("tsa", tsa_url).set("cert_chain", cert_chain))
signing_settings = SigningSettings(tsp_server=tsa_url, digest_algorithm=DigestAlgorithm.SHA256)
signer = TSPSigner()
message = bytes(data_to_sign, encoding="utf8")
# send TSQ and get TSR from the TSA server
signed = signer.sign(message=message, signing_settings=signing_settings)
# fail if there's any issue with the certificates, uses certifi list of trusted CAs
TSPVerifier(certifi.where()).verify(signed, message=message)
# download and verify timestamping certificate
cert_chain = self.download_and_verify_certificate(signed)
# continue with saving the timestamp token
tst_fn = os.path.join(tmp_dir, f"timestamp_token_{slugify(tsa_url)}")
with open(tst_fn, "wb") as f:
f.write(signed)
timestamp_tokens.append(Media(filename=tst_fn).set("tsa", tsa_url).set("cert_chain", cert_chain))
except Exception as e:
logger.warning(f"Error while timestamping {url=} with {tsa_url=}: {e}")
if len(timestamp_tokens):
hashes_media.set("timestamp_authority_files", timestamp_tokens)
hashes_media.set("certifi v", version("certifi"))
hashes_media.set("rfc3161-client v", version("rfc3161_client"))
hashes_media.set("cryptography v", version("cryptography"))
hashes_media.set("tsp_client v", version("tsp_client"))
hashes_media.set("certvalidator v", version("certvalidator"))
to_enrich.add_media(hashes_media, id="timestamped_hashes")
to_enrich.set("timestamped", True)
logger.success(f"{len(timestamp_tokens)} timestamp tokens created for {url=}")
else:
logger.warning(f"No successful timestamps for {url=}")
def save_timestamp_token(self, timestamp_token: bytes, tsa_url: str) -> str:
"""
Takes a timestamp token, and saves it to a file with the TSA URL as part of the filename.
"""
tst_path = os.path.join(self.tmp_dir, f"timestamp_token_{slugify(tsa_url)}")
with open(tst_path, "wb") as f:
f.write(timestamp_token)
return tst_path
def verify_signed(self, timestamp_response: TimeStampResponse, message: bytes) -> x509.Certificate:
"""
Verify a Signed Timestamp Response is trusted by a known Certificate Authority.
Args:
timestamp_response (TimeStampResponse): The signed timestamp response.
message (bytes): The message that was timestamped.
Returns:
x509.Certificate: A valid root certificate that was used to sign the timestamp response, or None
Raises:
ValueError: If no valid root certificate was found in the trusted root store.
"""
trusted_root_path = self.cert_authorities or certifi.where()
cert_authorities = []
with open(trusted_root_path, 'rb') as f:
cert_authorities = x509.load_pem_x509_certificates(f.read())
if not cert_authorities:
raise ValueError(f"No trusted roots found in {trusted_root_path}.")
timestamp_certs = self.tst_certs(timestamp_response)
intermediate_certs = timestamp_certs[1:-1]
message_hash = None
hash_algorithm = timestamp_response.tst_info.message_imprint.hash_algorithm
if hash_algorithm == x509.ObjectIdentifier(value="2.16.840.1.101.3.4.2.3"):
message_hash = hashlib.sha512(message).digest()
elif hash_algorithm == x509.ObjectIdentifier(value="2.16.840.1.101.3.4.2.1"):
message_hash = hashlib.sha256(message).digest()
else:
raise ValueError(f"Unsupported hash algorithm: {hash_algorithm}")
for certificate in cert_authorities:
builder = VerifierBuilder()
builder.add_root_certificate(certificate)
for intermediate_cert in intermediate_certs:
builder.add_intermediate_certificate(intermediate_cert)
verifier = builder.build()
try:
verifier.verify(timestamp_response, message_hash)
return certificate
except Rfc3161VerificationError:
continue
return None
def sign_data(self, tsa_url: str, bytes_data: bytes) -> TimeStampResponse:
# see https://github.com/sigstore/sigstore-python/blob/99948d5b80525a5a104e904ffea58169dc6e0629/sigstore/_internal/timestamp.py#L84-L121
timestamp_request = (
TimestampRequestBuilder().data(bytes_data).nonce(nonce=True).build()
)
try:
response = self.session.post(tsa_url, data=timestamp_request.as_bytes(), timeout=10)
response.raise_for_status()
except requests.RequestException as e:
logger.error(f"Error while sending request to {tsa_url=}: {e}")
raise
# Check that we can parse the response but do not *verify* it
try:
timestamp_response = decode_timestamp_response(response.content)
except ValueError as e:
logger.error(f"Invalid timestamp response from server {tsa_url}: {e}")
raise
return timestamp_response
def tst_certs(self, tsp_response: TimeStampResponse):
signed_data: SignedData = tsp_response.signed_data
certs = [x509.load_der_x509_certificate(c) for c in signed_data.certificates]
# reorder the certs to be in the correct order
ordered_certs = []
if len(certs) == 1:
return certs
while(len(ordered_certs) < len(certs)):
if len(ordered_certs) == 0:
for cert in certs:
if not [c for c in certs if cert.subject == c.issuer]:
ordered_certs.append(cert)
break
else:
for cert in certs:
if cert.subject == ordered_certs[-1].issuer:
ordered_certs.append(cert)
break
return ordered_certs
def save_certificate(self, tsp_response: TimeStampResponse, verified_root_cert: x509.Certificate) -> list[Media]:
def download_and_verify_certificate(self, signed: bytes) -> list[Media]:
# returns the leaf certificate URL, fails if not set
tst = ContentInfo.load(signed)
certificates = self.tst_certs(tsp_response)
trust_roots = []
with open(certifi.where(), "rb") as f:
for _, _, der_bytes in pem.unarmor(f.read(), multiple=True):
trust_roots.append(der_bytes)
context = ValidationContext(trust_roots=trust_roots)
if verified_root_cert:
# add the verified root certificate (if there is one - self signed certs will have None here)
certificates += [verified_root_cert]
certificates = tst["content"]["certificates"]
first_cert = certificates[0].dump()
intermediate_certs = []
for i in range(1, len(certificates)): # cannot use list comprehension [1:]
intermediate_certs.append(certificates[i].dump())
validator = CertificateValidator(first_cert, intermediate_certs=intermediate_certs, validation_context=context)
path = validator.validate_usage({"digital_signature"}, extended_key_usage={"time_stamping"})
cert_chain = []
for i, cert in enumerate(certificates):
cert_fn = os.path.join(self.tmp_dir, f"{i+1} {str(cert.serial_number)[:20]}.crt")
for cert in path:
cert_fn = os.path.join(self.tmp_dir, f"{str(cert.serial_number)[:20]}.crt")
with open(cert_fn, "wb") as f:
f.write(cert.public_bytes(encoding=serialization.Encoding.PEM))
cert_chain.append(Media(filename=cert_fn).set("subject", cert.subject.get_attributes_for_oid(x509.NameOID.COMMON_NAME)[0].value))
f.write(cert.dump())
cert_chain.append(Media(filename=cert_fn).set("subject", cert.subject.native["common_name"]))
return cert_chain

View File

@@ -40,31 +40,27 @@
Creates .WACZ archives of web pages using the `browsertrix-crawler` tool, with options for media extraction and screenshot saving.
[Browsertrix-crawler](https://crawler.docs.browsertrix.com/user-guide/) is a headless browser-based crawler that archives web pages in WACZ format.
## Setup
**Docker**
If you are using the Docker file to run Auto Archiver (recommended), then everything is set up and you can use WACZ out of the box!
Otherwise, if you are using a local install of Auto Archiver (e.g. pip or dev install), then you will need to install Docker and run
the docker daemon to be able to run the `browsertrix-crawler` tool.
**Browsertrix Profiles**
A browsertrix profile is a custom browser profile (login information, browser extensions, etc.) that can be used to archive private or dynamic content.
You can run the WACZ Enricher without a profile, but for more resilient archiving, it is recommended to create a profile. See the [Browsertrix documentation](https://crawler.docs.browsertrix.com/user-guide/browser-profiles/)
for more information.
** Docker in Docker **
If you are running Auto Archiver within a Docker container, you will need to enable Docker in Docker to run the `browsertrix-crawler` tool.
This can be done by setting the `WACZ_ENABLE_DOCKER` environment variable to `1`.
## Features
- Archives web pages into .WACZ format using Docker or direct invocation of `browsertrix-crawler`.
- Supports custom profiles for archiving private or dynamic content.
- Extracts media (images, videos, audio) and screenshots from the archive, optionally adding them to the enrichment pipeline.
- Generates metadata from the archived page's content and structure (e.g., titles, text).
## Setup
### Using Docker
If you are using the Auto Archiver [Docker image](https://auto-archiver.readthedocs.io/en/latest/installation/installation.html#installing-with-docker)
to run Auto Archiver (recommended), then everything is set up and you can use WACZ out of the box!
Otherwise, if you are using a local install of Auto Archiver (e.g. pip or dev install), then you will need to install Docker and run
the docker daemon to be able to run the `browsertrix-crawler` tool.
### Browsertrix Profiles
A browsertrix profile is a custom browser profile (login information, browser extensions, etc.) that can be used to archive private or dynamic content.
You can run the WACZ Enricher without a profile, but for more resilient archiving, it is recommended to create a profile.
See the [Browsertrix documentation](https://crawler.docs.browsertrix.com/user-guide/browser-profiles/) for more information on how to use the `create-login-profile` tool.
### Docker in Docker
If you are running Auto Archiver within a Docker container, you will need to enable Docker in Docker to run the `browsertrix-crawler` tool.
This can be done by setting the `WACZ_ENABLE_DOCKER` environment variable to `1`.
""",
}

View File

@@ -86,12 +86,6 @@ class WaczExtractorEnricher(Enricher, Extractor):
if self.docker_in_docker:
cmd.extend(["--cwd", self.cwd_dind])
if self.auth_for_site(url):
# there's an auth for this site, but browsertrix only supports username/password auth
logger.warning(
"The WACZ enricher / Browsertrix does not support using the 'authentication' information for logging in. You should consider creating a Browser Profile for WACZ archiving. More information: https://auto-archiver.readthedocs.io/en/latest/modules/autogen/extractor/wacz_extractor_enricher.html#browsertrix-profiles"
)
# call docker if explicitly enabled or we are running on the host (not in docker)
if self.use_docker:
logger.debug(f"generating WACZ in Docker for {url=}")

View File

@@ -10,31 +10,14 @@ from typing import Dict, Tuple
import hashlib
import pytest
from auto_archiver.core.metadata import Metadata, Media
from auto_archiver.core.metadata import Metadata
from auto_archiver.core.module import ModuleFactory
# Test names inserted into this list will be run last. This is useful for expensive/costly tests
# that you only want to run if everything else succeeds (e.g. API calls). The order here is important
# what comes first will be run first (at the end of all other tests not mentioned)
# format is the name of the module (python file) without the .py extension
TESTS_TO_RUN_LAST = ["test_generic_archiver", "test_twitter_api_archiver"]
# don't check for ytdlp updates in tests
@pytest.fixture(autouse=True)
def skip_check_for_update(mocker):
update_ytdlp = mocker.patch(
"auto_archiver.modules.generic_extractor.generic_extractor.GenericExtractor.update_package"
)
update_ytdlp.return_value = False
@pytest.fixture
def get_lazy_module():
def _get_lazy_module(module_name):
return ModuleFactory().get_module_lazy(module_name)
return _get_lazy_module
TESTS_TO_RUN_LAST = ["test_twitter_api_archiver"]
@pytest.fixture
@@ -151,21 +134,12 @@ def unpickle():
@pytest.fixture
def mock_binary_dependencies(mocker):
mocker.patch("subprocess.run").return_value = mocker.Mock(returncode=0)
mock_shutil_which = mocker.patch("shutil.which")
# Mock all binary dependencies as available
mock_shutil_which.return_value = "/usr/bin/fake_binary"
return mock_shutil_which
@pytest.fixture
def sample_media(tmp_path) -> Media:
"""Fixture creating a Media object with temporary source file"""
src_file = tmp_path / "source.txt"
src_file.write_text("test content")
return Media(_key="subdir/test.txt", filename=str(src_file))
@pytest.fixture
def sample_datetime():
return datetime(2023, 1, 1, 12, 0, tzinfo=timezone.utc)

View File

@@ -1,11 +1,6 @@
from auto_archiver.core import Extractor
from loguru import logger
class ExampleExtractor(Extractor):
def download(self, item):
logger.info("download")
def cleanup(self):
logger.info("cleanup")
print("download")

View File

@@ -1,29 +1,27 @@
from auto_archiver.core import Extractor, Enricher, Feeder, Database, Storage, Formatter, Metadata
from loguru import logger
class ExampleModule(Extractor, Enricher, Feeder, Database, Storage, Formatter):
def download(self, item):
logger.info("download")
print("download")
def __iter__(self):
yield Metadata().set_url("https://example.com")
def done(self, result):
logger.info("done")
print("done")
def enrich(self, to_enrich):
logger.info("enrich")
print("enrich")
def get_cdn_url(self, media):
return "nice_url"
def save(self, item):
logger.info("save")
print("save")
def uploadf(self, file, key, **kwargs):
logger.info("uploadf")
print("uploadf")
def format(self, item):
logger.info("format")
print("format")

Binary file not shown.

View File

@@ -1,215 +0,0 @@
from pathlib import Path
import pytest
from rfc3161_client import (
TimeStampResponse,
decode_timestamp_response,
)
import requests
from auto_archiver.modules.timestamping_enricher.timestamping_enricher import TimestampingEnricher
from auto_archiver.core import Metadata
@pytest.fixture
def timestamp_response() -> TimeStampResponse:
with open("tests/data/timestamping/valid_timestamp.tsr", "rb") as f:
return decode_timestamp_response(f.read())
@pytest.fixture
def wrong_order_timestamp_response() -> TimeStampResponse:
with open("tests/data/timestamping/rfc3161-client-issue-104.tsr", "rb") as f:
return decode_timestamp_response(f.read())
@pytest.fixture
def selfsigned_response() -> TimeStampResponse:
with open("tests/data/timestamping/self_signed.tsr", "rb") as f:
return decode_timestamp_response(f.read())
@pytest.fixture
def digicert_response() -> TimeStampResponse:
with open("tests/data/timestamping/digicert.tsr", "rb") as f:
return f.read()
@pytest.fixture
def filehash():
return "4b7b4e39f12b8c725e6e603e6d4422500316df94211070682ef10260ff5759ef"
@pytest.mark.download
def test_enriching(setup_module, sample_media):
tsp: TimestampingEnricher = setup_module("timestamping_enricher")
# tests the current TSAs set as default in the __manifest__ to make sure they are all still working
# test the enrich method
metadata = Metadata().set_url("https://example.com")
sample_media.set("hash", "4b7b4e39f12b8c725e6e603e6d4422500316df94211070682ef10260ff5759ef")
metadata.add_media(sample_media)
tsp.enrich(metadata)
def test_full_enriching_selfsigned(setup_module, sample_media, mocker, selfsigned_response, filehash):
mock_post = mocker.patch("requests.sessions.Session.post")
mock_post.return_value.status_code = 200
mock_decode_timestamp_response = mocker.patch(
"auto_archiver.modules.timestamping_enricher.timestamping_enricher.decode_timestamp_response"
)
mock_decode_timestamp_response.return_value = selfsigned_response
tsp: TimestampingEnricher = setup_module("timestamping_enricher", {"tsa_urls": ["http://timestamp.identrust.com"]})
metadata = Metadata().set_url("https://example.com")
sample_media.set("hash", filehash)
metadata.add_media(sample_media)
tsp.enrich(metadata)
assert len(metadata.media) == 1 # doesn't allow self-signed
# set self-signed on tsp
tsp.allow_selfsigned = True
tsp.enrich(metadata)
assert len(metadata.media) == 2
def test_full_enriching(setup_module, sample_media, mocker, timestamp_response, filehash):
mock_post = mocker.patch("requests.sessions.Session.post")
mock_post.return_value.status_code = 200
mock_decode_timestamp_response = mocker.patch(
"auto_archiver.modules.timestamping_enricher.timestamping_enricher.decode_timestamp_response"
)
mock_decode_timestamp_response.return_value = timestamp_response
tsp: TimestampingEnricher = setup_module("timestamping_enricher", {"tsa_urls": ["http://timestamp.identrust.com"]})
metadata = Metadata().set_url("https://example.com")
sample_media.set("hash", filehash)
metadata.add_media(sample_media)
tsp.enrich(metadata)
assert metadata.get("timestamped") is True
assert len(metadata.media) == 2 # the original 'sample_media' and the new 'timestamp_media'
timestamp_media = metadata.media[1]
assert timestamp_media.filename == f"{tsp.tmp_dir}/hashes.txt"
assert Path(timestamp_media.filename).read_text() == filehash
# we only have one authority file because we only used one TSA
assert len(timestamp_media.get("timestamp_authority_files")) == 1
timestamp_authority_file = timestamp_media.get("timestamp_authority_files")[0]
assert Path(timestamp_authority_file.filename).read_bytes() == timestamp_response.time_stamp_token()
cert_chain = timestamp_authority_file.get("cert_chain")
assert len(cert_chain) == 3
assert cert_chain[0].filename == f"{tsp.tmp_dir}/1 85078758028491331763.crt"
assert cert_chain[1].filename == f"{tsp.tmp_dir}/2 85078371663472981624.crt"
assert cert_chain[2].filename == f"{tsp.tmp_dir}/3 13298821034946342390.crt"
def test_full_enriching_multiple_tsa(setup_module, sample_media, mocker, timestamp_response, filehash):
mock_post = mocker.patch("requests.sessions.Session.post")
mock_post.return_value.status_code = 200
mock_decode_timestamp_response = mocker.patch(
"auto_archiver.modules.timestamping_enricher.timestamping_enricher.decode_timestamp_response"
)
mock_decode_timestamp_response.return_value = timestamp_response
tsp: TimestampingEnricher = setup_module(
"timestamping_enricher", {"tsa_urls": ["http://example.com/timestamp1", "http://example.com/timestamp2"]}
)
metadata = Metadata().set_url("https://example.com")
sample_media.set("hash", filehash)
metadata.add_media(sample_media)
tsp.enrich(metadata)
assert metadata.get("timestamped") is True
assert len(metadata.media) == 2 # the original 'sample_media' and the new 'timestamp_media'
timestamp_media = metadata.media[1]
assert len(timestamp_media.get("timestamp_authority_files")) == 2
for timestamp_token_media in timestamp_media.get("timestamp_authority_files"):
assert Path(timestamp_token_media.filename).read_bytes() == timestamp_response.time_stamp_token()
assert len(timestamp_token_media.get("cert_chain")) == 3
def test_fails_for_digicert(setup_module, mocker, digicert_response):
"""
Digicert TSRs are not compliant with RFC 3161.
See https://github.com/trailofbits/rfc3161-client/issues/104#issuecomment-2621960840
"""
mocker.patch("requests.sessions.Session.post", return_value=requests.Response())
mocker.patch("requests.Response.raise_for_status")
mocker.patch("requests.Response.content", new_callable=mocker.PropertyMock, return_value=digicert_response)
tsa_url = "http://timestamp.digicert.com"
tsp: TimestampingEnricher = setup_module("timestamping_enricher")
data = b"4b7b4e39f12b8c725e6e603e6d4422500316df94211070682ef10260ff5759ef"
with pytest.raises(ValueError) as e:
tsp.sign_data(tsa_url, data)
assert "ASN.1 parse error: ParseError" in str(e.value)
@pytest.mark.download
def test_download_tsr(setup_module):
tsa_url = "http://timestamp.identrust.com"
tsp: TimestampingEnricher = setup_module("timestamping_enricher")
data = b"4b7b4e39f12b8c725e6e603e6d4422500316df94211070682ef10260ff5759ef"
result: TimeStampResponse = tsp.sign_data(tsa_url, data)
assert isinstance(result, TimeStampResponse)
verified_root_cert = tsp.verify_signed(result, data)
assert verified_root_cert.subject.rfc4514_string() == "CN=IdenTrust Commercial Root CA 1,O=IdenTrust,C=US"
# test downloading the cert
cert_chain = tsp.save_certificate(result, verified_root_cert)
assert len(cert_chain) == 3
def test_verify_save(setup_module, timestamp_response):
tsp: TimestampingEnricher = setup_module("timestamping_enricher")
verified_root_cert = tsp.verify_signed(
timestamp_response, b"4b7b4e39f12b8c725e6e603e6d4422500316df94211070682ef10260ff5759ef"
)
assert verified_root_cert.subject.rfc4514_string() == "CN=IdenTrust Commercial Root CA 1,O=IdenTrust,C=US"
cert_chain = tsp.save_certificate(timestamp_response, verified_root_cert)
assert len(cert_chain) == 3
assert cert_chain[0].filename == f"{tsp.tmp_dir}/1 85078758028491331763.crt"
assert cert_chain[1].filename == f"{tsp.tmp_dir}/2 85078371663472981624.crt"
assert cert_chain[2].filename == f"{tsp.tmp_dir}/3 13298821034946342390.crt"
def test_order_crt_correctly(setup_module, wrong_order_timestamp_response):
# reference: https://github.com/trailofbits/rfc3161-client/issues/104#issuecomment-2711244010
tsp: TimestampingEnricher = setup_module("timestamping_enricher")
# get the certificates, make sure the reordering is working:
ordered_certs = tsp.tst_certs(wrong_order_timestamp_response)
assert len(ordered_certs) == 2
assert ordered_certs[0].subject.rfc4514_string() == "CN=TrustID Timestamp Authority,O=IdenTrust,C=US"
assert ordered_certs[1].subject.rfc4514_string() == "CN=TrustID Timestamping CA 3,O=IdenTrust,C=US"
def test_invalid_tsa_invalid_response(setup_module, mocker):
mocker.patch("requests.sessions.Session.post", return_value=requests.Response())
raise_for_status = mocker.patch("requests.Response.raise_for_status")
raise_for_status.side_effect = requests.exceptions.HTTPError("404 Client Error")
tsp = setup_module("timestamping_enricher")
with pytest.raises(requests.exceptions.HTTPError, match="404 Client Error"):
tsp.sign_data("http://bellingcat.com/page-not-found/", b"my-message")
def test_fail_on_selfsigned_cert(setup_module, selfsigned_response):
tsp = setup_module("timestamping_enricher")
root_cert = tsp.verify_signed(selfsigned_response, b"my-message")
assert root_cert is None

View File

@@ -25,5 +25,5 @@ class TestExtractorBase(object):
else:
assert status == test_response.status
assert title in test_response.get_title()
assert timestamp == test_response.get("timestamp")
assert title == test_response.get_title()
assert timestamp, test_response.get("timestamp")

View File

@@ -29,7 +29,6 @@ class TestGenericExtractor(TestExtractorBase):
"proxy": None,
"cookies_from_browser": False,
"cookie_file": None,
"pot_provider": False,
}
def test_load_dropin(self):
@@ -37,7 +36,7 @@ class TestGenericExtractor(TestExtractorBase):
package = "auto_archiver.modules.generic_extractor"
assert self.extractor.dropin_for_name("bluesky", package=package)
# test loading dropins via filepath
# test loading dropings via filepath
path = os.path.join(dirname(dirname(__file__)), "data/")
assert self.extractor.dropin_for_name("dropin", additional_paths=[path])
@@ -122,7 +121,7 @@ class TestGenericExtractor(TestExtractorBase):
== "Buy NEW Keyboard Cat Merch! https://keyboardcat.creator-spring.com\n\nxo Keyboard Cat memes make your day better!\nhttp://www.keyboardcatstore.com/\nhttps://www.facebook.com/thekeyboardcat\nhttp://www.charlieschmidt.com/"
)
assert len(result.media) == 2
assert "J---aiyznGQ" in Path(result.media[0].filename).name
assert Path(result.media[0].filename).name == "J---aiyznGQ.webm"
assert Path(result.media[1].filename).name == "hqdefault.jpg"
@pytest.mark.download
@@ -219,7 +218,7 @@ class TestGenericExtractor(TestExtractorBase):
post = self.extractor.download(make_item(url))
self.assertValidResponseMetadata(
post,
"Bellingcat - This month's Bellingchat Premium is with @KolinaKoltai",
"Bellingcat - This month's Bellingchat Premium is with @KolinaKoltai. She reveals how she investigated a platform allowing users to create AI-generated child sexual abuse material and explains why it's crucial to investigate the people behind these services",
datetime.datetime(2024, 12, 24, 13, 44, 46, tzinfo=datetime.timezone.utc),
)
@@ -292,42 +291,3 @@ class TestGenericExtractor(TestExtractorBase):
post = self.extractor.download(make_item(url))
assert "Bellingcat researcher Kolina Koltai delves deeper into Clothoff" in post.get("content")
assert post.get_title() == "Bellingcat"
class TestGenericExtractorPoToken:
@pytest.fixture
def extractor(self, mocker):
extractor = GenericExtractor()
extractor.extractor_args = {}
extractor.setup_token_generation_script = mocker.Mock()
return extractor
def test_po_token_disabled_does_not_call_setup(self, extractor):
extractor.bguils_po_token_method = "disabled"
extractor.in_docker = True
extractor.setup_po_tokens()
extractor.setup_token_generation_script.assert_not_called()
def test_po_token_default_in_docker_calls_setup(self, extractor, mocker):
extractor.bguils_po_token_method = "auto"
mocker.patch.dict(os.environ, {"RUNNING_IN_DOCKER": "1"})
extractor.setup_po_tokens()
extractor.setup_token_generation_script.assert_called_once()
def test_po_token_default_local_does_not_call_setup(self, extractor, caplog, mocker):
extractor.bguils_po_token_method = "auto"
# clears env vars for this test
mocker.patch.dict(os.environ, {}, clear=True)
extractor.setup_po_tokens()
extractor.setup_token_generation_script.assert_not_called()
assert "Proof of Origin Token method not explicitly set" in caplog.text
def test_po_token_script_always_calls_setup(self, extractor):
extractor.bguils_po_token_method = "script"
extractor.in_docker = False
extractor.setup_po_tokens()
extractor.setup_token_generation_script.assert_called_once()
extractor.setup_token_generation_script.reset_mock()
extractor.in_docker = True
extractor.setup_po_tokens()
extractor.setup_token_generation_script.assert_called_once()

View File

@@ -68,12 +68,6 @@ def test_download_invalid(extractor, metadata_sample, mocker):
assert extractor.download(metadata_sample) is False
def test_fails_with_empty_response(extractor, metadata_sample, mocker):
mocker.patch.object(extractor, "_send_url_to_bot", return_value=(mocker.MagicMock(), 101))
mocker.patch.object(extractor, "_process_messages", return_value="")
assert extractor.download(metadata_sample) is False
@pytest.mark.skip(reason="Requires authentication.")
class TestInstagramTbotExtractorReal(TestExtractorBase):
# To run these tests set the TELEGRAM_API_ID and TELEGRAM_API_HASH environment variables, and ensure the session file exists.

View File

@@ -1,26 +0,0 @@
import os
from datetime import date
import pytest
@pytest.fixture(autouse=True)
def mock_client_setup(mocker):
mocker.patch("telethon.client.auth.AuthMethods.start")
def test_setup_fails_clear_session_file(get_lazy_module, tmp_path, mocker):
start = mocker.patch("telethon.client.auth.AuthMethods.start")
start.side_effect = Exception("Test exception")
# make sure the default setup file is created
session_file = tmp_path / "test.session"
lazy_module = get_lazy_module("telethon_extractor")
with pytest.raises(Exception):
lazy_module.load({"telethon_extractor": {"session_file": str(session_file), "api_id": 123, "api_hash": "ABC"}})
assert session_file.exists()
assert f"telethon-{date.today().strftime('%Y-%m-%d')}" in lazy_module._instance.session_file
assert os.path.exists(lazy_module._instance.session_file + ".session")

View File

@@ -237,23 +237,3 @@ def test_wrong_step_type(test_args, caplog):
with pytest.raises(SetupError) as err:
orchestrator.setup(args)
assert "Module 'example_extractor' is not a feeder" in str(err.value)
def test_load_failed_extractor_cleanup(test_args, mocker, caplog):
orchestrator = ArchivingOrchestrator()
# hack to set up the paths so we can patch properly
orchestrator.module_factory.setup_paths([TEST_MODULES])
# patch example_module.setup to throw an exception
mocker.patch(
"auto_archiver.modules.example_extractor.example_extractor.ExampleExtractor.setup",
side_effect=Exception("Test exception"),
)
with pytest.raises(Exception):
orchestrator.setup(test_args + ["--extractors", "example_extractor"])
assert "Error during setup of modules: Test exception" in caplog.text
# make sure the 'cleanup' is called
assert "cleanup" in caplog.text