diff --git a/.github/workflows/docker-publish.yaml b/.github/workflows/docker-publish.yaml index 4d232e2..2031ad6 100644 --- a/.github/workflows/docker-publish.yaml +++ b/.github/workflows/docker-publish.yaml @@ -11,7 +11,7 @@ on: env: # Use docker.io for Docker Hub if empty - REGISTRY: ghcr.io + REGISTRY: docker.io # github.repository as / IMAGE_NAME: ${{ github.repository }} @@ -45,10 +45,12 @@ jobs: images: bellingcat/auto-archiver - name: Build and push Docker image - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v6 with: context: . platforms: linux/amd64,linux/arm64 push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:cache + cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:cache,mode=max diff --git a/.github/workflows/tests-core.yaml b/.github/workflows/tests-core.yaml index aab3713..57028bd 100644 --- a/.github/workflows/tests-core.yaml +++ b/.github/workflows/tests-core.yaml @@ -5,9 +5,13 @@ on: branches: [ main ] paths: - src/** + - poetry.lock + - pyproject.toml pull_request: paths: - src/** + - poetry.lock + - pyproject.toml jobs: tests: @@ -16,7 +20,8 @@ jobs: fail-fast: false matrix: python-version: ["3.10", "3.11", "3.12"] - os: [ubuntu-22.04, ubuntu-latest] + os: [ubuntu-22.04] + #TODO: re-enable ubuntu-latest, this is disabled as oscrypto cannot be pinned to github commit and pushed to pypi defaults: run: working-directory: ./ diff --git a/.gitignore b/.gitignore index 7c6bf08..f31bc6c 100644 --- a/.gitignore +++ b/.gitignore @@ -32,3 +32,6 @@ archived/ dist* docs/_build/ docs/source/autoapi/ +docs/source/modules/autogen/ +scripts/settings_page.html +.vite diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 9f67835..6dc9fe5 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -9,13 +9,19 @@ build: os: ubuntu-22.04 tools: python: "3.10" + nodejs: "22" jobs: post_install: - pip install poetry # https://python-poetry.org/docs/managing-dependencies/#dependency-groups # VIRTUAL_ENV needs to be set manually for now. # See https://github.com/readthedocs/readthedocs.org/pull/11152/ - - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH poetry install --only docs + - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH poetry install --with docs + + # generate the config editor page. Schema then HTML + - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH poetry run python scripts/generate_settings_schema.py + # install node dependencies and build the settings + - cd scripts/settings && npm install && npm run build && yes | cp dist/index.html ../../docs/source/installation/settings_base.html && cd ../.. sphinx: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..7e71f82 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,49 @@ +# Contributing to Auto Archiver + +Thank you for your interest in contributing to Auto Archiver! Your contributions help improve the project and make it more useful for everyone. Please follow the guidelines below to ensure a smooth collaboration. + +### 1. Reporting a Bug + +If you encounter a bug, please create an issue on GitHub with the following details: + +* Describe the bug: Provide a clear and concise description of the issue. +* Steps to reproduce: Include the steps needed to reproduce the bug. +* Expected behavior: Describe what you expected to happen. +* Actual behavior: Explain what actually happened. +* Screenshots/logs: If applicable, attach screenshots or logs to help diagnose the problem. +* Environment: Mention the OS, Ruby version, and any other relevant details. + +### 2. Writing a Patch/Fix and Submitting Pull Requests + +If you’d like to fix a bug or improve existing code: + +1. Open a pull request on GitHub and link it to the relevant issue. +2. Make sure to document your pull request with a clear description of what changes were made and why. +3. Wait for review and make any requested changes. + +### 3. Creating New Modules + +If you want to add a new module to Auto Archiver: + +1. Ensure your module follows the existing [coding style and project structure](https://auto-archiver.readthedocs.io/en/latest/development/creating_modules.html). +2. Write clear documentation explaining what your module does and how to use it. +3. Ideally, include unit tests for your module! +4. Follow the steps in Section 2 to submit a pull request. + +### 4. Do You Have Questions About the Source Code? + +If you have any questions about how the source code works or need help using Auto Archiver + +📝 Check the [Auto Archiver](https://auto-archiver.readthedocs.io/en/latest/) documentation. + +👉 Ask your questions in the [Bellingcat Discord](https://www.bellingcat.com/follow-bellingcat-on-social-media/). + +### 5. Do You Want to Contribute to the Documentation? + +We welcome contributions to the documentation! + +📖 Please read [Contributing to the Auto Archiver Documentation](https://auto-archiver.readthedocs.io/en/latest/development/docs.html) to learn how you can help improve the project's documentation. + +------------------ + +Thank you for contributing to Auto Archiver! 🚀 \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index cbcfdd4..68aed42 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,13 +7,24 @@ ENV RUNNING_IN_DOCKER=1 \ PYTHONFAULTHANDLER=1 \ PATH="/root/.local/bin:$PATH" + +ARG TARGETARCH + # Installing system dependencies RUN add-apt-repository ppa:mozillateam/ppa && \ apt-get update && \ apt-get install -y --no-install-recommends gcc ffmpeg fonts-noto exiftool && \ apt-get install -y --no-install-recommends firefox-esr && \ - ln -s /usr/bin/firefox-esr /usr/bin/firefox && \ - wget https://github.com/mozilla/geckodriver/releases/download/v0.35.0/geckodriver-v0.35.0-linux64.tar.gz && \ + ln -s /usr/bin/firefox-esr /usr/bin/firefox + +ARG GECKODRIVER_VERSION=0.36.0 + +RUN if [ $(uname -m) = "aarch64" ]; then \ + GECKODRIVER_ARCH=linux-aarch64; \ + else \ + GECKODRIVER_ARCH=linux64; \ + fi && \ + wget https://github.com/mozilla/geckodriver/releases/download/v${GECKODRIVER_VERSION}/geckodriver-v${GECKODRIVER_VERSION}-${GECKODRIVER_ARCH}.tar.gz && \ tar -xvzf geckodriver* -C /usr/local/bin && \ chmod +x /usr/local/bin/geckodriver && \ rm geckodriver-v* && \ diff --git a/README.md b/README.md index c52c464..8baa722 100644 --- a/README.md +++ b/README.md @@ -9,327 +9,33 @@ + +Auto Archiver is a Python tool to automatically archive content on the web in a secure and verifiable way. It takes URLs from different sources (e.g. a CSV file, Google Sheets, command line etc.) and archives the content of each one. It can archive social media posts, videos, images and webpages. Content can be enriched, then saved either locally or remotely (S3 bucket, Google Drive). The status of the archiving process can be appended to a CSV report, or if using Google Sheets – back to the original sheet. + +
+ +**[See the Auto Archiver documentation for more information.](https://auto-archiver.readthedocs.io/en/latest/)** + +
+ Read the [article about Auto Archiver on bellingcat.com](https://www.bellingcat.com/resources/2022/09/22/preserve-vital-online-content-with-bellingcats-auto-archiver-tool/). -Python tool to automatically archive social media posts, videos, and images from a Google Sheets, the console, and more. Uses different archivers depending on the platform, and can save content to local storage, S3 bucket (Digital Ocean Spaces, AWS, ...), and Google Drive. If using Google Sheets as the source for links, it will be updated with information about the archived content. It can be run manually or on an automated basis. +## Installation -There are 3 ways to use the auto-archiver: -1. (easiest installation) via docker -2. (local python install) `pip install auto-archiver` -3. (legacy/development) clone and manually install from repo (see legacy [tutorial video](https://youtu.be/VfAhcuV2tLQ)) +View the [Installation Guide](https://auto-archiver.readthedocs.io/en/latest/installation/installation.html) for full instructions -But **you always need a configuration/orchestration file**, which is where you'll configure where/what/how to archive. Make sure you read [orchestration](#orchestration). +**Advanced:** +To get started quickly using Docker: -## How to install and run the auto-archiver +`docker pull bellingcat/auto-archiver && docker run --rm -v secrets:/app/secrets bellingcat/auto-archiver --config secrets/orchestration.yaml` -### Option 1 - docker +Or pip: -[![dockeri.co](https://dockerico.blankenship.io/image/bellingcat/auto-archiver)](https://hub.docker.com/r/bellingcat/auto-archiver) +`pip install auto-archiver && auto-archiver --help` -Docker works like a virtual machine running inside your computer, it isolates everything and makes installation simple. Since it is an isolated environment when you need to pass it your orchestration file or get downloaded media out of docker you will need to connect folders on your machine with folders inside docker with the `-v` volume flag. +## Contributing +We welcome contributions to the Auto Archiver project! See the [Contributing Guide](https://auto-archiver.readthedocs.io/en/latest/contributing.html) for how to get involved! -1. install [docker](https://docs.docker.com/get-docker/) -2. pull the auto-archiver docker [image](https://hub.docker.com/r/bellingcat/auto-archiver) with `docker pull bellingcat/auto-archiver` -3. run the docker image locally in a container: `docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver --config secrets/orchestration.yaml` breaking this command down: - 1. `docker run` tells docker to start a new container (an instance of the image) - 2. `--rm` makes sure this container is removed after execution (less garbage locally) - 3. `-v $PWD/secrets:/app/secrets` - your secrets folder - 1. `-v` is a volume flag which means a folder that you have on your computer will be connected to a folder inside the docker container - 2. `$PWD/secrets` points to a `secrets/` folder in your current working directory (where your console points to), we use this folder as a best practice to hold all the secrets/tokens/passwords/... you use - 3. `/app/secrets` points to the path the docker container where this image can be found - 4. `-v $PWD/local_archive:/app/local_archive` - (optional) if you use local_storage - 1. `-v` same as above, this is a volume instruction - 2. `$PWD/local_archive` is a folder `local_archive/` in case you want to archive locally and have the files accessible outside docker - 3. `/app/local_archive` is a folder inside docker that you can reference in your orchestration.yml file - -### Option 2 - python package - -
Python package instructions - -1. make sure you have python 3.10 or higher installed -2. install the package with your preferred package manager: `pip/pipenv/conda install auto-archiver` or `poetry add auto-archiver` -3. test it's installed with `auto-archiver --help` -4. run it with your orchestration file and pass any flags you want in the command line `auto-archiver --config secrets/orchestration.yaml` if your orchestration file is inside a `secrets/`, which we advise - -You will also need [ffmpeg](https://www.ffmpeg.org/), [firefox](https://www.mozilla.org/en-US/firefox/new/) and [geckodriver](https://github.com/mozilla/geckodriver/releases), and optionally [fonts-noto](https://fonts.google.com/noto). Similar to the local installation. - -
- - -### Option 3 - local installation -This can also be used for development. - -
Legacy instructions, only use if docker/package is not an option - - -Install the following locally: -1. [ffmpeg](https://www.ffmpeg.org/) must also be installed locally for this tool to work. -2. [firefox](https://www.mozilla.org/en-US/firefox/new/) and [geckodriver](https://github.com/mozilla/geckodriver/releases) on a path folder like `/usr/local/bin`. -3. [Poetry](https://python-poetry.org/docs/#installation) for dependency management and packaging. -4. (optional) [fonts-noto](https://fonts.google.com/noto) to deal with multiple unicode characters during selenium/geckodriver's screenshots: `sudo apt install fonts-noto -y`. - -Clone and run: -1. `git clone https://github.com/bellingcat/auto-archiver` -2. `poetry install` -3. `poetry run python -m src.auto_archiver --config secrets/orchestration.yaml` - -Note: Add the plugin [poetry-shell-plugin](https://github.com/python-poetry/poetry-plugin-shell) and run `poetry shell` to activate the virtual environment. -This allows you to run the auto-archiver without the `poetry run` prefix. - -

- -# Orchestration -The archiver work is orchestrated by the following workflow (we call each a **step**): -1. **Feeder** gets the links (from a spreadsheet, from the console, ...) -2. **Archiver** tries to archive the link (twitter, youtube, ...) -3. **Enricher** adds more info to the content (hashes, thumbnails, ...) -4. **Formatter** creates a report from all the archived content (HTML, PDF, ...) -5. **Database** knows what's been archived and also stores the archive result (spreadsheet, CSV, or just the console) - -To setup an auto-archiver instance create an `orchestration.yaml` which contains the workflow you would like. We advise you put this file into a `secrets/` folder and do not share it with others because it will contain passwords and other secrets. - -The structure of orchestration file is split into 2 parts: `steps` (what **steps** to use) and `configurations` (how those steps should behave), here's a simplification: -```yaml -# orchestration.yaml content -steps: - feeder: gsheet_feeder - archivers: # order matters - - youtubedl_archiver - enrichers: - - thumbnail_enricher - formatter: html_formatter - storages: - - local_storage - databases: - - gsheet_db - -configurations: - gsheet_feeder: - sheet: "your google sheet name" - header: 2 # row with header for your sheet - # ... configurations for the other steps here ... -``` - -To see all available `steps` (which archivers, storages, databases, ...) exist check the [example.orchestration.yaml](example.orchestration.yaml). - -All the `configurations` in the `orchestration.yaml` file (you can name it differently but need to pass it in the `--config FILENAME` argument) can be seen in the console by using the `--help` flag. They can also be overwritten, for example if you are using the `cli_feeder` to archive from the command line and want to provide the URLs you should do: - -```bash -auto-archiver --config secrets/orchestration.yaml --cli_feeder.urls="url1,url2,url3" -``` - -Here's the complete workflow that the auto-archiver goes through: -```{mermaid} -graph TD - s((start)) --> F(fa:fa-table Feeder) - F -->|get and clean URL| D1{fa:fa-database Database} - D1 -->|is already archived| e((end)) - D1 -->|not yet archived| a(fa:fa-download Archivers) - a -->|got media| E(fa:fa-chart-line Enrichers) - E --> S[fa:fa-box-archive Storages] - E --> Fo(fa:fa-code Formatter) - Fo --> S - Fo -->|update database| D2(fa:fa-database Database) - D2 --> e -``` - -## Orchestration checklist -Use this to make sure you help making sure you did all the required steps: -* [ ] you have a `/secrets` folder with all your configuration files including - * [ ] a orchestration file eg: `orchestration.yaml` pointing to the correct location of other files - * [ ] (optional if you use GoogleSheets) you have a `service_account.json` (see [how-to](https://gspread.readthedocs.io/en/latest/oauth2.html#for-bots-using-service-account)) - * [ ] (optional for telegram) a `anon.session` which appears after the 1st run where you login to telegram - * if you use private channels you need to add `channel_invites` and set `join_channels=true` at least once - * [ ] (optional for VK) a `vk_config.v2.json` - * [ ] (optional for using GoogleDrive storage) `gd-token.json` (see [help script](scripts/create_update_gdrive_oauth_token.py)) - * [ ] (optional for instagram) `instaloader.session` file which appears after the 1st run and login in instagram - * [ ] (optional for browsertrix) `profile.tar.gz` file - -#### Example invocations -The recommended way to run the auto-archiver is through Docker. The invocations below will run the auto-archiver Docker image using a configuration file that you have specified - -```bash -# all the configurations come from ./secrets/orchestration.yaml -docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver --config secrets/orchestration.yaml -# uses the same configurations but for another google docs sheet -# with a header on row 2 and with some different column names -# notice that columns is a dictionary so you need to pass it as JSON and it will override only the values provided -docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver --config secrets/orchestration.yaml --gsheet_feeder.sheet="use it on another sheets doc" --gsheet_feeder.header=2 --gsheet_feeder.columns='{"url": "link"}' -# all the configurations come from orchestration.yaml and specifies that s3 files should be private -docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver --config secrets/orchestration.yaml --s3_storage.private=1 -``` - -The auto-archiver can also be run locally, if pre-requisites are correctly configured. Equivalent invocations are below. - -```bash -# all the configurations come from ./secrets/orchestration.yaml -auto-archiver --config secrets/orchestration.yaml -# uses the same configurations but for another google docs sheet -# with a header on row 2 and with some different column names -# notice that columns is a dictionary so you need to pass it as JSON and it will override only the values provided -auto-archiver --config secrets/orchestration.yaml --gsheet_feeder.sheet="use it on another sheets doc" --gsheet_feeder.header=2 --gsheet_feeder.columns='{"url": "link"}' -# all the configurations come from orchestration.yaml and specifies that s3 files should be private -auto-archiver --config secrets/orchestration.yaml --s3_storage.private=1 -``` - -### Extra notes on configuration -#### Google Drive -To use Google Drive storage you need the id of the shared folder in the `config.yaml` file which must be shared with the service account eg `autoarchiverservice@auto-archiver-111111.iam.gserviceaccount.com` and then you can use `--storage=gd` - -#### Telethon + Instagram with telegram bot -The first time you run, you will be prompted to do a authentication with the phone number associated, alternatively you can put your `anon.session` in the root. - -#### Atlos -When integrating with [Atlos](https://atlos.org), you will need to provide an API token in your configuration. You can learn more about Atlos and how to get an API token [here](https://docs.atlos.org/technical/api). You will have to provide this token to the `atlos_feeder`, `atlos_storage`, and `atlos_db` steps in your orchestration file. If you use a custom or self-hosted Atlos instance, you can also specify the `atlos_url` option to point to your custom instance's URL. For example: - -```yaml -# orchestration.yaml content -steps: - feeder: atlos_feeder - archivers: # order matters - - youtubedl_archiver - enrichers: - - thumbnail_enricher - - hash_enricher - formatter: html_formatter - storages: - - atlos_storage - databases: - - console_db - - atlos_db - -configurations: - atlos_feeder: - atlos_url: "https://platform.atlos.org" # optional - api_token: "...your API token..." - atlos_db: - atlos_url: "https://platform.atlos.org" # optional - api_token: "...your API token..." - atlos_storage: - atlos_url: "https://platform.atlos.org" # optional - api_token: "...your API token..." - hash_enricher: - algorithm: "SHA-256" -``` - -## Running on Google Sheets Feeder (gsheet_feeder) -The `--gsheet_feeder.sheet` property is the name of the Google Sheet to check for URLs. -This sheet must have been shared with the Google Service account used by `gspread`. -This sheet must also have specific columns (case-insensitive) in the `header` as specified in [gsheet_feeder.__manifest__.py](src/auto_archiver/modules/gsheet_feeder/__manifest__.py). The default names of these columns and their purpose is: - -Inputs: - -* **Link** *(required)*: the URL of the post to archive -* **Destination folder**: custom folder for archived file (regardless of storage) - -Outputs: -* **Archive status** *(required)*: Status of archive operation -* **Archive location**: URL of archived post -* **Archive date**: Date archived -* **Thumbnail**: Embeds a thumbnail for the post in the spreadsheet -* **Timestamp**: Timestamp of original post -* **Title**: Post title -* **Text**: Post text -* **Screenshot**: Link to screenshot of post -* **Hash**: Hash of archived HTML file (which contains hashes of post media) - for checksums/verification -* **Perceptual Hash**: Perceptual hashes of found images - these can be used for de-duplication of content -* **WACZ**: Link to a WACZ web archive of post -* **ReplayWebpage**: Link to a ReplayWebpage viewer of the WACZ archive - -For example, this is a spreadsheet configured with all of the columns for the auto archiver and a few URLs to archive. (Note that the column names are not case sensitive.) - -![A screenshot of a Google Spreadsheet with column headers defined as above, and several Youtube and Twitter URLs in the "Link" column](docs/demo-before.png) - -Now the auto archiver can be invoked, with this command in this example: `docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver:dockerize --config secrets/orchestration-global.yaml --gsheet_feeder.sheet "Auto archive test 2023-2"`. Note that the sheet name has been overridden/specified in the command line invocation. - -When the auto archiver starts running, it updates the "Archive status" column. - -![A screenshot of a Google Spreadsheet with column headers defined as above, and several Youtube and Twitter URLs in the "Link" column. The auto archiver has added "archive in progress" to one of the status columns.](docs/demo-progress.png) - -The links are downloaded and archived, and the spreadsheet is updated to the following: - -![A screenshot of a Google Spreadsheet with videos archived and metadata added per the description of the columns above.](docs/demo-after.png) - -Note that the first row is skipped, as it is assumed to be a header row (`--gsheet_feeder.header=1` and you can change it if you use more rows above). Rows with an empty URL column, or a non-empty archive column are also skipped. All sheets in the document will be checked. - -The "archive location" link contains the path of the archived file, in local storage, S3, or in Google Drive. - -![The archive result for a link in the demo sheet.](docs/demo-archive.png) - ---- -## Development -Use `python -m src.auto_archiver --config secrets/orchestration.yaml` to run from the local development environment. - -### Testing - -Tests are split using `pytest.mark` into 'core' and 'download' tests. Download tests will hit the network and make API calls (e.g. Twitter, Bluesky etc.) and should be run regularly to make sure that APIs have not changed. - -Tests can be run as follows: -``` -# run core tests -pytest -ra -v -m "not download" # or poetry run pytest -ra -v -m "not download" -# run download tests -pytest -ra -v -m "download" # or poetry run pytest -ra -v -m "download" -# run all tests -pytest -ra -v # or poetry run pytest -ra -v -``` - -#### Docker development -working with docker locally: - * `docker compose up` to build the first time and run a local image with the settings in `secrets/orchestration.yaml` - * To modify/pass additional command line args, use `docker compose run auto-archiver --config secrets/orchestration.yaml [OTHER ARGUMENTS]` - * To rebuild after code changes, just pass the `--build` flag, e.g. `docker compose up --build` - - -manual release to docker hub - * `docker image tag auto-archiver bellingcat/auto-archiver:latest` - * `docker push bellingcat/auto-archiver` - - -### Building the Docs - -The documentation is built using [Sphinx](https://www.sphinx-doc.org/en/master/) and [AutoAPI](https://sphinx-autoapi.readthedocs.io/en/latest/) and hosted on ReadTheDocs. -To build the documentation locally, run the following commands: - -**Install required dependencies:** -- Install the docs group of dependencies: -```shell -# only the docs dependencies -poetry install --only docs - -# or for all dependencies -poetry install -``` -- Either use [poetry-plugin-shell](https://github.com/python-poetry/poetry-plugin-shell) to activate the virtual environment: `poetry shell` -- Or prepend the following commands with `poetry run` - -**Create the documentation:** -- Build the documentation: -``` -# Using makefile (Linux/macOS): -make -C docs html - -# or using sphinx directly (Windows/Linux/macOS): -sphinx-build -b html docs/source docs/_build/html -``` -- If you make significant changes and want a fresh build run: `make -C docs clean` to remove the old build files. - -**Viewing the documentation:** -```shell -# to open the documentation in your browser. -open docs/_build/html/index.html - -# or run autobuild to automatically update the documentation when you make changes -sphinx-autobuild docs/source docs/_build/html -``` - - - -#### RELEASE -* update version in [version.py](src/auto_archiver/version.py) -* go to github releases > new release > use `vx.y.z` for matching version notation - * package is automatically updated in pypi - * docker image is automatically pushed to dockerhup diff --git a/docs/_static/custom.css b/docs/_static/custom.css new file mode 100644 index 0000000..d77ab9d --- /dev/null +++ b/docs/_static/custom.css @@ -0,0 +1,4 @@ +.hidden_rtd { + display:none; +} + diff --git a/docs/_templates/autoapi/index.rst b/docs/_templates/autoapi/index.rst new file mode 100644 index 0000000..a6f712b --- /dev/null +++ b/docs/_templates/autoapi/index.rst @@ -0,0 +1,46 @@ +API Reference +============= + +These pages are intended for developers of the `auto-archiver` package, +and include documentation on the core classes and functions used by +the auto-archiver + + +Core Classes +------------ + + +.. toctree:: + :titlesonly: + + {% for page in pages|selectattr("is_top_level_object") %} + {% if page.name == 'core' %} + {{ page.include_path }} + {% endif %} + {% endfor %} + +Util Functions +-------------- + +.. toctree:: + :titlesonly: + + {% for page in pages|selectattr("is_top_level_object") %} + {% if page.name == 'utils' %} + {{ page.include_path }} + {% endif %} + {% endfor %} + + +Core Modules +------------ + +.. toctree:: + :titlesonly: + + {% for page in pages|selectattr("is_top_level_object") %} + {% if page.name != 'core' and page.name != 'utils' %} + {{ page.include_path }} + {% endif %} + {% endfor %} + diff --git a/docs/_templates/autoapi/python/attribute.rst b/docs/_templates/autoapi/python/attribute.rst new file mode 100644 index 0000000..ebaba55 --- /dev/null +++ b/docs/_templates/autoapi/python/attribute.rst @@ -0,0 +1 @@ +{% extends "python/data.rst" %} diff --git a/docs/_templates/autoapi/python/class.rst b/docs/_templates/autoapi/python/class.rst new file mode 100644 index 0000000..379f83a --- /dev/null +++ b/docs/_templates/autoapi/python/class.rst @@ -0,0 +1,104 @@ +{% if obj.display %} + {% if is_own_page %} +{{ obj.id }} +{{ "=" * obj.id | length }} + + {% endif %} + {% set visible_children = obj.children|selectattr("display")|list %} + {% set own_page_children = visible_children|selectattr("type", "in", own_page_types)|list %} + {% if is_own_page and own_page_children %} +.. toctree:: + :hidden: + + {% for child in own_page_children %} + {{ child.include_path }} + {% endfor %} + + {% endif %} +.. py:{{ obj.type }}:: {% if is_own_page %}{{ obj.id }}{% else %}{{ obj.short_name }}{% endif %}{% if obj.args %}({{ obj.args }}){% endif %} + + {% for (args, return_annotation) in obj.overloads %} + {{ " " * (obj.type | length) }} {{ obj.short_name }}{% if args %}({{ args }}){% endif %} + + {% endfor %} + {% if obj.bases %} + {% if "show-inheritance" in autoapi_options %} + + Bases: {% for base in obj.bases %}{{ base|link_objs }}{% if not loop.last %}, {% endif %}{% endfor %} + {% endif %} + + + {% if "show-inheritance-diagram" in autoapi_options and obj.bases != ["object"] %} + .. autoapi-inheritance-diagram:: {{ obj.obj["full_name"] }} + :parts: 1 + {% if "private-members" in autoapi_options %} + :private-bases: + {% endif %} + + {% endif %} + {% endif %} + {% if obj.docstring %} + + {{ obj.docstring|indent(3) }} + {% endif %} + {% for obj_item in visible_children %} + {% if obj_item.type not in own_page_types %} + + {{ obj_item.render()|indent(3) }} + {% endif %} + {% endfor %} + {% if is_own_page and own_page_children %} + {% set visible_attributes = own_page_children|selectattr("type", "equalto", "attribute")|list %} + {% if visible_attributes %} +Attributes +---------- + +.. autoapisummary:: + + {% for attribute in visible_attributes %} + {{ attribute.id }} + {% endfor %} + + + {% endif %} + {% set visible_exceptions = own_page_children|selectattr("type", "equalto", "exception")|list %} + {% if visible_exceptions %} +Exceptions +---------- + +.. autoapisummary:: + + {% for exception in visible_exceptions %} + {{ exception.id }} + {% endfor %} + + + {% endif %} + {% set visible_classes = own_page_children|selectattr("type", "equalto", "class")|list %} + {% if visible_classes %} +Classes +------- + +.. autoapisummary:: + + {% for klass in visible_classes %} + {{ klass.id }} + {% endfor %} + + + {% endif %} + {% set visible_methods = own_page_children|selectattr("type", "equalto", "method")|list %} + {% if visible_methods %} +Methods +------- + +.. autoapisummary:: + + {% for method in visible_methods %} + {{ method.id }} + {% endfor %} + + + {% endif %} + {% endif %} +{% endif %} diff --git a/docs/_templates/autoapi/python/data.rst b/docs/_templates/autoapi/python/data.rst new file mode 100644 index 0000000..1a50250 --- /dev/null +++ b/docs/_templates/autoapi/python/data.rst @@ -0,0 +1,38 @@ +{% if obj.display %} + {% if is_own_page %} +{{ obj.id }} +{{ "=" * obj.id | length }} + + {% endif %} +.. py:{{ obj.type }}:: {% if is_own_page %}{{ obj.id }}{% else %}{{ obj.name }}{% endif %} + {% if obj.annotation is not none %} + + :type: {% if obj.annotation %} {{ obj.annotation }}{% endif %} + {% endif %} + {% if obj.value is not none %} + + {% if obj.value.splitlines()|count > 1 %} + :value: Multiline-String + + .. raw:: html + +
Show Value + + .. code-block:: python + + {{ obj.value|indent(width=6,blank=true) }} + + .. raw:: html + +
+ + {% else %} + :value: {{ obj.value|truncate(100) }} + {% endif %} + {% endif %} + + {% if obj.docstring %} + + {{ obj.docstring|indent(3) }} + {% endif %} +{% endif %} diff --git a/docs/_templates/autoapi/python/exception.rst b/docs/_templates/autoapi/python/exception.rst new file mode 100644 index 0000000..92f3d38 --- /dev/null +++ b/docs/_templates/autoapi/python/exception.rst @@ -0,0 +1 @@ +{% extends "python/class.rst" %} diff --git a/docs/_templates/autoapi/python/function.rst b/docs/_templates/autoapi/python/function.rst new file mode 100644 index 0000000..5dee5aa --- /dev/null +++ b/docs/_templates/autoapi/python/function.rst @@ -0,0 +1,21 @@ +{% if obj.display %} + {% if is_own_page %} +{{ obj.id }} +{{ "=" * obj.id | length }} + + {% endif %} +.. py:function:: {% if is_own_page %}{{ obj.id }}{% else %}{{ obj.short_name }}{% endif %}({{ obj.args }}){% if obj.return_annotation is not none %} -> {{ obj.return_annotation }}{% endif %} + {% for (args, return_annotation) in obj.overloads %} + + {%+ if is_own_page %}{{ obj.id }}{% else %}{{ obj.short_name }}{% endif %}({{ args }}){% if return_annotation is not none %} -> {{ return_annotation }}{% endif %} + {% endfor %} + {% for property in obj.properties %} + + :{{ property }}: + {% endfor %} + + {% if obj.docstring %} + + {{ obj.docstring|indent(3) }} + {% endif %} +{% endif %} diff --git a/docs/_templates/autoapi/python/method.rst b/docs/_templates/autoapi/python/method.rst new file mode 100644 index 0000000..12d42de --- /dev/null +++ b/docs/_templates/autoapi/python/method.rst @@ -0,0 +1,21 @@ +{% if obj.display %} + {% if is_own_page %} +{{ obj.id }} +{{ "=" * obj.id | length }} + + {% endif %} +.. py:method:: {% if is_own_page %}{{ obj.id }}{% else %}{{ obj.short_name }}{% endif %}({{ obj.args }}){% if obj.return_annotation is not none %} -> {{ obj.return_annotation }}{% endif %} + {% for (args, return_annotation) in obj.overloads %} + + {%+ if is_own_page %}{{ obj.id }}{% else %}{{ obj.short_name }}{% endif %}({{ args }}){% if return_annotation is not none %} -> {{ return_annotation }}{% endif %} + {% endfor %} + {% for property in obj.properties %} + + :{{ property }}: + {% endfor %} + + {% if obj.docstring %} + + {{ obj.docstring|indent(3) }} + {% endif %} +{% endif %} diff --git a/docs/_templates/autoapi/python/module.rst b/docs/_templates/autoapi/python/module.rst new file mode 100644 index 0000000..53cc65d --- /dev/null +++ b/docs/_templates/autoapi/python/module.rst @@ -0,0 +1,156 @@ +{% if obj.display %} + {% if is_own_page %} +{{ obj.id }} +{{ "=" * obj.id|length }} + +.. py:module:: {{ obj.name }} + + {% if obj.docstring %} +.. autoapi-nested-parse:: + + {{ obj.docstring|indent(3) }} + + {% endif %} + + {% block submodules %} + {% set visible_subpackages = obj.subpackages|selectattr("display")|list %} + {% set visible_submodules = obj.submodules|selectattr("display")|list %} + {% set visible_submodules = (visible_subpackages + visible_submodules)|sort %} + {% if visible_submodules %} +Submodules +---------- + +.. toctree:: + :maxdepth: 1 + + {% for submodule in visible_submodules %} + {{ submodule.include_path }} + {% endfor %} + + + {% endif %} + {% endblock %} + {% block content %} + {% set visible_children = obj.children|selectattr("display")|list %} + {% if visible_children %} + {% set visible_attributes = visible_children|selectattr("type", "equalto", "data")|list %} + {% if visible_attributes %} + {% if "attribute" in own_page_types or "show-module-summary" in autoapi_options %} +Attributes +---------- + + {% if "attribute" in own_page_types %} +.. toctree:: + :hidden: + + {% for attribute in visible_attributes %} + {{ attribute.include_path }} + {% endfor %} + + {% endif %} +.. autoapisummary:: + + {% for attribute in visible_attributes %} + {{ attribute.id }} + {% endfor %} + {% endif %} + + + {% endif %} + {% set visible_exceptions = visible_children|selectattr("type", "equalto", "exception")|list %} + {% if visible_exceptions %} + {% if "exception" in own_page_types or "show-module-summary" in autoapi_options %} +Exceptions +---------- + + {% if "exception" in own_page_types %} +.. toctree:: + :hidden: + + {% for exception in visible_exceptions %} + {{ exception.include_path }} + {% endfor %} + + {% endif %} +.. autoapisummary:: + + {% for exception in visible_exceptions %} + {{ exception.id }} + {% endfor %} + {% endif %} + + + {% endif %} + {% set visible_classes = visible_children|selectattr("type", "equalto", "class")|list %} + {% if visible_classes %} + {% if "class" in own_page_types or "show-module-summary" in autoapi_options %} +Classes +------- + + {% if "class" in own_page_types %} +.. toctree:: + :hidden: + + {% for klass in visible_classes %} + {{ klass.include_path }} + {% endfor %} + + {% endif %} +.. autoapisummary:: + + {% for klass in visible_classes %} + {{ klass.id }} + {% endfor %} + {% endif %} + + + {% endif %} + {% set visible_functions = visible_children|selectattr("type", "equalto", "function")|list %} + {% if visible_functions %} + {% if "function" in own_page_types or "show-module-summary" in autoapi_options %} +Functions +--------- + + {% if "function" in own_page_types %} +.. toctree:: + :hidden: + + {% for function in visible_functions %} + {{ function.include_path }} + {% endfor %} + + {% endif %} +.. autoapisummary:: + + {% for function in visible_functions %} + {{ function.id }} + {% endfor %} + {% endif %} + + + {% endif %} + {% set this_page_children = visible_children|rejectattr("type", "in", own_page_types)|list %} + {% if this_page_children %} +{{ obj.type|title }} Contents +{{ "-" * obj.type|length }}--------- + + {% for obj_item in this_page_children %} +{{ obj_item.render()|indent(0) }} + {% endfor %} + {% endif %} + {% endif %} + {% endblock %} + {% else %} +.. py:module:: {{ obj.name }} + + {% if obj.docstring %} + .. autoapi-nested-parse:: + + {{ obj.docstring|indent(6) }} + + {% endif %} + {% for obj_item in visible_children %} + {{ obj_item.render()|indent(3) }} + {% endfor %} + {% endif %} +{% endif %} diff --git a/docs/_templates/autoapi/python/package.rst b/docs/_templates/autoapi/python/package.rst new file mode 100644 index 0000000..fb9a649 --- /dev/null +++ b/docs/_templates/autoapi/python/package.rst @@ -0,0 +1 @@ +{% extends "python/module.rst" %} diff --git a/docs/_templates/autoapi/python/property.rst b/docs/_templates/autoapi/python/property.rst new file mode 100644 index 0000000..9165311 --- /dev/null +++ b/docs/_templates/autoapi/python/property.rst @@ -0,0 +1,21 @@ +{% if obj.display %} + {% if is_own_page %} +{{ obj.id }} +{{ "=" * obj.id | length }} + + {% endif %} +.. py:property:: {% if is_own_page %}{{ obj.id}}{% else %}{{ obj.short_name }}{% endif %} + {% if obj.annotation %} + + :type: {{ obj.annotation }} + {% endif %} + {% for property in obj.properties %} + + :{{ property }}: + {% endfor %} + + {% if obj.docstring %} + + {{ obj.docstring|indent(3) }} + {% endif %} +{% endif %} diff --git a/docs/scripts/__init__.py b/docs/scripts/__init__.py new file mode 100644 index 0000000..ba9737c --- /dev/null +++ b/docs/scripts/__init__.py @@ -0,0 +1 @@ +from scripts import generate_module_docs \ No newline at end of file diff --git a/docs/scripts/scripts.py b/docs/scripts/scripts.py new file mode 100644 index 0000000..66ba14d --- /dev/null +++ b/docs/scripts/scripts.py @@ -0,0 +1,147 @@ +# iterate through all the modules in auto_archiver.modules and turn the __manifest__.py file into a markdown table +from pathlib import Path +from auto_archiver.core.module import ModuleFactory +from auto_archiver.core.base_module import BaseModule +from ruamel.yaml import YAML +from ruamel.yaml.comments import CommentedMap +import io + +MODULES_FOLDER = Path(__file__).parent.parent.parent.parent / "src" / "auto_archiver" / "modules" +SAVE_FOLDER = Path(__file__).parent.parent / "source" / "modules" / "autogen" + +type_color = { + 'feeder': "[feeder](/core_modules.md#feeder-modules)", + 'extractor': "[extractor](/core_modules.md#extractor-modules)", + 'enricher': "[enricher](/core_modules.md#enricher-modules)", + 'database': "[database](/core_modules.md#database-modules)", + 'storage': "[storage](/core_modules.md#storage-modules)", + 'formatter': "[formatter](/core_modules.md#formatter-modules)", +} + +TABLE_HEADER = ("Option", "Description", "Default", "Type") + +EXAMPLE_YAML = """ +# steps configuration +steps: +... +{steps_str} +... + +# module configuration +... + +{config_string} + +""" + +def generate_module_docs(): + yaml = YAML() + SAVE_FOLDER.mkdir(exist_ok=True) + modules_by_type = {} + + header_row = "| " + " | ".join(TABLE_HEADER) + "|\n" + "| --- " * len(TABLE_HEADER) + "|\n" + global_table = "\n## Configuration Options\n" + header_row + + global_yaml = yaml.load("""\n# Module configuration\nplaceholder: {}""") + + for module in sorted(ModuleFactory().available_modules(), key=lambda x: (x.requires_setup, x.name)): + # generate the markdown file from the __manifest__.py file. + + manifest = module.manifest + for type in manifest['type']: + modules_by_type.setdefault(type, []).append(module) + + description = "\n".join(l.lstrip() for l in manifest['description'].split("\n")) + types = ", ".join(type_color[t] for t in manifest['type']) + readme_str = f""" +# {manifest['name']} +```{{admonition}} Module type + +{types} +``` +{description} +""" + steps_str = "\n".join(f" {t}s:\n - {module.name}" for t in manifest['type']) + + if not manifest['configs']: + config_string = f"# No configuration options for {module.name}.*\n" + else: + + config_table = header_row + config_yaml = {} + + global_yaml[module.name] = CommentedMap() + global_yaml.yaml_set_comment_before_after_key(module.name, f"\n\n{module.display_name} configuration options") + + + for key, value in manifest['configs'].items(): + type = value.get('type', 'string') + if type == 'json_loader': + value['type'] = 'json' + elif type == 'str': + type = "string" + + default = value.get('default', '') + config_yaml[key] = default + + global_yaml[module.name][key] = default + + if value.get('help', ''): + global_yaml[module.name].yaml_add_eol_comment(value.get('help', ''), key) + + help = "**Required**. " if value.get('required', False) else "Optional. " + help += value.get('help', '') + config_table += f"| `{module.name}.{key}` | {help} | {value.get('default', '')} | {type} |\n" + global_table += f"| `{module.name}.{key}` | {help} | {default} | {type} |\n" + readme_str += "\n## Configuration Options\n" + readme_str += "\n### YAML\n" + + config_string = io.BytesIO() + yaml.dump({module.name: config_yaml}, config_string) + config_string = config_string.getvalue().decode('utf-8') + yaml_string = EXAMPLE_YAML.format(steps_str=steps_str, config_string=config_string) + readme_str += f"```{{code}} yaml\n{yaml_string}\n```\n" + + if manifest['configs']: + readme_str += "\n### Command Line:\n" + readme_str += config_table + + # add a link to the autodoc refs + readme_str += f"\n[API Reference](../../../autoapi/{module.name}/index)\n" + # create the module.type folder, use the first type just for where to store the file + for type in manifest['type']: + type_folder = SAVE_FOLDER / type + type_folder.mkdir(exist_ok=True) + with open(type_folder / f"{module.name}.md", "w") as f: + print("writing", SAVE_FOLDER) + f.write(readme_str) + generate_index(modules_by_type) + + del global_yaml['placeholder'] + global_string = io.BytesIO() + global_yaml = yaml.dump(global_yaml, global_string) + global_string = global_string.getvalue().decode('utf-8') + global_yaml = f"```yaml\n{global_string}\n```" + with open(SAVE_FOLDER / "configs_cheatsheet.md", "w") as f: + f.write("### Configuration File\n" + global_yaml + "\n### Command Line\n" + global_table) + + +def generate_index(modules_by_type): + readme_str = "" + for type in BaseModule.MODULE_TYPES: + modules = modules_by_type.get(type, []) + module_str = f"## {type.capitalize()} Modules\n" + for module in modules: + module_str += f"\n[{module.manifest['name']}](/modules/autogen/{module.type[0]}/{module.name}.md)\n" + with open(SAVE_FOLDER / f"{type}.md", "w") as f: + print("writing", SAVE_FOLDER / f"{type}.md") + f.write(module_str) + readme_str += module_str + + with open(SAVE_FOLDER / "module_list.md", "w") as f: + print("writing", SAVE_FOLDER / "module_list.md") + f.write(readme_str) + + +if __name__ == "__main__": + generate_module_docs() \ No newline at end of file diff --git a/docs/source/_auto/configs.rst b/docs/source/_auto/configs.rst deleted file mode 100644 index f6e81f0..0000000 --- a/docs/source/_auto/configs.rst +++ /dev/null @@ -1,742 +0,0 @@ - -Configs -------- - -This section documents all configuration options available for various components. - -InstagramAPIArchiver --------------------- - -The following table lists all configuration options for this component: - -.. list-table:: Configuration Options - :header-rows: 1 - :widths: 25 20 55 - - * - **Key** - - **Default** - - **Description** - * - access_token - - None - - a valid instagrapi-api token - * - api_endpoint - - None - - API endpoint to use - * - full_profile - - False - - if true, will download all posts, tagged posts, stories, and highlights for a profile, if false, will only download the profile pic and information. - * - full_profile_max_posts - - 0 - - Use to limit the number of posts to download when full_profile is true. 0 means no limit. limit is applied softly since posts are fetched in batch, once to: posts, tagged posts, and highlights - * - minimize_json_output - - True - - if true, will remove empty values from the json output - -InstagramArchiver ------------------ - -The following table lists all configuration options for this component: - -.. list-table:: Configuration Options - :header-rows: 1 - :widths: 25 20 55 - - * - **Key** - - **Default** - - **Description** - * - username - - None - - a valid Instagram username - * - password - - None - - the corresponding Instagram account password - * - download_folder - - instaloader - - name of a folder to temporarily download content to - * - session_file - - secrets/instaloader.session - - path to the instagram session which saves session credentials - -InstagramTbotArchiver ---------------------- - -The following table lists all configuration options for this component: - -.. list-table:: Configuration Options - :header-rows: 1 - :widths: 25 20 55 - - * - **Key** - - **Default** - - **Description** - * - api_id - - None - - telegram API_ID value, go to https://my.telegram.org/apps - * - api_hash - - None - - telegram API_HASH value, go to https://my.telegram.org/apps - * - session_file - - secrets/anon-insta - - optional, records the telegram login session for future usage, '.session' will be appended to the provided value. - * - timeout - - 45 - - timeout to fetch the instagram content in seconds. - -TelethonArchiver ----------------- - -The following table lists all configuration options for this component: - -.. list-table:: Configuration Options - :header-rows: 1 - :widths: 25 20 55 - - * - **Key** - - **Default** - - **Description** - * - api_id - - None - - telegram API_ID value, go to https://my.telegram.org/apps - * - api_hash - - None - - telegram API_HASH value, go to https://my.telegram.org/apps - * - bot_token - - None - - optional, but allows access to more content such as large videos, talk to @botfather - * - session_file - - secrets/anon - - optional, records the telegram login session for future usage, '.session' will be appended to the provided value. - * - join_channels - - True - - disables the initial setup with channel_invites config, useful if you have a lot and get stuck - * - channel_invites - - {} - - (JSON string) private channel invite links (format: t.me/joinchat/HASH OR t.me/+HASH) and (optional but important to avoid hanging for minutes on startup) channel id (format: CHANNEL_ID taken from a post url like https://t.me/c/CHANNEL_ID/1), the telegram account will join any new channels on setup - -TwitterApiArchiver ------------------- - -The following table lists all configuration options for this component: - -.. list-table:: Configuration Options - :header-rows: 1 - :widths: 25 20 55 - - * - **Key** - - **Default** - - **Description** - * - bearer_token - - None - - [deprecated: see bearer_tokens] twitter API bearer_token which is enough for archiving, if not provided you will need consumer_key, consumer_secret, access_token, access_secret - * - bearer_tokens - - [] - - a list of twitter API bearer_token which is enough for archiving, if not provided you will need consumer_key, consumer_secret, access_token, access_secret, if provided you can still add those for better rate limits. CSV of bearer tokens if provided via the command line - * - consumer_key - - None - - twitter API consumer_key - * - consumer_secret - - None - - twitter API consumer_secret - * - access_token - - None - - twitter API access_token - * - access_secret - - None - - twitter API access_secret - -VkArchiver ----------- - -The following table lists all configuration options for this component: - -.. list-table:: Configuration Options - :header-rows: 1 - :widths: 25 20 55 - - * - **Key** - - **Default** - - **Description** - * - username - - None - - valid VKontakte username - * - password - - None - - valid VKontakte password - * - session_file - - secrets/vk_config.v2.json - - valid VKontakte password - -YoutubeDLArchiver ------------------ - -The following table lists all configuration options for this component: - -.. list-table:: Configuration Options - :header-rows: 1 - :widths: 25 20 55 - - * - **Key** - - **Default** - - **Description** - * - facebook_cookie - - None - - optional facebook cookie to have more access to content, from browser, looks like 'cookie: datr= xxxx' - * - subtitles - - True - - download subtitles if available - * - comments - - False - - download all comments if available, may lead to large metadata - * - livestreams - - False - - if set, will download live streams, otherwise will skip them; see --max-filesize for more control - * - live_from_start - - False - - if set, will download live streams from their earliest available moment, otherwise starts now. - * - proxy - - - - http/socks (https seems to not work atm) proxy to use for the webdriver, eg https://proxy- user:password@proxy-ip:port - * - end_means_success - - True - - if True, any archived content will mean a 'success', if False this archiver will not return a 'success' stage; this is useful for cases when the yt-dlp will archive a video but ignore other types of content like images or text only pages that the subsequent archivers can retrieve. - * - allow_playlist - - False - - If True will also download playlists, set to False if the expectation is to download a single video. - * - max_downloads - - inf - - Use to limit the number of videos to download when a channel or long page is being extracted. 'inf' means no limit. - * - cookies_from_browser - - None - - optional browser for ytdl to extract cookies from, can be one of: brave, chrome, chromium, edge, firefox, opera, safari, vivaldi, whale - * - cookie_file - - None - - optional cookie file to use for Youtube, see instructions here on how to export from your browser: https://github.com/yt-dlp/yt- dlp/wiki/FAQ#how-do-i-pass-cookies-to-yt-dlp - -AAApiDb -------- - -The following table lists all configuration options for this component: - -.. list-table:: Configuration Options - :header-rows: 1 - :widths: 25 20 55 - - * - **Key** - - **Default** - - **Description** - * - api_endpoint - - None - - API endpoint where calls are made to - * - api_token - - None - - API Bearer token. - * - public - - False - - whether the URL should be publicly available via the API - * - author_id - - None - - which email to assign as author - * - group_id - - None - - which group of users have access to the archive in case public=false as author - * - allow_rearchive - - True - - if False then the API database will be queried prior to any archiving operations and stop if the link has already been archived - * - store_results - - True - - when set, will send the results to the API database. - * - tags - - [] - - what tags to add to the archived URL - -AtlosDb -------- - -The following table lists all configuration options for this component: - -.. list-table:: Configuration Options - :header-rows: 1 - :widths: 25 20 55 - - * - **Key** - - **Default** - - **Description** - * - api_token - - None - - An Atlos API token. For more information, see https://docs.atlos.org/technical/api/ - * - atlos_url - - https://platform.atlos.org - - The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash. - -CSVDb ------ - -The following table lists all configuration options for this component: - -.. list-table:: Configuration Options - :header-rows: 1 - :widths: 25 20 55 - - * - **Key** - - **Default** - - **Description** - * - csv_file - - db.csv - - CSV file name - -HashEnricher ------------- - -The following table lists all configuration options for this component: - -.. list-table:: Configuration Options - :header-rows: 1 - :widths: 25 20 55 - - * - **Key** - - **Default** - - **Description** - * - algorithm - - SHA-256 - - hash algorithm to use - * - chunksize - - 16000000 - - number of bytes to use when reading files in chunks (if this value is too large you will run out of RAM), default is 16MB - -ScreenshotEnricher ------------------- - -The following table lists all configuration options for this component: - -.. list-table:: Configuration Options - :header-rows: 1 - :widths: 25 20 55 - - * - **Key** - - **Default** - - **Description** - * - width - - 1280 - - width of the screenshots - * - height - - 720 - - height of the screenshots - * - timeout - - 60 - - timeout for taking the screenshot - * - sleep_before_screenshot - - 4 - - seconds to wait for the pages to load before taking screenshot - * - http_proxy - - - - http proxy to use for the webdriver, eg http://proxy-user:password@proxy-ip:port - * - save_to_pdf - - False - - save the page as pdf along with the screenshot. PDF saving options can be adjusted with the 'print_options' parameter - * - print_options - - {} - - options to pass to the pdf printer - -SSLEnricher ------------ - -The following table lists all configuration options for this component: - -.. list-table:: Configuration Options - :header-rows: 1 - :widths: 25 20 55 - - * - **Key** - - **Default** - - **Description** - * - skip_when_nothing_archived - - True - - if true, will skip enriching when no media is archived - -ThumbnailEnricher ------------------ - -The following table lists all configuration options for this component: - -.. list-table:: Configuration Options - :header-rows: 1 - :widths: 25 20 55 - - * - **Key** - - **Default** - - **Description** - * - thumbnails_per_minute - - 60 - - how many thumbnails to generate per minute of video, can be limited by max_thumbnails - * - max_thumbnails - - 16 - - limit the number of thumbnails to generate per video, 0 means no limit - -TimestampingEnricher --------------------- - -The following table lists all configuration options for this component: - -.. list-table:: Configuration Options - :header-rows: 1 - :widths: 25 20 55 - - * - **Key** - - **Default** - - **Description** - * - tsa_urls - - ['http://timestamp.digicert.com', 'http://timestamp.identrust.com', 'http://timestamp.globalsign.com/tsa/r6advanced1', 'http://tss.accv.es:8318/tsa'] - - List of RFC3161 Time Stamp Authorities to use, separate with commas if passed via the command line. - -WaczArchiverEnricher --------------------- - -The following table lists all configuration options for this component: - -.. list-table:: Configuration Options - :header-rows: 1 - :widths: 25 20 55 - - * - **Key** - - **Default** - - **Description** - * - profile - - None - - browsertrix-profile (for profile generation see https://github.com/webrecorder/browsertrix- crawler#creating-and-using-browser-profiles). - * - docker_commands - - None - - if a custom docker invocation is needed - * - timeout - - 120 - - timeout for WACZ generation in seconds - * - extract_media - - False - - If enabled all the images/videos/audio present in the WACZ archive will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched. - * - extract_screenshot - - True - - If enabled the screenshot captured by browsertrix will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched. - * - socks_proxy_host - - None - - SOCKS proxy host for browsertrix-crawler, use in combination with socks_proxy_port. eg: user:password@host - * - socks_proxy_port - - None - - SOCKS proxy port for browsertrix-crawler, use in combination with socks_proxy_host. eg 1234 - * - proxy_server - - None - - SOCKS server proxy URL, in development - -WaybackArchiverEnricher ------------------------ - -The following table lists all configuration options for this component: - -.. list-table:: Configuration Options - :header-rows: 1 - :widths: 25 20 55 - - * - **Key** - - **Default** - - **Description** - * - timeout - - 15 - - seconds to wait for successful archive confirmation from wayback, if more than this passes the result contains the job_id so the status can later be checked manually. - * - if_not_archived_within - - None - - only tell wayback to archive if no archive is available before the number of seconds specified, use None to ignore this option. For more information: https://docs.google.com/document/d/1N sv52MvSjbLb2PCpHlat0gkzw0EvtSgpKHu4mk0MnrA - * - key - - None - - wayback API key. to get credentials visit https://archive.org/account/s3.php - * - secret - - None - - wayback API secret. to get credentials visit https://archive.org/account/s3.php - * - proxy_http - - None - - http proxy to use for wayback requests, eg http://proxy-user:password@proxy-ip:port - * - proxy_https - - None - - https proxy to use for wayback requests, eg https://proxy-user:password@proxy-ip:port - -WhisperEnricher ---------------- - -The following table lists all configuration options for this component: - -.. list-table:: Configuration Options - :header-rows: 1 - :widths: 25 20 55 - - * - **Key** - - **Default** - - **Description** - * - api_endpoint - - None - - WhisperApi api endpoint, eg: https://whisperbox- api.com/api/v1, a deployment of https://github.com/bellingcat/whisperbox- transcribe. - * - api_key - - None - - WhisperApi api key for authentication - * - include_srt - - False - - Whether to include a subtitle SRT (SubRip Subtitle file) for the video (can be used in video players). - * - timeout - - 90 - - How many seconds to wait at most for a successful job completion. - * - action - - translate - - which Whisper operation to execute - -AtlosFeeder ------------ - -The following table lists all configuration options for this component: - -.. list-table:: Configuration Options - :header-rows: 1 - :widths: 25 20 55 - - * - **Key** - - **Default** - - **Description** - * - api_token - - None - - An Atlos API token. For more information, see https://docs.atlos.org/technical/api/ - * - atlos_url - - https://platform.atlos.org - - The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash. - -CLIFeeder ---------- - -The following table lists all configuration options for this component: - -.. list-table:: Configuration Options - :header-rows: 1 - :widths: 25 20 55 - - * - **Key** - - **Default** - - **Description** - * - urls - - None - - URL(s) to archive, either a single URL or a list of urls, should not come from config.yaml - -GsheetsFeeder -------------- - -The following table lists all configuration options for this component: - -.. list-table:: Configuration Options - :header-rows: 1 - :widths: 25 20 55 - - * - **Key** - - **Default** - - **Description** - * - sheet - - None - - name of the sheet to archive - * - sheet_id - - None - - (alternative to sheet name) the id of the sheet to archive - * - header - - 1 - - index of the header row (starts at 1) - * - service_account - - secrets/service_account.json - - service account JSON file path - * - columns - - {'url': 'link', 'status': 'archive status', 'folder': 'destination folder', 'archive': 'archive location', 'date': 'archive date', 'thumbnail': 'thumbnail', 'timestamp': 'upload timestamp', 'title': 'upload title', 'text': 'text content', 'screenshot': 'screenshot', 'hash': 'hash', 'pdq_hash': 'perceptual hashes', 'wacz': 'wacz', 'replaywebpage': 'replaywebpage'} - - names of columns in the google sheet (stringified JSON object) - * - allow_worksheets - - set() - - (CSV) only worksheets whose name is included in allow are included (overrides worksheet_block), leave empty so all are allowed - * - block_worksheets - - set() - - (CSV) explicitly block some worksheets from being processed - * - use_sheet_names_in_stored_paths - - True - - if True the stored files path will include 'workbook_name/worksheet_name/...' - -HtmlFormatter -------------- - -The following table lists all configuration options for this component: - -.. list-table:: Configuration Options - :header-rows: 1 - :widths: 25 20 55 - - * - **Key** - - **Default** - - **Description** - * - detect_thumbnails - - True - - if true will group by thumbnails generated by thumbnail enricher by id 'thumbnail_00' - -AtlosStorage ------------- - -The following table lists all configuration options for this component: - -.. list-table:: Configuration Options - :header-rows: 1 - :widths: 25 20 55 - - * - **Key** - - **Default** - - **Description** - * - path_generator - - url - - how to store the file in terms of directory structure: 'flat' sets to root; 'url' creates a directory based on the provided URL; 'random' creates a random directory. - * - filename_generator - - random - - how to name stored files: 'random' creates a random string; 'static' uses a replicable strategy such as a hash. - * - api_token - - None - - An Atlos API token. For more information, see https://docs.atlos.org/technical/api/ - * - atlos_url - - https://platform.atlos.org - - The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash. - -GDriveStorage -------------- - -The following table lists all configuration options for this component: - -.. list-table:: Configuration Options - :header-rows: 1 - :widths: 25 20 55 - - * - **Key** - - **Default** - - **Description** - * - path_generator - - url - - how to store the file in terms of directory structure: 'flat' sets to root; 'url' creates a directory based on the provided URL; 'random' creates a random directory. - * - filename_generator - - random - - how to name stored files: 'random' creates a random string; 'static' uses a replicable strategy such as a hash. - * - root_folder_id - - None - - root google drive folder ID to use as storage, found in URL: 'https://drive.google.com/drive/folders/FOLDER_ID' - * - oauth_token - - None - - JSON filename with Google Drive OAuth token: check auto-archiver repository scripts folder for create_update_gdrive_oauth_token.py. NOTE: storage used will count towards owner of GDrive folder, therefore it is best to use oauth_token_filename over service_account. - * - service_account - - secrets/service_account.json - - service account JSON file path, same as used for Google Sheets. NOTE: storage used will count towards the developer account. - -LocalStorage ------------- - -The following table lists all configuration options for this component: - -.. list-table:: Configuration Options - :header-rows: 1 - :widths: 25 20 55 - - * - **Key** - - **Default** - - **Description** - * - path_generator - - url - - how to store the file in terms of directory structure: 'flat' sets to root; 'url' creates a directory based on the provided URL; 'random' creates a random directory. - * - filename_generator - - random - - how to name stored files: 'random' creates a random string; 'static' uses a replicable strategy such as a hash. - * - save_to - - ./archived - - folder where to save archived content - * - save_absolute - - False - - whether the path to the stored file is absolute or relative in the output result inc. formatters (WARN: leaks the file structure) - -S3Storage ---------- - -The following table lists all configuration options for this component: - -.. list-table:: Configuration Options - :header-rows: 1 - :widths: 25 20 55 - - * - **Key** - - **Default** - - **Description** - * - path_generator - - url - - how to store the file in terms of directory structure: 'flat' sets to root; 'url' creates a directory based on the provided URL; 'random' creates a random directory. - * - filename_generator - - random - - how to name stored files: 'random' creates a random string; 'static' uses a replicable strategy such as a hash. - * - bucket - - None - - S3 bucket name - * - region - - None - - S3 region name - * - key - - None - - S3 API key - * - secret - - None - - S3 API secret - * - random_no_duplicate - - False - - if set, it will override `path_generator`, `filename_generator` and `folder`. It will check if the file already exists and if so it will not upload it again. Creates a new root folder path `no-dups/` - * - endpoint_url - - https://{region}.digitaloceanspaces.com - - S3 bucket endpoint, {region} are inserted at runtime - * - cdn_url - - https://{bucket}.{region}.cdn.digitaloceanspaces.com/{key} - - S3 CDN url, {bucket}, {region} and {key} are inserted at runtime - * - private - - False - - if true S3 files will not be readable online - -Storage -------- - -The following table lists all configuration options for this component: - -.. list-table:: Configuration Options - :header-rows: 1 - :widths: 25 20 55 - - * - **Key** - - **Default** - - **Description** - * - path_generator - - url - - how to store the file in terms of directory structure: 'flat' sets to root; 'url' creates a directory based on the provided URL; 'random' creates a random directory. - * - filename_generator - - random - - how to name stored files: 'random' creates a random string; 'static' uses a replicable strategy such as a hash. - -Gsheets -------- - -The following table lists all configuration options for this component: - -.. list-table:: Configuration Options - :header-rows: 1 - :widths: 25 20 55 - - * - **Key** - - **Default** - - **Description** - * - sheet - - None - - name of the sheet to archive - * - sheet_id - - None - - (alternative to sheet name) the id of the sheet to archive - * - header - - 1 - - index of the header row (starts at 1) - * - service_account - - secrets/service_account.json - - service account JSON file path - * - columns - - {'url': 'link', 'status': 'archive status', 'folder': 'destination folder', 'archive': 'archive location', 'date': 'archive date', 'thumbnail': 'thumbnail', 'timestamp': 'upload timestamp', 'title': 'upload title', 'text': 'text content', 'screenshot': 'screenshot', 'hash': 'hash', 'pdq_hash': 'perceptual hashes', 'wacz': 'wacz', 'replaywebpage': 'replaywebpage'} - - names of columns in the google sheet (stringified JSON object) - diff --git a/docs/source/bc.png b/docs/source/bc.png new file mode 100644 index 0000000..766529b Binary files /dev/null and b/docs/source/bc.png differ diff --git a/docs/source/conf.py b/docs/source/conf.py index 3168b22..ee6416e 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -1,50 +1,64 @@ # Configuration file for the Sphinx documentation builder. # https://www.sphinx-doc.org/en/master/usage/configuration.html +import sys +import os +from importlib.metadata import metadata +from datetime import datetime + +sys.path.append(os.path.abspath('../scripts')) +from scripts import generate_module_docs +from auto_archiver.version import __version__ + +# -- Project Hooks ----------------------------------------------------------- +# convert the module __manifest__.py files into markdown files +generate_module_docs() + # -- Project information ----------------------------------------------------- -from importlib.metadata import metadata - package_metadata = metadata("auto-archiver") project = package_metadata["name"] -authors = package_metadata["authors"] +copyright = str(datetime.now().year) +author = "Bellingcat" release = package_metadata["version"] - +language = 'en' # -- General configuration --------------------------------------------------- extensions = [ - "autoapi.extension", # Generate API documentation from docstrings "myst_parser", # Markdown support - 'sphinxcontrib.mermaid', # Mermaid diagrams + "autoapi.extension", # Generate API documentation from docstrings + "sphinxcontrib.mermaid", # Mermaid diagrams "sphinx.ext.viewcode", # Source code links + "sphinx_copybutton", "sphinx.ext.napoleon", # Google-style and NumPy-style docstrings - # "sphinx.ext.autodoc", # Include custom docstrings + "sphinx.ext.autosectionlabel", # 'sphinx.ext.autosummary', # Summarize module/class/function docs ] templates_path = ['_templates'] -exclude_patterns = [] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", ""] # -- AutoAPI Configuration --------------------------------------------------- autoapi_type = 'python' -autoapi_dirs = ["../../src"] +autoapi_dirs = ["../../src/auto_archiver/core/", "../../src/auto_archiver/utils/"] +# get all the modules and add them to the autoapi_dirs +autoapi_dirs.extend([f"../../src/auto_archiver/modules/{m}" for m in os.listdir("../../src/auto_archiver/modules")]) autodoc_typehints = "signature" # Include type hints in the signature -autoapi_ignore = [] # Ignore specific modules +autoapi_ignore = ["*/version.py", ] # Ignore specific modules autoapi_keep_files = True # Option to retain intermediate JSON files for debugging autoapi_add_toctree_entry = True # Include API docs in the TOC -autoapi_template_dir = None # Use default templates +autoapi_python_use_implicit_namespaces = True +autoapi_template_dir = "../_templates/autoapi" autoapi_options = [ "members", "undoc-members", "show-inheritance", - "show-module-summary", "imported-members", ] # -- Markdown Support -------------------------------------------------------- myst_enable_extensions = [ - "colon_fence", # ::: fences "deflist", # Definition lists "html_admonition", # HTML-style admonitions "html_image", # Inline HTML images @@ -53,12 +67,27 @@ myst_enable_extensions = [ "linkify", # Auto-detect links "substitution", # Text substitutions ] +myst_heading_anchors = 2 +myst_fence_as_directive = ["mermaid"] + source_suffix = { ".rst": "restructuredtext", ".md": "markdown", } # -- Options for HTML output ------------------------------------------------- -html_theme = 'furo' -# html_static_path = ['_static'] +html_theme = 'sphinx_book_theme' +html_static_path = ["../_static"] +html_css_files = ["custom.css"] +html_title = f"Auto Archiver v{__version__}" +html_logo = "bc.png" +html_theme_options = { + "repository_url": "https://github.com/bellingcat/auto-archiver", + "use_repository_button": True, +} + + +copybutton_prompt_text = r">>> |\.\.\." +copybutton_prompt_is_regexp = True +copybutton_only_copy_prompt_lines = False \ No newline at end of file diff --git a/docs/source/configurations.rst b/docs/source/configurations.rst deleted file mode 100644 index 85d7922..0000000 --- a/docs/source/configurations.rst +++ /dev/null @@ -1,34 +0,0 @@ - -Configurations -============== - -This section of the documentation provides guidelines for configuring the tool. - -File Reference --------------- - - -Below is the content of the `example.orchestration.yaml` file: - -.. raw:: html - -
- View example.orchestration.yaml - -.. literalinclude:: ../../example.orchestration.yaml - :language: yaml - :caption: example.orchestration.yaml - -.. raw:: html - -
- - -Configs -------- - -This section of the documentation will show the custom configurations for the individual steps of the tool. - -.. include:: _auto/configs.rst - - diff --git a/docs/source/contributing.md b/docs/source/contributing.md new file mode 100644 index 0000000..e75758c --- /dev/null +++ b/docs/source/contributing.md @@ -0,0 +1,2 @@ +```{include} ../../CONTRIBUTING.md +``` \ No newline at end of file diff --git a/docs/source/core_modules.md b/docs/source/core_modules.md new file mode 100644 index 0000000..58eff08 --- /dev/null +++ b/docs/source/core_modules.md @@ -0,0 +1,28 @@ +# Module Documentation + +These pages describe the core modules that come with Auto Archiver and provide the main functionality for archiving websites on the internet. There are five core module types: + +1. Feeders - these 'feed' information (the URLs) from various sources to the Auto Archiver for processing +2. Extractors - these 'extract' the page data for a given URL that is fed in by a feeder +3. Enrichers - these 'enrich' the data extracted in the previous step with additional information +4. Storage - these 'store' the data in a persistent location (on disk, Google Drive etc.) +5. Databases - these 'store' the status of the entire archiving process in a log file or database. + + +```{include} modules/autogen/module_list.md +``` + + +```{toctree} +:maxdepth: 1 +:caption: Core Modules +:hidden: + +modules/config_cheatsheet +modules/feeder +modules/extractor +modules/enricher +modules/storage +modules/database +modules/formatter +``` \ No newline at end of file diff --git a/docs/source/developer_guidelines.rst b/docs/source/developer_guidelines.rst deleted file mode 100644 index c0fdee0..0000000 --- a/docs/source/developer_guidelines.rst +++ /dev/null @@ -1,6 +0,0 @@ - -Developer Guidelines -==================== - -This section of the documentation provides guidelines for developers who want to modify or contribute to the tool. - diff --git a/docs/source/development/creating_modules.md b/docs/source/development/creating_modules.md new file mode 100644 index 0000000..49468a4 --- /dev/null +++ b/docs/source/development/creating_modules.md @@ -0,0 +1,52 @@ +# Creating Your Own Modules + +Modules are what's used to extend Auto Archiver to process different websites or media, and/or transform the data in a way that suits your needs. In most cases, the [Core Modules](../core_modules.md) should be sufficient for every day use, but the most common use-cases for making your own Modules include: + +1. Extracting data from a website which doesn't work with the current core extractors. +2. Enriching or altering the data before saving with additional information that the core enrichers do not offer. +3. Storing your data in a different format/location from what the core storage providers offer. + +## Setting up the folder structure + +1. First, decide what type of module you wish to create. Check the types of modules on the [](../core_modules.md) page to decide what type you need. (Note: a module can be more than one type, more on that below) +2. Create a new python package (a folder) with the name of your module (in this tutorial, we'll call it `awesome_extractor`). +3. Create the `__manifest__.py` and an the `awesome_extractor.py` files in this folder. + +When done, you should have a module structure as follows: + +``` +. +├── awesome_extractor +│ ├── __manifest__.py +│ └── awesome_extractor.py +``` + +Check out the [core modules](https://github.com/bellingcat/auto-archiver/tree/main/src/auto_archiver/modules) in the Auto Archiver repository for examples of the folder structure for real-world modules. + +## Populating the Manifest File + +The manifest file is where you define the core information of your module. It is a python dict containing important information, here's an example file: + +```{include} ../../../tests/data/test_modules/example_module/__manifest__.py +:name: __manifest__.py +:literal: +:parser: python +``` + +## Creating the Python Code + +The next step is to create your module code. First, create a class which should subclass the base module types from `auto_archiver.core`, here's an example class for the `awesome_extractor` module which is an `extractor`: + +```{code-block} python +:filename: awesome_extractor.py + +from auto_archiver.core import Extractor, Metadata + +def AwesomeExtractor(Extractor): + + def download(self, item: Metadata) -> Metadata | False: + url = item.get_url() + # download the content and create the metadata object + metadata = ... + return metadata +``` diff --git a/docs/source/development/developer_guidelines.md b/docs/source/development/developer_guidelines.md new file mode 100644 index 0000000..0014d8f --- /dev/null +++ b/docs/source/development/developer_guidelines.md @@ -0,0 +1,35 @@ + +# Developer Guidelines + +This section of the documentation provides guidelines for developers who want to modify or contribute to the tool. + + +## Developer Install + +1. Clone the project using `git clone https://github.com/bellingcat/auto-archiver.git` +2. Install poetry using `curl -sSL https://install.python-poetry.org | python3 -` ([other installation methods](https://python-poetry.org/docs/#installation)) +3. Install dependencies with `poetry install` + +## Running +4. Run the code with `poetry run auto-archiver [my args]` + +```{note} +Add the plugin [poetry-shell-plugin](https://github.com/python-poetry/poetry-plugin-shell) and run `poetry shell` to activate the virtual environment. +This allows you to run the auto-archiver without the `poetry run` prefix. +``` + +### Optional Development Packages + +Install development packages (used for unit tests etc.) using: +`poetry install -with dev` + + +```{toctree} +:hidden: +creating_modules +docker_development +testing +docs +release +settings_page +``` \ No newline at end of file diff --git a/docs/source/development/docker_development.md b/docs/source/development/docker_development.md new file mode 100644 index 0000000..d770020 --- /dev/null +++ b/docs/source/development/docker_development.md @@ -0,0 +1,5 @@ +## Docker development +working with docker locally: + * `docker compose up` to build the first time and run a local image with the settings in `secrets/orchestration.yaml` + * To modify/pass additional command line args, use `docker compose run auto-archiver --config secrets/orchestration.yaml [OTHER ARGUMENTS]` + * To rebuild after code changes, just pass the `--build` flag, e.g. `docker compose up --build` \ No newline at end of file diff --git a/docs/source/development/docs.md b/docs/source/development/docs.md new file mode 100644 index 0000000..bb389ff --- /dev/null +++ b/docs/source/development/docs.md @@ -0,0 +1,38 @@ + +### Building the Docs + +The documentation is built using [Sphinx](https://www.sphinx-doc.org/en/master/) and [AutoAPI](https://sphinx-autoapi.readthedocs.io/en/latest/) and hosted on ReadTheDocs. +To build the documentation locally, run the following commands: + +**Install required dependencies:** +- Install the docs group of dependencies: +```shell +# only the docs dependencies +poetry install --only docs + +# or for all dependencies +poetry install +``` +- Either use [poetry-plugin-shell](https://github.com/python-poetry/poetry-plugin-shell) to activate the virtual environment: `poetry shell` +- Or prepend the following commands with `poetry run` + +**Create the documentation:** +- Build the documentation: +```shell +# Using makefile (Linux/macOS): +make -C docs html + +# or using sphinx directly (Windows/Linux/macOS): +sphinx-build -b html docs/source docs/_build/html +``` +- If you make significant changes and want a fresh build run: `make -C docs clean` to remove the old build files. + +**Viewing the documentation:** +```shell +# to open the documentation in your browser. +open docs/_build/html/index.html + +# or run autobuild to automatically update the documentation when you make changes +sphinx-autobuild docs/source docs/_build/html +``` + diff --git a/docs/source/development/release.md b/docs/source/development/release.md new file mode 100644 index 0000000..a2ed4c6 --- /dev/null +++ b/docs/source/development/release.md @@ -0,0 +1,33 @@ +# Release Process + +```{note} This is a work in progress. +``` +### Update the project version + +Update the version number in the project file: [pyproject.toml](../../pyproject.toml) following SemVer: +```toml +[project] +name = "auto-archiver" +version = "0.1.1" +``` +Then commit and push the changes. + +* The package version is automatically updated in PyPi using the workflow [python-publish.yml](../../.github/workflows/python-publish.yml) +* A Docker image is automatically pushed with the git tag to dockerhub using the workflow [docker-publish.yml](../../.github/workflows/docker-publish.yml) + +### Create the release on Git + +The release needs a git tag which should match the project version number, prefixed with a 'v'. For example, if the project version is `0.1.1`, the git tag should be `v0.1.1`. +This can be done the usual way, or created within the Github UI when you create the release. + +Go to GitHub releases > new release > create the release with the new tag and the release notes. + + +manual release to docker hub + * `docker image tag auto-archiver bellingcat/auto-archiver:latest` + * `docker push bellingcat/auto-archiver` + + +### Building the Settings Page + +The Settings page is built as part of the python-publish workflow and packaged within the app. \ No newline at end of file diff --git a/docs/source/development/settings_page.md b/docs/source/development/settings_page.md new file mode 100644 index 0000000..41271b9 --- /dev/null +++ b/docs/source/development/settings_page.md @@ -0,0 +1,31 @@ +# Configuration Editor + +The [configuration editor](../installation/config_editor.md), is an easy-to-use UI for users to edit their auto-archiver settings. + +The single-file app is built using React and vite. To get started developing the package, follow these steps: + +1. Make sure you have Node v22 installed. + +```{note} Tip: if you don't have node installed: + +Use `nvm` to manage your node installations. Use: +`curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash` to install `nvm` and then `nvm i 22` to install Node v22 +``` + +2. Generate the `schema.json` file for the currently installed modules using `python scripts/generate_settings_schema.py` +3. Go to the settings folder `cd scripts/settings/` and build your environment with `npm i` +4. Run a development version of the page with `npm run dev` and then open localhost:5173. +5. Build a release version of the page with `npm run build` + +A release version creates a single-file app called `dist/index.html`. This file should be copied to `docs/source/installation/settings_base.html` so that it can be integrated into the sphinx docs. + +```{note} + +The single-file app dist/index.html does not include any `` or `` tags as it is designed to be built into a RTD docs page. Edit `index.html` in the settings folder if you wish to modify the built page. +``` + +## Readthedocs Integration + +The configuration editor is built as part of the RTD deployment (see `.readthedocs.yaml` file). This command is run every time RTD is built: + +`cd scripts/settings && npm install && npm run build && yes | cp dist/index.html ../../docs/source/installation/settings_base.html && cd ../..` \ No newline at end of file diff --git a/docs/source/development/testing.md b/docs/source/development/testing.md new file mode 100644 index 0000000..5de9574 --- /dev/null +++ b/docs/source/development/testing.md @@ -0,0 +1,21 @@ +# Testing + +`pytest` is used for testing. There are two main types of tests: + +1. 'core' tests which should be run on every change +2. 'download' tests which hit the network. These tests will do things like make API calls (e.g. Twitter, Bluesky etc.) and should be run regularly to make sure that APIs have not changed. + + +## Running Tests + +1. Make sure you've installed the dev dependencies with `pytest install --with dev` +2. Tests can be run as follows: +``` +#### Command prefix of 'poetry run' removed here for simplicity +# run core tests +pytest -ra -v -m "not download" +# run download tests +pytest -ra -v -m "download" +# run all tests +pytest -ra -v +``` \ No newline at end of file diff --git a/docs/source/example.orchestration.yaml b/docs/source/example.orchestration.yaml new file mode 100644 index 0000000..48d354d --- /dev/null +++ b/docs/source/example.orchestration.yaml @@ -0,0 +1,79 @@ +# Auto Archiver Configuration +# Steps are the modules that will be run in the order they are defined + +steps: + feeders: + - cli_feeder + extractors: + - generic_extractor + - telegram_extractor + enrichers: + - thumbnail_enricher + - meta_enricher + - pdq_hash_enricher + - ssl_enricher + - hash_enricher + databases: + - console_db + - csv_db + storages: + - local_storage + formatters: + - html_formatter + +# Global configuration + +# Authentication +# a dictionary of authentication information that can be used by extractors to login to website. +# you can use a comma separated list for multiple domains on the same line (common usecase: x.com,twitter.com) +# Common login 'types' are username/password, cookie, api key/token. +# There are two special keys for using cookies, they are: cookies_file and cookies_from_browser. +# Some Examples: +# facebook.com: +# username: "my_username" +# password: "my_password" +# or for a site that uses an API key: +# twitter.com,x.com: +# api_key +# api_secret +# youtube.com: +# cookie: "login_cookie=value ; other_cookie=123" # multiple 'key=value' pairs should be separated by ; + +authentication: {} + +# Logging settings for your project. See the logging settings with --help + +logging: + level: INFO + +# These are the global configurations that are used by the modules + + file: + rotation: +local_storage: + path_generator: flat + filename_generator: static + save_to: ./local_archive + save_absolute: false +html_formatter: + detect_thumbnails: true +thumbnail_enricher: + thumbnails_per_minute: 60 + max_thumbnails: 16 +generic_extractor: + subtitles: true + comments: false + livestreams: false + live_from_start: false + proxy: '' + end_means_success: true + allow_playlist: false + max_downloads: inf +csv_db: + csv_file: db.csv +ssl_enricher: + skip_when_nothing_archived: true +hash_enricher: + algorithm: SHA-256 + chunksize: 16000000 + diff --git a/docs/source/flow_overview.md b/docs/source/flow_overview.md new file mode 100644 index 0000000..5ffa3a8 --- /dev/null +++ b/docs/source/flow_overview.md @@ -0,0 +1,30 @@ + +# Archiving Overview + +The archiver archives web pages using the following workflow +1. **Feeder** gets the links (from a spreadsheet, from the console, ...) +2. **Extractor** tries to extract content from the given link (e.g. videos from youtube, images from Twitter...) +3. **Enricher** adds more info to the content (hashes, thumbnails, ...) +4. **Formatter** creates a report from all the archived content (HTML, PDF, ...) +5. **Database** knows what's been archived and also stores the archive result (spreadsheet, CSV, or just the console) + +Each step in the workflow is handled by 'modules' that interact with the data in different ways. For example, the Twitter Extractor Module would extract information from the Twitter website. The Screenshot Enricher Module will take screenshots of the given page. See the [core modules page](core_modules.md) for an overview of all the modules that are available. + +Auto-archiver must have at least one module defined for each step of the workflow. This is done by setting the [configuration](installation/configurations.md) for your auto-archiver instance. + +Here's the complete workflow that the auto-archiver goes through: + +```mermaid + +graph TD + s((start)) --> F(fa:fa-table Feeder) + F -->|get and clean URL| D1{fa:fa-database Database} + D1 -->|is already archived| e((end)) + D1 -->|not yet archived| a(fa:fa-download Archivers) + a -->|got media| E(fa:fa-chart-line Enrichers) + E --> S[fa:fa-box-archive Storages] + E --> Fo(fa:fa-code Formatter) + Fo --> S + Fo -->|update database| D2(fa:fa-database Database) + D2 --> e +``` \ No newline at end of file diff --git a/docs/source/how_to.md b/docs/source/how_to.md new file mode 100644 index 0000000..e2238dd --- /dev/null +++ b/docs/source/how_to.md @@ -0,0 +1,12 @@ +# How-To Guides + +The follow pages contain helpful how-to guides for common use cases of the Auto Archiver. +--- + +```{toctree} +:maxdepth: 1 +:glob: + +how_to/* + +``` \ No newline at end of file diff --git a/docs/source/how_to/authentication_how_to.md b/docs/source/how_to/authentication_how_to.md new file mode 100644 index 0000000..0e842fb --- /dev/null +++ b/docs/source/how_to/authentication_how_to.md @@ -0,0 +1,110 @@ +# Logging in to sites + +This how-to guide shows you how you can use various authentication methods to allow you to login to a site you are trying to archive. This is useful for websites that require a user to be logged in to browse them, or for sites that restrict bots. + +In this How-To, we will authenticate on use Twitter/X.com using cookies, and on XXXX using username/password. + + + +## Using cookies to authenticate on Twitter/X + +It can be useful to archive tweets after logging in, since some tweets are only visible to authenticated users. One case is Tweets marked as 'Sensitive'. + +Take this tweet as an example: [https://x.com/SozinhoRamalho/status/1876710769913450647](https://x.com/SozinhoRamalho/status/1876710769913450647) + +This tweet has been marked as sensitive, so a normal run of Auto Archiver without a logged in session will fail to extract the tweet: + +```{code-block} console +:emphasize-lines: 3,4,5,6 + +>>> auto-archiver https://x.com/SozinhoRamalho/status/1876710769913450647 ✭ ✱ + ... +ERROR: [twitter] 1876710769913450647: NSFW tweet requires authentication. Use --cookies, +--cookies-from-browser, --username and --password, --netrc-cmd, or --netrc (twitter) to + provide account credentials. See https://github.com/yt-dlp/yt-dlp/wiki/FAQ#how-do-i-pass-cookies-to-yt-dlp + for how to manually pass cookies +[twitter] 1876710769913450647: Downloading guest token +[twitter] 1876710769913450647: Downloading GraphQL JSON +2025-02-20 15:06:13.362 | ERROR | auto_archiver.modules.generic_extractor.generic_extractor:download_for_extractor:248 - Error downloading metadata for post: NSFW tweet requires authentication. Use --cookies, --cookies-from-browser, --username and --password, --netrc-cmd, or --netrc (twitter) to provide account credentials. See https://github.com/yt-dlp/yt-dlp/wiki/FAQ#how-do-i-pass-cookies-to-yt-dlp for how to manually pass cookies +[generic] Extracting URL: https://x.com/SozinhoRamalho/status/1876710769913450647 +[generic] 1876710769913450647: Downloading webpage +WARNING: [generic] Falling back on generic information extractor +[generic] 1876710769913450647: Extracting information +ERROR: Unsupported URL: https://x.com/SozinhoRamalho/status/1876710769913450647 +2025-02-20 15:06:13.744 | INFO | auto_archiver.core.orchestrator:archive:483 - Trying extractor telegram_extractor for https://x.com/SozinhoRamalho/status/1876710769913450647 +2025-02-20 15:06:13.744 | SUCCESS | auto_archiver.modules.console_db.console_db:done:23 - DONE Metadata(status='nothing archived', metadata={'_processed_at': datetime.datetime(2025, 2, 20, 15, 6, 12, 473979, tzinfo=datetime.timezone.utc), 'url': 'https://x.com/SozinhoRamalho/status/1876710769913450647'}, media=[]) +... +``` + +To get round this limitation, we can use **cookies** (information about a logged in user) to mimic being logged in to Twitter. There are two ways to pass cookies to Auto Archiver. One is from a file, and the other is from a browser profile on your computer. + +In this tutorial, we will export the Twitter cookies from our browser and add them to Auto Archiver + +**1. Installing a cookie exporter extension** + +First, we need to install an extension in our browser to export the cookies for a certain site. The [FAQ on yt-dlp](https://github.com/yt-dlp/yt-dlp/wiki/FAQ#how-do-i-pass-cookies-to-yt-dlp) provides some suggestions: Get [cookies.txt LOCALLY](https://chromewebstore.google.com/detail/get-cookiestxt-locally/cclelndahbckbenkjhflpdbgdldlbecc) for Chrome or [cookies.txt](https://addons.mozilla.org/en-US/firefox/addon/cookies-txt/) for Firefox. + +**2. Export the cookies** + +```{note} See the note [here](../installation/authentication.md#recommendations-for-authentication) on why you shouldn't use your own personal account for archiving. +``` + +Once the extension is installed in your preferred browser, login to Twitter in this browser, and then activate the extension and export the cookies. You can choose to export all your cookies for your browser, or just cookies for this specific site. In the image below, we're only exporting cookies for Twitter/x.com: + +![extract cookies](extract_cookies.png) + + +**3. Adding the cookies file to Auto Archiver** + +You now will have a file called `cookies.txt` (tip: name it `twitter_cookies.txt` if you only exported cookies for Twitter), which needs to be added to Auto Archiver. + +Do this by going into your Auto Archiver configuration file, and editing the `authentication` section. We will add the `cookies_file` option for the site `x.com,twitter.com`. + +```{note} For websites that have multiple URLs (like x.com and twitter.com) you can 'reuse' the same login information without duplicating it using a comma separated list of domain names. +``` + +I've saved my `twitter_cookies.txt` file in a `secrets` folder, so here's how my authentication section looks now: + +```{code} yaml +:caption: orchestration.yaml + +... + +authentication: + x.com,twitter.com: + cookies_file: secrets/twitter_cookies.txt +... +``` + +**4. Re-run your archiving with the cookies enabled** + +Now, the next time we re-run Auto Archiver, the cookies from our logged-in session will be used by Auto Archiver, and restricted/sensitive tweets can be downloaded! + +```{code} console +>>> auto-archiver https://x.com/SozinhoRamalho/status/1876710769913450647 ✭ ✱ ◼ +... +2025-02-20 15:27:46.785 | WARNING | auto_archiver.modules.console_db.console_db:started:13 - STARTED Metadata(status='no archiver', metadata={'_processed_at': datetime.datetime(2025, 2, 20, 15, 27, 46, 785304, tzinfo=datetime.timezone.utc), 'url': 'https://x.com/SozinhoRamalho/status/1876710769913450647'}, media=[]) +2025-02-20 15:27:46.785 | INFO | auto_archiver.core.orchestrator:archive:483 - Trying extractor generic_extractor for https://x.com/SozinhoRamalho/status/1876710769913450647 +[twitter] Extracting URL: https://x.com/SozinhoRamalho/status/1876710769913450647 +... +2025-02-20 15:27:53.134 | INFO | auto_archiver.modules.local_storage.local_storage:upload:26 - ./local_archive/https-x-com-sozinhoramalho-status-1876710769913450647/06e8bacf27ac4bb983bf6280.html +2025-02-20 15:27:53.135 | SUCCESS | auto_archiver.modules.console_db.console_db:done:23 - DONE Metadata(status='yt-dlp_Twitter: success', +metadata={'_processed_at': datetime.datetime(2025, 2, 20, 15, 27, 48, 564738, tzinfo=datetime.timezone.utc), 'url': +'https://x.com/SozinhoRamalho/status/1876710769913450647', 'title': 'ignore tweet, testing sensitivity warning nudity https://t.co/t3u0hQsSB1', +... +``` + + +### Finishing Touches + +You've now successfully exported your cookies from a logged-in session in your browser, and used them to authenticate with Twitter and download a sensitive tweet. Congratulations! + +Finally,Some important things to remember: + +1. It's best not to use your own personal account for archiving. [Here's why](../installation/authentication.md#recommendations-for-authentication). +2. Cookies can be short-lived, so may need updating. Sometimes, a website session may 'expire' or a website may force you to login again. In these instances, you'll need to repeat the export step (step 2) after logging in again to update your cookies. + +## Authenticating on XXXX site with username/password + +```{note} This section is still under construction 🚧 +``` diff --git a/docs/source/how_to/extract_cookies.png b/docs/source/how_to/extract_cookies.png new file mode 100644 index 0000000..73b7917 Binary files /dev/null and b/docs/source/how_to/extract_cookies.png differ diff --git a/docs/source/how_to/gsheets_setup.md b/docs/source/how_to/gsheets_setup.md new file mode 100644 index 0000000..ade8024 --- /dev/null +++ b/docs/source/how_to/gsheets_setup.md @@ -0,0 +1,159 @@ +# Using Google Sheets + +This guide explains how to set up Google Sheets to process URLs automatically and then store the archiving status back into the Google sheet. It is broadly split into 3 steps: + +1. Setting up your Google Sheet +2. Setting up a service account so Auto Archiver can access the sheet +3. Setting the Auto Archiver settings + +### 1. Setting up your Google Sheet + +Any Google sheet must have at least *one* column, with the name 'link' (you can change this name afterwards). This is the column with the URLs that you want the Auto Archiver to archive. +Your sheet can have many other columns that the Auto Archiver can use, and you can also include any additional columns for your own personal use. The order of the columns does not matter, the naming just needs to be correctly assigned to its corresponding value in the configuration file. + +We recommend copying [this template Google Sheet](https://docs.google.com/spreadsheets/d/1NJZo_XZUBKTI1Ghlgi4nTPVvCfb0HXAs6j5tNGas72k/edit?usp=sharing) as a starting point for your project, as this matches the default column names. + +Here's an overview of all the columns, and what a complete sheet would look like. + +**Inputs:** + +These are processed by the Gsheet Feeder and passed to the Auto Archiver. + +* **Link** *(required)*: the URL of the post that is to be archived +* **Destination folder**: custom folder for archived file (regardless of storage) + +**Outputs:** + +These are updated by the Gsheet DB module during the archiving process. +Note the required columns are only required if you are using the Gsheet DB module as well as the feeder. + +* **Archive status** *(required)*: Status of archive operation +* **Archive location**: URL of archived post +* **Archive date**: Date archived +* **Thumbnail**: Embeds a thumbnail for the post in the spreadsheet +* **Timestamp**: Timestamp of original post +* **Title**: Post title +* **Text**: Post text +* **Screenshot**: Link to screenshot of post +* **Hash**: Hash of archived HTML file (which contains hashes of post media) - for checksums/verification +* **Perceptual Hash**: Perceptual hashes of found images - these can be used for de-duplication of content +* **WACZ**: Link to a WACZ web archive of post +* **ReplayWebpage**: Link to a ReplayWebpage viewer of the WACZ archive + +For example, this is a spreadsheet configured with all of the columns for the auto archiver and a few URLs to archive. +In this example the Ghseet Feeder and Gsheet DB are being used, and the archive is in progress. +(Note that the column names are not case sensitive.) + +![A screenshot of a Google Spreadsheet with column headers defined as above, and several Youtube and Twitter URLs in the "Link" column](../../demo-before.png) + +We'll change the name of the 'Destination Folder' column in step 3. + +## 2. Setting up your Service Account + +Once your Google Sheet is set up, you need to create what's called a 'service account' that will allow the Auto Archiver to access it. + +To do this, follow the steps in [this guide](https://gspread.readthedocs.io/en/latest/oauth2.html) all the way up until step 8. You should have downloaded a file called `service_account.json` and shared the Google Sheet with the log 'client_email' email address in this file. + +Once you've downloaded the file, save it to `secrets/service_account.json` + +## 3. Setting up the configuration file + +Now that you've set up your Google sheet, and you've set up the service account so Auto Archiver can access the sheet, the final step is to set your configuration. + +First, make sure you have `gsheet_feeder_db` set in the `steps.feeders` section of your config. If you wish to store the results of the archiving process back in your Google sheet, make sure to also set the `ghseet_db` settig in the `steps.databases` section. Here's how this might look: + +```{code} yaml +steps: + feeders: + - gsheet_feeder_db + ... + databases: + - gsheet_feeder_db # optional, if you also want to store the results in the Google sheet and tract the status of active archivals. + ... +``` + +Next, set up the `gsheet_feeder_db` configuration settings in the 'Configurations' part of the config `orchestration.yaml` file. Open up the file, and set the `gsheet_feeder_db.sheet` setting or the `gsheet_feeder_db.sheet_id` setting. The `sheet` should be the name of your sheet, as it shows in the top left of the sheet. +For example, the sheet [here](https://docs.google.com/spreadsheets/d/1NJZo_XZUBKTI1Ghlgi4nTPVvCfb0HXAs6j5tNGas72k/edit?gid=0#gid=0) is called 'Public Auto Archiver template'. + +Here's how this might look: + +```{code} yaml +... +gsheet_feeder_db: + sheet: 'My Awesome Sheet' + ... +``` + +You can also pass these settings directly on the command line without having to edit the file, here'a an example of how to do that (using docker): + +`docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver:dockerize --gsheet_feeder_db.sheet "My Awesome Sheet 2"`. + +Here, the sheet name has been overridden/specified in the command line invocation. + +### 3a. (Optional) Changing the column names + +In step 1, we said we would change the name of the 'Destination Folder'. Perhaps you don't like this name, or already have a sheet with a different name. In our example here, we want to name this column 'Save Folder'. To do this, we need to edit the `ghseet_feeder_db.column` setting in the configuration file. +For more information on this setting, see the [Gsheet Feeder Database docs](../modules/autogen/feeder/gsheet_feeder_db.md#configuration-options). We will first copy the default settings from the Gsheet Feeder docs for the 'column' settings, and then edit the 'Destination Folder' section to rename it 'Save Folder'. Our final configuration section looks like: + +```{code} yaml +... +gsheet_feeder_db: + sheet: 'My Awesome Sheet' + header: 1 + service_account: secrets/service_account.json + columns: + url: link + status: archive status + folder: save folder # <-- note how this value has been changed + archive: archive location + date: archive date + thumbnail: thumbnail + timestamp: upload timestamp + title: upload title + text: text content + screenshot: screenshot + hash: hash + pdq_hash: perceptual hashes + wacz: wacz + replaywebpage: replaywebpage + +``` +## 4. Running the Auto Archiver +### Feeding the URLs to the Auto Archiver + +The URLs to be archived should be added to the Google Sheet, and optionally a folder value. Leave all the other configured columns empty (but you may add additional columns for your own use, as long as they don't conflict with the column names mapped in the configuration file). +The Auto Archiver will archive any URLs which have an empty 'status' column + +### Viewing the Results after archiving + +With the `ghseet_feeder_db` installed, once you start running the Auto Archiver, it will update the "Archive status" column. +The status will be set to "Archive in progress" once the archival starts. If the archival is stopped during a run, either manually or because an error is raised the status value should be cleared. + +![A screenshot of a Google Spreadsheet with column headers defined as above, and several Youtube and Twitter URLs in the "Link" column. The auto archiver has added "archive in progress" to one of the status columns.](../../demo-progress.png) + +The links are downloaded and archived, and the spreadsheet is updated to the following: + +![A screenshot of a Google Spreadsheet with videos archived and metadata added per the description of the columns above.](../../demo-after.png) + +Note that the first row is skipped, as it is assumed to be a header row (`--gsheet_feeder_db.header=1` and you can change it if you use more rows above). Rows with an empty URL column, or a non-empty archive column are also skipped. All sheets in the document will be checked. + +The "archive location" link contains the path of the archived file, in local storage, S3, or in Google Drive. + +![The archive result for a link in the demo sheet.](../../demo-archive.png) + +### Troubleshooting + +**Hanging Archival in progress status** + +Occasionally system crashes or other unexpected events can cause the Auto Archiver to exit without cleaning up the status value. +If you are sure that all archival processes have stopped but you still see "Archive in progress" in the status column, you can manually clear the status column to allow the Auto Archiver to retry that archival on the next run. + +**Nothing archived status** + +Sometimes this means the tool is genuinely unable to extract the content at this point in time, but sometimes it can be resolved with different configurations. +Try: + - Turning on additional 'extractor' types in the configuration file (this can appear as 'no archiver' in the status column). + - Changing credentials or refreshing session files for extractors which require them + - Check if the extractors can accept any additional configurations such as adding a cookie file. + + diff --git a/docs/source/how_to/logging.md b/docs/source/how_to/logging.md new file mode 100644 index 0000000..d88882d --- /dev/null +++ b/docs/source/how_to/logging.md @@ -0,0 +1,71 @@ +# Keeping Logs + +Auto Archiver's logs can be helpful for debugging problematic archiving processes. This guide shows you how to use the logs to + +## Setting up logging + +Logging settings can be set on the command line or using the orchestration config file ([learn more](../installation/configuration)). A special `logging` section defines the logging options. + +#### Enabling or Disabling Logging + +Logging to the console is enabled by default. If you want to globally disable Auto Archiver's logging, then you can set `enabled: false` in your `logging` config: + +```{code} yaml + +... +logging: + enabled: false +... +``` + +```{note} +This will disable all logs from Auto Archiver, but it does not disable logs for other tools that the Auto Archiver uses (for example: yt-dlp, firefox or ffmpeg). These logs will still appear in your console. +``` + +#### Logging Level + +There are 7 logging levels in total, with 4 commonly used levels. They are: `DEBUG`, `INFO`, `WARNING` and `ERROR`. + +Change the warning level by setting the value in your orchestration config file: + +```{code} yaml +:caption: orchestration.yaml + +... +logging: + level: DEBUG # or INFO / WARNING / ERROR +... +``` + +For normal usage, it is recommended to use the `INFO` level, or if you prefer quieter logs with less information, you can use the `WARNING` level. If you encounter issues with the archiving, then it's recommended to enable the `DEBUG` level. + +```{note} To learn about all logging levels, see the [loguru documentation](https://loguru.readthedocs.io/en/stable/api/logger.html) +``` + +### Logging to a file + +As default, auto-archiver will log to the console. But if you wish to store your logs for future reference, or you are running the auto-archiver from within code a implementation, then you may with to enable file logging. This can be done by setting the `file:` config value in the logging settings. + +**Rotation:** For file logging, you can choose to 'rotate' your log files (creating new log files) so they do not get too large. Change this by setting the 'rotation' option in your logging settings. For a full list of rotation options, see the [loguru docs](https://loguru.readthedocs.io/en/stable/overview.html#easier-file-logging-with-rotation-retention-compression). + +```{code} yaml +:caption: orchestration.yaml + +logging: + ... + file: /my/log/file.log + rotation: 1 day +``` + +### Full logging example + +The below example logs only `WARNING` logs to the console and to the file `/my/file.log`, rotating that file once per week: + +```{code} yaml +:caption: orchestration.yaml + +logging: + level: WARNING + file: /my/file.log + rotation: 1 week +``` \ No newline at end of file diff --git a/docs/source/how_to/new_config_format.md b/docs/source/how_to/new_config_format.md new file mode 100644 index 0000000..5cef3c8 --- /dev/null +++ b/docs/source/how_to/new_config_format.md @@ -0,0 +1,146 @@ +# Upgrading from v0.12 + +```{note} This how-to is only relevant for people who used Auto Archiver before February 2025 (versions prior to 0.13). + +If you are new to Auto Archiver, then you are already using the latest configuration format and this how-to is not relevant for you. +``` + +Versions 0.13+ of Auto Archiver has breaking changes in the configuration format, which means earlier configuration formats will not work without slight modifications. + +## How do I know if I need to update my configuration format? + +There are two simple ways to check if you need to update your format: + +1. When you try and run auto-archiver using your existing configuration file, you get an error about no feeders or formatters being configured, like: + +```{code} console +AssertionError: No feeders were configured. Make sure to set at least one feeder in +your configuration file or on the command line (using --feeders) +``` + +2. Within your configuration file, you have a `feeder:` option. This is the old format. An example old format: +```{code} yaml + +steps: + feeder: cli_feeder +... +``` + +The next two sections outline the two methods you have for updating your file. + +## 1. Manually edit the configuration file and change the values. + +This is recommended if you want to keep all your old settings. Follow the steps below to change the relevant settings: + +#### a) Feeder & Formatter Steps Settings + +The feeder and formatter settings have been changed from a single string to a list. + +- `steps.feeder (string)` → `steps.feeders (list)` +- `steps.formatter (string)` → `steps.formatters (list)` + +Example: + +```{code} yaml + +steps: + feeder: cli_feeder + ... + formatter: html_formatter + +# the above should be changed to: +steps: + feeders: + - cli_feeder + ... + formatters: + - html_formatter +``` + +```{note} Auto Archiver still only supports one feeder and formatter, but from v0.13 onwards they must be added to the configuration file as a list. +``` + +#### b) Extractor (formerly Archiver) Steps Settings + +With v0.13 of Auto Archiver, `archivers` have been renamed to `extractors` to better reflect what they actually do - extract information from a URL. Change the configuration by renaming: + +- `steps.archivers` → `steps.extractors` + +The names of the actual modules have also changed, so for any extractor modules you have enabled, you will need to rename the `archiver` part to `extractor`. Some examples: + +- `telethon_archiver` → `telethon_extractor` +- `wacz_archiver_enricher` → `wacz_extractor_enricher` +- `wayback_archiver_enricher` → `wayback_extractor_enricher` +- `vk_archiver` → `vk_extractor` + + +#### c) Module Renaming + + +The `youtube_archiver` has been renamed to `generic_extractor` as it is considered the default/fallback extractor. Read more about the [generic extractor](../modules/autogen/extractor/generic_extractor.md). + +The `atlos` modules have been merged into one, as have the `gsheets` feeder and database. + +- `atlos_feeder` → `atlos_feeder_db_storage` +- `atlos_storage` → `atlos_feeder_db_storage` +- `atlos_db` → `atlos_feeder_db_storage` +- `gsheet_feeder` → `gsheet_feeder_db` +- `gsheet_db` → `gsheet_feeder_db` + + +Example: +```{code} yaml +steps: + feeders: + - gsheet_feeder_db # formerly gsheet_feeder + ... + extractors: # formerly 'archivers' + - telethon_extractor # formerly telethon_archiver + - generic_extractor # formerly youtube_archiver + - vk_extractor # formerly vk_archiver + databases: + - gsheet_feeder_db # formerly gsheet_db + ... + +``` + +```{note} + +Don't forget to also rename the configuration settings. For example: + +```{code} yaml +gsheet_feeder_db: # formerly gsheet_feeder + service_account: secrets/service_account.json + sheet: My Google Sheet +... +``` + +#### d) Redundant / Obsolete Modules + +With v0.13 of Auto Archiver, the following modules have been removed and their features have been built in to the generic_extractor. You should remove them from the 'steps' section of your configuration file: + +* `twitter_archiver` - use the `generic_extractor` for general extraction, or the `twitter_api_extractor` for API access. +* `tiktok_archiver` - use the `generic_extractor` to extract TikTok videos. + + +## 2. Auto-generate a new config, then copy over your settings. + +Using this method, you can have Auto Archiver auto-generate a configuration file for you, then you can copy over the desired settings from your old config file. This is probably the easiest method and quickest to setup, but it may require some trial and error as you copy over your settings. + +First, move your existing `orchestration.yaml` file to a different folder or rename it. + +Then, you can generate a `simple` or `full` config using: + +```{code} console +>>> # generate a simple config +>>> auto-archiver +>>> # config will be written to orchestration.yaml +>>> +>>> # generate a full config +>>> auto-archiver --mode=full +>>> +``` + +After this, copy over any settings from your old config to the new config. + + diff --git a/docs/source/index.md b/docs/source/index.md new file mode 100644 index 0000000..74b7969 --- /dev/null +++ b/docs/source/index.md @@ -0,0 +1,17 @@ + +```{include} ../../README.md +``` + +```{toctree} +:maxdepth: 2 +:hidden: +:caption: Contents: + +Overview +installation/setup +core_modules.md +how_to +contributing +development/developer_guidelines +autoapi/index.rst +``` \ No newline at end of file diff --git a/docs/source/index.rst b/docs/source/index.rst deleted file mode 100644 index 52449b8..0000000 --- a/docs/source/index.rst +++ /dev/null @@ -1,26 +0,0 @@ -.. auto-archiver documentation master file, created by - sphinx-quickstart on Sun Jan 12 20:35:50 2025. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -Auto Archiver documentation -=========================== - -.. note:: - - This is a work in progress. - - -.. include:: ../../README.md - :parser: myst - - -.. toctree:: - :maxdepth: 1 - :hidden: - :caption: Contents: - - user_guidelines - developer_guidelines - configurations - diff --git a/docs/source/installation/authentication.md b/docs/source/installation/authentication.md new file mode 100644 index 0000000..be30425 --- /dev/null +++ b/docs/source/installation/authentication.md @@ -0,0 +1,72 @@ +# Authentication + +The Authentication framework for auto-archiver allows you to add login details for various websites in a flexible way, directly from the configuration file. + +There are two main use cases for authentication: +* Some websites require some kind of authentication in order to view the content. Examples include Facebook, Telegram etc. +* Some websites use anti-bot systems to block bot-like tools from accessing the website. Adding real login information to auto-archiver can sometimes bypass this. + +## The Authentication Config + +You can save your authentication information directly inside your orchestration config file, or as a separate file (for security/multi-deploy purposes). Whether storing your settings inside the orchestration file, or as a separate file, the configuration format is the same. Currently, auto-archiver supports the following authentication types: + +**Username & Password:** +- `username`: str - the username to use for login +- `password`: str - the password to use for login + +**API** +- `api_key`: str - the API key to use for login +- `api_secret`: str - the API secret to use for login + +**Cookies** +- `cookie`: str - a cookie string to use for login (specific to this site) +- `cookies_from_browser`: str - load cookies from this browser, for this site only. +- `cookies_file`: str - load cookies from this file, for this site only. + +```{note} + +The Username & Password, and API settings only work with the Generic Extractor. Other modules (like the screenshot enricher) can only use the `cookies` options. Furthermore, many sites can still detect bots and block username/password logins. Twitter/X and YouTube are two prominent ones that block username/password logging. + +One of the 'Cookies' options is recommended for the most robust archiving. +``` + +```{code} yaml +authentication: + # optional file to load authentication information from, for security or multi-system deploy purposes + load_from_file: path/to/authentication/file.txt + # optional setting to load cookies from the named browser on the system, for **ALL** websites + cookies_from_browser: firefox + # optional setting to load cookies from a cookies.txt/cookies.jar file, for **ALL** websites. See note below on extracting these + cookies_file: path/to/cookies.jar + + mysite.com: + username: myusername + password: 123 + + facebook.com: + cookie: single_cookie + + othersite.com: + api_key: 123 + api_secret: 1234 + +``` + + +### Recommendations for authentication + +1. **Store authentication information separately:** +The authentication part of your configuration contains sensitive information. You should make efforts not to share this with others. For extra security, use the `load_from_file` option to keep your authentication settings out of your configuration file, ideally in a different folder. + +2. **Don't use your own personal credentials** +Depending on the website you are extracting information from, there may be rules (Terms of Service) that prohibit you from scraping or extracting information using a bot. If you use your own personal account, there's a possibility it might get blocked/disabled. It's recommended to set up a separate, 'throwaway' account. In that way, if it gets blocked you can easily create another one to continue your archiving. + + +### How to create a cookies.jar or pass cookies directly to auto-archiver + +auto-archiver uses yt-dlp's powerful cookies features under the hood. For instructions on how to extract a cookies.jar (or cookies.txt) file directly from your browser, see the FAQ in the [yt-dlp documentation](https://github.com/yt-dlp/yt-dlp/wiki/FAQ#how-do-i-pass-cookies-to-yt-dlp) + +```{note} For developers: + +For information on how to access and use authentication settings from within your module, see the `{generic_extractor}` for an example, or view the [`auth_for_site()` function in BaseModule](../autoapi/core/base_module/index.rst) +``` \ No newline at end of file diff --git a/docs/source/installation/config_cheatsheet.md b/docs/source/installation/config_cheatsheet.md new file mode 100644 index 0000000..820e30c --- /dev/null +++ b/docs/source/installation/config_cheatsheet.md @@ -0,0 +1,6 @@ +# Configuration Cheat Sheet + +Below is a list of all configurations for the core modules in Auto Archiver + +```{include} ../modules/autogen/configs_cheatsheet.md +``` diff --git a/docs/source/installation/config_editor.md b/docs/source/installation/config_editor.md new file mode 100644 index 0000000..a23ebce --- /dev/null +++ b/docs/source/installation/config_editor.md @@ -0,0 +1,5 @@ +# Configuration Editor + +```{raw} html +:file: settings.html +``` \ No newline at end of file diff --git a/docs/source/installation/configurations.md b/docs/source/installation/configurations.md new file mode 100644 index 0000000..e3aa76e --- /dev/null +++ b/docs/source/installation/configurations.md @@ -0,0 +1,103 @@ + +# Configuration + +The recommended way to configure auto-archiver for first-time users is to [run the Auto Archiver](setup.md#running) and have it auto-generate a default configuration for you. Then, if needed, you can edit the configuration file using one of the following methods. + + +## 1. Configuration file + +The configuration file is typically called `orchestration.yaml` and stored in the `secrets` folder on your desktop. The configuration file contains all the settings for your entire Auto Archiver workflow in one easy-to-find place. + +If you want to have Auto Archiver run with the recommended 'basic' setup, + +### Advanced Configuration + +The structure of orchestration file is split into 2 parts: `steps` (what [steps](../flow_overview.md) to use) and `configurations` (settings for individual modules). + +A default `orchestration.yaml` will be created for you the first time you run auto-archiver (without any arguments). Here's what it looks like: + +
+View exampleorchestration.yaml + +```{literalinclude} ../example.orchestration.yaml + :language: yaml + :caption: orchestration.yaml +``` + +
+ +## 2. Command Line configuration + +You can run auto-archiver directly from the command line, without the need for a configuration file, command line arguments are parsed using the format `module_name.config_value`. For example, a config value of `api_key` in the `instagram_extractor` module would be passed on the command line with the flag `--instagram_extractor.api_key=API_KEY`. + +The command line arguments are useful for testing or editing config values and enabling/disabling modules on the fly. When you are happy with your settings, you can store them back in your configuration file by passing the `-s/--store` flag on the command line. + +```bash +auto-archiver --instagram_extractor.api_key=123 --other_module.setting --store +# will store the new settings into the configuration file (default: orchestration.yaml) +``` + +```{note} Arguments passed on the command line override those saved in your settings file. Save them to your config file using the -s or --store flag +``` + +## Seeing all Configuration Options + +View the configurable settings for the core modules on the individual doc pages for each [](../core_modules.md). +You can also view all settings available for the modules you have on your system using the `--help` flag in auto-archiver. + +```{code-block} console +:caption: Example output when using the --help flag with auto-archiver +$ auto-archiver --help +... +Positional Arguments: + urls URL(s) to archive, either a single URL or a list of urls, should not come from config.yaml + +Options: + --help, -h show a full help message and exit + --version show program's version number and exit + --config CONFIG_FILE the filename of the YAML configuration file (defaults to 'config.yaml') + --mode {simple,full} the mode to run the archiver in + -s, --store, --no-store + Store the created config in the config file + --module_paths MODULE_PATHS [MODULE_PATHS ...] + additional paths to search for modules + --feeders STEPS.FEEDERS [STEPS.FEEDERS ...] + the feeders to use + --enrichers STEPS.ENRICHERS [STEPS.ENRICHERS ...] + the enrichers to use + --extractors STEPS.EXTRACTORS [STEPS.EXTRACTORS ...] + the extractors to use + --databases STEPS.DATABASES [STEPS.DATABASES ...] + the databases to use + --storages STEPS.STORAGES [STEPS.STORAGES ...] + the storages to use + --formatters STEPS.FORMATTERS [STEPS.FORMATTERS ...] + the formatter to use + --authentication AUTHENTICATION + A dictionary of sites and their authentication methods (token, username etc.) that extractors can use to log into a website. If passing this on the command line, use a JSON string. You may + also pass a path to a valid JSON/YAML file which will be parsed. + --logging.level {INFO,DEBUG,ERROR,WARNING} + the logging level to use + --logging.file LOGGING.FILE + the logging file to write to + --logging.rotation LOGGING.ROTATION + the logging rotation to use + +Wayback Machine Enricher: + Submits the current URL to the Wayback Machine for archiving and returns either a job ID or the... + + --wayback_extractor_enricher.timeout TIMEOUT + seconds to wait for successful archive confirmation from wayback, if more than this passes the result contains the job_id so the status can later be checked manually. + --wayback_extractor_enricher.if_not_archived_within IF_NOT_ARCHIVED_WITHIN + only tell wayback to archive if no archive is available before the number of seconds specified, use None to ignore this option. For more information: + https://docs.google.com/document/d/1Nsv52MvSjbLb2PCpHlat0gkzw0EvtSgpKHu4mk0MnrA + --wayback_extractor_enricher.key KEY + wayback API key. to get credentials visit https://archive.org/account/s3.php + --wayback_extractor_enricher.secret SECRET + wayback API secret. to get credentials visit https://archive.org/account/s3.php + --wayback_extractor_enricher.proxy_http PROXY_HTTP + http proxy to use for wayback requests, eg http://proxy-user:password@proxy-ip:port + --wayback_extractor_enricher.proxy_https PROXY_HTTPS + https proxy to use for wayback requests, eg https://proxy-user:password@proxy-ip:port +``` + diff --git a/docs/source/installation/installation.md b/docs/source/installation/installation.md new file mode 100644 index 0000000..eff0720 --- /dev/null +++ b/docs/source/installation/installation.md @@ -0,0 +1,56 @@ +# Installation + +There are 3 main ways to use the auto-archiver. We recommend the 'docker' method for most uses. This installs all the requirements in one command. + +1. Easiest (recommended): [via docker](#installing-with-docker) +2. Local Install: [using pip](#installing-locally-with-pip) +3. Developer Install: [see the developer guidelines](../development/developer_guidelines) + +## 1. Installing with Docker + +[![dockeri.co](https://dockerico.blankenship.io/image/bellingcat/auto-archiver)](https://hub.docker.com/r/bellingcat/auto-archiver) + +Docker works like a virtual machine running inside your computer, making installation simple. You'll need to first set up Docker, and then download the Auto Archiver 'image': + + +**a) Download and install docker** + +Go to the [Docker website](https://docs.docker.com/get-docker/) and download right version for your operating system. + +**b) Pull the Auto Archiver docker image** + +Open your command line terminal, and copy-paste / type: + +```bash +docker pull bellingcat/auto-archiver +``` + +This will download the docker image, which may take a while. + +That's it, all done! You're now ready to set up [your configuration file](configurations.md). Or, if you want to use the recommended defaults, then you can [run Auto Archiver immediately](setup.md#running-a-docker-install). + +------------ + +## 2. Installing Locally with Pip + +1. Make sure you have python 3.10 or higher installed +2. Install the package with your preferred package manager: `pip/pipenv/conda install auto-archiver` or `poetry add auto-archiver` +3. Test it's installed with `auto-archiver --help` +4. Install other local dependency requirements (for example `ffmpeg`, `firefox`) + +After this, you're ready to set up your [your configuration file](configurations.md), or if you want to use the recommended defaults, then you can [run Auto Archiver immediately](setup.md#running-a-local-install). + +### Installing Local Requirements + +If using the local installation method, you will also need to install the following dependencies locally: + +1.[ffmpeg](https://www.ffmpeg.org/) - for handling of downloaded videos +2. [firefox](https://www.mozilla.org/en-US/firefox/new/) and [geckodriver](https://github.com/mozilla/geckodriver/releases) on a path folder like `/usr/local/bin` - for taking webpage screenshots with the screenshot enricher +3. (optional) [fonts-noto](https://fonts.google.com/noto) to deal with multiple unicode characters during selenium/geckodriver's screenshots: `sudo apt install fonts-noto -y`. +4. [Browsertrix Crawler docker image](https://hub.docker.com/r/webrecorder/browsertrix-crawler) for the WACZ enricher/archiver + + + +## Developer Install + +[See the developer guidelines](../development/developer_guidelines) \ No newline at end of file diff --git a/docs/source/installation/requirements.md b/docs/source/installation/requirements.md new file mode 100644 index 0000000..b820272 --- /dev/null +++ b/docs/source/installation/requirements.md @@ -0,0 +1,14 @@ +# Requirements + +Using the Auto Archiver is very simple, but ideally you have some familiarity with using the command line to run programs. ([Command line crash course](https://developer.mozilla.org/en-US/docs/Learn_web_development/Getting_started/Environment_setup/Command_line)). + +### System Requirements + +* Auto Archiver works on any Windows, macOS and Linux computer +* If you're using the **local install** method, then you should make sure to have python3.10+ installed + +### Storage Requirements + +By default, Auto Archiver uses your local computer storage for any downloaded media (videos, images etc.). If you're downloading large files, this may take up a lot of your local computer's space (more than 5GB of space). + +If your storage space is limited, then you may want to set up an [alternative storage method](../modules/storage.md) for your media. \ No newline at end of file diff --git a/docs/source/installation/settings.html b/docs/source/installation/settings.html new file mode 100644 index 0000000..915ee8d --- /dev/null +++ b/docs/source/installation/settings.html @@ -0,0 +1,48685 @@ + + +
diff --git a/docs/source/installation/setup.md b/docs/source/installation/setup.md new file mode 100644 index 0000000..e5c96a6 --- /dev/null +++ b/docs/source/installation/setup.md @@ -0,0 +1,77 @@ +# Getting Started + +```{toctree} +:maxdepth: 1 +:hidden: + +installation.md +configurations.md +config_editor.md +authentication.md +requirements.md +config_cheatsheet.md +``` + +## Getting Started + +To get started with Auto Archiver, there are 3 main steps you need to complete. + +1. [Install Auto Archiver](installation.md) +2. [Setup up your configuration](configurations.md) (if you are ok with the default settings, you can skip this step) +3. Run the archiving process + +The way you run the Auto Archiver depends on how you installed it (docker install or local install) + +### Running a Docker Install + +If you installed Auto Archiver using docker, open up your terminal, and copy-paste / type the following command: + +```bash +docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver + ``` + +breaking this command down: + 1. `docker run` tells docker to start a new container (an instance of the image) + 2. `--rm` makes sure this container is removed after execution (less garbage locally) + 3. `-v $PWD/secrets:/app/secrets` - your secrets folder with settings + 1. `-v` is a volume flag which means a folder that you have on your computer will be connected to a folder inside the docker container + 2. `$PWD/secrets` points to a `secrets/` folder in your current working directory (where your console points to), we use this folder as a best practice to hold all the secrets/tokens/passwords/... you use + 3. `/app/secrets` points to the path the docker container where this image can be found + 4. `-v $PWD/local_archive:/app/local_archive` - (optional) if you use local_storage + 1. `-v` same as above, this is a volume instruction + 2. `$PWD/local_archive` is a folder `local_archive/` in case you want to archive locally and have the files accessible outside docker + 3. `/app/local_archive` is a folder inside docker that you can reference in your orchestration.yml file + +### Example invocations + +The invocations below will run the auto-archiver Docker image using a configuration file that you have specified + +```bash +# Have auto-archiver run with the default settings, generating a settings file in ./secrets/orchestration.yaml +docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver + +# uses the same configuration, but with the `gsheet_feeder`, a header on row 2 and with some different column names +# notice that columns is a dictionary so you need to pass it as JSON and it will override only the values provided +docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver --feeders=gsheet_feeder --gsheet_feeder.sheet="use it on another sheets doc" --gsheet_feeder.header=2 --gsheet_feeder.columns='{"url": "link"}' +# Runs auto-archiver for the first time, but in 'full' mode, enabling all modules to get a full settings file +docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver --mode full +``` + +------------ + +### Running a Local Install + +### Example invocations + +Once all your [local requirements](#installing-local-requirements) are correctly installed, the + +```bash +# all the configurations come from ./secrets/orchestration.yaml +auto-archiver --config secrets/orchestration.yaml +# uses the same configurations but for another google docs sheet +# with a header on row 2 and with some different column names +# notice that columns is a dictionary so you need to pass it as JSON and it will override only the values provided +auto-archiver --config secrets/orchestration.yaml --gsheet_feeder.sheet="use it on another sheets doc" --gsheet_feeder.header=2 --gsheet_feeder.columns='{"url": "link"}' +# all the configurations come from orchestration.yaml and specifies that s3 files should be private +auto-archiver --config secrets/orchestration.yaml --s3_storage.private=1 +``` diff --git a/docs/source/modules/database.md b/docs/source/modules/database.md new file mode 100644 index 0000000..3ecd2e2 --- /dev/null +++ b/docs/source/modules/database.md @@ -0,0 +1,15 @@ +# Database Modules + +Database modules are used to store the status and results of the extraction and enrichment processes somewhere. The database modules are responsible for creating and managing entires for each item that has been processed. + +The default (enabled) databases are the CSV Database and the Console Database. + +```{include} autogen/database.md +``` + +```{toctree} +:maxdepth: 1 +:hidden: +:glob: +autogen/database/* +``` \ No newline at end of file diff --git a/docs/source/modules/enricher.md b/docs/source/modules/enricher.md new file mode 100644 index 0000000..a145a1d --- /dev/null +++ b/docs/source/modules/enricher.md @@ -0,0 +1,14 @@ +# Enricher Modules + +Enricher modules are used to add additional information to the items that have been extracted. Common enrichment tasks include adding metadata to items, such as the hash of the item, a screenshot of the webpage when the item was extracted, or general metadata like the date and time the item was extracted. + + +```{include} autogen/enricher.md +``` + +```{toctree} +:maxdepth: 1 +:hidden: +:glob: +autogen/enricher/* +``` \ No newline at end of file diff --git a/docs/source/modules/extractor.md b/docs/source/modules/extractor.md new file mode 100644 index 0000000..e6375db --- /dev/null +++ b/docs/source/modules/extractor.md @@ -0,0 +1,18 @@ +# Extractor Modules + +Extractor modules are used to extract the content of a given URL. Typically, one extractor will work for one website or platform (e.g. a Telegram extractor or an Instagram), however, there are several wide-ranging extractors which work for a wide range of websites. + +Extractors that are able to extract content from a wide range of websites include: +1. Generic Extractor: parses videos and images on sites using the powerful yt-dlp library. +2. Wayback Machine Extractor: sends pages to the Wayback machine for archiving, and stores the link. +3. WACZ Extractor: runs a web browser to 'browse' the URL and save a copy of the page in WACZ format. + +```{include} autogen/extractor.md +``` + +```{toctree} +:maxdepth: 1 +:hidden: +:glob: +autogen/extractor/* +``` \ No newline at end of file diff --git a/docs/source/modules/feeder.md b/docs/source/modules/feeder.md new file mode 100644 index 0000000..dcac749 --- /dev/null +++ b/docs/source/modules/feeder.md @@ -0,0 +1,20 @@ +# Feeder Modules + +Feeder modules are used to feed URLs into the Auto Archiver for processing. Feeders can take these URLs from a variety of sources, such as a file, a database, or the command line. + +The default feeder is the command line feeder (`cli_feeder`), which allows you to input URLs directly into `auto-archiver` from the command line. + +Command line feeder usage: +```{code} bash +auto-archiver [options] -- URL1 URL2 ... +``` + +```{include} autogen/feeder.md +``` + +```{toctree} +:maxdepth: 1 +:glob: +:hidden: +autogen/feeder/* +``` \ No newline at end of file diff --git a/docs/source/modules/formatter.md b/docs/source/modules/formatter.md new file mode 100644 index 0000000..7d5713c --- /dev/null +++ b/docs/source/modules/formatter.md @@ -0,0 +1,13 @@ +# Formatter Modules + +Formatter modules are used to format the data extracted from a URL into a specific format. Currently the most widely-used formatter is the HTML formatter, which formats the data into an easily viewable HTML page. + +```{include} autogen/formatter.md +``` + +```{toctree} +:maxdepth: 1 +:hidden: +:glob: +autogen/formatter/* +``` \ No newline at end of file diff --git a/docs/source/modules/storage.md b/docs/source/modules/storage.md new file mode 100644 index 0000000..d4a2f99 --- /dev/null +++ b/docs/source/modules/storage.md @@ -0,0 +1,15 @@ +# Storage Modules + +Storage modules are used to store the data extracted from a URL in a persistent location. This can be on your local hard disk, or on a remote server (e.g. S3 or Google Drive). + +The default is to store the files downloaded (e.g. images, videos) in a local directory. + +```{include} autogen/storage.md +``` + +```{toctree} +:maxdepth: 1 +:hidden: +:glob: +autogen/storage/* +``` \ No newline at end of file diff --git a/docs/source/overview.md b/docs/source/overview.md new file mode 100644 index 0000000..57e8d60 --- /dev/null +++ b/docs/source/overview.md @@ -0,0 +1,16 @@ + +```{include} ../../README.md +``` + +```{toctree} +:maxdepth: 2 +:hidden: +:caption: Contents: + +Overview +installation/installation.rst +core_modules.md +how_to +development/developer_guidelines +autoapi/index.rst +``` \ No newline at end of file diff --git a/docs/source/user_guidelines.rst b/docs/source/user_guidelines.rst deleted file mode 100644 index 93fb2f2..0000000 --- a/docs/source/user_guidelines.rst +++ /dev/null @@ -1,11 +0,0 @@ - -User Guidelines -=============== - -This section of the documentation provides guidelines for users who want to use the tool, -without needing to modify the code. -To see the developer guidelines, see :ref:`developer_guidelines`. - -.. note:: - - This is a work in progress. diff --git a/example.orchestration.yaml b/example.orchestration.yaml deleted file mode 100644 index f1eed2a..0000000 --- a/example.orchestration.yaml +++ /dev/null @@ -1,156 +0,0 @@ -steps: - # only 1 feeder allowed - feeder: gsheet_feeder # defaults to cli_feeder - archivers: # order matters, uncomment to activate - - bluesky_archiver - # - vk_archiver - # - telethon_archiver - # - telegram_archiver - # - twitter_archiver - # - twitter_api_archiver - # - instagram_api_archiver - # - instagram_tbot_archiver - # - instagram_archiver - # - tiktok_archiver - - youtubedl_archiver - # - wayback_archiver_enricher - # - wacz_archiver_enricher - enrichers: - - hash_enricher - # - meta_enricher - # - metadata_enricher - # - screenshot_enricher - # - pdq_hash_enricher - # - ssl_enricher - # - timestamping_enricher - # - whisper_enricher - # - thumbnail_enricher - # - wayback_archiver_enricher - # - wacz_archiver_enricher - # - pdq_hash_enricher # if you want to calculate hashes for thumbnails, include this after thumbnail_enricher - formatter: html_formatter # defaults to mute_formatter - storages: - - local_storage - # - s3_storage - # - gdrive_storage - databases: - - console_db - # - csv_db - # - gsheet_db - # - mongo_db - -configurations: - gsheet_feeder: - sheet: "your sheet name" - header: 1 - service_account: "secrets/service_account.json" - # allow_worksheets: "only parse this worksheet" - # block_worksheets: "blocked sheet 1,blocked sheet 2" - use_sheet_names_in_stored_paths: false - columns: - url: link - status: archive status - folder: destination folder - archive: archive location - date: archive date - thumbnail: thumbnail - timestamp: upload timestamp - title: upload title - text: textual content - screenshot: screenshot - hash: hash - pdq_hash: perceptual hashes - wacz: wacz - replaywebpage: replaywebpage - instagram_tbot_archiver: - api_id: "TELEGRAM_BOT_API_ID" - api_hash: "TELEGRAM_BOT_API_HASH" - # session_file: "secrets/anon" - telethon_archiver: - api_id: "TELEGRAM_BOT_API_ID" - api_hash: "TELEGRAM_BOT_API_HASH" - # session_file: "secrets/anon" - join_channels: false - channel_invites: # if you want to archive from private channels - - invite: https://t.me/+123456789 - id: 0000000001 - - invite: https://t.me/+123456788 - id: 0000000002 - - twitter_api_archiver: - # either bearer_token only - bearer_token: "TWITTER_BEARER_TOKEN" - # OR all of the below - # consumer_key: "" - # consumer_secret: "" - # access_token: "" - # access_secret: "" - instagram_archiver: - username: "INSTAGRAM_USERNAME" - password: "INSTAGRAM_PASSWORD" - # session_file: "secrets/instaloader.session" - - vk_archiver: - username: "or phone number" - password: "vk pass" - session_file: "secrets/vk_config.v2.json" - - youtubedl_archiver: - subtitles: true - # use one of the following two methods to authenticate in youtube - either provide a cookies file or use the cookies of the given browser - # for more information, see https://github.com/yt-dlp/yt-dlp/wiki/FAQ#how-do-i-pass-cookies-to-yt-dlp - # cookie_file: "secrets/youtube_cookies.txt" - # cookies_from_browser: firefox - # proxy: socks5://proxy-user:password@proxy-ip:port - - screenshot_enricher: - width: 1280 - height: 2300 - # to save as pdf, uncomment the following lines and adjust the print options - # save_to_pdf: true - # print_options: - # for all options see https://www.selenium.dev/selenium/docs/api/py/webdriver/selenium.webdriver.common.print_page_options.html - # background: true - # orientation: "portrait" - # scale: 1 - # page_width: 8.5in - # page_height: 11in - # margin_top: 0.4in - # margin_bottom: 0.4in - # margin_left: 0.4in - # margin_right: 0.4in - # page_ranges: "" - # shrink_to_fit: true - - wayback_archiver_enricher: - timeout: 10 - key: "wayback key" - secret: "wayback secret" - hash_enricher: - algorithm: "SHA3-512" # can also be SHA-256 - wacz_archiver_enricher: - profile: secrets/profile.tar.gz - local_storage: - save_to: "./local_archive" - save_absolute: true - filename_generator: static - path_generator: flat - s3_storage: - bucket: your-bucket-name - region: reg1 - key: S3_KEY - secret: S3_SECRET - endpoint_url: "https://{region}.digitaloceanspaces.com" - cdn_url: "https://{bucket}.{region}.cdn.digitaloceanspaces.com/{key}" - # if private:true S3 urls will not be readable online - private: false - # with 'random' you can generate a random UUID for the URL instead of a predictable path, useful to still have public but unlisted files, alternative is 'default' or not omitted from config - key_path: random - gdrive_storage: - path_generator: url - filename_generator: random - root_folder_id: folder_id_from_url - oauth_token: secrets/gd-token.json # needs to be generated with scripts/create_update_gdrive_oauth_token.py - service_account: "secrets/service_account.json" - csv_db: - csv_file: "./local_archive/db.csv" diff --git a/poetry.lock b/poetry.lock index 8fb48ec..f59d5c9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,5 +1,24 @@ # This file is automatically @generated by Poetry 2.0.1 and should not be changed by hand. +[[package]] +name = "accessible-pygments" +version = "0.0.5" +description = "A collection of accessible pygments styles" +optional = false +python-versions = ">=3.9" +groups = ["docs"] +files = [ + {file = "accessible_pygments-0.0.5-py3-none-any.whl", hash = "sha256:88ae3211e68a1d0b011504b2ffc1691feafce124b845bd072ab6f9f66f34d4b7"}, + {file = "accessible_pygments-0.0.5.tar.gz", hash = "sha256:40918d3e6a2b619ad424cb91e556bd3bd8865443d9f22f1dcdf79e33c8046872"}, +] + +[package.dependencies] +pygments = ">=1.5" + +[package.extras] +dev = ["pillow", "pkginfo (>=1.10)", "playwright", "pre-commit", "setuptools", "twine (>=5.0)"] +tests = ["hypothesis", "pytest"] + [[package]] name = "alabaster" version = "1.0.0" @@ -84,14 +103,14 @@ tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"] [[package]] name = "authlib" -version = "1.4.0" +version = "1.5.1" description = "The ultimate Python library in building OAuth and OpenID Connect servers and clients." optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "Authlib-1.4.0-py2.py3-none-any.whl", hash = "sha256:4bb20b978c8b636222b549317c1815e1fe62234fc1c5efe8855d84aebf3a74e3"}, - {file = "authlib-1.4.0.tar.gz", hash = "sha256:1c1e6608b5ed3624aeeee136ca7f8c120d6f51f731aa152b153d54741840e1f2"}, + {file = "authlib-1.5.1-py2.py3-none-any.whl", hash = "sha256:8408861cbd9b4ea2ff759b00b6f02fd7d81ac5a56d0b2b22c08606c6049aae11"}, + {file = "authlib-1.5.1.tar.gz", hash = "sha256:5cbc85ecb0667312c1cdc2f9095680bb735883b123fb509fde1e65b1c5df972e"}, ] [package.dependencies] @@ -115,33 +134,34 @@ tomli = {version = "*", markers = "python_version < \"3.11\""} [[package]] name = "babel" -version = "2.16.0" +version = "2.17.0" description = "Internationalization utilities" optional = false python-versions = ">=3.8" groups = ["docs"] files = [ - {file = "babel-2.16.0-py3-none-any.whl", hash = "sha256:368b5b98b37c06b7daf6696391c3240c938b37767d4584413e8438c5c435fa8b"}, - {file = "babel-2.16.0.tar.gz", hash = "sha256:d1f3554ca26605fe173f3de0c65f750f5a42f924499bf134de6423582298e316"}, + {file = "babel-2.17.0-py3-none-any.whl", hash = "sha256:4d0b53093fdfb4b21c92b5213dba5a1b23885afa8383709427046b21c366e5f2"}, + {file = "babel-2.17.0.tar.gz", hash = "sha256:0c54cffb19f690cdcc52a3b50bcbf71e07a808d1c80d549f2459b9d2cf0afb9d"}, ] [package.extras] -dev = ["freezegun (>=1.0,<2.0)", "pytest (>=6.0)", "pytest-cov"] +dev = ["backports.zoneinfo", "freezegun (>=1.0,<2.0)", "jinja2 (>=3.0)", "pytest (>=6.0)", "pytest-cov", "pytz", "setuptools", "tzdata"] [[package]] name = "beautifulsoup4" -version = "4.12.3" +version = "4.13.3" description = "Screen-scraping library" optional = false -python-versions = ">=3.6.0" +python-versions = ">=3.7.0" groups = ["main", "docs"] files = [ - {file = "beautifulsoup4-4.12.3-py3-none-any.whl", hash = "sha256:b80878c9f40111313e55da8ba20bdba06d8fa3969fc68304167741bbf9e082ed"}, - {file = "beautifulsoup4-4.12.3.tar.gz", hash = "sha256:74e3d1928edc070d21748185c46e3fb33490f22f52a3addee9aee0f4f7781051"}, + {file = "beautifulsoup4-4.13.3-py3-none-any.whl", hash = "sha256:99045d7d3f08f91f0d656bc9b7efbae189426cd913d830294a15eefa0ea4df16"}, + {file = "beautifulsoup4-4.13.3.tar.gz", hash = "sha256:1bd32405dacc920b42b83ba01644747ed77456a65760e285fbc47633ceddaf8b"}, ] [package.dependencies] soupsieve = ">1.2" +typing-extensions = ">=4.0.0" [package.extras] cchardet = ["cchardet"] @@ -152,18 +172,18 @@ lxml = ["lxml"] [[package]] name = "boto3" -version = "1.36.6" +version = "1.37.8" description = "The AWS SDK for Python" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "boto3-1.36.6-py3-none-any.whl", hash = "sha256:6d473f0f340d02b4e9ad5b8e68786a09728101a8b950231b89ebdaf72b6dca21"}, - {file = "boto3-1.36.6.tar.gz", hash = "sha256:b36feae061dc0793cf311468956a0a9e99215ce38bc99a1a4e55a5b105f16297"}, + {file = "boto3-1.37.8-py3-none-any.whl", hash = "sha256:b9f506e08c9f54687d6c073ef1c550a24a62cc2d1e0bc7cda9f13112a38818bf"}, + {file = "boto3-1.37.8.tar.gz", hash = "sha256:9448f4a079189e19c3253cfdc5b8ef6dc51a3b82431e8347a51f4c1b2d9dab42"}, ] [package.dependencies] -botocore = ">=1.36.6,<1.37.0" +botocore = ">=1.37.8,<1.38.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.11.0,<0.12.0" @@ -172,14 +192,14 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.36.6" +version = "1.37.8" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "botocore-1.36.6-py3-none-any.whl", hash = "sha256:f77bbbb03fb420e260174650fb5c0cc142ec20a96967734eed2b0ef24334ef34"}, - {file = "botocore-1.36.6.tar.gz", hash = "sha256:4864c53d638da191a34daf3ede3ff1371a3719d952cc0c6bd24ce2836a38dd77"}, + {file = "botocore-1.37.8-py3-none-any.whl", hash = "sha256:a6c94f33de12f4b10b10684019e554c980469b8394c6d82448a738cbd8452cef"}, + {file = "botocore-1.37.8.tar.gz", hash = "sha256:b5825e08dd3e25642aa22a0d7d92bf81fef1ef857117e4155f923bbccf5aba63"}, ] [package.dependencies] @@ -188,7 +208,7 @@ python-dateutil = ">=2.1,<3.0.0" urllib3 = {version = ">=1.25.4,<2.2.0 || >2.2.0,<3", markers = "python_version >= \"3.10\""} [package.extras] -crt = ["awscrt (==0.23.4)"] +crt = ["awscrt (==0.23.8)"] [[package]] name = "brotli" @@ -343,26 +363,26 @@ beautifulsoup4 = "*" [[package]] name = "cachetools" -version = "5.5.1" +version = "5.5.2" description = "Extensible memoizing collections and decorators" optional = false python-versions = ">=3.7" groups = ["main"] files = [ - {file = "cachetools-5.5.1-py3-none-any.whl", hash = "sha256:b76651fdc3b24ead3c648bbdeeb940c1b04d365b38b4af66788f9ec4a81d42bb"}, - {file = "cachetools-5.5.1.tar.gz", hash = "sha256:70f238fbba50383ef62e55c6aff6d9673175fe59f7c6782c7a0b9e38f4a9df95"}, + {file = "cachetools-5.5.2-py3-none-any.whl", hash = "sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a"}, + {file = "cachetools-5.5.2.tar.gz", hash = "sha256:1a661caa9175d26759571b2e19580f9d6393969e5dfca11fdb1f947a23e640d4"}, ] [[package]] name = "certifi" -version = "2024.12.14" +version = "2025.1.31" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.6" groups = ["main", "docs"] files = [ - {file = "certifi-2024.12.14-py3-none-any.whl", hash = "sha256:1275f7a45be9464efc1173084eaa30f866fe2e47d389406136d332ed4967ec56"}, - {file = "certifi-2024.12.14.tar.gz", hash = "sha256:b650d30f370c2b724812bee08008be0c4163b163ddaec3f2546c1caf65f191db"}, + {file = "certifi-2025.1.31-py3-none-any.whl", hash = "sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe"}, + {file = "certifi-2025.1.31.tar.gz", hash = "sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651"}, ] [[package]] @@ -655,26 +675,26 @@ typing-inspect = ">=0.4.0,<1" [[package]] name = "dateparser" -version = "1.2.0" +version = "1.2.1" description = "Date parsing library designed to parse dates from HTML pages" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" groups = ["main"] files = [ - {file = "dateparser-1.2.0-py2.py3-none-any.whl", hash = "sha256:0b21ad96534e562920a0083e97fd45fa959882d4162acc358705144520a35830"}, - {file = "dateparser-1.2.0.tar.gz", hash = "sha256:7975b43a4222283e0ae15be7b4999d08c9a70e2d378ac87385b1ccf2cffbbb30"}, + {file = "dateparser-1.2.1-py3-none-any.whl", hash = "sha256:bdcac262a467e6260030040748ad7c10d6bacd4f3b9cdb4cfd2251939174508c"}, + {file = "dateparser-1.2.1.tar.gz", hash = "sha256:7e4919aeb48481dbfc01ac9683c8e20bfe95bb715a38c1e9f6af889f4f30ccc3"}, ] [package.dependencies] -python-dateutil = "*" -pytz = "*" -regex = "<2019.02.19 || >2019.02.19,<2021.8.27 || >2021.8.27" -tzlocal = "*" +python-dateutil = ">=2.7.0" +pytz = ">=2024.2" +regex = ">=2015.06.24,<2019.02.19 || >2019.02.19,<2021.8.27 || >2021.8.27" +tzlocal = ">=0.2" [package.extras] -calendars = ["convertdate", "hijri-converter"] -fasttext = ["fasttext"] -langdetect = ["langdetect"] +calendars = ["convertdate (>=2.2.1)", "hijridate"] +fasttext = ["fasttext (>=0.9.1)", "numpy (>=1.19.3,<2)"] +langdetect = ["langdetect (>=1.0.0)"] [[package]] name = "docutils" @@ -722,24 +742,6 @@ future = "*" [package.extras] dev = ["Sphinx (==2.1.0)", "future (==0.17.1)", "numpy (==1.16.4)", "pytest (==4.6.1)", "pytest-mock (==1.10.4)", "tox (==3.12.1)"] -[[package]] -name = "furo" -version = "2024.8.6" -description = "A clean customisable Sphinx documentation theme." -optional = false -python-versions = ">=3.8" -groups = ["docs"] -files = [ - {file = "furo-2024.8.6-py3-none-any.whl", hash = "sha256:6cd97c58b47813d3619e63e9081169880fbe331f0ca883c871ff1f3f11814f5c"}, - {file = "furo-2024.8.6.tar.gz", hash = "sha256:b63e4cee8abfc3136d3bc03a3d45a76a850bada4d6374d24c1716b0e01394a01"}, -] - -[package.dependencies] -beautifulsoup4 = "*" -pygments = ">=2.7" -sphinx = ">=6.0,<9.0" -sphinx-basic-ng = ">=1.0.0.beta2" - [[package]] name = "future" version = "1.0.0" @@ -754,14 +756,14 @@ files = [ [[package]] name = "google-api-core" -version = "2.24.0" +version = "2.24.1" description = "Google API client core library" optional = false python-versions = ">=3.7" groups = ["main"] files = [ - {file = "google_api_core-2.24.0-py3-none-any.whl", hash = "sha256:10d82ac0fca69c82a25b3efdeefccf6f28e02ebb97925a8cce8edbfe379929d9"}, - {file = "google_api_core-2.24.0.tar.gz", hash = "sha256:e255640547a597a4da010876d333208ddac417d60add22b6851a0c66a831fcaf"}, + {file = "google_api_core-2.24.1-py3-none-any.whl", hash = "sha256:bc78d608f5a5bf853b80bd70a795f703294de656c096c0968320830a4bc280f1"}, + {file = "google_api_core-2.24.1.tar.gz", hash = "sha256:f8b36f5456ab0dd99a1b693a40a31d1e7757beea380ad1b38faaf8941eae9d8a"}, ] [package.dependencies] @@ -779,14 +781,14 @@ grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"] [[package]] name = "google-api-python-client" -version = "2.159.0" +version = "2.163.0" description = "Google API Client Library for Python" optional = false python-versions = ">=3.7" groups = ["main"] files = [ - {file = "google_api_python_client-2.159.0-py2.py3-none-any.whl", hash = "sha256:baef0bb631a60a0bd7c0bf12a5499e3a40cd4388484de7ee55c1950bf820a0cf"}, - {file = "google_api_python_client-2.159.0.tar.gz", hash = "sha256:55197f430f25c907394b44fa078545ffef89d33fd4dca501b7db9f0d8e224bd6"}, + {file = "google_api_python_client-2.163.0-py2.py3-none-any.whl", hash = "sha256:080e8bc0669cb4c1fb8efb8da2f5b91a2625d8f0e7796cfad978f33f7016c6c4"}, + {file = "google_api_python_client-2.163.0.tar.gz", hash = "sha256:88dee87553a2d82176e2224648bf89272d536c8f04dcdda37ef0a71473886dd7"}, ] [package.dependencies] @@ -858,14 +860,14 @@ tool = ["click (>=6.0.0)"] [[package]] name = "googleapis-common-protos" -version = "1.66.0" +version = "1.69.1" description = "Common protobufs used in Google APIs" optional = false python-versions = ">=3.7" groups = ["main"] files = [ - {file = "googleapis_common_protos-1.66.0-py2.py3-none-any.whl", hash = "sha256:d7abcd75fabb2e0ec9f74466401f6c119a0b498e27370e9be4c94cb7e382b8ed"}, - {file = "googleapis_common_protos-1.66.0.tar.gz", hash = "sha256:c3e7b33d15fdca5374cc0a7346dd92ffa847425cc4ea941d970f13680052ec8c"}, + {file = "googleapis_common_protos-1.69.1-py2.py3-none-any.whl", hash = "sha256:4077f27a6900d5946ee5a369fab9c8ded4c0ef1c6e880458ea2f70c14f7b70d5"}, + {file = "googleapis_common_protos-1.69.1.tar.gz", hash = "sha256:e20d2d8dda87da6fe7340afbbdf4f0bcb4c8fae7e6cadf55926c31f946b0b9b1"}, ] [package.dependencies] @@ -876,14 +878,14 @@ grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"] [[package]] name = "gspread" -version = "6.1.4" +version = "6.2.0" description = "Google Spreadsheets Python API" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "gspread-6.1.4-py3-none-any.whl", hash = "sha256:c34781c426031a243ad154952b16f21ac56a5af90687885fbee3d1fba5280dcd"}, - {file = "gspread-6.1.4.tar.gz", hash = "sha256:b8eec27de7cadb338bb1b9f14a9be168372dee8965c0da32121816b5050ac1de"}, + {file = "gspread-6.2.0-py3-none-any.whl", hash = "sha256:7fa1a11e1ecacc6c5946fa016be05941baca8540404314f59aec963dd8ae5db3"}, + {file = "gspread-6.2.0.tar.gz", hash = "sha256:bc3d02d1c39e0b40bfc8035b4fec407aa71a17f343fc81cc7e3f75bfa6555de6"}, ] [package.dependencies] @@ -976,14 +978,14 @@ browser-cookie3 = ["browser_cookie3 (>=0.19.1)"] [[package]] name = "jinja2" -version = "3.1.5" +version = "3.1.6" description = "A very fast and expressive template engine." optional = false python-versions = ">=3.7" groups = ["main", "docs"] files = [ - {file = "jinja2-3.1.5-py3-none-any.whl", hash = "sha256:aba0f4dc9ed8013c424088f68a5c226f7d6097ed89b246d7749c2ec4175c6adb"}, - {file = "jinja2-3.1.5.tar.gz", hash = "sha256:8fefff8dc3034e27bb80d67c671eb8a9bc424c0ef4c0826edbff304cceff43bb"}, + {file = "jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67"}, + {file = "jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d"}, ] [package.dependencies] @@ -1019,6 +1021,27 @@ files = [ [package.dependencies] attrs = ">=19.2.0" +[[package]] +name = "linkify-it-py" +version = "2.0.3" +description = "Links recognition library with FULL unicode support." +optional = false +python-versions = ">=3.7" +groups = ["docs"] +files = [ + {file = "linkify-it-py-2.0.3.tar.gz", hash = "sha256:68cda27e162e9215c17d786649d1da0021a451bdc436ef9e0fa0ba5234b9b048"}, + {file = "linkify_it_py-2.0.3-py3-none-any.whl", hash = "sha256:6bcbc417b0ac14323382aef5c5192c0075bf8a9d6b41820a2b66371eac6b6d79"}, +] + +[package.dependencies] +uc-micro-py = "*" + +[package.extras] +benchmark = ["pytest", "pytest-benchmark"] +dev = ["black", "flake8", "isort", "pre-commit", "pyproject-flake8"] +doc = ["myst-parser", "sphinx", "sphinx-book-theme"] +test = ["coverage", "pytest", "pytest-cov"] + [[package]] name = "loguru" version = "0.7.3" @@ -1136,14 +1159,14 @@ files = [ [[package]] name = "marshmallow" -version = "3.26.0" +version = "3.26.1" description = "A lightweight library for converting complex datatypes to and from native Python datatypes." optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "marshmallow-3.26.0-py3-none-any.whl", hash = "sha256:1287bca04e6a5f4094822ac153c03da5e214a0a60bcd557b140f3e66991b8ca1"}, - {file = "marshmallow-3.26.0.tar.gz", hash = "sha256:eb36762a1cc76d7abf831e18a3a1b26d3d481bbc74581b8e532a3d3a8115e1cb"}, + {file = "marshmallow-3.26.1-py3-none-any.whl", hash = "sha256:3350409f20a70a7e4e11a27661187b77cdcaeb20abca41c1454fe33636bea09c"}, + {file = "marshmallow-3.26.1.tar.gz", hash = "sha256:e6d8affb6cb61d39d26402096dc0aee12d5a26d490a121f118d2e81dc0719dc6"}, ] [package.dependencies] @@ -1212,14 +1235,14 @@ files = [ [[package]] name = "myst-parser" -version = "4.0.0" +version = "4.0.1" description = "An extended [CommonMark](https://spec.commonmark.org/) compliant parser," optional = false python-versions = ">=3.10" groups = ["docs"] files = [ - {file = "myst_parser-4.0.0-py3-none-any.whl", hash = "sha256:b9317997552424448c6096c2558872fdb6f81d3ecb3a40ce84a7518798f3f28d"}, - {file = "myst_parser-4.0.0.tar.gz", hash = "sha256:851c9dfb44e36e56d15d05e72f02b80da21a9e0d07cba96baf5e2d476bb91531"}, + {file = "myst_parser-4.0.1-py3-none-any.whl", hash = "sha256:9134e88959ec3b5780aedf8a99680ea242869d012e8821db3126d427edc9c95d"}, + {file = "myst_parser-4.0.1.tar.gz", hash = "sha256:5cfea715e4f3574138aecbf7d54132296bfd72bb614d31168f48c477a830a7c4"}, ] [package.dependencies] @@ -1231,10 +1254,10 @@ pyyaml = "*" sphinx = ">=7,<9" [package.extras] -code-style = ["pre-commit (>=3.0,<4.0)"] +code-style = ["pre-commit (>=4.0,<5.0)"] linkify = ["linkify-it-py (>=2.0,<3.0)"] rtd = ["ipython", "sphinx (>=7)", "sphinx-autodoc2 (>=0.5.0,<0.6.0)", "sphinx-book-theme (>=1.1,<2.0)", "sphinx-copybutton", "sphinx-design", "sphinx-pyscript", "sphinx-tippy (>=0.4.3)", "sphinx-togglebutton", "sphinxext-opengraph (>=0.9.0,<0.10.0)", "sphinxext-rediraffe (>=0.2.7,<0.3.0)"] -testing = ["beautifulsoup4", "coverage[toml]", "defusedxml", "pytest (>=8,<9)", "pytest-cov", "pytest-param-files (>=0.6.0,<0.7.0)", "pytest-regressions", "sphinx-pytest"] +testing = ["beautifulsoup4", "coverage[toml]", "defusedxml", "pygments (<2.19)", "pytest (>=8,<9)", "pytest-cov", "pytest-param-files (>=0.6.0,<0.7.0)", "pytest-regressions", "sphinx-pytest"] testing-docutils = ["pygments", "pytest (>=8,<9)", "pytest-param-files (>=0.6.0,<0.7.0)"] [[package]] @@ -1345,18 +1368,14 @@ description = "TLS (SSL) sockets, key generation, encryption, decryption, signin optional = false python-versions = "*" groups = ["main"] -files = [] -develop = false +files = [ + {file = "oscrypto-1.3.0-py2.py3-none-any.whl", hash = "sha256:2b2f1d2d42ec152ca90ccb5682f3e051fb55986e1b170ebde472b133713e7085"}, + {file = "oscrypto-1.3.0.tar.gz", hash = "sha256:6f5fef59cb5b3708321db7cca56aed8ad7e662853351e7991fcf60ec606d47a4"}, +] [package.dependencies] asn1crypto = ">=1.5.1" -[package.source] -type = "git" -url = "https://github.com/wbond/oscrypto.git" -reference = "d5f3437ed24257895ae1edd9e503cfb352e635a8" -resolved_reference = "d5f3437ed24257895ae1edd9e503cfb352e635a8" - [[package]] name = "outcome" version = "1.3.0.post0" @@ -1512,14 +1531,14 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "proto-plus" -version = "1.25.0" -description = "Beautiful, Pythonic protocol buffers." +version = "1.26.0" +description = "Beautiful, Pythonic protocol buffers" optional = false python-versions = ">=3.7" groups = ["main"] files = [ - {file = "proto_plus-1.25.0-py3-none-any.whl", hash = "sha256:c91fc4a65074ade8e458e95ef8bac34d4008daa7cce4a12d6707066fca648961"}, - {file = "proto_plus-1.25.0.tar.gz", hash = "sha256:fbb17f57f7bd05a68b7707e745e26528b0b3c34e378db91eef93912c54982d91"}, + {file = "proto_plus-1.26.0-py3-none-any.whl", hash = "sha256:bf2dfaa3da281fc3187d12d224c707cb57214fb2c22ba854eb0c105a3fb2d4d7"}, + {file = "proto_plus-1.26.0.tar.gz", hash = "sha256:6e93d5f5ca267b54300880fff156b6a3386b3fa3f43b1da62e680fc0c586ef22"}, ] [package.dependencies] @@ -1653,6 +1672,35 @@ files = [ {file = "pycryptodomex-3.21.0.tar.gz", hash = "sha256:222d0bd05381dd25c32dd6065c071ebf084212ab79bab4599ba9e6a3e0009e6c"}, ] +[[package]] +name = "pydata-sphinx-theme" +version = "0.15.4" +description = "Bootstrap-based Sphinx theme from the PyData community" +optional = false +python-versions = ">=3.9" +groups = ["docs"] +files = [ + {file = "pydata_sphinx_theme-0.15.4-py3-none-any.whl", hash = "sha256:2136ad0e9500d0949f96167e63f3e298620040aea8f9c74621959eda5d4cf8e6"}, + {file = "pydata_sphinx_theme-0.15.4.tar.gz", hash = "sha256:7762ec0ac59df3acecf49fd2f889e1b4565dbce8b88b2e29ee06fdd90645a06d"}, +] + +[package.dependencies] +accessible-pygments = "*" +Babel = "*" +beautifulsoup4 = "*" +docutils = "!=0.17.0" +packaging = "*" +pygments = ">=2.7" +sphinx = ">=5" +typing-extensions = "*" + +[package.extras] +a11y = ["pytest-playwright"] +dev = ["pandoc", "pre-commit", "pydata-sphinx-theme[doc,test]", "pyyaml", "sphinx-theme-builder[cli]", "tox"] +doc = ["ablog (>=0.11.8)", "colorama", "graphviz", "ipykernel", "ipyleaflet", "ipywidgets", "jupyter_sphinx", "jupyterlite-sphinx", "linkify-it-py", "matplotlib", "myst-parser", "nbsphinx", "numpy", "numpydoc", "pandas", "plotly", "rich", "sphinx-autoapi (>=3.0.0)", "sphinx-copybutton", "sphinx-design", "sphinx-favicon (>=1.0.1)", "sphinx-sitemap", "sphinx-togglebutton", "sphinxcontrib-youtube (>=1.4.1)", "sphinxext-rediraffe", "xarray"] +i18n = ["Babel", "jinja2"] +test = ["pytest", "pytest-cov", "pytest-regressions", "sphinx[test]"] + [[package]] name = "pygments" version = "2.19.1" @@ -1729,14 +1777,14 @@ files = [ [[package]] name = "pytest" -version = "8.3.4" +version = "8.3.5" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.8" groups = ["dev"] files = [ - {file = "pytest-8.3.4-py3-none-any.whl", hash = "sha256:50e16d954148559c9a74109af1eaf0c945ba2d8f30f0a3d3335edde19788b6f6"}, - {file = "pytest-8.3.4.tar.gz", hash = "sha256:965370d062bce11e73868e0335abac31b4d3de0e82f4007408d242b4f8610761"}, + {file = "pytest-8.3.5-py3-none-any.whl", hash = "sha256:c69214aa47deac29fad6c2a4f590b9c4a9fdb16a403176fe154b79c0b4d4d820"}, + {file = "pytest-8.3.5.tar.gz", hash = "sha256:f4efe70cc14e511565ac476b57c279e12a855b11f48f212af1080ef2263d3845"}, ] [package.dependencies] @@ -1768,6 +1816,24 @@ loguru = "*" [package.extras] test = ["pytest", "pytest-cov"] +[[package]] +name = "pytest-mock" +version = "3.14.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = false +python-versions = ">=3.8" +groups = ["dev"] +files = [ + {file = "pytest-mock-3.14.0.tar.gz", hash = "sha256:2719255a1efeceadbc056d6bf3df3d1c5015530fb40cf347c0f9afac88410bd0"}, + {file = "pytest_mock-3.14.0-py3-none-any.whl", hash = "sha256:0b72c38033392a5f4621342fe11e9219ac11ec9d375f8e2a0c164539e0d70f6f"}, +] + +[package.dependencies] +pytest = ">=6.2.5" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -1820,14 +1886,14 @@ requests = ">=2.28" [[package]] name = "pytz" -version = "2024.2" +version = "2025.1" description = "World timezone definitions, modern and historical" optional = false python-versions = "*" groups = ["main"] files = [ - {file = "pytz-2024.2-py2.py3-none-any.whl", hash = "sha256:31c7c1817eb7fae7ca4b8c7ee50c72f93aa2dd863de768e1ef4245d426aa0725"}, - {file = "pytz-2024.2.tar.gz", hash = "sha256:2aa355083c50a0f93fa581709deac0c9ad65cca8a9e9beac660adcbd493c798a"}, + {file = "pytz-2025.1-py2.py3-none-any.whl", hash = "sha256:89dd22dca55b46eac6eda23b2d72721bf1bdfef212645d81513ef5d03038de57"}, + {file = "pytz-2025.1.tar.gz", hash = "sha256:c2db42be2a2518b28e65f9207c4d05e6ff547d1efa4086469ef855e4ab70178e"}, ] [[package]] @@ -2076,14 +2142,14 @@ jupyter = ["ipywidgets (>=7.5.1,<9)"] [[package]] name = "rich-argparse" -version = "1.6.0" +version = "1.7.0" description = "Rich help formatters for argparse and optparse" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "rich_argparse-1.6.0-py3-none-any.whl", hash = "sha256:fbe70a1d821b3f2fa8958cddf0cae131870a6e9faa04ab52b409cb1eda809bd7"}, - {file = "rich_argparse-1.6.0.tar.gz", hash = "sha256:092083c30da186f25bcdff8b1d47fdfb571288510fb051e0488a72cc3128de13"}, + {file = "rich_argparse-1.7.0-py3-none-any.whl", hash = "sha256:b8ec8943588e9731967f4f97b735b03dc127c416f480a083060433a97baf2fd3"}, + {file = "rich_argparse-1.7.0.tar.gz", hash = "sha256:f31d809c465ee43f367d599ccaf88b73bc2c4d75d74ed43f2d538838c53544ba"}, ] [package.dependencies] @@ -2182,32 +2248,32 @@ files = [ [[package]] name = "s3transfer" -version = "0.11.2" +version = "0.11.4" description = "An Amazon S3 Transfer Manager" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "s3transfer-0.11.2-py3-none-any.whl", hash = "sha256:be6ecb39fadd986ef1701097771f87e4d2f821f27f6071c872143884d2950fbc"}, - {file = "s3transfer-0.11.2.tar.gz", hash = "sha256:3b39185cb72f5acc77db1a58b6e25b977f28d20496b6e58d6813d75f464d632f"}, + {file = "s3transfer-0.11.4-py3-none-any.whl", hash = "sha256:ac265fa68318763a03bf2dc4f39d5cbd6a9e178d81cc9483ad27da33637e320d"}, + {file = "s3transfer-0.11.4.tar.gz", hash = "sha256:559f161658e1cf0a911f45940552c696735f5c74e64362e515f333ebed87d679"}, ] [package.dependencies] -botocore = ">=1.36.0,<2.0a.0" +botocore = ">=1.37.4,<2.0a.0" [package.extras] -crt = ["botocore[crt] (>=1.36.0,<2.0a.0)"] +crt = ["botocore[crt] (>=1.37.4,<2.0a.0)"] [[package]] name = "selenium" -version = "4.28.1" +version = "4.29.0" description = "Official Python bindings for Selenium WebDriver" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "selenium-4.28.1-py3-none-any.whl", hash = "sha256:4238847e45e24e4472cfcf3554427512c7aab9443396435b1623ef406fff1cc1"}, - {file = "selenium-4.28.1.tar.gz", hash = "sha256:0072d08670d7ec32db901bd0107695a330cecac9f196e3afb3fa8163026e022a"}, + {file = "selenium-4.29.0-py3-none-any.whl", hash = "sha256:ce5d26f1ddc1111641113653af33694c13947dd36c2df09cdd33f554351d372e"}, + {file = "selenium-4.29.0.tar.gz", hash = "sha256:3a62f7ec33e669364a6c0562a701deb69745b569c50d55f1a912bf8eb33358ba"}, ] [package.dependencies] @@ -2316,24 +2382,24 @@ test = ["cython (>=3.0)", "defusedxml (>=0.7.1)", "pytest (>=8.0)", "setuptools [[package]] name = "sphinx-autoapi" -version = "3.4.0" +version = "3.6.0" description = "Sphinx API documentation generator" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["docs"] files = [ - {file = "sphinx_autoapi-3.4.0-py3-none-any.whl", hash = "sha256:4027fef2875a22c5f2a57107c71641d82f6166bf55beb407a47aaf3ef14e7b92"}, - {file = "sphinx_autoapi-3.4.0.tar.gz", hash = "sha256:e6d5371f9411bbb9fca358c00a9e57aef3ac94cbfc5df4bab285946462f69e0c"}, + {file = "sphinx_autoapi-3.6.0-py3-none-any.whl", hash = "sha256:f3b66714493cab140b0e896d33ce7137654a16ac1edb6563edcbd47bf975f711"}, + {file = "sphinx_autoapi-3.6.0.tar.gz", hash = "sha256:c685f274e41d0842ae7e199460c322c4bd7fec816ccc2da8d806094b4f64af06"}, ] [package.dependencies] astroid = [ {version = ">=2.7", markers = "python_version < \"3.12\""}, - {version = ">=3.0.0a1", markers = "python_version >= \"3.12\""}, + {version = ">=3", markers = "python_version >= \"3.12\""}, ] Jinja2 = "*" PyYAML = "*" -sphinx = ">=6.1.0" +sphinx = ">=7.4.0" [[package]] name = "sphinx-autobuild" @@ -2359,22 +2425,25 @@ websockets = ">=11" test = ["httpx", "pytest (>=6)"] [[package]] -name = "sphinx-basic-ng" -version = "1.0.0b2" -description = "A modern skeleton for Sphinx themes." +name = "sphinx-book-theme" +version = "1.1.4" +description = "A clean book theme for scientific explanations and documentation with Sphinx" optional = false -python-versions = ">=3.7" +python-versions = ">=3.9" groups = ["docs"] files = [ - {file = "sphinx_basic_ng-1.0.0b2-py3-none-any.whl", hash = "sha256:eb09aedbabfb650607e9b4b68c9d240b90b1e1be221d6ad71d61c52e29f7932b"}, - {file = "sphinx_basic_ng-1.0.0b2.tar.gz", hash = "sha256:9ec55a47c90c8c002b5960c57492ec3021f5193cb26cebc2dc4ea226848651c9"}, + {file = "sphinx_book_theme-1.1.4-py3-none-any.whl", hash = "sha256:843b3f5c8684640f4a2d01abd298beb66452d1b2394cd9ef5be5ebd5640ea0e1"}, + {file = "sphinx_book_theme-1.1.4.tar.gz", hash = "sha256:73efe28af871d0a89bd05856d300e61edce0d5b2fbb7984e84454be0fedfe9ed"}, ] [package.dependencies] -sphinx = ">=4.0" +pydata-sphinx-theme = "0.15.4" +sphinx = ">=6.1" [package.extras] -docs = ["furo", "ipython", "myst-parser", "sphinx-copybutton", "sphinx-inline-tabs"] +code-style = ["pre-commit"] +doc = ["ablog", "folium", "ipywidgets", "matplotlib", "myst-nb", "nbclient", "numpy", "numpydoc", "pandas", "plotly", "sphinx-copybutton", "sphinx-design", "sphinx-examples", "sphinx-tabs", "sphinx-thebe", "sphinx-togglebutton", "sphinxcontrib-bibtex", "sphinxcontrib-youtube", "sphinxext-opengraph"] +test = ["beautifulsoup4", "coverage", "defusedxml", "myst-nb", "pytest", "pytest-cov", "pytest-regressions", "sphinx_thebe"] [[package]] name = "sphinx-copybutton" @@ -2516,14 +2585,14 @@ test = ["pytest"] [[package]] name = "starlette" -version = "0.45.3" +version = "0.46.0" description = "The little ASGI library that shines." optional = false python-versions = ">=3.9" groups = ["docs"] files = [ - {file = "starlette-0.45.3-py3-none-any.whl", hash = "sha256:dfb6d332576f136ec740296c7e8bb8c8a7125044e7c6da30744718880cdd059d"}, - {file = "starlette-0.45.3.tar.gz", hash = "sha256:2cbcba2a75806f8a41c722141486f37c28e30a0921c5f6fe4346cb0dcee1302f"}, + {file = "starlette-0.46.0-py3-none-any.whl", hash = "sha256:913f0798bd90ba90a9156383bcf1350a17d6259451d0d8ee27fc0cf2db609038"}, + {file = "starlette-0.46.0.tar.gz", hash = "sha256:b359e4567456b28d473d0193f34c0de0ed49710d75ef183a74a5ce0499324f50"}, ] [package.dependencies] @@ -2534,14 +2603,14 @@ full = ["httpx (>=0.27.0,<0.29.0)", "itsdangerous", "jinja2", "python-multipart [[package]] name = "telethon" -version = "1.38.1" +version = "1.39.0" description = "Full-featured Telegram client library for Python 3" optional = false python-versions = ">=3.5" groups = ["main"] files = [ - {file = "Telethon-1.38.1-py3-none-any.whl", hash = "sha256:30c187017501bfb982b8af5659f864dda4108f77ea49cfce61e8f6fdb8a18d6e"}, - {file = "Telethon-1.38.1.tar.gz", hash = "sha256:f9866c1e37197a0894e0c02aa56a6359bffb14a585e88e18e3e819df4fda399a"}, + {file = "Telethon-1.39.0-py3-none-any.whl", hash = "sha256:aa9f394b94be144799a6f6a93ab463867bc7c63503ede9631751940a98f6c703"}, + {file = "telethon-1.39.0.tar.gz", hash = "sha256:35d4795d8c91deac515fb0bcb3723866b924de1c724e1d5c230460e96f284a63"}, ] [package.dependencies] @@ -2630,14 +2699,14 @@ telegram = ["requests"] [[package]] name = "trio" -version = "0.28.0" +version = "0.29.0" description = "A friendly Python library for async concurrency and I/O" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "trio-0.28.0-py3-none-any.whl", hash = "sha256:56d58977acc1635735a96581ec70513cc781b8b6decd299c487d3be2a721cd94"}, - {file = "trio-0.28.0.tar.gz", hash = "sha256:4e547896fe9e8a5658e54e4c7c5fa1db748cbbbaa7c965e7d40505b928c73c05"}, + {file = "trio-0.29.0-py3-none-any.whl", hash = "sha256:d8c463f1a9cc776ff63e331aba44c125f423a5a13c684307e828d930e625ba66"}, + {file = "trio-0.29.0.tar.gz", hash = "sha256:ea0d3967159fc130acb6939a0be0e558e364fee26b5deeecc893a6b08c361bdf"}, ] [package.dependencies] @@ -2651,18 +2720,19 @@ sortedcontainers = "*" [[package]] name = "trio-websocket" -version = "0.11.1" +version = "0.12.2" description = "WebSocket library for Trio" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" groups = ["main"] files = [ - {file = "trio-websocket-0.11.1.tar.gz", hash = "sha256:18c11793647703c158b1f6e62de638acada927344d534e3c7628eedcb746839f"}, - {file = "trio_websocket-0.11.1-py3-none-any.whl", hash = "sha256:520d046b0d030cf970b8b2b2e00c4c2245b3807853ecd44214acd33d74581638"}, + {file = "trio_websocket-0.12.2-py3-none-any.whl", hash = "sha256:df605665f1db533f4a386c94525870851096a223adcb97f72a07e8b4beba45b6"}, + {file = "trio_websocket-0.12.2.tar.gz", hash = "sha256:22c72c436f3d1e264d0910a3951934798dcc5b00ae56fc4ee079d46c7cf20fae"}, ] [package.dependencies] exceptiongroup = {version = "*", markers = "python_version < \"3.11\""} +outcome = ">=1.2.0" trio = ">=0.11" wsproto = ">=0.14" @@ -2729,14 +2799,14 @@ files = [ [[package]] name = "tzlocal" -version = "5.2" +version = "5.3.1" description = "tzinfo object for the local timezone" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" groups = ["main"] files = [ - {file = "tzlocal-5.2-py3-none-any.whl", hash = "sha256:49816ef2fe65ea8ac19d19aa7a1ae0551c834303d5014c6d5a62e4cbda8047b8"}, - {file = "tzlocal-5.2.tar.gz", hash = "sha256:8d399205578f1a9342816409cc1e46a93ebd5755e39ea2d85334bea911bf0e6e"}, + {file = "tzlocal-5.3.1-py3-none-any.whl", hash = "sha256:eb1a66c3ef5847adf7a834f1be0800581b683b5608e74f86ecbcef8ab91bb85d"}, + {file = "tzlocal-5.3.1.tar.gz", hash = "sha256:cceffc7edecefea1f595541dbd6e990cb1ea3d19bf01b2809f362a03dd7921fd"}, ] [package.dependencies] @@ -2745,6 +2815,21 @@ tzdata = {version = "*", markers = "platform_system == \"Windows\""} [package.extras] devenv = ["check-manifest", "pytest (>=4.3)", "pytest-cov", "pytest-mock (>=3.3)", "zest.releaser"] +[[package]] +name = "uc-micro-py" +version = "1.0.3" +description = "Micro subset of unicode data files for linkify-it-py projects." +optional = false +python-versions = ">=3.7" +groups = ["docs"] +files = [ + {file = "uc-micro-py-1.0.3.tar.gz", hash = "sha256:d321b92cff673ec58027c04015fcaa8bb1e005478643ff4a500882eaab88c48a"}, + {file = "uc_micro_py-1.0.3-py3-none-any.whl", hash = "sha256:db1dffff340817673d7b466ec86114a9dc0e9d4d9b5ba229d9d60e5c12600cd5"}, +] + +[package.extras] +test = ["coverage", "pytest", "pytest-cov"] + [[package]] name = "uritemplate" version = "4.1.1" @@ -2967,81 +3052,81 @@ test = ["websockets"] [[package]] name = "websockets" -version = "14.2" +version = "15.0.1" description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)" optional = false python-versions = ">=3.9" groups = ["main", "docs"] files = [ - {file = "websockets-14.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e8179f95323b9ab1c11723e5d91a89403903f7b001828161b480a7810b334885"}, - {file = "websockets-14.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0d8c3e2cdb38f31d8bd7d9d28908005f6fa9def3324edb9bf336d7e4266fd397"}, - {file = "websockets-14.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:714a9b682deb4339d39ffa674f7b674230227d981a37d5d174a4a83e3978a610"}, - {file = "websockets-14.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2e53c72052f2596fb792a7acd9704cbc549bf70fcde8a99e899311455974ca3"}, - {file = "websockets-14.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e3fbd68850c837e57373d95c8fe352203a512b6e49eaae4c2f4088ef8cf21980"}, - {file = "websockets-14.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b27ece32f63150c268593d5fdb82819584831a83a3f5809b7521df0685cd5d8"}, - {file = "websockets-14.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4daa0faea5424d8713142b33825fff03c736f781690d90652d2c8b053345b0e7"}, - {file = "websockets-14.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:bc63cee8596a6ec84d9753fd0fcfa0452ee12f317afe4beae6b157f0070c6c7f"}, - {file = "websockets-14.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:7a570862c325af2111343cc9b0257b7119b904823c675b22d4ac547163088d0d"}, - {file = "websockets-14.2-cp310-cp310-win32.whl", hash = "sha256:75862126b3d2d505e895893e3deac0a9339ce750bd27b4ba515f008b5acf832d"}, - {file = "websockets-14.2-cp310-cp310-win_amd64.whl", hash = "sha256:cc45afb9c9b2dc0852d5c8b5321759cf825f82a31bfaf506b65bf4668c96f8b2"}, - {file = "websockets-14.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3bdc8c692c866ce5fefcaf07d2b55c91d6922ac397e031ef9b774e5b9ea42166"}, - {file = "websockets-14.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c93215fac5dadc63e51bcc6dceca72e72267c11def401d6668622b47675b097f"}, - {file = "websockets-14.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1c9b6535c0e2cf8a6bf938064fb754aaceb1e6a4a51a80d884cd5db569886910"}, - {file = "websockets-14.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a52a6d7cf6938e04e9dceb949d35fbdf58ac14deea26e685ab6368e73744e4c"}, - {file = "websockets-14.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9f05702e93203a6ff5226e21d9b40c037761b2cfb637187c9802c10f58e40473"}, - {file = "websockets-14.2-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22441c81a6748a53bfcb98951d58d1af0661ab47a536af08920d129b4d1c3473"}, - {file = "websockets-14.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:efd9b868d78b194790e6236d9cbc46d68aba4b75b22497eb4ab64fa640c3af56"}, - {file = "websockets-14.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1a5a20d5843886d34ff8c57424cc65a1deda4375729cbca4cb6b3353f3ce4142"}, - {file = "websockets-14.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:34277a29f5303d54ec6468fb525d99c99938607bc96b8d72d675dee2b9f5bf1d"}, - {file = "websockets-14.2-cp311-cp311-win32.whl", hash = "sha256:02687db35dbc7d25fd541a602b5f8e451a238ffa033030b172ff86a93cb5dc2a"}, - {file = "websockets-14.2-cp311-cp311-win_amd64.whl", hash = "sha256:862e9967b46c07d4dcd2532e9e8e3c2825e004ffbf91a5ef9dde519ee2effb0b"}, - {file = "websockets-14.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:1f20522e624d7ffbdbe259c6b6a65d73c895045f76a93719aa10cd93b3de100c"}, - {file = "websockets-14.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:647b573f7d3ada919fd60e64d533409a79dcf1ea21daeb4542d1d996519ca967"}, - {file = "websockets-14.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6af99a38e49f66be5a64b1e890208ad026cda49355661549c507152113049990"}, - {file = "websockets-14.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:091ab63dfc8cea748cc22c1db2814eadb77ccbf82829bac6b2fbe3401d548eda"}, - {file = "websockets-14.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b374e8953ad477d17e4851cdc66d83fdc2db88d9e73abf755c94510ebddceb95"}, - {file = "websockets-14.2-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a39d7eceeea35db85b85e1169011bb4321c32e673920ae9c1b6e0978590012a3"}, - {file = "websockets-14.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0a6f3efd47ffd0d12080594f434faf1cd2549b31e54870b8470b28cc1d3817d9"}, - {file = "websockets-14.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:065ce275e7c4ffb42cb738dd6b20726ac26ac9ad0a2a48e33ca632351a737267"}, - {file = "websockets-14.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e9d0e53530ba7b8b5e389c02282f9d2aa47581514bd6049d3a7cffe1385cf5fe"}, - {file = "websockets-14.2-cp312-cp312-win32.whl", hash = "sha256:20e6dd0984d7ca3037afcb4494e48c74ffb51e8013cac71cf607fffe11df7205"}, - {file = "websockets-14.2-cp312-cp312-win_amd64.whl", hash = "sha256:44bba1a956c2c9d268bdcdf234d5e5ff4c9b6dc3e300545cbe99af59dda9dcce"}, - {file = "websockets-14.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:6f1372e511c7409a542291bce92d6c83320e02c9cf392223272287ce55bc224e"}, - {file = "websockets-14.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4da98b72009836179bb596a92297b1a61bb5a830c0e483a7d0766d45070a08ad"}, - {file = "websockets-14.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f8a86a269759026d2bde227652b87be79f8a734e582debf64c9d302faa1e9f03"}, - {file = "websockets-14.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:86cf1aaeca909bf6815ea714d5c5736c8d6dd3a13770e885aafe062ecbd04f1f"}, - {file = "websockets-14.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9b0f6c3ba3b1240f602ebb3971d45b02cc12bd1845466dd783496b3b05783a5"}, - {file = "websockets-14.2-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:669c3e101c246aa85bc8534e495952e2ca208bd87994650b90a23d745902db9a"}, - {file = "websockets-14.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:eabdb28b972f3729348e632ab08f2a7b616c7e53d5414c12108c29972e655b20"}, - {file = "websockets-14.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:2066dc4cbcc19f32c12a5a0e8cc1b7ac734e5b64ac0a325ff8353451c4b15ef2"}, - {file = "websockets-14.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ab95d357cd471df61873dadf66dd05dd4709cae001dd6342edafc8dc6382f307"}, - {file = "websockets-14.2-cp313-cp313-win32.whl", hash = "sha256:a9e72fb63e5f3feacdcf5b4ff53199ec8c18d66e325c34ee4c551ca748623bbc"}, - {file = "websockets-14.2-cp313-cp313-win_amd64.whl", hash = "sha256:b439ea828c4ba99bb3176dc8d9b933392a2413c0f6b149fdcba48393f573377f"}, - {file = "websockets-14.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7cd5706caec1686c5d233bc76243ff64b1c0dc445339bd538f30547e787c11fe"}, - {file = "websockets-14.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ec607328ce95a2f12b595f7ae4c5d71bf502212bddcea528290b35c286932b12"}, - {file = "websockets-14.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:da85651270c6bfb630136423037dd4975199e5d4114cae6d3066641adcc9d1c7"}, - {file = "websockets-14.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c3ecadc7ce90accf39903815697917643f5b7cfb73c96702318a096c00aa71f5"}, - {file = "websockets-14.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1979bee04af6a78608024bad6dfcc0cc930ce819f9e10342a29a05b5320355d0"}, - {file = "websockets-14.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2dddacad58e2614a24938a50b85969d56f88e620e3f897b7d80ac0d8a5800258"}, - {file = "websockets-14.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:89a71173caaf75fa71a09a5f614f450ba3ec84ad9fca47cb2422a860676716f0"}, - {file = "websockets-14.2-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:6af6a4b26eea4fc06c6818a6b962a952441e0e39548b44773502761ded8cc1d4"}, - {file = "websockets-14.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:80c8efa38957f20bba0117b48737993643204645e9ec45512579132508477cfc"}, - {file = "websockets-14.2-cp39-cp39-win32.whl", hash = "sha256:2e20c5f517e2163d76e2729104abc42639c41cf91f7b1839295be43302713661"}, - {file = "websockets-14.2-cp39-cp39-win_amd64.whl", hash = "sha256:b4c8cef610e8d7c70dea92e62b6814a8cd24fbd01d7103cc89308d2bfe1659ef"}, - {file = "websockets-14.2-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:d7d9cafbccba46e768be8a8ad4635fa3eae1ffac4c6e7cb4eb276ba41297ed29"}, - {file = "websockets-14.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:c76193c1c044bd1e9b3316dcc34b174bbf9664598791e6fb606d8d29000e070c"}, - {file = "websockets-14.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fd475a974d5352390baf865309fe37dec6831aafc3014ffac1eea99e84e83fc2"}, - {file = "websockets-14.2-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2c6c0097a41968b2e2b54ed3424739aab0b762ca92af2379f152c1aef0187e1c"}, - {file = "websockets-14.2-pp310-pypy310_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d7ff794c8b36bc402f2e07c0b2ceb4a2424147ed4785ff03e2a7af03711d60a"}, - {file = "websockets-14.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:dec254fcabc7bd488dab64846f588fc5b6fe0d78f641180030f8ea27b76d72c3"}, - {file = "websockets-14.2-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:bbe03eb853e17fd5b15448328b4ec7fb2407d45fb0245036d06a3af251f8e48f"}, - {file = "websockets-14.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:a3c4aa3428b904d5404a0ed85f3644d37e2cb25996b7f096d77caeb0e96a3b42"}, - {file = "websockets-14.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:577a4cebf1ceaf0b65ffc42c54856214165fb8ceeba3935852fc33f6b0c55e7f"}, - {file = "websockets-14.2-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ad1c1d02357b7665e700eca43a31d52814ad9ad9b89b58118bdabc365454b574"}, - {file = "websockets-14.2-pp39-pypy39_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f390024a47d904613577df83ba700bd189eedc09c57af0a904e5c39624621270"}, - {file = "websockets-14.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:3c1426c021c38cf92b453cdf371228d3430acd775edee6bac5a4d577efc72365"}, - {file = "websockets-14.2-py3-none-any.whl", hash = "sha256:7a6ceec4ea84469f15cf15807a747e9efe57e369c384fa86e022b3bea679b79b"}, - {file = "websockets-14.2.tar.gz", hash = "sha256:5059ed9c54945efb321f097084b4c7e52c246f2c869815876a69d1efc4ad6eb5"}, + {file = "websockets-15.0.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d63efaa0cd96cf0c5fe4d581521d9fa87744540d4bc999ae6e08595a1014b45b"}, + {file = "websockets-15.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ac60e3b188ec7574cb761b08d50fcedf9d77f1530352db4eef1707fe9dee7205"}, + {file = "websockets-15.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5756779642579d902eed757b21b0164cd6fe338506a8083eb58af5c372e39d9a"}, + {file = "websockets-15.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0fdfe3e2a29e4db3659dbd5bbf04560cea53dd9610273917799f1cde46aa725e"}, + {file = "websockets-15.0.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c2529b320eb9e35af0fa3016c187dffb84a3ecc572bcee7c3ce302bfeba52bf"}, + {file = "websockets-15.0.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac1e5c9054fe23226fb11e05a6e630837f074174c4c2f0fe442996112a6de4fb"}, + {file = "websockets-15.0.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5df592cd503496351d6dc14f7cdad49f268d8e618f80dce0cd5a36b93c3fc08d"}, + {file = "websockets-15.0.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:0a34631031a8f05657e8e90903e656959234f3a04552259458aac0b0f9ae6fd9"}, + {file = "websockets-15.0.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3d00075aa65772e7ce9e990cab3ff1de702aa09be3940d1dc88d5abf1ab8a09c"}, + {file = "websockets-15.0.1-cp310-cp310-win32.whl", hash = "sha256:1234d4ef35db82f5446dca8e35a7da7964d02c127b095e172e54397fb6a6c256"}, + {file = "websockets-15.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:39c1fec2c11dc8d89bba6b2bf1556af381611a173ac2b511cf7231622058af41"}, + {file = "websockets-15.0.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:823c248b690b2fd9303ba00c4f66cd5e2d8c3ba4aa968b2779be9532a4dad431"}, + {file = "websockets-15.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678999709e68425ae2593acf2e3ebcbcf2e69885a5ee78f9eb80e6e371f1bf57"}, + {file = "websockets-15.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d50fd1ee42388dcfb2b3676132c78116490976f1300da28eb629272d5d93e905"}, + {file = "websockets-15.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d99e5546bf73dbad5bf3547174cd6cb8ba7273062a23808ffea025ecb1cf8562"}, + {file = "websockets-15.0.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:66dd88c918e3287efc22409d426c8f729688d89a0c587c88971a0faa2c2f3792"}, + {file = "websockets-15.0.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8dd8327c795b3e3f219760fa603dcae1dcc148172290a8ab15158cf85a953413"}, + {file = "websockets-15.0.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8fdc51055e6ff4adeb88d58a11042ec9a5eae317a0a53d12c062c8a8865909e8"}, + {file = "websockets-15.0.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:693f0192126df6c2327cce3baa7c06f2a117575e32ab2308f7f8216c29d9e2e3"}, + {file = "websockets-15.0.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:54479983bd5fb469c38f2f5c7e3a24f9a4e70594cd68cd1fa6b9340dadaff7cf"}, + {file = "websockets-15.0.1-cp311-cp311-win32.whl", hash = "sha256:16b6c1b3e57799b9d38427dda63edcbe4926352c47cf88588c0be4ace18dac85"}, + {file = "websockets-15.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:27ccee0071a0e75d22cb35849b1db43f2ecd3e161041ac1ee9d2352ddf72f065"}, + {file = "websockets-15.0.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:3e90baa811a5d73f3ca0bcbf32064d663ed81318ab225ee4f427ad4e26e5aff3"}, + {file = "websockets-15.0.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:592f1a9fe869c778694f0aa806ba0374e97648ab57936f092fd9d87f8bc03665"}, + {file = "websockets-15.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0701bc3cfcb9164d04a14b149fd74be7347a530ad3bbf15ab2c678a2cd3dd9a2"}, + {file = "websockets-15.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8b56bdcdb4505c8078cb6c7157d9811a85790f2f2b3632c7d1462ab5783d215"}, + {file = "websockets-15.0.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0af68c55afbd5f07986df82831c7bff04846928ea8d1fd7f30052638788bc9b5"}, + {file = "websockets-15.0.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64dee438fed052b52e4f98f76c5790513235efaa1ef7f3f2192c392cd7c91b65"}, + {file = "websockets-15.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d5f6b181bb38171a8ad1d6aa58a67a6aa9d4b38d0f8c5f496b9e42561dfc62fe"}, + {file = "websockets-15.0.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:5d54b09eba2bada6011aea5375542a157637b91029687eb4fdb2dab11059c1b4"}, + {file = "websockets-15.0.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3be571a8b5afed347da347bfcf27ba12b069d9d7f42cb8c7028b5e98bbb12597"}, + {file = "websockets-15.0.1-cp312-cp312-win32.whl", hash = "sha256:c338ffa0520bdb12fbc527265235639fb76e7bc7faafbb93f6ba80d9c06578a9"}, + {file = "websockets-15.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:fcd5cf9e305d7b8338754470cf69cf81f420459dbae8a3b40cee57417f4614a7"}, + {file = "websockets-15.0.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ee443ef070bb3b6ed74514f5efaa37a252af57c90eb33b956d35c8e9c10a1931"}, + {file = "websockets-15.0.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5a939de6b7b4e18ca683218320fc67ea886038265fd1ed30173f5ce3f8e85675"}, + {file = "websockets-15.0.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:746ee8dba912cd6fc889a8147168991d50ed70447bf18bcda7039f7d2e3d9151"}, + {file = "websockets-15.0.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:595b6c3969023ecf9041b2936ac3827e4623bfa3ccf007575f04c5a6aa318c22"}, + {file = "websockets-15.0.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c714d2fc58b5ca3e285461a4cc0c9a66bd0e24c5da9911e30158286c9b5be7f"}, + {file = "websockets-15.0.1-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f3c1e2ab208db911594ae5b4f79addeb3501604a165019dd221c0bdcabe4db8"}, + {file = "websockets-15.0.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:229cf1d3ca6c1804400b0a9790dc66528e08a6a1feec0d5040e8b9eb14422375"}, + {file = "websockets-15.0.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:756c56e867a90fb00177d530dca4b097dd753cde348448a1012ed6c5131f8b7d"}, + {file = "websockets-15.0.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:558d023b3df0bffe50a04e710bc87742de35060580a293c2a984299ed83bc4e4"}, + {file = "websockets-15.0.1-cp313-cp313-win32.whl", hash = "sha256:ba9e56e8ceeeedb2e080147ba85ffcd5cd0711b89576b83784d8605a7df455fa"}, + {file = "websockets-15.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:e09473f095a819042ecb2ab9465aee615bd9c2028e4ef7d933600a8401c79561"}, + {file = "websockets-15.0.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5f4c04ead5aed67c8a1a20491d54cdfba5884507a48dd798ecaf13c74c4489f5"}, + {file = "websockets-15.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:abdc0c6c8c648b4805c5eacd131910d2a7f6455dfd3becab248ef108e89ab16a"}, + {file = "websockets-15.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a625e06551975f4b7ea7102bc43895b90742746797e2e14b70ed61c43a90f09b"}, + {file = "websockets-15.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d591f8de75824cbb7acad4e05d2d710484f15f29d4a915092675ad3456f11770"}, + {file = "websockets-15.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:47819cea040f31d670cc8d324bb6435c6f133b8c7a19ec3d61634e62f8d8f9eb"}, + {file = "websockets-15.0.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac017dd64572e5c3bd01939121e4d16cf30e5d7e110a119399cf3133b63ad054"}, + {file = "websockets-15.0.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:4a9fac8e469d04ce6c25bb2610dc535235bd4aa14996b4e6dbebf5e007eba5ee"}, + {file = "websockets-15.0.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:363c6f671b761efcb30608d24925a382497c12c506b51661883c3e22337265ed"}, + {file = "websockets-15.0.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:2034693ad3097d5355bfdacfffcbd3ef5694f9718ab7f29c29689a9eae841880"}, + {file = "websockets-15.0.1-cp39-cp39-win32.whl", hash = "sha256:3b1ac0d3e594bf121308112697cf4b32be538fb1444468fb0a6ae4feebc83411"}, + {file = "websockets-15.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:b7643a03db5c95c799b89b31c036d5f27eeb4d259c798e878d6937d71832b1e4"}, + {file = "websockets-15.0.1-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0c9e74d766f2818bb95f84c25be4dea09841ac0f734d1966f415e4edfc4ef1c3"}, + {file = "websockets-15.0.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:1009ee0c7739c08a0cd59de430d6de452a55e42d6b522de7aa15e6f67db0b8e1"}, + {file = "websockets-15.0.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76d1f20b1c7a2fa82367e04982e708723ba0e7b8d43aa643d3dcd404d74f1475"}, + {file = "websockets-15.0.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f29d80eb9a9263b8d109135351caf568cc3f80b9928bccde535c235de55c22d9"}, + {file = "websockets-15.0.1-pp310-pypy310_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b359ed09954d7c18bbc1680f380c7301f92c60bf924171629c5db97febb12f04"}, + {file = "websockets-15.0.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:cad21560da69f4ce7658ca2cb83138fb4cf695a2ba3e475e0559e05991aa8122"}, + {file = "websockets-15.0.1-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7f493881579c90fc262d9cdbaa05a6b54b3811c2f300766748db79f098db9940"}, + {file = "websockets-15.0.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:47b099e1f4fbc95b701b6e85768e1fcdaf1630f3cbe4765fa216596f12310e2e"}, + {file = "websockets-15.0.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:67f2b6de947f8c757db2db9c71527933ad0019737ec374a8a6be9a956786aaf9"}, + {file = "websockets-15.0.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d08eb4c2b7d6c41da6ca0600c077e93f5adcfd979cd777d747e9ee624556da4b"}, + {file = "websockets-15.0.1-pp39-pypy39_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b826973a4a2ae47ba357e4e82fa44a463b8f168e1ca775ac64521442b19e87f"}, + {file = "websockets-15.0.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:21c1fa28a6a7e3cbdc171c694398b6df4744613ce9b36b1a498e816787e28123"}, + {file = "websockets-15.0.1-py3-none-any.whl", hash = "sha256:f7a866fbc1e97b5c617ee4116daaa09b722101d4a3c170c787450ba409f9736f"}, + {file = "websockets-15.0.1.tar.gz", hash = "sha256:82544de02076bafba038ce055ee6412d68da13ab47f0c60cab827346de828dee"}, ] [[package]] @@ -3077,14 +3162,14 @@ h11 = ">=0.9.0,<1" [[package]] name = "yt-dlp" -version = "2025.1.26" +version = "2025.2.19" description = "A feature-rich command-line audio/video downloader" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "yt_dlp-2025.1.26-py3-none-any.whl", hash = "sha256:3e76bd896b9f96601021ca192ca0fbdd195e3c3dcc28302a3a34c9bc4979da7b"}, - {file = "yt_dlp-2025.1.26.tar.gz", hash = "sha256:1c9738266921ad43c568ad01ac3362fb7c7af549276fbec92bd72f140da16240"}, + {file = "yt_dlp-2025.2.19-py3-none-any.whl", hash = "sha256:3ed218eaeece55e9d715afd41abc450dc406ee63bf79355169dfde312d38fdb8"}, + {file = "yt_dlp-2025.2.19.tar.gz", hash = "sha256:f33ca76df2e4db31880f2fe408d44f5058d9f135015b13e50610dfbe78245bea"}, ] [package.extras] @@ -3100,4 +3185,4 @@ test = ["pytest (>=8.1,<9.0)", "pytest-rerunfailures (>=14.0,<15.0)"] [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.13" -content-hash = "9ca114395e73af8982abbccc25b385bbca62e50ba7cca8239e52e5c1227cb4b0" +content-hash = "2d0a953383901fe12e97f6f56a76a9d8008788695425792eedbf739a18585188" diff --git a/pyproject.toml b/pyproject.toml index f1be273..30693d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [project] name = "auto-archiver" -version = "0.13.0" +version = "0.13.5" description = "Automatically archive links to videos, images, and social media content from Google Sheets (and more)." requires-python = ">=3.10,<3.13" @@ -22,7 +22,6 @@ classifiers = [ ] dependencies = [ - "oscrypto @ git+https://github.com/wbond/oscrypto.git@d5f3437ed24257895ae1edd9e503cfb352e635a8", "gspread (>=0.0.0)", "beautifulsoup4 (>=0.0.0)", "bs4 (>=0.0.0)", @@ -64,6 +63,7 @@ dependencies = [ pytest = "^8.3.4" autopep8 = "^2.3.1" pytest-loguru = "^0.4.0" +pytest-mock = "^3.14.0" [tool.poetry.group.docs.dependencies] sphinx = "^8.1.3" @@ -72,7 +72,8 @@ sphinxcontrib-mermaid = "^1.0.0" sphinx-autobuild = "^2024.10.3" sphinx-copybutton = "^0.5.2" myst-parser = "^4.0.0" -furo = "^2024.8.6" +sphinx-book-theme = "^1.1.3" +linkify-it-py = "^2.0.3" [project.scripts] diff --git a/scripts/generate_settings_schema.py b/scripts/generate_settings_schema.py new file mode 100644 index 0000000..16cb22f --- /dev/null +++ b/scripts/generate_settings_schema.py @@ -0,0 +1,52 @@ +import json +import os +import io + +from ruamel.yaml import YAML + +from auto_archiver.core.module import ModuleFactory +from auto_archiver.core.consts import MODULE_TYPES +from auto_archiver.core.config import EMPTY_CONFIG + +class SchemaEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, set): + return list(obj) + return json.JSONEncoder.default(self, obj) + +# Get available modules +module_factory = ModuleFactory() +available_modules = module_factory.available_modules() + +modules_by_type = {} +# Categorize modules by type +for module in available_modules: + for type in module.manifest.get('type', []): + modules_by_type.setdefault(type, []).append(module) + +all_modules_ordered_by_type = sorted(available_modules, key=lambda x: (MODULE_TYPES.index(x.type[0]), not x.requires_setup)) + +yaml: YAML = YAML() + +config_string = io.BytesIO() +yaml.dump(EMPTY_CONFIG, config_string) +config_string = config_string.getvalue().decode('utf-8') +output_schema = { + 'modules': dict((module.name, + { + 'name': module.name, + 'display_name': module.display_name, + 'manifest': module.manifest, + 'configs': module.configs or None + } + ) for module in all_modules_ordered_by_type), + 'steps': dict((f"{module_type}s", [module.name for module in modules_by_type[module_type]]) for module_type in MODULE_TYPES), + 'configs': [m.name for m in all_modules_ordered_by_type if m.configs], + 'module_types': MODULE_TYPES, + 'empty_config': config_string +} + +current_file_dir = os.path.dirname(os.path.abspath(__file__)) +output_file = os.path.join(current_file_dir, 'settings/src/schema.json') +with open(output_file, 'w') as file: + json.dump(output_schema, file, indent=4, cls=SchemaEncoder) \ No newline at end of file diff --git a/scripts/settings/.gitignore b/scripts/settings/.gitignore new file mode 100644 index 0000000..a547bf3 --- /dev/null +++ b/scripts/settings/.gitignore @@ -0,0 +1,24 @@ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* +lerna-debug.log* + +node_modules +dist +dist-ssr +*.local + +# Editor directories and files +.vscode/* +!.vscode/extensions.json +.idea +.DS_Store +*.suo +*.ntvs* +*.njsproj +*.sln +*.sw? diff --git a/scripts/settings/index.html b/scripts/settings/index.html new file mode 100644 index 0000000..22ff169 --- /dev/null +++ b/scripts/settings/index.html @@ -0,0 +1,3 @@ + +
+ diff --git a/scripts/settings/package-lock.json b/scripts/settings/package-lock.json new file mode 100644 index 0000000..cd40c14 --- /dev/null +++ b/scripts/settings/package-lock.json @@ -0,0 +1,3743 @@ +{ + "name": "material-ui-vite-ts", + "version": "5.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "material-ui-vite-ts", + "version": "5.0.0", + "dependencies": { + "@dnd-kit/core": "^6.3.1", + "@dnd-kit/sortable": "^10.0.0", + "@emotion/react": "latest", + "@emotion/styled": "latest", + "@mui/icons-material": "latest", + "@mui/material": "latest", + "react": "19.0.0", + "react-dom": "19.0.0", + "react-markdown": "^10.0.0", + "yaml": "^2.7.0" + }, + "devDependencies": { + "@types/react": "latest", + "@types/react-dom": "latest", + "@vitejs/plugin-react": "latest", + "typescript": "latest", + "vite": "latest", + "vite-plugin-singlefile": "^2.1.0" + } + }, + "node_modules/@ampproject/remapping": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.3.0.tgz", + "integrity": "sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/code-frame": { + "version": "7.26.2", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.26.2.tgz", + "integrity": "sha512-RJlIHRueQgwWitWgF8OdFYGZX328Ax5BCemNGlqHfplnRT9ESi8JkFlvaVYbS+UubVY6dpv87Fs2u5M29iNFVQ==", + "license": "MIT", + "dependencies": { + "@babel/helper-validator-identifier": "^7.25.9", + "js-tokens": "^4.0.0", + "picocolors": "^1.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/compat-data": { + "version": "7.26.8", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.26.8.tgz", + "integrity": "sha512-oH5UPLMWR3L2wEFLnFJ1TZXqHufiTKAiLfqw5zkhS4dKXLJ10yVztfil/twG8EDTA4F/tvVNw9nOl4ZMslB8rQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/core": { + "version": "7.26.9", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.26.9.tgz", + "integrity": "sha512-lWBYIrF7qK5+GjY5Uy+/hEgp8OJWOD/rpy74GplYRhEauvbHDeFB8t5hPOZxCZ0Oxf4Cc36tK51/l3ymJysrKw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@ampproject/remapping": "^2.2.0", + "@babel/code-frame": "^7.26.2", + "@babel/generator": "^7.26.9", + "@babel/helper-compilation-targets": "^7.26.5", + "@babel/helper-module-transforms": "^7.26.0", + "@babel/helpers": "^7.26.9", + "@babel/parser": "^7.26.9", + "@babel/template": "^7.26.9", + "@babel/traverse": "^7.26.9", + "@babel/types": "^7.26.9", + "convert-source-map": "^2.0.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.2", + "json5": "^2.2.3", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/babel" + } + }, + "node_modules/@babel/core/node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@babel/generator": { + "version": "7.26.9", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.26.9.tgz", + "integrity": "sha512-kEWdzjOAUMW4hAyrzJ0ZaTOu9OmpyDIQicIh0zg0EEcEkYXZb2TjtBhnHi2ViX7PKwZqF4xwqfAm299/QMP3lg==", + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.26.9", + "@babel/types": "^7.26.9", + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.25", + "jsesc": "^3.0.2" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-compilation-targets": { + "version": "7.26.5", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.26.5.tgz", + "integrity": "sha512-IXuyn5EkouFJscIDuFF5EsiSolseme1s0CZB+QxVugqJLYmKdxI1VfIBOst0SUu4rnk2Z7kqTwmoO1lp3HIfnA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/compat-data": "^7.26.5", + "@babel/helper-validator-option": "^7.25.9", + "browserslist": "^4.24.0", + "lru-cache": "^5.1.1", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-imports": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.25.9.tgz", + "integrity": "sha512-tnUA4RsrmflIM6W6RFTLFSXITtl0wKjgpnLgXyowocVPrbYrLUXSBXDgTs8BlbmIzIdlBySRQjINYs2BAkiLtw==", + "license": "MIT", + "dependencies": { + "@babel/traverse": "^7.25.9", + "@babel/types": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-transforms": { + "version": "7.26.0", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.26.0.tgz", + "integrity": "sha512-xO+xu6B5K2czEnQye6BHA7DolFFmS3LB7stHZFaOLb1pAwO1HWLS8fXA+eh0A2yIvltPVmx3eNNDBJA2SLHXFw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-module-imports": "^7.25.9", + "@babel/helper-validator-identifier": "^7.25.9", + "@babel/traverse": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-plugin-utils": { + "version": "7.26.5", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.26.5.tgz", + "integrity": "sha512-RS+jZcRdZdRFzMyr+wcsaqOmld1/EqTghfaBGQQd/WnRdzdlvSZ//kF7U8VQTxf1ynZ4cjUcYgjVGx13ewNPMg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.25.9.tgz", + "integrity": "sha512-4A/SCr/2KLd5jrtOMFzaKjVtAei3+2r/NChoBNoZ3EyP/+GlhoaEGoWOZUmFmoITP7zOJyHIMm+DYRd8o3PvHA==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.25.9.tgz", + "integrity": "sha512-Ed61U6XJc3CVRfkERJWDz4dJwKe7iLmmJsbOGu9wSloNSFttHV0I8g6UAgb7qnK5ly5bGLPd4oXZlxCdANBOWQ==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-option": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.25.9.tgz", + "integrity": "sha512-e/zv1co8pp55dNdEcCynfj9X7nyUKUXoUEwfXqaZt0omVOmDe9oOTdKStH4GmAw6zxMFs50ZayuMfHDKlO7Tfw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helpers": { + "version": "7.26.9", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.26.9.tgz", + "integrity": "sha512-Mz/4+y8udxBKdmzt/UjPACs4G3j5SshJJEFFKxlCGPydG4JAHXxjWjAwjd09tf6oINvl1VfMJo+nB7H2YKQ0dA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/template": "^7.26.9", + "@babel/types": "^7.26.9" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.26.9", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.26.9.tgz", + "integrity": "sha512-81NWa1njQblgZbQHxWHpxxCzNsa3ZwvFqpUg7P+NNUU6f3UU2jBEg4OlF/J6rl8+PQGh1q6/zWScd001YwcA5A==", + "license": "MIT", + "dependencies": { + "@babel/types": "^7.26.9" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/plugin-transform-react-jsx-self": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-self/-/plugin-transform-react-jsx-self-7.25.9.tgz", + "integrity": "sha512-y8quW6p0WHkEhmErnfe58r7x0A70uKphQm8Sp8cV7tjNQwK56sNVK0M73LK3WuYmsuyrftut4xAkjjgU0twaMg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-react-jsx-source": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-source/-/plugin-transform-react-jsx-source-7.25.9.tgz", + "integrity": "sha512-+iqjT8xmXhhYv4/uiYd8FNQsraMFZIfxVSqxxVSZP0WbbSAWvBXAul0m/zu+7Vv4O/3WtApy9pmaTMiumEZgfg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/runtime": { + "version": "7.26.9", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.26.9.tgz", + "integrity": "sha512-aA63XwOkcl4xxQa3HjPMqOP6LiK0ZDv3mUPYEFXkpHbaFjtGggE1A61FjFzJnB+p7/oy2gA8E+rcBNl/zC1tMg==", + "license": "MIT", + "dependencies": { + "regenerator-runtime": "^0.14.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/template": { + "version": "7.26.9", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.26.9.tgz", + "integrity": "sha512-qyRplbeIpNZhmzOysF/wFMuP9sctmh2cFzRAZOn1YapxBsE1i9bJIY586R/WBLfLcmcBlM8ROBiQURnnNy+zfA==", + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.26.2", + "@babel/parser": "^7.26.9", + "@babel/types": "^7.26.9" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/traverse": { + "version": "7.26.9", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.26.9.tgz", + "integrity": "sha512-ZYW7L+pL8ahU5fXmNbPF+iZFHCv5scFak7MZ9bwaRPLUhHh7QQEMjZUg0HevihoqCM5iSYHN61EyCoZvqC+bxg==", + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.26.2", + "@babel/generator": "^7.26.9", + "@babel/parser": "^7.26.9", + "@babel/template": "^7.26.9", + "@babel/types": "^7.26.9", + "debug": "^4.3.1", + "globals": "^11.1.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/types": { + "version": "7.26.9", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.26.9.tgz", + "integrity": "sha512-Y3IR1cRnOxOCDvMmNiym7XpXQ93iGDDPHx+Zj+NM+rg0fBaShfQLkg+hKPaZCEvg5N/LeCo4+Rj/i3FuJsIQaw==", + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.25.9", + "@babel/helper-validator-identifier": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@dnd-kit/accessibility": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/@dnd-kit/accessibility/-/accessibility-3.1.1.tgz", + "integrity": "sha512-2P+YgaXF+gRsIihwwY1gCsQSYnu9Zyj2py8kY5fFvUM1qm2WA2u639R6YNVfU4GWr+ZM5mqEsfHZZLoRONbemw==", + "license": "MIT", + "dependencies": { + "tslib": "^2.0.0" + }, + "peerDependencies": { + "react": ">=16.8.0" + } + }, + "node_modules/@dnd-kit/core": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/@dnd-kit/core/-/core-6.3.1.tgz", + "integrity": "sha512-xkGBRQQab4RLwgXxoqETICr6S5JlogafbhNsidmrkVv2YRs5MLwpjoF2qpiGjQt8S9AoxtIV603s0GIUpY5eYQ==", + "license": "MIT", + "dependencies": { + "@dnd-kit/accessibility": "^3.1.1", + "@dnd-kit/utilities": "^3.2.2", + "tslib": "^2.0.0" + }, + "peerDependencies": { + "react": ">=16.8.0", + "react-dom": ">=16.8.0" + } + }, + "node_modules/@dnd-kit/sortable": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/@dnd-kit/sortable/-/sortable-10.0.0.tgz", + "integrity": "sha512-+xqhmIIzvAYMGfBYYnbKuNicfSsk4RksY2XdmJhT+HAC01nix6fHCztU68jooFiMUB01Ky3F0FyOvhG/BZrWkg==", + "license": "MIT", + "dependencies": { + "@dnd-kit/utilities": "^3.2.2", + "tslib": "^2.0.0" + }, + "peerDependencies": { + "@dnd-kit/core": "^6.3.0", + "react": ">=16.8.0" + } + }, + "node_modules/@dnd-kit/utilities": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/@dnd-kit/utilities/-/utilities-3.2.2.tgz", + "integrity": "sha512-+MKAJEOfaBe5SmV6t34p80MMKhjvUz0vRrvVJbPT0WElzaOJ/1xs+D+KDv+tD/NE5ujfrChEcshd4fLn0wpiqg==", + "license": "MIT", + "dependencies": { + "tslib": "^2.0.0" + }, + "peerDependencies": { + "react": ">=16.8.0" + } + }, + "node_modules/@emotion/babel-plugin": { + "version": "11.13.5", + "resolved": "https://registry.npmjs.org/@emotion/babel-plugin/-/babel-plugin-11.13.5.tgz", + "integrity": "sha512-pxHCpT2ex+0q+HH91/zsdHkw/lXd468DIN2zvfvLtPKLLMo6gQj7oLObq8PhkrxOZb/gGCq03S3Z7PDhS8pduQ==", + "license": "MIT", + "dependencies": { + "@babel/helper-module-imports": "^7.16.7", + "@babel/runtime": "^7.18.3", + "@emotion/hash": "^0.9.2", + "@emotion/memoize": "^0.9.0", + "@emotion/serialize": "^1.3.3", + "babel-plugin-macros": "^3.1.0", + "convert-source-map": "^1.5.0", + "escape-string-regexp": "^4.0.0", + "find-root": "^1.1.0", + "source-map": "^0.5.7", + "stylis": "4.2.0" + } + }, + "node_modules/@emotion/cache": { + "version": "11.14.0", + "resolved": "https://registry.npmjs.org/@emotion/cache/-/cache-11.14.0.tgz", + "integrity": "sha512-L/B1lc/TViYk4DcpGxtAVbx0ZyiKM5ktoIyafGkH6zg/tj+mA+NE//aPYKG0k8kCHSHVJrpLpcAlOBEXQ3SavA==", + "license": "MIT", + "dependencies": { + "@emotion/memoize": "^0.9.0", + "@emotion/sheet": "^1.4.0", + "@emotion/utils": "^1.4.2", + "@emotion/weak-memoize": "^0.4.0", + "stylis": "4.2.0" + } + }, + "node_modules/@emotion/hash": { + "version": "0.9.2", + "resolved": "https://registry.npmjs.org/@emotion/hash/-/hash-0.9.2.tgz", + "integrity": "sha512-MyqliTZGuOm3+5ZRSaaBGP3USLw6+EGykkwZns2EPC5g8jJ4z9OrdZY9apkl3+UP9+sdz76YYkwCKP5gh8iY3g==", + "license": "MIT" + }, + "node_modules/@emotion/is-prop-valid": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@emotion/is-prop-valid/-/is-prop-valid-1.3.1.tgz", + "integrity": "sha512-/ACwoqx7XQi9knQs/G0qKvv5teDMhD7bXYns9N/wM8ah8iNb8jZ2uNO0YOgiq2o2poIvVtJS2YALasQuMSQ7Kw==", + "license": "MIT", + "dependencies": { + "@emotion/memoize": "^0.9.0" + } + }, + "node_modules/@emotion/memoize": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/@emotion/memoize/-/memoize-0.9.0.tgz", + "integrity": "sha512-30FAj7/EoJ5mwVPOWhAyCX+FPfMDrVecJAM+Iw9NRoSl4BBAQeqj4cApHHUXOVvIPgLVDsCFoz/hGD+5QQD1GQ==", + "license": "MIT" + }, + "node_modules/@emotion/react": { + "version": "11.14.0", + "resolved": "https://registry.npmjs.org/@emotion/react/-/react-11.14.0.tgz", + "integrity": "sha512-O000MLDBDdk/EohJPFUqvnp4qnHeYkVP5B0xEG0D/L7cOKP9kefu2DXn8dj74cQfsEzUqh+sr1RzFqiL1o+PpA==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.18.3", + "@emotion/babel-plugin": "^11.13.5", + "@emotion/cache": "^11.14.0", + "@emotion/serialize": "^1.3.3", + "@emotion/use-insertion-effect-with-fallbacks": "^1.2.0", + "@emotion/utils": "^1.4.2", + "@emotion/weak-memoize": "^0.4.0", + "hoist-non-react-statics": "^3.3.1" + }, + "peerDependencies": { + "react": ">=16.8.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@emotion/serialize": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@emotion/serialize/-/serialize-1.3.3.tgz", + "integrity": "sha512-EISGqt7sSNWHGI76hC7x1CksiXPahbxEOrC5RjmFRJTqLyEK9/9hZvBbiYn70dw4wuwMKiEMCUlR6ZXTSWQqxA==", + "license": "MIT", + "dependencies": { + "@emotion/hash": "^0.9.2", + "@emotion/memoize": "^0.9.0", + "@emotion/unitless": "^0.10.0", + "@emotion/utils": "^1.4.2", + "csstype": "^3.0.2" + } + }, + "node_modules/@emotion/sheet": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@emotion/sheet/-/sheet-1.4.0.tgz", + "integrity": "sha512-fTBW9/8r2w3dXWYM4HCB1Rdp8NLibOw2+XELH5m5+AkWiL/KqYX6dc0kKYlaYyKjrQ6ds33MCdMPEwgs2z1rqg==", + "license": "MIT" + }, + "node_modules/@emotion/styled": { + "version": "11.14.0", + "resolved": "https://registry.npmjs.org/@emotion/styled/-/styled-11.14.0.tgz", + "integrity": "sha512-XxfOnXFffatap2IyCeJyNov3kiDQWoR08gPUQxvbL7fxKryGBKUZUkG6Hz48DZwVrJSVh9sJboyV1Ds4OW6SgA==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.18.3", + "@emotion/babel-plugin": "^11.13.5", + "@emotion/is-prop-valid": "^1.3.0", + "@emotion/serialize": "^1.3.3", + "@emotion/use-insertion-effect-with-fallbacks": "^1.2.0", + "@emotion/utils": "^1.4.2" + }, + "peerDependencies": { + "@emotion/react": "^11.0.0-rc.0", + "react": ">=16.8.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@emotion/unitless": { + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/@emotion/unitless/-/unitless-0.10.0.tgz", + "integrity": "sha512-dFoMUuQA20zvtVTuxZww6OHoJYgrzfKM1t52mVySDJnMSEa08ruEvdYQbhvyu6soU+NeLVd3yKfTfT0NeV6qGg==", + "license": "MIT" + }, + "node_modules/@emotion/use-insertion-effect-with-fallbacks": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@emotion/use-insertion-effect-with-fallbacks/-/use-insertion-effect-with-fallbacks-1.2.0.tgz", + "integrity": "sha512-yJMtVdH59sxi/aVJBpk9FQq+OR8ll5GT8oWd57UpeaKEVGab41JWaCFA7FRLoMLloOZF/c/wsPoe+bfGmRKgDg==", + "license": "MIT", + "peerDependencies": { + "react": ">=16.8.0" + } + }, + "node_modules/@emotion/utils": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/@emotion/utils/-/utils-1.4.2.tgz", + "integrity": "sha512-3vLclRofFziIa3J2wDh9jjbkUz9qk5Vi3IZ/FSTKViB0k+ef0fPV7dYrUIugbgupYDx7v9ud/SjrtEP8Y4xLoA==", + "license": "MIT" + }, + "node_modules/@emotion/weak-memoize": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/@emotion/weak-memoize/-/weak-memoize-0.4.0.tgz", + "integrity": "sha512-snKqtPW01tN0ui7yu9rGv69aJXr/a/Ywvl11sUjNtEcRc+ng/mQriFL0wLXMef74iHa/EkftbDzU9F8iFbH+zg==", + "license": "MIT" + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.0.tgz", + "integrity": "sha512-O7vun9Sf8DFjH2UtqK8Ku3LkquL9SZL8OLY1T5NZkA34+wG3OQF7cl4Ql8vdNzM6fzBbYfLaiRLIOZ+2FOCgBQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.0.tgz", + "integrity": "sha512-PTyWCYYiU0+1eJKmw21lWtC+d08JDZPQ5g+kFyxP0V+es6VPPSUhM6zk8iImp2jbV6GwjX4pap0JFbUQN65X1g==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.0.tgz", + "integrity": "sha512-grvv8WncGjDSyUBjN9yHXNt+cq0snxXbDxy5pJtzMKGmmpPxeAmAhWxXI+01lU5rwZomDgD3kJwulEnhTRUd6g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.0.tgz", + "integrity": "sha512-m/ix7SfKG5buCnxasr52+LI78SQ+wgdENi9CqyCXwjVR2X4Jkz+BpC3le3AoBPYTC9NHklwngVXvbJ9/Akhrfg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.0.tgz", + "integrity": "sha512-mVwdUb5SRkPayVadIOI78K7aAnPamoeFR2bT5nszFUZ9P8UpK4ratOdYbZZXYSqPKMHfS1wdHCJk1P1EZpRdvw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.0.tgz", + "integrity": "sha512-DgDaYsPWFTS4S3nWpFcMn/33ZZwAAeAFKNHNa1QN0rI4pUjgqf0f7ONmXf6d22tqTY+H9FNdgeaAa+YIFUn2Rg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.0.tgz", + "integrity": "sha512-VN4ocxy6dxefN1MepBx/iD1dH5K8qNtNe227I0mnTRjry8tj5MRk4zprLEdG8WPyAPb93/e4pSgi1SoHdgOa4w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.0.tgz", + "integrity": "sha512-mrSgt7lCh07FY+hDD1TxiTyIHyttn6vnjesnPoVDNmDfOmggTLXRv8Id5fNZey1gl/V2dyVK1VXXqVsQIiAk+A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.0.tgz", + "integrity": "sha512-vkB3IYj2IDo3g9xX7HqhPYxVkNQe8qTK55fraQyTzTX/fxaDtXiEnavv9geOsonh2Fd2RMB+i5cbhu2zMNWJwg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.0.tgz", + "integrity": "sha512-9QAQjTWNDM/Vk2bgBl17yWuZxZNQIF0OUUuPZRKoDtqF2k4EtYbpyiG5/Dk7nqeK6kIJWPYldkOcBqjXjrUlmg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.0.tgz", + "integrity": "sha512-43ET5bHbphBegyeqLb7I1eYn2P/JYGNmzzdidq/w0T8E2SsYL1U6un2NFROFRg1JZLTzdCoRomg8Rvf9M6W6Gg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.0.tgz", + "integrity": "sha512-fC95c/xyNFueMhClxJmeRIj2yrSMdDfmqJnyOY4ZqsALkDrrKJfIg5NTMSzVBr5YW1jf+l7/cndBfP3MSDpoHw==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.0.tgz", + "integrity": "sha512-nkAMFju7KDW73T1DdH7glcyIptm95a7Le8irTQNO/qtkoyypZAnjchQgooFUDQhNAy4iu08N79W4T4pMBwhPwQ==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.0.tgz", + "integrity": "sha512-NhyOejdhRGS8Iwv+KKR2zTq2PpysF9XqY+Zk77vQHqNbo/PwZCzB5/h7VGuREZm1fixhs4Q/qWRSi5zmAiO4Fw==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.0.tgz", + "integrity": "sha512-5S/rbP5OY+GHLC5qXp1y/Mx//e92L1YDqkiBbO9TQOvuFXM+iDqUNG5XopAnXoRH3FjIUDkeGcY1cgNvnXp/kA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.0.tgz", + "integrity": "sha512-XM2BFsEBz0Fw37V0zU4CXfcfuACMrppsMFKdYY2WuTS3yi8O1nFOhil/xhKTmE1nPmVyvQJjJivgDT+xh8pXJA==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.0.tgz", + "integrity": "sha512-9yl91rHw/cpwMCNytUDxwj2XjFpxML0y9HAOH9pNVQDpQrBxHy01Dx+vaMu0N1CKa/RzBD2hB4u//nfc+Sd3Cw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.0.tgz", + "integrity": "sha512-RuG4PSMPFfrkH6UwCAqBzauBWTygTvb1nxWasEJooGSJ/NwRw7b2HOwyRTQIU97Hq37l3npXoZGYMy3b3xYvPw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.0.tgz", + "integrity": "sha512-jl+qisSB5jk01N5f7sPCsBENCOlPiS/xptD5yxOx2oqQfyourJwIKLRA2yqWdifj3owQZCL2sn6o08dBzZGQzA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.0.tgz", + "integrity": "sha512-21sUNbq2r84YE+SJDfaQRvdgznTD8Xc0oc3p3iW/a1EVWeNj/SdUCbm5U0itZPQYRuRTW20fPMWMpcrciH2EJw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.0.tgz", + "integrity": "sha512-2gwwriSMPcCFRlPlKx3zLQhfN/2WjJ2NSlg5TKLQOJdV0mSxIcYNTMhk3H3ulL/cak+Xj0lY1Ym9ysDV1igceg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.0.tgz", + "integrity": "sha512-bxI7ThgLzPrPz484/S9jLlvUAHYMzy6I0XiU1ZMeAEOBcS0VePBFxh1JjTQt3Xiat5b6Oh4x7UC7IwKQKIJRIg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.0.tgz", + "integrity": "sha512-ZUAc2YK6JW89xTbXvftxdnYy3m4iHIkDtK3CLce8wg8M2L+YZhIvO1DKpxrd0Yr59AeNNkTiic9YLf6FTtXWMw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.0.tgz", + "integrity": "sha512-eSNxISBu8XweVEWG31/JzjkIGbGIJN/TrRoiSVZwZ6pkC6VX4Im/WV2cz559/TXLcYbcrDN8JtKgd9DJVIo8GA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.0.tgz", + "integrity": "sha512-ZENoHJBxA20C2zFzh6AI4fT6RraMzjYw4xKWemRTRmRVtN9c5DcH9r/f2ihEkMjOW5eGgrwCslG/+Y/3bL+DHQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.8", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.8.tgz", + "integrity": "sha512-imAbBGkb+ebQyxKgzv5Hu2nmROxoDOXHh80evxdoXNOrvAnVx7zimzc1Oo5h9RlfV4vPXaE2iM5pOFbvOCClWA==", + "license": "MIT", + "dependencies": { + "@jridgewell/set-array": "^1.2.1", + "@jridgewell/sourcemap-codec": "^1.4.10", + "@jridgewell/trace-mapping": "^0.3.24" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/set-array": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.2.1.tgz", + "integrity": "sha512-R8gLRTZeyp03ymzP/6Lil/28tGeGEzhx1q2k703KGWRAI1VdvPIXdG70VJc2pAMw3NA6JKL5hhFu1sJX0Mnn/A==", + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.0.tgz", + "integrity": "sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ==", + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.25", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.25.tgz", + "integrity": "sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ==", + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@mui/core-downloads-tracker": { + "version": "6.4.6", + "resolved": "https://registry.npmjs.org/@mui/core-downloads-tracker/-/core-downloads-tracker-6.4.6.tgz", + "integrity": "sha512-rho5Q4IscbrVmK9rCrLTJmjLjfH6m/NcqKr/mchvck0EIXlyYUB9+Z0oVmkt/+Mben43LMRYBH8q/Uzxj/c4Vw==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mui-org" + } + }, + "node_modules/@mui/icons-material": { + "version": "6.4.6", + "resolved": "https://registry.npmjs.org/@mui/icons-material/-/icons-material-6.4.6.tgz", + "integrity": "sha512-rGJBvIQQbQAlyKYljHQ8wAQS/K2/uYwvemcpygnAmCizmCI4zSF9HQPuiG8Ql4YLZ6V/uKjA3WHIYmF/8sV+pQ==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.26.0" + }, + "engines": { + "node": ">=14.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mui-org" + }, + "peerDependencies": { + "@mui/material": "^6.4.6", + "@types/react": "^17.0.0 || ^18.0.0 || ^19.0.0", + "react": "^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@mui/material": { + "version": "6.4.6", + "resolved": "https://registry.npmjs.org/@mui/material/-/material-6.4.6.tgz", + "integrity": "sha512-6UyAju+DBOdMogfYmLiT3Nu7RgliorimNBny1pN/acOjc+THNFVE7hlxLyn3RDONoZJNDi/8vO4AQQr6dLAXqA==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.26.0", + "@mui/core-downloads-tracker": "^6.4.6", + "@mui/system": "^6.4.6", + "@mui/types": "^7.2.21", + "@mui/utils": "^6.4.6", + "@popperjs/core": "^2.11.8", + "@types/react-transition-group": "^4.4.12", + "clsx": "^2.1.1", + "csstype": "^3.1.3", + "prop-types": "^15.8.1", + "react-is": "^19.0.0", + "react-transition-group": "^4.4.5" + }, + "engines": { + "node": ">=14.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mui-org" + }, + "peerDependencies": { + "@emotion/react": "^11.5.0", + "@emotion/styled": "^11.3.0", + "@mui/material-pigment-css": "^6.4.6", + "@types/react": "^17.0.0 || ^18.0.0 || ^19.0.0", + "react": "^17.0.0 || ^18.0.0 || ^19.0.0", + "react-dom": "^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@emotion/react": { + "optional": true + }, + "@emotion/styled": { + "optional": true + }, + "@mui/material-pigment-css": { + "optional": true + }, + "@types/react": { + "optional": true + } + } + }, + "node_modules/@mui/private-theming": { + "version": "6.4.6", + "resolved": "https://registry.npmjs.org/@mui/private-theming/-/private-theming-6.4.6.tgz", + "integrity": "sha512-T5FxdPzCELuOrhpA2g4Pi6241HAxRwZudzAuL9vBvniuB5YU82HCmrARw32AuCiyTfWzbrYGGpZ4zyeqqp9RvQ==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.26.0", + "@mui/utils": "^6.4.6", + "prop-types": "^15.8.1" + }, + "engines": { + "node": ">=14.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mui-org" + }, + "peerDependencies": { + "@types/react": "^17.0.0 || ^18.0.0 || ^19.0.0", + "react": "^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@mui/styled-engine": { + "version": "6.4.6", + "resolved": "https://registry.npmjs.org/@mui/styled-engine/-/styled-engine-6.4.6.tgz", + "integrity": "sha512-vSWYc9ZLX46be5gP+FCzWVn5rvDr4cXC5JBZwSIkYk9xbC7GeV+0kCvB8Q6XLFQJy+a62bbqtmdwS4Ghi9NBlQ==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.26.0", + "@emotion/cache": "^11.13.5", + "@emotion/serialize": "^1.3.3", + "@emotion/sheet": "^1.4.0", + "csstype": "^3.1.3", + "prop-types": "^15.8.1" + }, + "engines": { + "node": ">=14.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mui-org" + }, + "peerDependencies": { + "@emotion/react": "^11.4.1", + "@emotion/styled": "^11.3.0", + "react": "^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@emotion/react": { + "optional": true + }, + "@emotion/styled": { + "optional": true + } + } + }, + "node_modules/@mui/system": { + "version": "6.4.6", + "resolved": "https://registry.npmjs.org/@mui/system/-/system-6.4.6.tgz", + "integrity": "sha512-FQjWwPec7pMTtB/jw5f9eyLynKFZ6/Ej9vhm5kGdtmts1z5b7Vyn3Rz6kasfYm1j2TfrfGnSXRvvtwVWxjpz6g==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.26.0", + "@mui/private-theming": "^6.4.6", + "@mui/styled-engine": "^6.4.6", + "@mui/types": "^7.2.21", + "@mui/utils": "^6.4.6", + "clsx": "^2.1.1", + "csstype": "^3.1.3", + "prop-types": "^15.8.1" + }, + "engines": { + "node": ">=14.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mui-org" + }, + "peerDependencies": { + "@emotion/react": "^11.5.0", + "@emotion/styled": "^11.3.0", + "@types/react": "^17.0.0 || ^18.0.0 || ^19.0.0", + "react": "^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@emotion/react": { + "optional": true + }, + "@emotion/styled": { + "optional": true + }, + "@types/react": { + "optional": true + } + } + }, + "node_modules/@mui/types": { + "version": "7.2.21", + "resolved": "https://registry.npmjs.org/@mui/types/-/types-7.2.21.tgz", + "integrity": "sha512-6HstngiUxNqLU+/DPqlUJDIPbzUBxIVHb1MmXP0eTWDIROiCR2viugXpEif0PPe2mLqqakPzzRClWAnK+8UJww==", + "license": "MIT", + "peerDependencies": { + "@types/react": "^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@mui/utils": { + "version": "6.4.6", + "resolved": "https://registry.npmjs.org/@mui/utils/-/utils-6.4.6.tgz", + "integrity": "sha512-43nZeE1pJF2anGafNydUcYFPtHwAqiBiauRtaMvurdrZI3YrUjHkAu43RBsxef7OFtJMXGiHFvq43kb7lig0sA==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.26.0", + "@mui/types": "^7.2.21", + "@types/prop-types": "^15.7.14", + "clsx": "^2.1.1", + "prop-types": "^15.8.1", + "react-is": "^19.0.0" + }, + "engines": { + "node": ">=14.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mui-org" + }, + "peerDependencies": { + "@types/react": "^17.0.0 || ^18.0.0 || ^19.0.0", + "react": "^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@popperjs/core": { + "version": "2.11.8", + "resolved": "https://registry.npmjs.org/@popperjs/core/-/core-2.11.8.tgz", + "integrity": "sha512-P1st0aksCrn9sGZhp8GMYwBnQsbvAWsZAX44oXNNvLHGqAOcoVxmjZiohstwQ7SqKnbR47akdNi+uleWD8+g6A==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/popperjs" + } + }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.34.8", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.34.8.tgz", + "integrity": "sha512-q217OSE8DTp8AFHuNHXo0Y86e1wtlfVrXiAlwkIvGRQv9zbc6mE3sjIVfwI8sYUyNxwOg0j/Vm1RKM04JcWLJw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.34.8", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.34.8.tgz", + "integrity": "sha512-Gigjz7mNWaOL9wCggvoK3jEIUUbGul656opstjaUSGC3eT0BM7PofdAJaBfPFWWkXNVAXbaQtC99OCg4sJv70Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.34.8", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.34.8.tgz", + "integrity": "sha512-02rVdZ5tgdUNRxIUrFdcMBZQoaPMrxtwSb+/hOfBdqkatYHR3lZ2A2EGyHq2sGOd0Owk80oV3snlDASC24He3Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.34.8", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.34.8.tgz", + "integrity": "sha512-qIP/elwR/tq/dYRx3lgwK31jkZvMiD6qUtOycLhTzCvrjbZ3LjQnEM9rNhSGpbLXVJYQ3rq39A6Re0h9tU2ynw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.34.8", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.34.8.tgz", + "integrity": "sha512-IQNVXL9iY6NniYbTaOKdrlVP3XIqazBgJOVkddzJlqnCpRi/yAeSOa8PLcECFSQochzqApIOE1GHNu3pCz+BDA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.34.8", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.34.8.tgz", + "integrity": "sha512-TYXcHghgnCqYFiE3FT5QwXtOZqDj5GmaFNTNt3jNC+vh22dc/ukG2cG+pi75QO4kACohZzidsq7yKTKwq/Jq7Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.34.8", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.34.8.tgz", + "integrity": "sha512-A4iphFGNkWRd+5m3VIGuqHnG3MVnqKe7Al57u9mwgbyZ2/xF9Jio72MaY7xxh+Y87VAHmGQr73qoKL9HPbXj1g==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.34.8", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.34.8.tgz", + "integrity": "sha512-S0lqKLfTm5u+QTxlFiAnb2J/2dgQqRy/XvziPtDd1rKZFXHTyYLoVL58M/XFwDI01AQCDIevGLbQrMAtdyanpA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.34.8", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.34.8.tgz", + "integrity": "sha512-jpz9YOuPiSkL4G4pqKrus0pn9aYwpImGkosRKwNi+sJSkz+WU3anZe6hi73StLOQdfXYXC7hUfsQlTnjMd3s1A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.34.8", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.34.8.tgz", + "integrity": "sha512-KdSfaROOUJXgTVxJNAZ3KwkRc5nggDk+06P6lgi1HLv1hskgvxHUKZ4xtwHkVYJ1Rep4GNo+uEfycCRRxht7+Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loongarch64-gnu": { + "version": "4.34.8", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loongarch64-gnu/-/rollup-linux-loongarch64-gnu-4.34.8.tgz", + "integrity": "sha512-NyF4gcxwkMFRjgXBM6g2lkT58OWztZvw5KkV2K0qqSnUEqCVcqdh2jN4gQrTn/YUpAcNKyFHfoOZEer9nwo6uQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-powerpc64le-gnu": { + "version": "4.34.8", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.34.8.tgz", + "integrity": "sha512-LMJc999GkhGvktHU85zNTDImZVUCJ1z/MbAJTnviiWmmjyckP5aQsHtcujMjpNdMZPT2rQEDBlJfubhs3jsMfw==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.34.8", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.34.8.tgz", + "integrity": "sha512-xAQCAHPj8nJq1PI3z8CIZzXuXCstquz7cIOL73HHdXiRcKk8Ywwqtx2wrIy23EcTn4aZ2fLJNBB8d0tQENPCmw==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.34.8", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.34.8.tgz", + "integrity": "sha512-DdePVk1NDEuc3fOe3dPPTb+rjMtuFw89gw6gVWxQFAuEqqSdDKnrwzZHrUYdac7A7dXl9Q2Vflxpme15gUWQFA==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.34.8", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.34.8.tgz", + "integrity": "sha512-8y7ED8gjxITUltTUEJLQdgpbPh1sUQ0kMTmufRF/Ns5tI9TNMNlhWtmPKKHCU0SilX+3MJkZ0zERYYGIVBYHIA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.34.8", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.34.8.tgz", + "integrity": "sha512-SCXcP0ZpGFIe7Ge+McxY5zKxiEI5ra+GT3QRxL0pMMtxPfpyLAKleZODi1zdRHkz5/BhueUrYtYVgubqe9JBNQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.34.8", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.34.8.tgz", + "integrity": "sha512-YHYsgzZgFJzTRbth4h7Or0m5O74Yda+hLin0irAIobkLQFRQd1qWmnoVfwmKm9TXIZVAD0nZ+GEb2ICicLyCnQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.34.8", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.34.8.tgz", + "integrity": "sha512-r3NRQrXkHr4uWy5TOjTpTYojR9XmF0j/RYgKCef+Ag46FWUTltm5ziticv8LdNsDMehjJ543x/+TJAek/xBA2w==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.34.8", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.34.8.tgz", + "integrity": "sha512-U0FaE5O1BCpZSeE6gBl3c5ObhePQSfk9vDRToMmTkbhCOgW4jqvtS5LGyQ76L1fH8sM0keRp4uDTsbjiUyjk0g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@types/babel__core": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", + "integrity": "sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.20.7", + "@babel/types": "^7.20.7", + "@types/babel__generator": "*", + "@types/babel__template": "*", + "@types/babel__traverse": "*" + } + }, + "node_modules/@types/babel__generator": { + "version": "7.6.8", + "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.6.8.tgz", + "integrity": "sha512-ASsj+tpEDsEiFr1arWrlN6V3mdfjRMZt6LtK/Vp/kreFLnr5QH5+DhvD5nINYZXzwJvXeGq+05iUXcAzVrqWtw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__template": { + "version": "7.4.4", + "resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.4.4.tgz", + "integrity": "sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.1.0", + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__traverse": { + "version": "7.20.6", + "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.20.6.tgz", + "integrity": "sha512-r1bzfrm0tomOI8g1SzvCaQHo6Lcv6zu0EA+W2kHrt8dyrHQxGzBBL4kdkzIS+jBMV+EYcMAEAqXqYaLJq5rOZg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.20.7" + } + }, + "node_modules/@types/debug": { + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.12.tgz", + "integrity": "sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==", + "license": "MIT", + "dependencies": { + "@types/ms": "*" + } + }, + "node_modules/@types/estree": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.6.tgz", + "integrity": "sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw==", + "license": "MIT" + }, + "node_modules/@types/estree-jsx": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@types/estree-jsx/-/estree-jsx-1.0.5.tgz", + "integrity": "sha512-52CcUVNFyfb1A2ALocQw/Dd1BQFNmSdkuC3BkZ6iqhdMfQz7JWOFRuJFloOzjk+6WijU56m9oKXFAXc7o3Towg==", + "license": "MIT", + "dependencies": { + "@types/estree": "*" + } + }, + "node_modules/@types/hast": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/hast/-/hast-3.0.4.tgz", + "integrity": "sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ==", + "license": "MIT", + "dependencies": { + "@types/unist": "*" + } + }, + "node_modules/@types/mdast": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-4.0.4.tgz", + "integrity": "sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==", + "license": "MIT", + "dependencies": { + "@types/unist": "*" + } + }, + "node_modules/@types/ms": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@types/ms/-/ms-2.1.0.tgz", + "integrity": "sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==", + "license": "MIT" + }, + "node_modules/@types/parse-json": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.2.tgz", + "integrity": "sha512-dISoDXWWQwUquiKsyZ4Ng+HX2KsPL7LyHKHQwgGFEA3IaKac4Obd+h2a/a6waisAoepJlBcx9paWqjA8/HVjCw==", + "license": "MIT" + }, + "node_modules/@types/prop-types": { + "version": "15.7.14", + "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.14.tgz", + "integrity": "sha512-gNMvNH49DJ7OJYv+KAKn0Xp45p8PLl6zo2YnvDIbTd4J6MER2BmWN49TG7n9LvkyihINxeKW8+3bfS2yDC9dzQ==", + "license": "MIT" + }, + "node_modules/@types/react": { + "version": "19.0.10", + "resolved": "https://registry.npmjs.org/@types/react/-/react-19.0.10.tgz", + "integrity": "sha512-JuRQ9KXLEjaUNjTWpzuR231Z2WpIwczOkBEIvbHNCzQefFIT0L8IqE6NV6ULLyC1SI/i234JnDoMkfg+RjQj2g==", + "license": "MIT", + "dependencies": { + "csstype": "^3.0.2" + } + }, + "node_modules/@types/react-dom": { + "version": "19.0.4", + "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.0.4.tgz", + "integrity": "sha512-4fSQ8vWFkg+TGhePfUzVmat3eC14TXYSsiiDSLI0dVLsrm9gZFABjPy/Qu6TKgl1tq1Bu1yDsuQgY3A3DOjCcg==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "^19.0.0" + } + }, + "node_modules/@types/react-transition-group": { + "version": "4.4.12", + "resolved": "https://registry.npmjs.org/@types/react-transition-group/-/react-transition-group-4.4.12.tgz", + "integrity": "sha512-8TV6R3h2j7a91c+1DXdJi3Syo69zzIZbz7Lg5tORM5LEJG7X/E6a1V3drRyBRZq7/utz7A+c4OgYLiLcYGHG6w==", + "license": "MIT", + "peerDependencies": { + "@types/react": "*" + } + }, + "node_modules/@types/unist": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", + "integrity": "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==", + "license": "MIT" + }, + "node_modules/@ungap/structured-clone": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.3.0.tgz", + "integrity": "sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==", + "license": "ISC" + }, + "node_modules/@vitejs/plugin-react": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-4.3.4.tgz", + "integrity": "sha512-SCCPBJtYLdE8PX/7ZQAs1QAZ8Jqwih+0VBLum1EGqmCCQal+MIUqLCzj3ZUy8ufbC0cAM4LRlSTm7IQJwWT4ug==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/core": "^7.26.0", + "@babel/plugin-transform-react-jsx-self": "^7.25.9", + "@babel/plugin-transform-react-jsx-source": "^7.25.9", + "@types/babel__core": "^7.20.5", + "react-refresh": "^0.14.2" + }, + "engines": { + "node": "^14.18.0 || >=16.0.0" + }, + "peerDependencies": { + "vite": "^4.2.0 || ^5.0.0 || ^6.0.0" + } + }, + "node_modules/babel-plugin-macros": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/babel-plugin-macros/-/babel-plugin-macros-3.1.0.tgz", + "integrity": "sha512-Cg7TFGpIr01vOQNODXOOaGz2NpCU5gl8x1qJFbb6hbZxR7XrcE2vtbAsTAbJ7/xwJtUuJEw8K8Zr/AE0LHlesg==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.12.5", + "cosmiconfig": "^7.0.0", + "resolve": "^1.19.0" + }, + "engines": { + "node": ">=10", + "npm": ">=6" + } + }, + "node_modules/bail": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/bail/-/bail-2.0.2.tgz", + "integrity": "sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/braces": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fill-range": "^7.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/browserslist": { + "version": "4.24.4", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.24.4.tgz", + "integrity": "sha512-KDi1Ny1gSePi1vm0q4oxSF8b4DR44GF4BbmS2YdhPLOEqd8pDviZOGH/GsmRwoWJ2+5Lr085X7naowMwKHDG1A==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "caniuse-lite": "^1.0.30001688", + "electron-to-chromium": "^1.5.73", + "node-releases": "^2.0.19", + "update-browserslist-db": "^1.1.1" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, + "node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001701", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001701.tgz", + "integrity": "sha512-faRs/AW3jA9nTwmJBSO1PQ6L/EOgsB5HMQQq4iCu5zhPgVVgO/pZRHlmatwijZKetFw8/Pr4q6dEN8sJuq8qTw==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "CC-BY-4.0" + }, + "node_modules/ccount": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/ccount/-/ccount-2.0.1.tgz", + "integrity": "sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-entities": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-2.0.2.tgz", + "integrity": "sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-entities-html4": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/character-entities-html4/-/character-entities-html4-2.1.0.tgz", + "integrity": "sha512-1v7fgQRj6hnSwFpq1Eu0ynr/CDEw0rXo2B61qXrLNdHZmPKgb7fqS1a2JwF0rISo9q77jDI8VMEHoApn8qDoZA==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-entities-legacy": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-3.0.0.tgz", + "integrity": "sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-reference-invalid": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/character-reference-invalid/-/character-reference-invalid-2.0.1.tgz", + "integrity": "sha512-iBZ4F4wRbyORVsu0jPV7gXkOsGYjGHPmAyv+HiHG8gi5PtC9KI2j1+v8/tlibRvjoWX027ypmG/n0HtO5t7unw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/clsx": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz", + "integrity": "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/comma-separated-tokens": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-2.0.3.tgz", + "integrity": "sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/convert-source-map": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.9.0.tgz", + "integrity": "sha512-ASFBup0Mz1uyiIjANan1jzLQami9z1PoYSZCiiYW2FczPbenXc45FZdBZLzOT+r6+iciuEModtmCti+hjaAk0A==", + "license": "MIT" + }, + "node_modules/cosmiconfig": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-7.1.0.tgz", + "integrity": "sha512-AdmX6xUzdNASswsFtmwSt7Vj8po9IuqXm0UXz7QKPuEUmPB4XyjGfaAr2PSuELMwkRMVH1EpIkX5bTZGRB3eCA==", + "license": "MIT", + "dependencies": { + "@types/parse-json": "^4.0.0", + "import-fresh": "^3.2.1", + "parse-json": "^5.0.0", + "path-type": "^4.0.0", + "yaml": "^1.10.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/cosmiconfig/node_modules/yaml": { + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-1.10.2.tgz", + "integrity": "sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==", + "license": "ISC", + "engines": { + "node": ">= 6" + } + }, + "node_modules/csstype": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.3.tgz", + "integrity": "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==", + "license": "MIT" + }, + "node_modules/debug": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", + "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/decode-named-character-reference": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/decode-named-character-reference/-/decode-named-character-reference-1.0.2.tgz", + "integrity": "sha512-O8x12RzrUF8xyVcY0KJowWsmaJxQbmy0/EtnNtHRpsOcT7dFk5W598coHqBVpmWo1oQQfsCqfCmkZN5DJrZVdg==", + "license": "MIT", + "dependencies": { + "character-entities": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/devlop": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/devlop/-/devlop-1.1.0.tgz", + "integrity": "sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==", + "license": "MIT", + "dependencies": { + "dequal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/dom-helpers": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/dom-helpers/-/dom-helpers-5.2.1.tgz", + "integrity": "sha512-nRCa7CK3VTrM2NmGkIy4cbK7IZlgBE/PYMn55rrXefr5xXDP0LdtfPnblFDoVdcAfslJ7or6iqAUnx0CCGIWQA==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.8.7", + "csstype": "^3.0.2" + } + }, + "node_modules/electron-to-chromium": { + "version": "1.5.107", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.107.tgz", + "integrity": "sha512-dJr1o6yCntRkXElnhsHh1bAV19bo/hKyFf7tCcWgpXbuFIF0Lakjgqv5LRfSDaNzAII8Fnxg2tqgHkgCvxdbxw==", + "dev": true, + "license": "ISC" + }, + "node_modules/error-ex": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", + "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==", + "license": "MIT", + "dependencies": { + "is-arrayish": "^0.2.1" + } + }, + "node_modules/esbuild": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.0.tgz", + "integrity": "sha512-BXq5mqc8ltbaN34cDqWuYKyNhX8D/Z0J1xdtdQ8UcIIIyJyz+ZMKUt58tF3SrZ85jcfN/PZYhjR5uDQAYNVbuw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.25.0", + "@esbuild/android-arm": "0.25.0", + "@esbuild/android-arm64": "0.25.0", + "@esbuild/android-x64": "0.25.0", + "@esbuild/darwin-arm64": "0.25.0", + "@esbuild/darwin-x64": "0.25.0", + "@esbuild/freebsd-arm64": "0.25.0", + "@esbuild/freebsd-x64": "0.25.0", + "@esbuild/linux-arm": "0.25.0", + "@esbuild/linux-arm64": "0.25.0", + "@esbuild/linux-ia32": "0.25.0", + "@esbuild/linux-loong64": "0.25.0", + "@esbuild/linux-mips64el": "0.25.0", + "@esbuild/linux-ppc64": "0.25.0", + "@esbuild/linux-riscv64": "0.25.0", + "@esbuild/linux-s390x": "0.25.0", + "@esbuild/linux-x64": "0.25.0", + "@esbuild/netbsd-arm64": "0.25.0", + "@esbuild/netbsd-x64": "0.25.0", + "@esbuild/openbsd-arm64": "0.25.0", + "@esbuild/openbsd-x64": "0.25.0", + "@esbuild/sunos-x64": "0.25.0", + "@esbuild/win32-arm64": "0.25.0", + "@esbuild/win32-ia32": "0.25.0", + "@esbuild/win32-x64": "0.25.0" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/estree-util-is-identifier-name": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/estree-util-is-identifier-name/-/estree-util-is-identifier-name-3.0.0.tgz", + "integrity": "sha512-hFtqIDZTIUZ9BXLb8y4pYGyk6+wekIivNVTcmvk8NoOh+VeRn5y6cEHzbURrWbfp1fIqdVipilzj+lfaadNZmg==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", + "license": "MIT" + }, + "node_modules/fill-range": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "dev": true, + "license": "MIT", + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/find-root": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/find-root/-/find-root-1.1.0.tgz", + "integrity": "sha512-NKfW6bec6GfKc0SGx1e07QZY9PE99u0Bft/0rzSD5k3sO/vwkVUpDUKVm5Gpp5Ue3YfShPFTX2070tDs5kB9Ng==", + "license": "MIT" + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/gensync": { + "version": "1.0.0-beta.2", + "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", + "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/globals": { + "version": "11.12.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", + "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/hast-util-to-jsx-runtime": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/hast-util-to-jsx-runtime/-/hast-util-to-jsx-runtime-2.3.5.tgz", + "integrity": "sha512-gHD+HoFxOMmmXLuq9f2dZDMQHVcplCVpMfBNRpJsF03yyLZvJGzsFORe8orVuYDX9k2w0VH0uF8oryFd1whqKQ==", + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "@types/hast": "^3.0.0", + "@types/unist": "^3.0.0", + "comma-separated-tokens": "^2.0.0", + "devlop": "^1.0.0", + "estree-util-is-identifier-name": "^3.0.0", + "hast-util-whitespace": "^3.0.0", + "mdast-util-mdx-expression": "^2.0.0", + "mdast-util-mdx-jsx": "^3.0.0", + "mdast-util-mdxjs-esm": "^2.0.0", + "property-information": "^7.0.0", + "space-separated-tokens": "^2.0.0", + "style-to-object": "^1.0.0", + "unist-util-position": "^5.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-whitespace": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/hast-util-whitespace/-/hast-util-whitespace-3.0.0.tgz", + "integrity": "sha512-88JUN06ipLwsnv+dVn+OIYOvAuvBMy/Qoi6O7mQHxdPXpjy+Cd6xRkWwux7DKO+4sYILtLBRIKgsdpS2gQc7qw==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hoist-non-react-statics": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/hoist-non-react-statics/-/hoist-non-react-statics-3.3.2.tgz", + "integrity": "sha512-/gGivxi8JPKWNm/W0jSmzcMPpfpPLc3dY/6GxhX2hQ9iGj3aDfklV4ET7NjKpSinLpJ5vafa9iiGIEZg10SfBw==", + "license": "BSD-3-Clause", + "dependencies": { + "react-is": "^16.7.0" + } + }, + "node_modules/hoist-non-react-statics/node_modules/react-is": { + "version": "16.13.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz", + "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==", + "license": "MIT" + }, + "node_modules/html-url-attributes": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/html-url-attributes/-/html-url-attributes-3.0.1.tgz", + "integrity": "sha512-ol6UPyBWqsrO6EJySPz2O7ZSr856WDrEzM5zMqp+FJJLGMW35cLYmmZnl0vztAZxRUoNZJFTCohfjuIJ8I4QBQ==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/import-fresh": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", + "integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==", + "license": "MIT", + "dependencies": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/inline-style-parser": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/inline-style-parser/-/inline-style-parser-0.2.4.tgz", + "integrity": "sha512-0aO8FkhNZlj/ZIbNi7Lxxr12obT7cL1moPfE4tg1LkX7LlLfC6DeX4l2ZEud1ukP9jNQyNnfzQVqwbwmAATY4Q==", + "license": "MIT" + }, + "node_modules/is-alphabetical": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-alphabetical/-/is-alphabetical-2.0.1.tgz", + "integrity": "sha512-FWyyY60MeTNyeSRpkM2Iry0G9hpr7/9kD40mD/cGQEuilcZYS4okz8SN2Q6rLCJ8gbCt6fN+rC+6tMGS99LaxQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-alphanumerical": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-alphanumerical/-/is-alphanumerical-2.0.1.tgz", + "integrity": "sha512-hmbYhX/9MUMF5uh7tOXyK/n0ZvWpad5caBA17GsC6vyuCqaWliRG5K1qS9inmUhEMaOBIW7/whAnSwveW/LtZw==", + "license": "MIT", + "dependencies": { + "is-alphabetical": "^2.0.0", + "is-decimal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-arrayish": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", + "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", + "license": "MIT" + }, + "node_modules/is-core-module": { + "version": "2.16.1", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.1.tgz", + "integrity": "sha512-UfoeMA6fIJ8wTYFEUjelnaGI67v6+N7qXJEvQuIGa99l4xsCruSYOVSQ0uPANn4dAzm8lkYPaKLrrijLq7x23w==", + "license": "MIT", + "dependencies": { + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-decimal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-decimal/-/is-decimal-2.0.1.tgz", + "integrity": "sha512-AAB9hiomQs5DXWcRB1rqsxGUstbRroFOPPVAomNk/3XHR5JyEZChOyTWe2oayKnsSsr/kcGqF+z6yuH6HHpN0A==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-hexadecimal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-hexadecimal/-/is-hexadecimal-2.0.1.tgz", + "integrity": "sha512-DgZQp241c8oO6cA1SbTEWiXeoxV42vlcJxgH+B3hi1AiqqKruZR3ZGF8In3fj4+/y/7rHvlOZLZtgJ/4ttYGZg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/is-plain-obj": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-4.1.0.tgz", + "integrity": "sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "license": "MIT" + }, + "node_modules/jsesc": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", + "integrity": "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==", + "license": "MIT", + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/json-parse-even-better-errors": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", + "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", + "license": "MIT" + }, + "node_modules/json5": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", + "dev": true, + "license": "MIT", + "bin": { + "json5": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/lines-and-columns": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", + "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", + "license": "MIT" + }, + "node_modules/longest-streak": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/longest-streak/-/longest-streak-3.1.0.tgz", + "integrity": "sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/loose-envify": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", + "integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==", + "license": "MIT", + "dependencies": { + "js-tokens": "^3.0.0 || ^4.0.0" + }, + "bin": { + "loose-envify": "cli.js" + } + }, + "node_modules/lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "dev": true, + "license": "ISC", + "dependencies": { + "yallist": "^3.0.2" + } + }, + "node_modules/mdast-util-from-markdown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/mdast-util-from-markdown/-/mdast-util-from-markdown-2.0.2.tgz", + "integrity": "sha512-uZhTV/8NBuw0WHkPTrCqDOl0zVe1BIng5ZtHoDk49ME1qqcjYmmLmOf0gELgcRMxN4w2iuIeVso5/6QymSrgmA==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "@types/unist": "^3.0.0", + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "mdast-util-to-string": "^4.0.0", + "micromark": "^4.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-decode-string": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0", + "unist-util-stringify-position": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-mdx-expression": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-mdx-expression/-/mdast-util-mdx-expression-2.0.1.tgz", + "integrity": "sha512-J6f+9hUp+ldTZqKRSg7Vw5V6MqjATc+3E4gf3CFNcuZNWD8XdyI6zQ8GqH7f8169MM6P7hMBRDVGnn7oHB9kXQ==", + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-mdx-jsx": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/mdast-util-mdx-jsx/-/mdast-util-mdx-jsx-3.2.0.tgz", + "integrity": "sha512-lj/z8v0r6ZtsN/cGNNtemmmfoLAFZnjMbNyLzBafjzikOM+glrjNHPlf6lQDOTccj9n5b0PPihEBbhneMyGs1Q==", + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "@types/unist": "^3.0.0", + "ccount": "^2.0.0", + "devlop": "^1.1.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0", + "parse-entities": "^4.0.0", + "stringify-entities": "^4.0.0", + "unist-util-stringify-position": "^4.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-mdxjs-esm": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-mdxjs-esm/-/mdast-util-mdxjs-esm-2.0.1.tgz", + "integrity": "sha512-EcmOpxsZ96CvlP03NghtH1EsLtr0n9Tm4lPUJUBccV9RwUOneqSycg19n5HGzCf+10LozMRSObtVr3ee1WoHtg==", + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-phrasing": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/mdast-util-phrasing/-/mdast-util-phrasing-4.1.0.tgz", + "integrity": "sha512-TqICwyvJJpBwvGAMZjj4J2n0X8QWp21b9l0o7eXyVJ25YNWYbJDVIyD1bZXE6WtV6RmKJVYmQAKWa0zWOABz2w==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "unist-util-is": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-to-hast": { + "version": "13.2.0", + "resolved": "https://registry.npmjs.org/mdast-util-to-hast/-/mdast-util-to-hast-13.2.0.tgz", + "integrity": "sha512-QGYKEuUsYT9ykKBCMOEDLsU5JRObWQusAolFMeko/tYPufNkRffBAQjIE+99jbA87xv6FgmjLtwjh9wBWajwAA==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "@ungap/structured-clone": "^1.0.0", + "devlop": "^1.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "trim-lines": "^3.0.0", + "unist-util-position": "^5.0.0", + "unist-util-visit": "^5.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-to-markdown": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/mdast-util-to-markdown/-/mdast-util-to-markdown-2.1.2.tgz", + "integrity": "sha512-xj68wMTvGXVOKonmog6LwyJKrYXZPvlwabaryTjLh9LuvovB/KAH+kvi8Gjj+7rJjsFi23nkUxRQv1KqSroMqA==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "@types/unist": "^3.0.0", + "longest-streak": "^3.0.0", + "mdast-util-phrasing": "^4.0.0", + "mdast-util-to-string": "^4.0.0", + "micromark-util-classify-character": "^2.0.0", + "micromark-util-decode-string": "^2.0.0", + "unist-util-visit": "^5.0.0", + "zwitch": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-to-string": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-to-string/-/mdast-util-to-string-4.0.0.tgz", + "integrity": "sha512-0H44vDimn51F0YwvxSJSm0eCDOJTRlmN0R1yBh4HLj9wiV1Dn0QoXGbvFAWj2hSItVTlCmBF1hqKlIyUBVFLPg==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/micromark/-/micromark-4.0.1.tgz", + "integrity": "sha512-eBPdkcoCNvYcxQOAKAlceo5SNdzZWfF+FcSupREAzdAh9rRmE239CEQAiTwIgblwnoM8zzj35sZ5ZwvSEOF6Kw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "@types/debug": "^4.0.0", + "debug": "^4.0.0", + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "micromark-core-commonmark": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-combine-extensions": "^2.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-encode": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-subtokenize": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-core-commonmark": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/micromark-core-commonmark/-/micromark-core-commonmark-2.0.2.tgz", + "integrity": "sha512-FKjQKbxd1cibWMM1P9N+H8TwlgGgSkWZMmfuVucLCHaYqeSvJ0hFeHsIa65pA2nYbes0f8LDHPMrd9X7Ujxg9w==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "micromark-factory-destination": "^2.0.0", + "micromark-factory-label": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-factory-title": "^2.0.0", + "micromark-factory-whitespace": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-classify-character": "^2.0.0", + "micromark-util-html-tag-name": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-subtokenize": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-destination": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-destination/-/micromark-factory-destination-2.0.1.tgz", + "integrity": "sha512-Xe6rDdJlkmbFRExpTOmRj9N3MaWmbAgdpSrBQvCFqhezUn4AHqJHbaEnfbVYYiexVSs//tqOdY/DxhjdCiJnIA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-label": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-label/-/micromark-factory-label-2.0.1.tgz", + "integrity": "sha512-VFMekyQExqIW7xIChcXn4ok29YE3rnuyveW3wZQWWqF4Nv9Wk5rgJ99KzPvHjkmPXF93FXIbBp6YdW3t71/7Vg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-space": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-space/-/micromark-factory-space-2.0.1.tgz", + "integrity": "sha512-zRkxjtBxxLd2Sc0d+fbnEunsTj46SWXgXciZmHq0kDYGnck/ZSGj9/wULTV95uoeYiK5hRXP2mJ98Uo4cq/LQg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-title": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-title/-/micromark-factory-title-2.0.1.tgz", + "integrity": "sha512-5bZ+3CjhAd9eChYTHsjy6TGxpOFSKgKKJPJxr293jTbfry2KDoWkhBb6TcPVB4NmzaPhMs1Frm9AZH7OD4Cjzw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-whitespace": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-whitespace/-/micromark-factory-whitespace-2.0.1.tgz", + "integrity": "sha512-Ob0nuZ3PKt/n0hORHyvoD9uZhr+Za8sFoP+OnMcnWK5lngSzALgQYKMr9RJVOWLqQYuyn6ulqGWSXdwf6F80lQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-character": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-2.1.1.tgz", + "integrity": "sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-chunked": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-chunked/-/micromark-util-chunked-2.0.1.tgz", + "integrity": "sha512-QUNFEOPELfmvv+4xiNg2sRYeS/P84pTW0TCgP5zc9FpXetHY0ab7SxKyAQCNCc1eK0459uoLI1y5oO5Vc1dbhA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-classify-character": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-classify-character/-/micromark-util-classify-character-2.0.1.tgz", + "integrity": "sha512-K0kHzM6afW/MbeWYWLjoHQv1sgg2Q9EccHEDzSkxiP/EaagNzCm7T/WMKZ3rjMbvIpvBiZgwR3dKMygtA4mG1Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-combine-extensions": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-combine-extensions/-/micromark-util-combine-extensions-2.0.1.tgz", + "integrity": "sha512-OnAnH8Ujmy59JcyZw8JSbK9cGpdVY44NKgSM7E9Eh7DiLS2E9RNQf0dONaGDzEG9yjEl5hcqeIsj4hfRkLH/Bg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-chunked": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-decode-numeric-character-reference": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/micromark-util-decode-numeric-character-reference/-/micromark-util-decode-numeric-character-reference-2.0.2.tgz", + "integrity": "sha512-ccUbYk6CwVdkmCQMyr64dXz42EfHGkPQlBj5p7YVGzq8I7CtjXZJrubAYezf7Rp+bjPseiROqe7G6foFd+lEuw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-decode-string": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-decode-string/-/micromark-util-decode-string-2.0.1.tgz", + "integrity": "sha512-nDV/77Fj6eH1ynwscYTOsbK7rR//Uj0bZXBwJZRfaLEJ1iGBR6kIfNmlNqaqJf649EP0F3NWNdeJi03elllNUQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "decode-named-character-reference": "^1.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-encode": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-encode/-/micromark-util-encode-2.0.1.tgz", + "integrity": "sha512-c3cVx2y4KqUnwopcO9b/SCdo2O67LwJJ/UyqGfbigahfegL9myoEFoDYZgkT7f36T0bLrM9hZTAaAyH+PCAXjw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-html-tag-name": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-html-tag-name/-/micromark-util-html-tag-name-2.0.1.tgz", + "integrity": "sha512-2cNEiYDhCWKI+Gs9T0Tiysk136SnR13hhO8yW6BGNyhOC4qYFnwF1nKfD3HFAIXA5c45RrIG1ub11GiXeYd1xA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-normalize-identifier": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-normalize-identifier/-/micromark-util-normalize-identifier-2.0.1.tgz", + "integrity": "sha512-sxPqmo70LyARJs0w2UclACPUUEqltCkJ6PhKdMIDuJ3gSf/Q+/GIe3WKl0Ijb/GyH9lOpUkRAO2wp0GVkLvS9Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-resolve-all": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-resolve-all/-/micromark-util-resolve-all-2.0.1.tgz", + "integrity": "sha512-VdQyxFWFT2/FGJgwQnJYbe1jjQoNTS4RjglmSjTUlpUMa95Htx9NHeYW4rGDJzbjvCsl9eLjMQwGeElsqmzcHg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-sanitize-uri": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-sanitize-uri/-/micromark-util-sanitize-uri-2.0.1.tgz", + "integrity": "sha512-9N9IomZ/YuGGZZmQec1MbgxtlgougxTodVwDzzEouPKo3qFWvymFHWcnDi2vzV1ff6kas9ucW+o3yzJK9YB1AQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-encode": "^2.0.0", + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-subtokenize": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/micromark-util-subtokenize/-/micromark-util-subtokenize-2.0.4.tgz", + "integrity": "sha512-N6hXjrin2GTJDe3MVjf5FuXpm12PGm80BrUAeub9XFXca8JZbP+oIwY4LJSVwFUCL1IPm/WwSVUN7goFHmSGGQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-symbol": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz", + "integrity": "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-types": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-types/-/micromark-util-types-2.0.1.tgz", + "integrity": "sha512-534m2WhVTddrcKVepwmVEVnUAmtrx9bfIjNoQHRqfnvdaHQiFytEhJoTgpWJvDEXCO5gLTQh3wYC1PgOJA4NSQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromatch": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", + "dev": true, + "license": "MIT", + "dependencies": { + "braces": "^3.0.3", + "picomatch": "^2.3.1" + }, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/nanoid": { + "version": "3.3.8", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.8.tgz", + "integrity": "sha512-WNLf5Sd8oZxOm+TzppcYk8gVOgP+l58xNy58D0nbUnOxOWRWvlcCV4kUF7ltmI6PsrLl/BgKEyS4mqsGChFN0w==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/node-releases": { + "version": "2.0.19", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.19.tgz", + "integrity": "sha512-xxOWJsBKtzAq7DY0J+DTzuz58K8e7sJbdgwkbMWQe8UYB6ekmsQ45q0M/tJDsGaZmbC+l7n57UV8Hl5tHxO9uw==", + "dev": true, + "license": "MIT" + }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "license": "MIT", + "dependencies": { + "callsites": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/parse-entities": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/parse-entities/-/parse-entities-4.0.2.tgz", + "integrity": "sha512-GG2AQYWoLgL877gQIKeRPGO1xF9+eG1ujIb5soS5gPvLQ1y2o8FL90w2QWNdf9I361Mpp7726c+lj3U0qK1uGw==", + "license": "MIT", + "dependencies": { + "@types/unist": "^2.0.0", + "character-entities-legacy": "^3.0.0", + "character-reference-invalid": "^2.0.0", + "decode-named-character-reference": "^1.0.0", + "is-alphanumerical": "^2.0.0", + "is-decimal": "^2.0.0", + "is-hexadecimal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/parse-entities/node_modules/@types/unist": { + "version": "2.0.11", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.11.tgz", + "integrity": "sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==", + "license": "MIT" + }, + "node_modules/parse-json": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", + "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.0.0", + "error-ex": "^1.3.1", + "json-parse-even-better-errors": "^2.3.0", + "lines-and-columns": "^1.1.6" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", + "license": "MIT" + }, + "node_modules/path-type": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", + "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/postcss": { + "version": "8.5.3", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.3.tgz", + "integrity": "sha512-dle9A3yYxlBSrt8Fu+IpjGT8SY8hN0mlaA6GY8t0P5PjIOZemULz/E2Bnm/2dcUOena75OTNkHI76uZBNUUq3A==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.8", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/prop-types": { + "version": "15.8.1", + "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz", + "integrity": "sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==", + "license": "MIT", + "dependencies": { + "loose-envify": "^1.4.0", + "object-assign": "^4.1.1", + "react-is": "^16.13.1" + } + }, + "node_modules/prop-types/node_modules/react-is": { + "version": "16.13.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz", + "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==", + "license": "MIT" + }, + "node_modules/property-information": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/property-information/-/property-information-7.0.0.tgz", + "integrity": "sha512-7D/qOz/+Y4X/rzSB6jKxKUsQnphO046ei8qxG59mtM3RG3DHgTK81HrxrmoDVINJb8NKT5ZsRbwHvQ6B68Iyhg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/react": { + "version": "19.0.0", + "resolved": "https://registry.npmjs.org/react/-/react-19.0.0.tgz", + "integrity": "sha512-V8AVnmPIICiWpGfm6GLzCR/W5FXLchHop40W4nXBmdlEceh16rCN8O8LNWm5bh5XUX91fh7KpA+W0TgMKmgTpQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-dom": { + "version": "19.0.0", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.0.0.tgz", + "integrity": "sha512-4GV5sHFG0e/0AD4X+ySy6UJd3jVl1iNsNHdpad0qhABJ11twS3TTBnseqsKurKcsNqCEFeGL3uLpVChpIO3QfQ==", + "license": "MIT", + "dependencies": { + "scheduler": "^0.25.0" + }, + "peerDependencies": { + "react": "^19.0.0" + } + }, + "node_modules/react-is": { + "version": "19.0.0", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-19.0.0.tgz", + "integrity": "sha512-H91OHcwjZsbq3ClIDHMzBShc1rotbfACdWENsmEf0IFvZ3FgGPtdHMcsv45bQ1hAbgdfiA8SnxTKfDS+x/8m2g==", + "license": "MIT" + }, + "node_modules/react-markdown": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/react-markdown/-/react-markdown-10.0.0.tgz", + "integrity": "sha512-4mTz7Sya/YQ1jYOrkwO73VcFdkFJ8L8I9ehCxdcV0XrClHyOJGKbBk5FR4OOOG+HnyKw5u+C/Aby9TwinCteYA==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "hast-util-to-jsx-runtime": "^2.0.0", + "html-url-attributes": "^3.0.0", + "mdast-util-to-hast": "^13.0.0", + "remark-parse": "^11.0.0", + "remark-rehype": "^11.0.0", + "unified": "^11.0.0", + "unist-util-visit": "^5.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + }, + "peerDependencies": { + "@types/react": ">=18", + "react": ">=18" + } + }, + "node_modules/react-refresh": { + "version": "0.14.2", + "resolved": "https://registry.npmjs.org/react-refresh/-/react-refresh-0.14.2.tgz", + "integrity": "sha512-jCvmsr+1IUSMUyzOkRcvnVbX3ZYC6g9TDrDbFuFmRDq7PD4yaGbLKNQL6k2jnArV8hjYxh7hVhAZB6s9HDGpZA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-transition-group": { + "version": "4.4.5", + "resolved": "https://registry.npmjs.org/react-transition-group/-/react-transition-group-4.4.5.tgz", + "integrity": "sha512-pZcd1MCJoiKiBR2NRxeCRg13uCXbydPnmB4EOeRrY7480qNWO8IIgQG6zlDkm6uRMsURXPuKq0GWtiM59a5Q6g==", + "license": "BSD-3-Clause", + "dependencies": { + "@babel/runtime": "^7.5.5", + "dom-helpers": "^5.0.1", + "loose-envify": "^1.4.0", + "prop-types": "^15.6.2" + }, + "peerDependencies": { + "react": ">=16.6.0", + "react-dom": ">=16.6.0" + } + }, + "node_modules/regenerator-runtime": { + "version": "0.14.1", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz", + "integrity": "sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==", + "license": "MIT" + }, + "node_modules/remark-parse": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/remark-parse/-/remark-parse-11.0.0.tgz", + "integrity": "sha512-FCxlKLNGknS5ba/1lmpYijMUzX2esxW5xQqjWxw2eHFfS2MSdaHVINFmhjo+qN1WhZhNimq0dZATN9pH0IDrpA==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-from-markdown": "^2.0.0", + "micromark-util-types": "^2.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remark-rehype": { + "version": "11.1.1", + "resolved": "https://registry.npmjs.org/remark-rehype/-/remark-rehype-11.1.1.tgz", + "integrity": "sha512-g/osARvjkBXb6Wo0XvAeXQohVta8i84ACbenPpoSsxTOQH/Ae0/RGP4WZgnMH5pMLpsj4FG7OHmcIcXxpza8eQ==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "mdast-util-to-hast": "^13.0.0", + "unified": "^11.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/resolve": { + "version": "1.22.10", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.10.tgz", + "integrity": "sha512-NPRy+/ncIMeDlTAsuqwKIiferiawhefFJtkNSW0qZJEqMEb+qBt/77B/jGeeek+F0uOeN05CDa6HXbbIgtVX4w==", + "license": "MIT", + "dependencies": { + "is-core-module": "^2.16.0", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + }, + "bin": { + "resolve": "bin/resolve" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/rollup": { + "version": "4.34.8", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.34.8.tgz", + "integrity": "sha512-489gTVMzAYdiZHFVA/ig/iYFllCcWFHMvUHI1rpFmkoUtRlQxqh6/yiNqnYibjMZ2b/+FUQwldG+aLsEt6bglQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "1.0.6" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@rollup/rollup-android-arm-eabi": "4.34.8", + "@rollup/rollup-android-arm64": "4.34.8", + "@rollup/rollup-darwin-arm64": "4.34.8", + "@rollup/rollup-darwin-x64": "4.34.8", + "@rollup/rollup-freebsd-arm64": "4.34.8", + "@rollup/rollup-freebsd-x64": "4.34.8", + "@rollup/rollup-linux-arm-gnueabihf": "4.34.8", + "@rollup/rollup-linux-arm-musleabihf": "4.34.8", + "@rollup/rollup-linux-arm64-gnu": "4.34.8", + "@rollup/rollup-linux-arm64-musl": "4.34.8", + "@rollup/rollup-linux-loongarch64-gnu": "4.34.8", + "@rollup/rollup-linux-powerpc64le-gnu": "4.34.8", + "@rollup/rollup-linux-riscv64-gnu": "4.34.8", + "@rollup/rollup-linux-s390x-gnu": "4.34.8", + "@rollup/rollup-linux-x64-gnu": "4.34.8", + "@rollup/rollup-linux-x64-musl": "4.34.8", + "@rollup/rollup-win32-arm64-msvc": "4.34.8", + "@rollup/rollup-win32-ia32-msvc": "4.34.8", + "@rollup/rollup-win32-x64-msvc": "4.34.8", + "fsevents": "~2.3.2" + } + }, + "node_modules/scheduler": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.25.0.tgz", + "integrity": "sha512-xFVuu11jh+xcO7JOAGJNOXld8/TcEHK/4CituBUeUb5hqxJLj9YuemAEuvm9gQ/+pgXYfbQuqAkiYu+u7YEsNA==", + "license": "MIT" + }, + "node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/source-map": { + "version": "0.5.7", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", + "integrity": "sha512-LbrmJOMUSdEVxIKvdcJzQC+nQhe8FUZQTXQy6+I75skNgn3OoQ0DZA8YnFa7gp8tqtL3KPf1kmo0R5DoApeSGQ==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/space-separated-tokens": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/space-separated-tokens/-/space-separated-tokens-2.0.2.tgz", + "integrity": "sha512-PEGlAwrG8yXGXRjW32fGbg66JAlOAwbObuqVoJpv/mRgoWDQfgH1wDPvtzWyUSNAXBGSk8h755YDbbcEy3SH2Q==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/stringify-entities": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/stringify-entities/-/stringify-entities-4.0.4.tgz", + "integrity": "sha512-IwfBptatlO+QCJUo19AqvrPNqlVMpW9YEL2LIVY+Rpv2qsjCGxaDLNRgeGsQWJhfItebuJhsGSLjaBbNSQ+ieg==", + "license": "MIT", + "dependencies": { + "character-entities-html4": "^2.0.0", + "character-entities-legacy": "^3.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/style-to-object": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/style-to-object/-/style-to-object-1.0.8.tgz", + "integrity": "sha512-xT47I/Eo0rwJmaXC4oilDGDWLohVhR6o/xAQcPQN8q6QBuZVL8qMYL85kLmST5cPjAorwvqIA4qXTRQoYHaL6g==", + "license": "MIT", + "dependencies": { + "inline-style-parser": "0.2.4" + } + }, + "node_modules/stylis": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/stylis/-/stylis-4.2.0.tgz", + "integrity": "sha512-Orov6g6BB1sDfYgzWfTHDOxamtX1bE/zo104Dh9e6fqJ3PooipYyfJ0pUmrZO2wAvO8YbEyeFrkV91XTsGMSrw==", + "license": "MIT" + }, + "node_modules/supports-preserve-symlinks-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/trim-lines": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/trim-lines/-/trim-lines-3.0.1.tgz", + "integrity": "sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/trough": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/trough/-/trough-2.2.0.tgz", + "integrity": "sha512-tmMpK00BjZiUyVyvrBK7knerNgmgvcV/KLVyuma/SC+TQN167GrMRciANTz09+k3zW8L8t60jWO1GpfkZdjTaw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, + "node_modules/typescript": { + "version": "5.7.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.7.3.tgz", + "integrity": "sha512-84MVSjMEHP+FQRPy3pX9sTVV/INIex71s9TL2Gm5FG/WG1SqXeKyZ0k7/blY/4FdOzI12CBy1vGc4og/eus0fw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/unified": { + "version": "11.0.5", + "resolved": "https://registry.npmjs.org/unified/-/unified-11.0.5.tgz", + "integrity": "sha512-xKvGhPWw3k84Qjh8bI3ZeJjqnyadK+GEFtazSfZv/rKeTkTjOJho6mFqh2SM96iIcZokxiOpg78GazTSg8+KHA==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "bail": "^2.0.0", + "devlop": "^1.0.0", + "extend": "^3.0.0", + "is-plain-obj": "^4.0.0", + "trough": "^2.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-is": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-6.0.0.tgz", + "integrity": "sha512-2qCTHimwdxLfz+YzdGfkqNlH0tLi9xjTnHddPmJwtIG9MGsdbutfTc4P+haPD7l7Cjxf/WZj+we5qfVPvvxfYw==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-position": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/unist-util-position/-/unist-util-position-5.0.0.tgz", + "integrity": "sha512-fucsC7HjXvkB5R3kTCO7kUjRdrS0BJt3M/FPxmHMBOm8JQi2BsHAHFsy27E0EolP8rp0NzXsJ+jNPyDWvOJZPA==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-stringify-position": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/unist-util-stringify-position/-/unist-util-stringify-position-4.0.0.tgz", + "integrity": "sha512-0ASV06AAoKCDkS2+xw5RXJywruurpbC4JZSm7nr7MOt1ojAzvyyaO+UxZf18j8FCF6kmzCZKcAgN/yu2gm2XgQ==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-visit": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-5.0.0.tgz", + "integrity": "sha512-MR04uvD+07cwl/yhVuVWAtw+3GOR/knlL55Nd/wAdblk27GCVt3lqpTivy/tkJcZoNPzTwS1Y+KMojlLDhoTzg==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-is": "^6.0.0", + "unist-util-visit-parents": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-visit-parents": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/unist-util-visit-parents/-/unist-util-visit-parents-6.0.1.tgz", + "integrity": "sha512-L/PqWzfTP9lzzEa6CKs0k2nARxTdZduw3zyh8d2NVBnsyvHjSX4TWse388YrrQKbvI8w20fGjGlhgT96WwKykw==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-is": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/update-browserslist-db": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.3.tgz", + "integrity": "sha512-UxhIZQ+QInVdunkDAaiazvvT/+fXL5Osr0JZlJulepYu6Jd7qJtDZjlur0emRlT71EN3ScPoE7gvsuIKKNavKw==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "escalade": "^3.2.0", + "picocolors": "^1.1.1" + }, + "bin": { + "update-browserslist-db": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, + "node_modules/vfile": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/vfile/-/vfile-6.0.3.tgz", + "integrity": "sha512-KzIbH/9tXat2u30jf+smMwFCsno4wHVdNmzFyL+T/L3UGqqk6JKfVqOFOZEpZSHADH1k40ab6NUIXZq422ov3Q==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/vfile-message": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/vfile-message/-/vfile-message-4.0.2.tgz", + "integrity": "sha512-jRDZ1IMLttGj41KcZvlrYAaI3CfqpLpfpf+Mfig13viT6NKvRzWZ+lXz0Y5D60w6uJIBAOGq9mSHf0gktF0duw==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-stringify-position": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/vite": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/vite/-/vite-6.2.0.tgz", + "integrity": "sha512-7dPxoo+WsT/64rDcwoOjk76XHj+TqNTIvHKcuMQ1k4/SeHDaQt5GFAeLYzrimZrMpn/O6DtdI03WUjdxuPM0oQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "^0.25.0", + "postcss": "^8.5.3", + "rollup": "^4.30.1" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^18.0.0 || ^20.0.0 || >=22.0.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", + "jiti": ">=1.21.0", + "less": "*", + "lightningcss": "^1.21.0", + "sass": "*", + "sass-embedded": "*", + "stylus": "*", + "sugarss": "*", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/vite-plugin-singlefile": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/vite-plugin-singlefile/-/vite-plugin-singlefile-2.1.0.tgz", + "integrity": "sha512-7tJo+UgZABlKpY/nubth/wxJ4+pUGREPnEwNOknxwl2MM0zTvF14KTU4Ln1lc140gjLLV5mjDrvuoquU7OZqCg==", + "dev": true, + "license": "MIT", + "dependencies": { + "micromatch": "^4.0.8" + }, + "engines": { + "node": ">18.0.0" + }, + "peerDependencies": { + "rollup": "^4.28.1", + "vite": "^5.4.11 || ^6.0.0" + } + }, + "node_modules/yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", + "dev": true, + "license": "ISC" + }, + "node_modules/yaml": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.7.0.tgz", + "integrity": "sha512-+hSoy/QHluxmC9kCIJyL/uyFmLmc+e5CFR5Wa+bpIhIj85LVb9ZH2nVnqrHoSvKogwODv0ClqZkmiSSaIH5LTA==", + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/zwitch": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/zwitch/-/zwitch-2.0.4.tgz", + "integrity": "sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + } + } +} diff --git a/scripts/settings/package.json b/scripts/settings/package.json new file mode 100644 index 0000000..fc7bb7b --- /dev/null +++ b/scripts/settings/package.json @@ -0,0 +1,31 @@ +{ + "name": "material-ui-vite-ts", + "private": true, + "version": "5.0.0", + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "preview": "vite preview" + }, + "dependencies": { + "@dnd-kit/core": "^6.3.1", + "@dnd-kit/sortable": "^10.0.0", + "@emotion/react": "latest", + "@emotion/styled": "latest", + "@mui/icons-material": "latest", + "@mui/material": "latest", + "react": "19.0.0", + "react-dom": "19.0.0", + "react-markdown": "^10.0.0", + "yaml": "^2.7.0" + }, + "devDependencies": { + "@types/react": "latest", + "@types/react-dom": "latest", + "@vitejs/plugin-react": "latest", + "typescript": "latest", + "vite": "latest", + "vite-plugin-singlefile": "^2.1.0" + } +} diff --git a/scripts/settings/src/App.tsx b/scripts/settings/src/App.tsx new file mode 100644 index 0000000..4d98528 --- /dev/null +++ b/scripts/settings/src/App.tsx @@ -0,0 +1,450 @@ +import * as React from 'react'; +import { useEffect, useState, useRef } from 'react'; +import Container from '@mui/material/Container'; +import Typography from '@mui/material/Typography'; +import Box from '@mui/material/Box'; +import FileUploadIcon from '@mui/icons-material/FileUpload'; +// +import { + DndContext, + closestCenter, + KeyboardSensor, + PointerSensor, + useSensor, + useSensors, + DragOverlay +} from "@dnd-kit/core"; +import { + arrayMove, + SortableContext, + sortableKeyboardCoordinates, + rectSortingStrategy +} from "@dnd-kit/sortable"; + +import type { DragStartEvent, DragEndEvent, UniqueIdentifier } from "@dnd-kit/core"; + + +import { Module } from './types'; + +import { modules, steps, module_types, empty_config } from './schema.json'; +import { + Stack, + Button, +} from '@mui/material'; +import Grid from '@mui/material/Grid2'; + +import { parseDocument, Document, YAMLSeq, YAMLMap, Scalar } from 'yaml' +import StepCard from './StepCard'; + + +function FileDrop({ setYamlFile }: { setYamlFile: React.Dispatch> }) { + + const [showError, setShowError] = useState(false); + const [label, setLabel] = useState(<>Drag and drop your orchestration.yaml file here, or click to select a file.); + const wrapperRef = useRef(null); + + function openYAMLFile(event: any) { + let file = event.target.files[0]; + if (file.type.indexOf('yaml') === -1) { + setShowError(true); + setLabel(<>Invalid type, only YAML files are accepted.) + return; + } + let reader = new FileReader(); + reader.onload = function (e) { + let contents = e.target ? e.target.result : ''; + try { + let document = parseDocument(contents as string); + if (document.errors.length > 0) { + // not a valid yaml file + setShowError(true); + setLabel(<>Invalid file. Make sure your Orchestration is a valid YAML file with a 'steps' section in it.) + return; + } else { + setShowError(false); + setLabel(<>File loaded successfully.) + } + // do some basic validation of 'steps' + let steps = document.get('steps'); + if (!steps) { + setShowError(true); + setLabel(<>Invalid file. Your orchestration file must have a 'steps' section in it.) + return; + } + const replacements = { + feeder: 'feeders', + formatter: 'formatters', + archivers: 'extractors', + }; + + let error = false; + for (let stepType of Object.keys(replacements)) { + if (steps.get(stepType) !== undefined) { + setShowError(true); + setLabel(<>Invalid file. Your orchestration file appears to be in the old (v0.12) format with a '{stepType}' section.
You should manually update your orchestration file first (hint: {stepType} → {replacements[stepType]})); + error = true; + return; + } + }; + setYamlFile(document); + } catch (e) { + console.error(e); + } + } + reader.readAsText(file); + } + return ( + <> +
{ + e.currentTarget.style.backgroundColor = 'var(--mui-palette-LinearProgress-infoBg)'; + }} + onDragLeave={(e) => { + e.currentTarget.style.backgroundColor = ''; + }} + onDrop={(e) => { + e.currentTarget.style.backgroundColor = ''; + }} + > + + + + {label} + +
+ + ); +} + +function ModuleTypes({ stepType, setEnabledModules, enabledModules, configValues }: { stepType: string, setEnabledModules: any, enabledModules: any, configValues: any }) { + const [showError, setShowError] = useState(false); + const [activeId, setActiveId] = useState(); + const [items, setItems] = useState([]); + + useEffect(() => { + setItems(enabledModules[stepType].map(([name, enabled]: [string, boolean]) => name)); + } + , [enabledModules]); + + const toggleModule = (event: any) => { + // make sure that 'feeder' and 'formatter' types only have one value + let name = event.target.id; + let checked = event.target.checked; + if (stepType === 'feeders' || stepType === 'formatters') { + // check how many modules of this type are enabled + const checkedModules = enabledModules[stepType].filter(([m, enabled]: [string, boolean]) => { + return (m !== name && enabled) || (checked && m === name) + }); + if (checkedModules.length > 1) { + setShowError(true); + } else { + setShowError(false); + } + } else { + setShowError(false); + } + let newEnabledModules = { ...enabledModules }; + newEnabledModules[stepType] = enabledModules[stepType].map(([m, enabled]: [string, boolean]) => { + return (m === name) ? [m, checked] : [m, enabled]; + }); + setEnabledModules(newEnabledModules); + } + + const sensors = useSensors( + useSensor(PointerSensor), + useSensor(KeyboardSensor, { + coordinateGetter: sortableKeyboardCoordinates + }) + ); + + const handleDragStart = (event: DragStartEvent) => { + setActiveId(event.active.id); + }; + + const handleDragEnd = (event: DragEndEvent) => { + setActiveId(undefined); + const { active, over } = event; + + if (active.id !== over?.id) { + const oldIndex = items.indexOf(active.id as string); + const newIndex = items.indexOf(over?.id as string); + + let newArray = arrayMove(items, oldIndex, newIndex); + // set it also on steps + let newEnabledModules = { ...enabledModules }; + newEnabledModules[stepType] = enabledModules[stepType].sort((a, b) => { + return newArray.indexOf(a[0]) - newArray.indexOf(b[0]); + }) + setEnabledModules(newEnabledModules); + } + }; + return ( + <> + + + {stepType} + + + Select the {stepType} you wish to enable. Drag to reorder. + + + {showError ? Only one {stepType.slice(0,-1)} can be enabled at a time. : null} + + + + + {items.map((name: string) => { + let m: Module = modules[name]; + return ( + + ); + })} + + {activeId ? ( +
+ + ) : null} + + + + + + ); +} + + +export default function App() { + const [yamlFile, setYamlFile] = useState(new Document()); + const [enabledModules, setEnabledModules] = useState<{}>(Object.fromEntries(Object.keys(steps).map(type => [type, steps[type].map((name: string) => [name, false])]))); + const [configValues, setConfigValues] = useState<{ + [key: string]: { + [key: string + ]: any + } + }>( + Object.keys(modules).reduce((acc, module) => { + acc[module] = {}; + return acc; + }, {}) + ); + + const saveSettings = function (copy: boolean = false) { + // edit the yamlFile + + // generate the steps config + let stepsConfig = enabledModules; + + let finalYamlFile: Document = null; + if (!yamlFile || yamlFile.contents == null) { + // create the yaml file from + finalYamlFile = parseDocument(empty_config as string); + } else { + finalYamlFile = yamlFile; + } + + // set the steps + module_types.forEach((type: string) => { + let stepType = type + 's'; + let existingSteps = finalYamlFile.getIn(['steps', stepType]) as YAMLSeq; + stepsConfig[stepType].forEach(([name, enabled]: [string, boolean]) => { + let index = existingSteps.items.findIndex((item) => { + return (item.value || item) === name + }); + let stepItem = finalYamlFile.getIn(['steps', stepType], true) as YAMLSeq; + + if (enabled && index === -1) { + finalYamlFile.addIn(['steps', stepType], name); + stepItem.commentBefore = stepItem.commentBefore?.replace("\n - " + name, ''); + stepItem.comment = stepItem.comment?.replace("\n - " + name, ''); + } else if (!enabled && index !== -1) { + // set the value to empty and add a comment before with the commented value + finalYamlFile.deleteIn(['steps', stepType, index]); + stepItem.commentBefore += "\n - " + name; + finalYamlFile.setIn(['steps', stepType], stepItem); + } + }); + // sort the items + existingSteps.items.sort((a: Scalar | string, b: Scalar | string) => { + return (stepsConfig[stepType].findIndex((val: [string, boolean]) => {return val[0] === (a.value || a)}) - + stepsConfig[stepType].findIndex((val: [string, boolean]) => {return val[0] === (b.value || b)})) + }); + existingSteps.flow = existingSteps.items.length ? false : true; + }); + + // set all other settings + // loop through each item that isn't 'steps' in the finalYamlFile and check if it exists in configValues + + Object.keys(configValues).forEach((module_name: string) => { + // get an existing key + let existingConfig = finalYamlFile.get(module_name, true) as YAMLMap; + if (existingConfig) { + Object.keys(configValues[module_name]).forEach((config_name: string) => { + let existingConfigYAML = existingConfig.get(config_name, true) as Scalar; + if (existingConfigYAML) { + existingConfigYAML.value = configValues[module_name][config_name]; + existingConfig.set(config_name, existingConfigYAML); + } else { + existingConfig.set(config_name, configValues[module_name][config_name]); + } + }); + finalYamlFile.set(module_name, existingConfig); + } else { + if (configValues[module_name] && Object.keys(configValues[module_name]).length > 0) { + finalYamlFile.set(module_name, configValues[module_name]); + } + } + }); + + if (copy) { + navigator.clipboard.writeText(String(finalYamlFile)).then(() => { + alert("Settings copied to clipboard."); + }); + } else { + // offer the file for download + const blob = new Blob([String(finalYamlFile)], { type: 'application/x-yaml' }); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = 'orchestration.yaml'; + a.click(); + } + } + + useEffect(() => { + // load the configs, and set the default values if they exist + let newConfigValues = {}; + Object.keys(modules).map((module: string) => { + let m = modules[module]; + let configs = m.configs; + if (!configs) { + return; + } + newConfigValues[module] = {}; + Object.keys(configs).map((config: string) => { + let config_args = configs[config]; + if (config_args.default !== undefined) { + newConfigValues[module][config] = config_args.default; + } + }); + }) + setConfigValues(newConfigValues); + }, []); + + useEffect(() => { + if (!yamlFile || yamlFile.contents == null) { + return; + } + + let settings = yamlFile.toJS(); + // make a deep copy of settings + let stepSettings = settings['steps']; + + let newEnabledModules = Object.fromEntries(Object.keys(steps).map((type: string) => { + return [type, steps[type].map((name: string) => { + return [name, stepSettings[type].indexOf(name) !== -1]; + }).sort((a, b) => { + let aIndex = stepSettings[type].indexOf(a[0]); + let bIndex = stepSettings[type].indexOf(b[0]); + if (aIndex === -1 && bIndex === -1) { + return a - b; + } + if (bIndex === -1) { + return -1; + } + if (aIndex === -1) { + return 1; + } + return aIndex - bIndex; + })]; + }).sort((a, b) => { + return module_types.indexOf(a[0]) - module_types.indexOf(b[0]); + })); + setEnabledModules(newEnabledModules); + + // set the config values + let newConfigValues = settings; + delete newConfigValues['steps']; + + + setConfigValues(Object.keys(modules).reduce((acc, module) => { + acc[module] = newConfigValues[module] || {}; + return acc; + }, {})); + }, [yamlFile]); + + + + return ( + + + + + 1. Select your orchestration.yaml settings file. + + Or skip this step to start from scratch + + + + + 2. Choose the Modules you wish to enable/disable + + {Object.keys(steps).map((stepType: string) => { + return ( + + + + ); + })} + + + + 3. Configure your Enabled Modules + + + Next to each module you've enabled, you can click 'Configure' to set the module's settings. + + + + + 4. Save your settings + + + + + + + + + ); +} diff --git a/scripts/settings/src/StepCard.tsx b/scripts/settings/src/StepCard.tsx new file mode 100644 index 0000000..52ee76b --- /dev/null +++ b/scripts/settings/src/StepCard.tsx @@ -0,0 +1,258 @@ +import { useState } from "react"; +import { useSortable } from "@dnd-kit/sortable"; +import ReactMarkdown from 'react-markdown'; + +import { CSS } from "@dnd-kit/utilities"; + +import { + Card, + CardActions, + CardHeader, + Button, + Dialog, + DialogTitle, + DialogContent, + Box, + IconButton, + Checkbox, + Select, + MenuItem, + FormControl, + FormControlLabel, + FormHelperText, + TextField, + Stack, + Typography, + InputAdornment, +} from '@mui/material'; +import Grid from '@mui/material/Grid2'; +import DragIndicatorIcon from '@mui/icons-material/DragIndicator'; +import Visibility from '@mui/icons-material/Visibility'; +import VisibilityOff from '@mui/icons-material/VisibilityOff'; +import HelpIconOutlined from '@mui/icons-material/HelpOutline'; +import { Module, Config } from "./types"; + + +// adds 'capitalize' method to String prototype +declare global { + interface String { + capitalize(): string; + } +} +String.prototype.capitalize = function (this: string) { + return this.charAt(0).toUpperCase() + this.slice(1); +}; + +const StepCard = ({ + type, + module, + toggleModule, + enabledModules, + configValues +}: { + type: string, + module: Module, + toggleModule: any, + enabledModules: any, + configValues: any +}) => { + const { + attributes, + listeners, + setNodeRef, + transform, + transition, + isDragging + } = useSortable({ id: module.name }); + + + const style = { + ...Card.style, + transform: CSS.Transform.toString(transform), + transition, + zIndex: isDragging ? "100" : "auto", + opacity: isDragging ? 0.3 : 1 + }; + + let name = module.name; + const [helpOpen, setHelpOpen] = useState(false); + const [configOpen, setConfigOpen] = useState(false); + const enabled = enabledModules[type].find((m: any) => m[0] === name)[1]; + + return ( + + + } + label={module.display_name} /> + } + /> + + + + setHelpOpen(true)}> + + + {enabled && module.configs && name != 'cli_feeder' ? ( + + ) : null} + + + + + + + + setHelpOpen(false)} + maxWidth="lg" + > + + {module.display_name} + + + + {module.manifest.description.split("\n").map((line: string) => line.trim()).join("\n")} + + + + {module.configs && name != 'cli_feeder' && } + + ) +} + +function ConfigField({ config_value, module, configValues }: { config_value: any, module: Module, configValues: any }) { + const [showPassword, setShowPassword] = useState(false); + const handleClickShowPassword = () => setShowPassword((show) => !show); + + const handleMouseDownPassword = (event: React.MouseEvent) => { + event.preventDefault(); + }; + + const handleMouseUpPassword = (event: React.MouseEvent) => { + event.preventDefault(); + }; + + function setConfigValue(config: any, value: any) { + configValues[module.name][config] = value; + } + const config_args: Config = module.configs[config_value]; + const config_name: string = config_value.replace(/_/g, " "); + const config_display_name = config_name.capitalize(); + const value = configValues[module.name][config_value] || config_args.default; + + + const config_value_lower = config_value.toLowerCase(); + const is_password = config_value_lower.includes('password') || + config_value_lower.includes('secret') || + config_value_lower.includes('token') || + config_value_lower.includes('key') || + config_value_lower.includes('api_hash') || + config_args.type === 'password'; + + const text_input_type = is_password ? 'password' : (config_args.type === 'int' ? 'number' : 'text'); + + return ( + + {config_display_name} {config_args.required && (`(required)`)} + + {config_args.type === 'bool' ? + { + setConfigValue(config_value, e.target.checked); + }} + />} label={config_args.help.capitalize()} + /> + : + ( + config_args.choices !== undefined ? + + : + (config_args.type === 'json_loader' ? + { + try { + let val = JSON.parse(e.target.value); + setConfigValue(config_value, val); + } catch (e) { + console.log(e); + } + } + } /> + : + { + setConfigValue(config_value, e.target.value); + }} + required={config_args.required} + slotProps={ is_password ? { + input: { endAdornment: ( + + + {showPassword ? : } + + + )} + } : {}} + /> + ) + ) + } + {config_args.type !== 'bool' && ( + {config_args.help.capitalize()} + )} + + + ) +} + +function ConfigPanel({ module, open, setOpen, configValues }: { module: Module, open: boolean, setOpen: any, configValues: any }) { + + return ( + <> + setOpen(false)} + maxWidth="lg" + > + + {module.display_name} + + + + {Object.keys(module.configs).map((config_value: any) => { + return ( + + ); + })} + + + + + ); +} + +export default StepCard; \ No newline at end of file diff --git a/scripts/settings/src/main.tsx b/scripts/settings/src/main.tsx new file mode 100644 index 0000000..44c7951 --- /dev/null +++ b/scripts/settings/src/main.tsx @@ -0,0 +1,44 @@ +import * as React from 'react'; +import * as ReactDOM from 'react-dom/client'; +import { ThemeProvider } from '@mui/material/styles'; +import { CssBaseline } from '@mui/material'; +import App from './App'; +import { createTheme } from '@mui/material/styles'; +import { red } from '@mui/material/colors'; +import { useState, useEffect } from 'react'; + +function RootApp() { + const [mode, setMode] = useState('light'); + +useEffect(() => { + setMode(window.localStorage.getItem('theme') || 'light'); +}, []); + +var observer = new MutationObserver(function(mutations) { + setMode(window.localStorage.getItem('theme') || 'light'); + +}) +observer.observe(document.documentElement, {attributes: true, attributeFilter: ['data-theme']}); + +// A custom theme for this app +const theme = createTheme({ + palette: { + mode: mode == 'light' ? 'light' : 'dark', + }, + cssVariables: true +}); + + return ( + + + + + ); +} + +ReactDOM.createRoot(document.getElementById('root')!).render( + + + , +); + diff --git a/scripts/settings/src/schema.json b/scripts/settings/src/schema.json new file mode 100644 index 0000000..64a903a --- /dev/null +++ b/scripts/settings/src/schema.json @@ -0,0 +1,2118 @@ +{ + "modules": { + "gsheet_feeder": { + "name": "gsheet_feeder", + "display_name": "Google Sheets Feeder", + "manifest": { + "name": "Google Sheets Feeder", + "author": "Bellingcat", + "type": [ + "feeder" + ], + "requires_setup": true, + "description": "\n GsheetsFeeder \n A Google Sheets-based feeder for the Auto Archiver.\n\n This reads data from Google Sheets and filters rows based on user-defined rules.\n The filtered rows are processed into `Metadata` objects.\n\n ### Features\n - Validates the sheet structure and filters rows based on input configurations.\n - Processes only worksheets allowed by the `allow_worksheets` and `block_worksheets` configurations.\n - Ensures only rows with valid URLs and unprocessed statuses are included for archival.\n - Supports organizing stored files into folder paths based on sheet and worksheet names.\n\n ### Setup\n - Requires a Google Service Account JSON file for authentication, which should be stored in `secrets/gsheets_service_account.json`.\n To set up a service account, follow the instructions [here](https://gspread.readthedocs.io/en/latest/oauth2.html).\n - Define the `sheet` or `sheet_id` configuration to specify the sheet to archive.\n - Customize the column names in your Google sheet using the `columns` configuration.\n ", + "dependencies": { + "python": [ + "loguru", + "gspread", + "slugify" + ] + }, + "entry_point": "gsheet_feeder::GsheetsFeeder", + "version": "1.0", + "configs": { + "sheet": { + "default": null, + "help": "name of the sheet to archive" + }, + "sheet_id": { + "default": null, + "help": "the id of the sheet to archive (alternative to 'sheet' config)" + }, + "header": { + "default": 1, + "type": "int", + "help": "index of the header row (starts at 1)" + }, + "service_account": { + "default": "secrets/service_account.json", + "help": "service account JSON file path. Learn how to create one: https://gspread.readthedocs.io/en/latest/oauth2.html", + "required": true + }, + "columns": { + "default": { + "url": "link", + "status": "archive status", + "folder": "destination folder", + "archive": "archive location", + "date": "archive date", + "thumbnail": "thumbnail", + "timestamp": "upload timestamp", + "title": "upload title", + "text": "text content", + "screenshot": "screenshot", + "hash": "hash", + "pdq_hash": "perceptual hashes", + "wacz": "wacz", + "replaywebpage": "replaywebpage" + }, + "help": "Custom names for the columns in your Google sheet. If you don't want to use the default column names, change them with this setting", + "type": "json_loader" + }, + "allow_worksheets": { + "default": [], + "help": "A list of worksheet names that should be processed (overrides worksheet_block), leave empty so all are allowed" + }, + "block_worksheets": { + "default": [], + "help": "A list of worksheet names for worksheets that should be explicitly blocked from being processed" + }, + "use_sheet_names_in_stored_paths": { + "default": true, + "help": "if True the stored files path will include 'workbook_name/worksheet_name/...'", + "type": "bool" + } + } + }, + "configs": { + "sheet": { + "default": null, + "help": "name of the sheet to archive" + }, + "sheet_id": { + "default": null, + "help": "the id of the sheet to archive (alternative to 'sheet' config)" + }, + "header": { + "default": 1, + "type": "int", + "help": "index of the header row (starts at 1)" + }, + "service_account": { + "default": "secrets/service_account.json", + "help": "service account JSON file path. Learn how to create one: https://gspread.readthedocs.io/en/latest/oauth2.html", + "required": true + }, + "columns": { + "default": { + "url": "link", + "status": "archive status", + "folder": "destination folder", + "archive": "archive location", + "date": "archive date", + "thumbnail": "thumbnail", + "timestamp": "upload timestamp", + "title": "upload title", + "text": "text content", + "screenshot": "screenshot", + "hash": "hash", + "pdq_hash": "perceptual hashes", + "wacz": "wacz", + "replaywebpage": "replaywebpage" + }, + "help": "Custom names for the columns in your Google sheet. If you don't want to use the default column names, change them with this setting", + "type": "json_loader" + }, + "allow_worksheets": { + "default": [], + "help": "A list of worksheet names that should be processed (overrides worksheet_block), leave empty so all are allowed" + }, + "block_worksheets": { + "default": [], + "help": "A list of worksheet names for worksheets that should be explicitly blocked from being processed" + }, + "use_sheet_names_in_stored_paths": { + "default": true, + "help": "if True the stored files path will include 'workbook_name/worksheet_name/...'", + "type": "bool" + } + } + }, + "atlos_feeder": { + "name": "atlos_feeder", + "display_name": "Atlos Feeder", + "manifest": { + "name": "Atlos Feeder", + "author": "Bellingcat", + "type": [ + "feeder" + ], + "requires_setup": true, + "description": "\n AtlosFeeder: A feeder module that integrates with the Atlos API to fetch source material URLs for archival.\n\n ### Features\n - Connects to the Atlos API to retrieve a list of source material URLs.\n - Filters source materials based on visibility, processing status, and metadata.\n - Converts filtered source materials into `Metadata` objects with the relevant `atlos_id` and URL.\n - Iterates through paginated results using a cursor for efficient API interaction.\n\n ### Notes\n - Requires an Atlos API endpoint and a valid API token for authentication.\n - Ensures only unprocessed, visible, and ready-to-archive URLs are returned.\n - Handles pagination transparently when retrieving data from the Atlos API.\n ", + "dependencies": { + "python": [ + "loguru", + "requests" + ] + }, + "entry_point": "", + "version": "1.0", + "configs": { + "api_token": { + "type": "str", + "required": true, + "help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/" + }, + "atlos_url": { + "default": "https://platform.atlos.org", + "help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.", + "type": "str" + } + } + }, + "configs": { + "api_token": { + "type": "str", + "required": true, + "help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/" + }, + "atlos_url": { + "default": "https://platform.atlos.org", + "help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.", + "type": "str" + } + } + }, + "csv_feeder": { + "name": "csv_feeder", + "display_name": "CSV Feeder", + "manifest": { + "name": "CSV Feeder", + "author": "Bellingcat", + "type": [ + "feeder" + ], + "requires_setup": true, + "description": "\n Reads URLs from CSV files and feeds them into the archiving process.\n\n ### Features\n - Supports reading URLs from multiple input files, specified as a comma-separated list.\n - Allows specifying the column number or name to extract URLs from.\n - Skips header rows if the first value is not a valid URL.\n\n ### Setup\n - Input files should be formatted with one URL per line, with or without a header row.\n - If you have a header row, you can specify the column number or name to read URLs from using the 'column' config option.\n ", + "dependencies": { + "python": [ + "loguru" + ], + "bin": [ + "" + ] + }, + "entry_point": "csv_feeder::CSVFeeder", + "version": "1.0", + "configs": { + "files": { + "default": null, + "help": "Path to the input file(s) to read the URLs from, comma separated. Input files should be formatted with one URL per line", + "required": true, + "type": "valid_file", + "nargs": "+" + }, + "column": { + "default": null, + "help": "Column number or name to read the URLs from, 0-indexed" + } + } + }, + "configs": { + "files": { + "default": null, + "help": "Path to the input file(s) to read the URLs from, comma separated. Input files should be formatted with one URL per line", + "required": true, + "type": "valid_file", + "nargs": "+" + }, + "column": { + "default": null, + "help": "Column number or name to read the URLs from, 0-indexed" + } + } + }, + "cli_feeder": { + "name": "cli_feeder", + "display_name": "Command Line Feeder", + "manifest": { + "name": "Command Line Feeder", + "author": "Bellingcat", + "type": [ + "feeder" + ], + "requires_setup": false, + "description": "\nThe Command Line Feeder is the default enabled feeder for the Auto Archiver. It allows you to pass URLs directly to the orchestrator from the command line \nwithout the need to specify any additional configuration or command line arguments:\n\n`auto-archiver --feeder cli_feeder -- \"https://example.com/1/,https://example.com/2/\"`\n\nYou can pass multiple URLs by separating them with a space. The URLs will be processed in the order they are provided.\n\n`auto-archiver --feeder cli_feeder -- https://example.com/1/ https://example.com/2/`\n", + "dependencies": {}, + "entry_point": "cli_feeder::CLIFeeder", + "version": "1.0", + "configs": { + "urls": { + "default": null, + "help": "URL(s) to archive, either a single URL or a list of urls, should not come from config.yaml" + } + } + }, + "configs": { + "urls": { + "default": null, + "help": "URL(s) to archive, either a single URL or a list of urls, should not come from config.yaml" + } + } + }, + "instagram_api_extractor": { + "name": "instagram_api_extractor", + "display_name": "Instagram API Extractor", + "manifest": { + "name": "Instagram API Extractor", + "author": "Bellingcat", + "type": [ + "extractor" + ], + "requires_setup": true, + "description": "\nArchives various types of Instagram content using the Instagrapi API.\n\nRequires setting up an Instagrapi API deployment and providing an access token and API endpoint.\n\n### Features\n- Connects to an Instagrapi API deployment to fetch Instagram profiles, posts, stories, highlights, reels, and tagged content.\n- Supports advanced configuration options, including:\n - Full profile download (all posts, stories, highlights, and tagged content).\n - Limiting the number of posts to fetch for large profiles.\n - Minimising JSON output to remove empty fields and redundant data.\n- Provides robust error handling and retries for API calls.\n- Ensures efficient media scraping, including handling nested or carousel media items.\n- Adds downloaded media and metadata to the result for further processing.\n\n### Notes\n- Requires a valid Instagrapi API token (`access_token`) and API endpoint (`api_endpoint`).\n- Full-profile downloads can be limited by setting `full_profile_max_posts`.\n- Designed to fetch content in batches for large profiles, minimising API load.\n", + "dependencies": { + "python": [ + "requests", + "loguru", + "retrying", + "tqdm" + ] + }, + "entry_point": "instagram_api_extractor::InstagramAPIExtractor", + "version": "1.0", + "configs": { + "access_token": { + "default": null, + "help": "a valid instagrapi-api token" + }, + "api_endpoint": { + "required": true, + "help": "API endpoint to use" + }, + "full_profile": { + "default": false, + "type": "bool", + "help": "if true, will download all posts, tagged posts, stories, and highlights for a profile, if false, will only download the profile pic and information." + }, + "full_profile_max_posts": { + "default": 0, + "type": "int", + "help": "Use to limit the number of posts to download when full_profile is true. 0 means no limit. limit is applied softly since posts are fetched in batch, once to: posts, tagged posts, and highlights" + }, + "minimize_json_output": { + "default": true, + "type": "bool", + "help": "if true, will remove empty values from the json output" + } + } + }, + "configs": { + "access_token": { + "default": null, + "help": "a valid instagrapi-api token" + }, + "api_endpoint": { + "required": true, + "help": "API endpoint to use" + }, + "full_profile": { + "default": false, + "type": "bool", + "help": "if true, will download all posts, tagged posts, stories, and highlights for a profile, if false, will only download the profile pic and information." + }, + "full_profile_max_posts": { + "default": 0, + "type": "int", + "help": "Use to limit the number of posts to download when full_profile is true. 0 means no limit. limit is applied softly since posts are fetched in batch, once to: posts, tagged posts, and highlights" + }, + "minimize_json_output": { + "default": true, + "type": "bool", + "help": "if true, will remove empty values from the json output" + } + } + }, + "instagram_tbot_extractor": { + "name": "instagram_tbot_extractor", + "display_name": "Instagram Telegram Bot Extractor", + "manifest": { + "name": "Instagram Telegram Bot Extractor", + "author": "Bellingcat", + "type": [ + "extractor" + ], + "requires_setup": true, + "description": "\nThe `InstagramTbotExtractor` module uses a Telegram bot (`instagram_load_bot`) to fetch and archive Instagram content,\nsuch as posts and stories. It leverages the Telethon library to interact with the Telegram API, sending Instagram URLs\nto the bot and downloading the resulting media and metadata. The downloaded content is stored as `Media` objects and\nreturned as part of a `Metadata` object.\n\n### Features\n- Supports archiving Instagram posts and stories through the Telegram bot.\n- Downloads and saves media files (e.g., images, videos) in a temporary directory.\n- Captures and returns metadata, including titles and descriptions, as a `Metadata` object.\n- Automatically manages Telegram session files for secure access.\n\n### Setup\n\nTo use the `InstagramTbotExtractor`, you need to provide the following configuration settings:\n- **API ID and Hash**: Telegram API credentials obtained from [my.telegram.org/apps](https://my.telegram.org/apps).\n- **Session File**: Optional path to store the Telegram session file for future use.\n- The session file is created automatically and should be unique for each instance.\n- You may need to enter your Telegram credentials (phone) and use the a 2FA code sent to you the first time you run the extractor.:\n```2025-01-30 00:43:49.348 | INFO | auto_archiver.modules.instagram_tbot_extractor.instagram_tbot_extractor:setup:36 - SETUP instagram_tbot_extractor checking login...\nPlease enter your phone (or bot token): +447123456789\nPlease enter the code you received: 00000\nSigned in successfully as E C; remember to not break the ToS or you will risk an account ban!\n```\n ", + "dependencies": { + "python": [ + "loguru", + "telethon" + ] + }, + "entry_point": "", + "version": "1.0", + "configs": { + "api_id": { + "default": null, + "help": "telegram API_ID value, go to https://my.telegram.org/apps" + }, + "api_hash": { + "default": null, + "help": "telegram API_HASH value, go to https://my.telegram.org/apps" + }, + "session_file": { + "default": "secrets/anon-insta", + "help": "optional, records the telegram login session for future usage, '.session' will be appended to the provided value." + }, + "timeout": { + "default": 45, + "type": "int", + "help": "timeout to fetch the instagram content in seconds." + } + } + }, + "configs": { + "api_id": { + "default": null, + "help": "telegram API_ID value, go to https://my.telegram.org/apps" + }, + "api_hash": { + "default": null, + "help": "telegram API_HASH value, go to https://my.telegram.org/apps" + }, + "session_file": { + "default": "secrets/anon-insta", + "help": "optional, records the telegram login session for future usage, '.session' will be appended to the provided value." + }, + "timeout": { + "default": 45, + "type": "int", + "help": "timeout to fetch the instagram content in seconds." + } + } + }, + "twitter_api_extractor": { + "name": "twitter_api_extractor", + "display_name": "Twitter API Extractor", + "manifest": { + "name": "Twitter API Extractor", + "author": "Bellingcat", + "type": [ + "extractor" + ], + "requires_setup": true, + "description": "\n The `TwitterApiExtractor` fetches tweets and associated media using the Twitter API. \n It supports multiple API configurations for extended rate limits and reliable access. \n Features include URL expansion, media downloads (e.g., images, videos), and structured output \n via `Metadata` and `Media` objects. Requires Twitter API credentials such as bearer tokens \n or consumer key/secret and access token/secret.\n \n ### Features\n - Fetches tweets and their metadata, including text, creation timestamp, and author information.\n - Downloads media attachments (e.g., images, videos) in high quality.\n - Supports multiple API configurations for improved rate limiting.\n - Expands shortened URLs (e.g., `t.co` links).\n - Outputs structured metadata and media using `Metadata` and `Media` objects.\n \n ### Setup\n To use the `TwitterApiExtractor`, you must provide valid Twitter API credentials via configuration:\n - **Bearer Token(s)**: A single token or a list for rate-limited API access.\n - **Consumer Key and Secret**: Required for user-authenticated API access.\n - **Access Token and Secret**: Complements the consumer key for enhanced API capabilities.\n \n Credentials can be obtained by creating a Twitter developer account at [Twitter Developer Platform](https://developer.twitter.com/en).\n ", + "dependencies": { + "python": [ + "requests", + "loguru", + "pytwitter", + "slugify" + ], + "bin": [ + "" + ] + }, + "entry_point": "", + "version": "1.0", + "configs": { + "bearer_token": { + "default": null, + "help": "[deprecated: see bearer_tokens] twitter API bearer_token which is enough for archiving, if not provided you will need consumer_key, consumer_secret, access_token, access_secret" + }, + "bearer_tokens": { + "default": [], + "help": " a list of twitter API bearer_token which is enough for archiving, if not provided you will need consumer_key, consumer_secret, access_token, access_secret, if provided you can still add those for better rate limits. CSV of bearer tokens if provided via the command line" + }, + "consumer_key": { + "default": null, + "help": "twitter API consumer_key" + }, + "consumer_secret": { + "default": null, + "help": "twitter API consumer_secret" + }, + "access_token": { + "default": null, + "help": "twitter API access_token" + }, + "access_secret": { + "default": null, + "help": "twitter API access_secret" + } + } + }, + "configs": { + "bearer_token": { + "default": null, + "help": "[deprecated: see bearer_tokens] twitter API bearer_token which is enough for archiving, if not provided you will need consumer_key, consumer_secret, access_token, access_secret" + }, + "bearer_tokens": { + "default": [], + "help": " a list of twitter API bearer_token which is enough for archiving, if not provided you will need consumer_key, consumer_secret, access_token, access_secret, if provided you can still add those for better rate limits. CSV of bearer tokens if provided via the command line" + }, + "consumer_key": { + "default": null, + "help": "twitter API consumer_key" + }, + "consumer_secret": { + "default": null, + "help": "twitter API consumer_secret" + }, + "access_token": { + "default": null, + "help": "twitter API access_token" + }, + "access_secret": { + "default": null, + "help": "twitter API access_secret" + } + } + }, + "instagram_extractor": { + "name": "instagram_extractor", + "display_name": "Instagram Extractor", + "manifest": { + "name": "Instagram Extractor", + "author": "Bellingcat", + "type": [ + "extractor" + ], + "requires_setup": true, + "description": "\n Uses the [Instaloader library](https://instaloader.github.io/as-module.html) to download content from Instagram. This class handles both individual posts\n and user profiles, downloading as much information as possible, including images, videos, text, stories,\n highlights, and tagged posts. \n Authentication is required via username/password or a session file.\n \n ", + "dependencies": { + "python": [ + "instaloader", + "loguru" + ] + }, + "entry_point": "", + "version": "1.0", + "configs": { + "username": { + "required": true, + "help": "a valid Instagram username" + }, + "password": { + "required": true, + "help": "the corresponding Instagram account password" + }, + "download_folder": { + "default": "instaloader", + "help": "name of a folder to temporarily download content to" + }, + "session_file": { + "default": "secrets/instaloader.session", + "help": "path to the instagram session which saves session credentials" + } + } + }, + "configs": { + "username": { + "required": true, + "help": "a valid Instagram username" + }, + "password": { + "required": true, + "help": "the corresponding Instagram account password" + }, + "download_folder": { + "default": "instaloader", + "help": "name of a folder to temporarily download content to" + }, + "session_file": { + "default": "secrets/instaloader.session", + "help": "path to the instagram session which saves session credentials" + } + } + }, + "telethon_extractor": { + "name": "telethon_extractor", + "display_name": "Telethon Extractor", + "manifest": { + "name": "Telethon Extractor", + "author": "Bellingcat", + "type": [ + "extractor" + ], + "requires_setup": true, + "description": "\nThe `TelethonExtractor` uses the Telethon library to archive posts and media from Telegram channels and groups. \nIt supports private and public channels, downloading grouped posts with media, and can join channels using invite links \nif provided in the configuration. \n\n### Features\n- Fetches posts and metadata from Telegram channels and groups, including private channels.\n- Downloads media attachments (e.g., images, videos, audio) from individual posts or grouped posts.\n- Handles channel invites to join channels dynamically during setup.\n- Utilizes Telethon's capabilities for reliable Telegram interactions.\n- Outputs structured metadata and media using `Metadata` and `Media` objects.\n\n### Setup\nTo use the `TelethonExtractor`, you must configure the following:\n- **API ID and API Hash**: Obtain these from [my.telegram.org](https://my.telegram.org/apps).\n- **Session File**: Optional, but records login sessions for future use (default: `secrets/anon.session`).\n- **Bot Token**: Optional, allows access to additional content (e.g., large videos) but limits private channel archiving.\n- **Channel Invites**: Optional, specify a JSON string of invite links to join channels during setup.\n\n### First Time Login\nThe first time you run, you will be prompted to do a authentication with the phone number associated, alternatively you can put your `anon.session` in the root.\n\n\n", + "dependencies": { + "python": [ + "telethon", + "loguru", + "tqdm" + ], + "bin": [ + "" + ] + }, + "entry_point": "", + "version": "1.0", + "configs": { + "api_id": { + "default": null, + "help": "telegram API_ID value, go to https://my.telegram.org/apps" + }, + "api_hash": { + "default": null, + "help": "telegram API_HASH value, go to https://my.telegram.org/apps" + }, + "bot_token": { + "default": null, + "help": "optional, but allows access to more content such as large videos, talk to @botfather" + }, + "session_file": { + "default": "secrets/anon", + "help": "optional, records the telegram login session for future usage, '.session' will be appended to the provided value." + }, + "join_channels": { + "default": true, + "type": "bool", + "help": "disables the initial setup with channel_invites config, useful if you have a lot and get stuck" + }, + "channel_invites": { + "default": {}, + "help": "(JSON string) private channel invite links (format: t.me/joinchat/HASH OR t.me/+HASH) and (optional but important to avoid hanging for minutes on startup) channel id (format: CHANNEL_ID taken from a post url like https://t.me/c/CHANNEL_ID/1), the telegram account will join any new channels on setup", + "type": "json_loader" + } + } + }, + "configs": { + "api_id": { + "default": null, + "help": "telegram API_ID value, go to https://my.telegram.org/apps" + }, + "api_hash": { + "default": null, + "help": "telegram API_HASH value, go to https://my.telegram.org/apps" + }, + "bot_token": { + "default": null, + "help": "optional, but allows access to more content such as large videos, talk to @botfather" + }, + "session_file": { + "default": "secrets/anon", + "help": "optional, records the telegram login session for future usage, '.session' will be appended to the provided value." + }, + "join_channels": { + "default": true, + "type": "bool", + "help": "disables the initial setup with channel_invites config, useful if you have a lot and get stuck" + }, + "channel_invites": { + "default": {}, + "help": "(JSON string) private channel invite links (format: t.me/joinchat/HASH OR t.me/+HASH) and (optional but important to avoid hanging for minutes on startup) channel id (format: CHANNEL_ID taken from a post url like https://t.me/c/CHANNEL_ID/1), the telegram account will join any new channels on setup", + "type": "json_loader" + } + } + }, + "vk_extractor": { + "name": "vk_extractor", + "display_name": "VKontakte Extractor", + "manifest": { + "name": "VKontakte Extractor", + "author": "Bellingcat", + "type": [ + "extractor" + ], + "requires_setup": true, + "description": "\nThe `VkExtractor` fetches posts, text, and images from VK (VKontakte) social media pages. \nThis archiver is specialized for `/wall` posts and uses the `VkScraper` library to extract \nand download content. Note that VK videos are handled separately by the `YTDownloader`.\n\n### Features\n- Extracts text, timestamps, and metadata from VK `/wall` posts.\n- Downloads associated images and attaches them to the resulting `Metadata` object.\n- Processes multiple segments of VK URLs that contain mixed content (e.g., wall, photo).\n- Outputs structured metadata and media using `Metadata` and `Media` objects.\n\n### Setup\nTo use the `VkArchiver`, you must provide valid VKontakte login credentials and session information:\n- **Username**: A valid VKontakte account username.\n- **Password**: The corresponding password for the VKontakte account.\n- **Session File**: Optional. Path to a session configuration file (`.json`) for persistent VK login.\n\nCredentials can be set in the configuration file or directly via environment variables. Ensure you \nhave access to the VKontakte API by creating an account at [VKontakte](https://vk.com/).\n", + "dependencies": { + "python": [ + "loguru", + "vk_url_scraper" + ] + }, + "entry_point": "", + "version": "1.0", + "configs": { + "username": { + "required": true, + "help": "valid VKontakte username" + }, + "password": { + "required": true, + "help": "valid VKontakte password" + }, + "session_file": { + "default": "secrets/vk_config.v2.json", + "help": "valid VKontakte password" + } + }, + "depends": [ + "core", + "utils" + ] + }, + "configs": { + "username": { + "required": true, + "help": "valid VKontakte username" + }, + "password": { + "required": true, + "help": "valid VKontakte password" + }, + "session_file": { + "default": "secrets/vk_config.v2.json", + "help": "valid VKontakte password" + } + } + }, + "generic_extractor": { + "name": "generic_extractor", + "display_name": "Generic Extractor", + "manifest": { + "name": "Generic Extractor", + "author": "Bellingcat", + "type": [ + "extractor" + ], + "requires_setup": false, + "description": "\nThis is the generic extractor used by auto-archiver, which uses `yt-dlp` under the hood.\n\nThis module is responsible for downloading and processing media content from platforms\nsupported by `yt-dlp`, such as YouTube, Facebook, and others. It provides functionality\nfor retrieving videos, subtitles, comments, and other metadata, and it integrates with\nthe broader archiving framework.\n\n### Features\n- Supports downloading videos and playlists.\n- Retrieves metadata like titles, descriptions, upload dates, and durations.\n- Downloads subtitles and comments when enabled.\n- Configurable options for handling live streams, proxies, and more.\n- Supports authentication of websites using the 'authentication' settings from your orchestration.\n\n### Dropins\n- For websites supported by `yt-dlp` that also contain posts in addition to videos\n (e.g. Facebook, Twitter, Bluesky), dropins can be created to extract post data and create \n metadata objects. Some dropins are included in this generic_archiver by default, but\ncustom dropins can be created to handle additional websites and passed to the archiver\nvia the command line using the `--dropins` option (TODO!).\n", + "dependencies": { + "python": [ + "yt_dlp", + "requests", + "loguru", + "slugify" + ] + }, + "entry_point": "", + "version": "0.1.0", + "configs": { + "subtitles": { + "default": true, + "help": "download subtitles if available", + "type": "bool" + }, + "comments": { + "default": false, + "help": "download all comments if available, may lead to large metadata", + "type": "bool" + }, + "livestreams": { + "default": false, + "help": "if set, will download live streams, otherwise will skip them; see --max-filesize for more control", + "type": "bool" + }, + "live_from_start": { + "default": false, + "help": "if set, will download live streams from their earliest available moment, otherwise starts now.", + "type": "bool" + }, + "proxy": { + "default": "", + "help": "http/socks (https seems to not work atm) proxy to use for the webdriver, eg https://proxy-user:password@proxy-ip:port" + }, + "end_means_success": { + "default": true, + "help": "if True, any archived content will mean a 'success', if False this archiver will not return a 'success' stage; this is useful for cases when the yt-dlp will archive a video but ignore other types of content like images or text only pages that the subsequent archivers can retrieve.", + "type": "bool" + }, + "allow_playlist": { + "default": false, + "help": "If True will also download playlists, set to False if the expectation is to download a single video.", + "type": "bool" + }, + "max_downloads": { + "default": "inf", + "help": "Use to limit the number of videos to download when a channel or long page is being extracted. 'inf' means no limit." + } + } + }, + "configs": { + "subtitles": { + "default": true, + "help": "download subtitles if available", + "type": "bool" + }, + "comments": { + "default": false, + "help": "download all comments if available, may lead to large metadata", + "type": "bool" + }, + "livestreams": { + "default": false, + "help": "if set, will download live streams, otherwise will skip them; see --max-filesize for more control", + "type": "bool" + }, + "live_from_start": { + "default": false, + "help": "if set, will download live streams from their earliest available moment, otherwise starts now.", + "type": "bool" + }, + "proxy": { + "default": "", + "help": "http/socks (https seems to not work atm) proxy to use for the webdriver, eg https://proxy-user:password@proxy-ip:port" + }, + "end_means_success": { + "default": true, + "help": "if True, any archived content will mean a 'success', if False this archiver will not return a 'success' stage; this is useful for cases when the yt-dlp will archive a video but ignore other types of content like images or text only pages that the subsequent archivers can retrieve.", + "type": "bool" + }, + "allow_playlist": { + "default": false, + "help": "If True will also download playlists, set to False if the expectation is to download a single video.", + "type": "bool" + }, + "max_downloads": { + "default": "inf", + "help": "Use to limit the number of videos to download when a channel or long page is being extracted. 'inf' means no limit." + } + } + }, + "telegram_extractor": { + "name": "telegram_extractor", + "display_name": "Telegram Extractor", + "manifest": { + "name": "Telegram Extractor", + "author": "Bellingcat", + "type": [ + "extractor" + ], + "requires_setup": false, + "description": " \n The `TelegramExtractor` retrieves publicly available media content from Telegram message links without requiring login credentials. \n It processes URLs to fetch images and videos embedded in Telegram messages, ensuring a structured output using `Metadata` \n and `Media` objects. Recommended for scenarios where login-based archiving is not viable, although `telethon_archiver` \n is advised for more comprehensive functionality, and higher quality media extraction.\n \n ### Features\n- Extracts images and videos from public Telegram message links (`t.me`).\n- Processes HTML content of messages to retrieve embedded media.\n- Sets structured metadata, including timestamps, content, and media details.\n- Does not require user authentication for Telegram.\n\n ", + "dependencies": { + "python": [ + "requests", + "bs4", + "loguru" + ] + }, + "entry_point": "", + "version": "1.0", + "configs": {} + }, + "configs": null + }, + "wayback_extractor_enricher": { + "name": "wayback_extractor_enricher", + "display_name": "Wayback Machine Enricher (and Extractor)", + "manifest": { + "name": "Wayback Machine Enricher (and Extractor)", + "author": "Bellingcat", + "type": [ + "enricher", + "extractor" + ], + "requires_setup": true, + "description": "\n Submits the current URL to the Wayback Machine for archiving and returns either a job ID or the completed archive URL.\n\n ### Features\n - Archives URLs using the Internet Archive's Wayback Machine API.\n - Supports conditional archiving based on the existence of prior archives within a specified time range.\n - Provides proxies for HTTP and HTTPS requests.\n - Fetches and confirms the archive URL or provides a job ID for later status checks.\n\n ### Notes\n - Requires a valid Wayback Machine API key and secret.\n - Handles rate-limiting by Wayback Machine and retries status checks with exponential backoff.\n \n ### Steps to Get an Wayback API Key:\n - Sign up for an account at [Internet Archive](https://archive.org/account/signup).\n - Log in to your account.\n - Navigte to your [account settings](https://archive.org/account).\n - or: https://archive.org/developers/tutorial-get-ia-credentials.html\n - Under Wayback Machine API Keys, generate a new key.\n - Note down your API key and secret, as they will be required for authentication.\n ", + "dependencies": { + "python": [ + "loguru", + "requests" + ] + }, + "entry_point": "wayback_extractor_enricher::WaybackExtractorEnricher", + "version": "1.0", + "configs": { + "timeout": { + "default": 15, + "type": "int", + "help": "seconds to wait for successful archive confirmation from wayback, if more than this passes the result contains the job_id so the status can later be checked manually." + }, + "if_not_archived_within": { + "default": null, + "help": "only tell wayback to archive if no archive is available before the number of seconds specified, use None to ignore this option. For more information: https://docs.google.com/document/d/1Nsv52MvSjbLb2PCpHlat0gkzw0EvtSgpKHu4mk0MnrA" + }, + "key": { + "required": true, + "help": "wayback API key. to get credentials visit https://archive.org/account/s3.php" + }, + "secret": { + "required": true, + "help": "wayback API secret. to get credentials visit https://archive.org/account/s3.php" + }, + "proxy_http": { + "default": null, + "help": "http proxy to use for wayback requests, eg http://proxy-user:password@proxy-ip:port" + }, + "proxy_https": { + "default": null, + "help": "https proxy to use for wayback requests, eg https://proxy-user:password@proxy-ip:port" + } + } + }, + "configs": { + "timeout": { + "default": 15, + "type": "int", + "help": "seconds to wait for successful archive confirmation from wayback, if more than this passes the result contains the job_id so the status can later be checked manually." + }, + "if_not_archived_within": { + "default": null, + "help": "only tell wayback to archive if no archive is available before the number of seconds specified, use None to ignore this option. For more information: https://docs.google.com/document/d/1Nsv52MvSjbLb2PCpHlat0gkzw0EvtSgpKHu4mk0MnrA" + }, + "key": { + "required": true, + "help": "wayback API key. to get credentials visit https://archive.org/account/s3.php" + }, + "secret": { + "required": true, + "help": "wayback API secret. to get credentials visit https://archive.org/account/s3.php" + }, + "proxy_http": { + "default": null, + "help": "http proxy to use for wayback requests, eg http://proxy-user:password@proxy-ip:port" + }, + "proxy_https": { + "default": null, + "help": "https proxy to use for wayback requests, eg https://proxy-user:password@proxy-ip:port" + } + } + }, + "wacz_extractor_enricher": { + "name": "wacz_extractor_enricher", + "display_name": "WACZ Enricher (and Extractor)", + "manifest": { + "name": "WACZ Enricher (and Extractor)", + "author": "Bellingcat", + "type": [ + "enricher", + "extractor" + ], + "requires_setup": true, + "description": "\n Creates .WACZ archives of web pages using the `browsertrix-crawler` tool, with options for media extraction and screenshot saving.\n [Browsertrix-crawler](https://crawler.docs.browsertrix.com/user-guide/) is a headless browser-based crawler that archives web pages in WACZ format.\n\n ### Features\n - Archives web pages into .WACZ format using Docker or direct invocation of `browsertrix-crawler`.\n - Supports custom profiles for archiving private or dynamic content.\n - Extracts media (images, videos, audio) and screenshots from the archive, optionally adding them to the enrichment pipeline.\n - Generates metadata from the archived page's content and structure (e.g., titles, text).\n\n ### Notes\n - Requires Docker for running `browsertrix-crawler` .\n - Configurable via parameters for timeout, media extraction, screenshots, and proxy settings.\n ", + "dependencies": { + "python": [ + "loguru", + "jsonlines", + "warcio" + ], + "bin": [ + "docker" + ] + }, + "entry_point": "wacz_extractor_enricher::WaczExtractorEnricher", + "version": "1.0", + "configs": { + "profile": { + "default": null, + "help": "browsertrix-profile (for profile generation see https://github.com/webrecorder/browsertrix-crawler#creating-and-using-browser-profiles)." + }, + "docker_commands": { + "default": null, + "help": "if a custom docker invocation is needed" + }, + "timeout": { + "default": 120, + "type": "int", + "help": "timeout for WACZ generation in seconds" + }, + "extract_media": { + "default": false, + "type": "bool", + "help": "If enabled all the images/videos/audio present in the WACZ archive will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched." + }, + "extract_screenshot": { + "default": true, + "type": "bool", + "help": "If enabled the screenshot captured by browsertrix will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched." + }, + "socks_proxy_host": { + "default": null, + "help": "SOCKS proxy host for browsertrix-crawler, use in combination with socks_proxy_port. eg: user:password@host" + }, + "socks_proxy_port": { + "default": null, + "type": "int", + "help": "SOCKS proxy port for browsertrix-crawler, use in combination with socks_proxy_host. eg 1234" + }, + "proxy_server": { + "default": null, + "help": "SOCKS server proxy URL, in development" + } + } + }, + "configs": { + "profile": { + "default": null, + "help": "browsertrix-profile (for profile generation see https://github.com/webrecorder/browsertrix-crawler#creating-and-using-browser-profiles)." + }, + "docker_commands": { + "default": null, + "help": "if a custom docker invocation is needed" + }, + "timeout": { + "default": 120, + "type": "int", + "help": "timeout for WACZ generation in seconds" + }, + "extract_media": { + "default": false, + "type": "bool", + "help": "If enabled all the images/videos/audio present in the WACZ archive will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched." + }, + "extract_screenshot": { + "default": true, + "type": "bool", + "help": "If enabled the screenshot captured by browsertrix will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched." + }, + "socks_proxy_host": { + "default": null, + "help": "SOCKS proxy host for browsertrix-crawler, use in combination with socks_proxy_port. eg: user:password@host" + }, + "socks_proxy_port": { + "default": null, + "type": "int", + "help": "SOCKS proxy port for browsertrix-crawler, use in combination with socks_proxy_host. eg 1234" + }, + "proxy_server": { + "default": null, + "help": "SOCKS server proxy URL, in development" + } + } + }, + "metadata_enricher": { + "name": "metadata_enricher", + "display_name": "Media Metadata Enricher", + "manifest": { + "name": "Media Metadata Enricher", + "author": "Bellingcat", + "type": [ + "enricher" + ], + "requires_setup": true, + "description": "\n Extracts metadata information from files using ExifTool.\n\n ### Features\n - Uses ExifTool to extract detailed metadata from media files.\n - Processes file-specific data like camera settings, geolocation, timestamps, and other embedded metadata.\n - Adds extracted metadata to the corresponding `Media` object within the `Metadata`.\n\n ### Notes\n - Requires ExifTool to be installed and accessible via the system's PATH.\n - Skips enrichment for files where metadata extraction fails.\n ", + "dependencies": { + "python": [ + "loguru" + ], + "bin": [ + "exiftool" + ] + }, + "entry_point": "", + "version": "1.0", + "configs": {} + }, + "configs": null + }, + "timestamping_enricher": { + "name": "timestamping_enricher", + "display_name": "Timestamping Enricher", + "manifest": { + "name": "Timestamping Enricher", + "author": "Bellingcat", + "type": [ + "enricher" + ], + "requires_setup": true, + "description": "\n Generates RFC3161-compliant timestamp tokens using Time Stamp Authorities (TSA) for archived files.\n\n ### Features\n - Creates timestamp tokens to prove the existence of files at a specific time, useful for legal and authenticity purposes.\n - Aggregates file hashes into a text file and timestamps the concatenated data.\n - Uses multiple Time Stamp Authorities (TSAs) to ensure reliability and redundancy.\n - Validates timestamping certificates against trusted Certificate Authorities (CAs) using the `certifi` trust store.\n\n ### Notes\n - Should be run after the `hash_enricher` to ensure file hashes are available.\n - Requires internet access to interact with the configured TSAs.\n ", + "dependencies": { + "python": [ + "loguru", + "slugify", + "tsp_client", + "asn1crypto", + "certvalidator", + "certifi" + ] + }, + "entry_point": "", + "version": "1.0", + "configs": { + "tsa_urls": { + "default": [ + "http://timestamp.digicert.com", + "http://timestamp.identrust.com", + "http://timestamp.globalsign.com/tsa/r6advanced1", + "http://tss.accv.es:8318/tsa" + ], + "help": "List of RFC3161 Time Stamp Authorities to use, separate with commas if passed via the command line." + } + } + }, + "configs": { + "tsa_urls": { + "default": [ + "http://timestamp.digicert.com", + "http://timestamp.identrust.com", + "http://timestamp.globalsign.com/tsa/r6advanced1", + "http://tss.accv.es:8318/tsa" + ], + "help": "List of RFC3161 Time Stamp Authorities to use, separate with commas if passed via the command line." + } + } + }, + "screenshot_enricher": { + "name": "screenshot_enricher", + "display_name": "Screenshot Enricher", + "manifest": { + "name": "Screenshot Enricher", + "author": "Bellingcat", + "type": [ + "enricher" + ], + "requires_setup": true, + "description": "\n Captures screenshots and optionally saves web pages as PDFs using a WebDriver.\n\n ### Features\n - Takes screenshots of web pages, with configurable width, height, and timeout settings.\n - Optionally saves pages as PDFs, with additional configuration for PDF printing options.\n - Bypasses URLs detected as authentication walls.\n - Integrates seamlessly with the metadata enrichment pipeline, adding screenshots and PDFs as media.\n\n ### Notes\n - Requires a WebDriver (e.g., ChromeDriver) installed and accessible via the system's PATH.\n ", + "dependencies": { + "python": [ + "loguru", + "selenium" + ] + }, + "entry_point": "", + "version": "1.0", + "configs": { + "width": { + "default": 1280, + "type": "int", + "help": "width of the screenshots" + }, + "height": { + "default": 720, + "type": "int", + "help": "height of the screenshots" + }, + "timeout": { + "default": 60, + "type": "int", + "help": "timeout for taking the screenshot" + }, + "sleep_before_screenshot": { + "default": 4, + "type": "int", + "help": "seconds to wait for the pages to load before taking screenshot" + }, + "http_proxy": { + "default": "", + "help": "http proxy to use for the webdriver, eg http://proxy-user:password@proxy-ip:port" + }, + "save_to_pdf": { + "default": false, + "type": "bool", + "help": "save the page as pdf along with the screenshot. PDF saving options can be adjusted with the 'print_options' parameter" + }, + "print_options": { + "default": {}, + "help": "options to pass to the pdf printer, in JSON format. See https://www.selenium.dev/documentation/webdriver/interactions/print_page/ for more information", + "type": "json_loader" + } + } + }, + "configs": { + "width": { + "default": 1280, + "type": "int", + "help": "width of the screenshots" + }, + "height": { + "default": 720, + "type": "int", + "help": "height of the screenshots" + }, + "timeout": { + "default": 60, + "type": "int", + "help": "timeout for taking the screenshot" + }, + "sleep_before_screenshot": { + "default": 4, + "type": "int", + "help": "seconds to wait for the pages to load before taking screenshot" + }, + "http_proxy": { + "default": "", + "help": "http proxy to use for the webdriver, eg http://proxy-user:password@proxy-ip:port" + }, + "save_to_pdf": { + "default": false, + "type": "bool", + "help": "save the page as pdf along with the screenshot. PDF saving options can be adjusted with the 'print_options' parameter" + }, + "print_options": { + "default": {}, + "help": "options to pass to the pdf printer, in JSON format. See https://www.selenium.dev/documentation/webdriver/interactions/print_page/ for more information", + "type": "json_loader" + } + } + }, + "whisper_enricher": { + "name": "whisper_enricher", + "display_name": "Whisper Enricher", + "manifest": { + "name": "Whisper Enricher", + "author": "Bellingcat", + "type": [ + "enricher" + ], + "requires_setup": true, + "description": "\n Integrates with a Whisper API service to transcribe, translate, or detect the language of audio and video files.\n\n ### Features\n - Submits audio or video files to a Whisper API deployment for processing.\n - Supports operations such as transcription, translation, and language detection.\n - Optionally generates SRT subtitle files for video content.\n - Integrates with S3-compatible storage systems to make files publicly accessible for processing.\n - Handles job submission, status checking, artifact retrieval, and cleanup.\n\n ### Notes\n - Requires a Whisper API endpoint and API key for authentication.\n - Only compatible with S3-compatible storage systems for media file accessibility.\n - ** This stores the media files in S3 prior to enriching them as Whisper requires public URLs to access the media files.\n - Handles multiple jobs and retries for failed or incomplete processing.\n ", + "dependencies": { + "python": [ + "s3_storage", + "loguru", + "requests" + ] + }, + "entry_point": "", + "version": "1.0", + "configs": { + "api_endpoint": { + "required": true, + "help": "WhisperApi api endpoint, eg: https://whisperbox-api.com/api/v1, a deployment of https://github.com/bellingcat/whisperbox-transcribe." + }, + "api_key": { + "required": true, + "help": "WhisperApi api key for authentication" + }, + "include_srt": { + "default": false, + "type": "bool", + "help": "Whether to include a subtitle SRT (SubRip Subtitle file) for the video (can be used in video players)." + }, + "timeout": { + "default": 90, + "type": "int", + "help": "How many seconds to wait at most for a successful job completion." + }, + "action": { + "default": "translate", + "help": "which Whisper operation to execute", + "choices": [ + "transcribe", + "translate", + "language_detection" + ] + } + } + }, + "configs": { + "api_endpoint": { + "required": true, + "help": "WhisperApi api endpoint, eg: https://whisperbox-api.com/api/v1, a deployment of https://github.com/bellingcat/whisperbox-transcribe." + }, + "api_key": { + "required": true, + "help": "WhisperApi api key for authentication" + }, + "include_srt": { + "default": false, + "type": "bool", + "help": "Whether to include a subtitle SRT (SubRip Subtitle file) for the video (can be used in video players)." + }, + "timeout": { + "default": 90, + "type": "int", + "help": "How many seconds to wait at most for a successful job completion." + }, + "action": { + "default": "translate", + "help": "which Whisper operation to execute", + "choices": [ + "transcribe", + "translate", + "language_detection" + ] + } + } + }, + "thumbnail_enricher": { + "name": "thumbnail_enricher", + "display_name": "Thumbnail Enricher", + "manifest": { + "name": "Thumbnail Enricher", + "author": "Bellingcat", + "type": [ + "enricher" + ], + "requires_setup": false, + "description": "\n Generates thumbnails for video files to provide visual previews.\n\n ### Features\n - Processes video files and generates evenly distributed thumbnails.\n - Calculates the number of thumbnails based on video duration, `thumbnails_per_minute`, and `max_thumbnails`.\n - Distributes thumbnails equally across the video's duration and stores them as media objects.\n - Adds metadata for each thumbnail, including timestamps and IDs.\n\n ### Notes\n - Requires `ffmpeg` to be installed and accessible via the system's PATH.\n - Handles videos without pre-existing duration metadata by probing with `ffmpeg`.\n - Skips enrichment for non-video media files.\n ", + "dependencies": { + "python": [ + "loguru", + "ffmpeg" + ], + "bin": [ + "ffmpeg" + ] + }, + "entry_point": "", + "version": "1.0", + "configs": { + "thumbnails_per_minute": { + "default": 60, + "type": "int", + "help": "how many thumbnails to generate per minute of video, can be limited by max_thumbnails" + }, + "max_thumbnails": { + "default": 16, + "type": "int", + "help": "limit the number of thumbnails to generate per video, 0 means no limit" + } + } + }, + "configs": { + "thumbnails_per_minute": { + "default": 60, + "type": "int", + "help": "how many thumbnails to generate per minute of video, can be limited by max_thumbnails" + }, + "max_thumbnails": { + "default": 16, + "type": "int", + "help": "limit the number of thumbnails to generate per video, 0 means no limit" + } + } + }, + "meta_enricher": { + "name": "meta_enricher", + "display_name": "Archive Metadata Enricher", + "manifest": { + "name": "Archive Metadata Enricher", + "author": "Bellingcat", + "type": [ + "enricher" + ], + "requires_setup": false, + "description": " \n Adds metadata information about the archive operations, Adds metadata about archive operations, including file sizes and archive duration./\n To be included at the end of all enrichments.\n \n ### Features\n- Calculates the total size of all archived media files, storing the result in human-readable and byte formats.\n- Computes the duration of the archival process, storing the elapsed time in seconds.\n- Ensures all enrichments are performed only if the `Metadata` object contains valid data.\n- Adds detailed metadata to provide insights into file sizes and archival performance.\n\n### Notes\n- Skips enrichment if no media or metadata is available in the `Metadata` object.\n- File sizes are calculated using the `os.stat` module, ensuring accurate byte-level reporting.\n", + "dependencies": { + "python": [ + "loguru" + ] + }, + "entry_point": "", + "version": "1.0", + "configs": {} + }, + "configs": null + }, + "pdq_hash_enricher": { + "name": "pdq_hash_enricher", + "display_name": "PDQ Hash Enricher", + "manifest": { + "name": "PDQ Hash Enricher", + "author": "Bellingcat", + "type": [ + "enricher" + ], + "requires_setup": false, + "description": "\n PDQ Hash Enricher for generating perceptual hashes of media files.\n\n ### Features\n - Calculates perceptual hashes for image files using the PDQ hashing algorithm.\n - Enables detection of duplicate or near-duplicate visual content.\n - Processes images stored in `Metadata` objects, adding computed hashes to the corresponding `Media` entries.\n - Skips non-image media or files unsuitable for hashing (e.g., corrupted or unsupported formats).\n\n ### Notes\n - Best used after enrichers like `thumbnail_enricher` or `screenshot_enricher` to ensure images are available.\n - Uses the `pdqhash` library to compute 256-bit perceptual hashes, which are stored as hexadecimal strings.\n ", + "dependencies": { + "python": [ + "loguru", + "pdqhash", + "numpy", + "PIL" + ] + }, + "entry_point": "", + "version": "1.0", + "configs": {} + }, + "configs": null + }, + "ssl_enricher": { + "name": "ssl_enricher", + "display_name": "SSL Certificate Enricher", + "manifest": { + "name": "SSL Certificate Enricher", + "author": "Bellingcat", + "type": [ + "enricher" + ], + "requires_setup": false, + "description": "\n Retrieves SSL certificate information for a domain and stores it as a file.\n\n ### Features\n - Fetches SSL certificates for domains using the HTTPS protocol.\n - Stores certificates in PEM format and adds them as media to the metadata.\n - Skips enrichment if no media has been archived, based on the `skip_when_nothing_archived` configuration.\n\n ### Notes\n - Requires the target URL to use the HTTPS scheme; other schemes are not supported.\n ", + "dependencies": { + "python": [ + "loguru", + "slugify" + ] + }, + "entry_point": "ssl_enricher::SSLEnricher", + "version": "1.0", + "configs": { + "skip_when_nothing_archived": { + "default": true, + "type": "bool", + "help": "if true, will skip enriching when no media is archived" + } + } + }, + "configs": { + "skip_when_nothing_archived": { + "default": true, + "type": "bool", + "help": "if true, will skip enriching when no media is archived" + } + } + }, + "hash_enricher": { + "name": "hash_enricher", + "display_name": "Hash Enricher", + "manifest": { + "name": "Hash Enricher", + "author": "Bellingcat", + "type": [ + "enricher" + ], + "requires_setup": false, + "description": "\nGenerates cryptographic hashes for media files to ensure data integrity and authenticity.\n\n### Features\n- Calculates cryptographic hashes (SHA-256 or SHA3-512) for media files stored in `Metadata` objects.\n- Ensures content authenticity, integrity validation, and duplicate identification.\n- Efficiently processes large files by reading file bytes in configurable chunk sizes.\n- Supports dynamic configuration of hash algorithms and chunk sizes.\n- Updates media metadata with the computed hash value in the format `:`.\n\n### Notes\n- Default hash algorithm is SHA-256, but SHA3-512 is also supported.\n- Chunk size defaults to 16 MB but can be adjusted based on memory requirements.\n- Useful for workflows requiring hash-based content validation or deduplication.\n", + "dependencies": { + "python": [ + "loguru" + ] + }, + "entry_point": "", + "version": "1.0", + "configs": { + "algorithm": { + "default": "SHA-256", + "help": "hash algorithm to use", + "choices": [ + "SHA-256", + "SHA3-512" + ] + }, + "chunksize": { + "default": 16000000, + "help": "number of bytes to use when reading files in chunks (if this value is too large you will run out of RAM), default is 16MB", + "type": "int" + } + } + }, + "configs": { + "algorithm": { + "default": "SHA-256", + "help": "hash algorithm to use", + "choices": [ + "SHA-256", + "SHA3-512" + ] + }, + "chunksize": { + "default": 16000000, + "help": "number of bytes to use when reading files in chunks (if this value is too large you will run out of RAM), default is 16MB", + "type": "int" + } + } + }, + "atlos_db": { + "name": "atlos_db", + "display_name": "Atlos Database", + "manifest": { + "name": "Atlos Database", + "author": "Bellingcat", + "type": [ + "database" + ], + "requires_setup": true, + "description": "\nHandles integration with the Atlos platform for managing archival results.\n\n### Features\n- Outputs archival results to the Atlos API for storage and tracking.\n- Updates failure status with error details when archiving fails.\n- Processes and formats metadata, including ISO formatting for datetime fields.\n- Skips processing for items without an Atlos ID.\n\n### Setup\nRequired configs:\n- atlos_url: Base URL for the Atlos API.\n- api_token: Authentication token for API access.\n", + "dependencies": { + "python": [ + "loguru", + "" + ], + "bin": [ + "" + ] + }, + "entry_point": "atlos_db::AtlosDb", + "version": "1.0", + "configs": { + "api_token": { + "default": null, + "help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/", + "required": true, + "type": "str" + }, + "atlos_url": { + "default": "https://platform.atlos.org", + "help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.", + "type": "str" + } + } + }, + "configs": { + "api_token": { + "default": null, + "help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/", + "required": true, + "type": "str" + }, + "atlos_url": { + "default": "https://platform.atlos.org", + "help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.", + "type": "str" + } + } + }, + "api_db": { + "name": "api_db", + "display_name": "Auto Archiver API Database", + "manifest": { + "name": "Auto Archiver API Database", + "author": "Bellingcat", + "type": [ + "database" + ], + "requires_setup": true, + "description": "\n Provides integration with the Auto Archiver API for querying and storing archival data.\n\n### Features\n- **API Integration**: Supports querying for existing archives and submitting results.\n- **Duplicate Prevention**: Avoids redundant archiving when `use_api_cache` is disabled.\n- **Configurable**: Supports settings like API endpoint, authentication token, tags, and permissions.\n- **Tagging and Metadata**: Adds tags and manages metadata for archives.\n- **Optional Storage**: Archives results conditionally based on configuration.\n\n### Setup\nRequires access to an Auto Archiver API instance and a valid API token.\n ", + "dependencies": { + "python": [ + "requests", + "loguru" + ] + }, + "entry_point": "api_db::AAApiDb", + "version": "1.0", + "configs": { + "api_endpoint": { + "required": true, + "help": "API endpoint where calls are made to" + }, + "api_token": { + "default": null, + "help": "API Bearer token." + }, + "public": { + "default": false, + "type": "bool", + "help": "whether the URL should be publicly available via the API" + }, + "author_id": { + "default": null, + "help": "which email to assign as author" + }, + "group_id": { + "default": null, + "help": "which group of users have access to the archive in case public=false as author" + }, + "use_api_cache": { + "default": true, + "type": "bool", + "help": "if False then the API database will be queried prior to any archiving operations and stop if the link has already been archived" + }, + "store_results": { + "default": true, + "type": "bool", + "help": "when set, will send the results to the API database." + }, + "tags": { + "default": [], + "help": "what tags to add to the archived URL" + } + } + }, + "configs": { + "api_endpoint": { + "required": true, + "help": "API endpoint where calls are made to" + }, + "api_token": { + "default": null, + "help": "API Bearer token." + }, + "public": { + "default": false, + "type": "bool", + "help": "whether the URL should be publicly available via the API" + }, + "author_id": { + "default": null, + "help": "which email to assign as author" + }, + "group_id": { + "default": null, + "help": "which group of users have access to the archive in case public=false as author" + }, + "use_api_cache": { + "default": true, + "type": "bool", + "help": "if False then the API database will be queried prior to any archiving operations and stop if the link has already been archived" + }, + "store_results": { + "default": true, + "type": "bool", + "help": "when set, will send the results to the API database." + }, + "tags": { + "default": [], + "help": "what tags to add to the archived URL" + } + } + }, + "gsheet_db": { + "name": "gsheet_db", + "display_name": "Google Sheets Database", + "manifest": { + "name": "Google Sheets Database", + "author": "Bellingcat", + "type": [ + "database" + ], + "requires_setup": true, + "description": "\n GsheetsDatabase:\n Handles integration with Google Sheets for tracking archival tasks.\n\n### Features\n- Updates a Google Sheet with the status of the archived URLs, including in progress, success or failure, and method used.\n- Saves metadata such as title, text, timestamp, hashes, screenshots, and media URLs to designated columns.\n- Formats media-specific metadata, such as thumbnails and PDQ hashes for the sheet.\n- Skips redundant updates for empty or invalid data fields.\n\n### Notes\n- Currently works only with metadata provided by GsheetFeeder. \n- Requires configuration of a linked Google Sheet and appropriate API credentials.\n ", + "dependencies": { + "python": [ + "loguru", + "gspread", + "slugify" + ] + }, + "entry_point": "gsheet_db::GsheetsDb", + "version": "1.0", + "configs": { + "allow_worksheets": { + "default": [], + "help": "(CSV) only worksheets whose name is included in allow are included (overrides worksheet_block), leave empty so all are allowed" + }, + "block_worksheets": { + "default": [], + "help": "(CSV) explicitly block some worksheets from being processed" + }, + "use_sheet_names_in_stored_paths": { + "default": true, + "type": "bool", + "help": "if True the stored files path will include 'workbook_name/worksheet_name/...'" + } + } + }, + "configs": { + "allow_worksheets": { + "default": [], + "help": "(CSV) only worksheets whose name is included in allow are included (overrides worksheet_block), leave empty so all are allowed" + }, + "block_worksheets": { + "default": [], + "help": "(CSV) explicitly block some worksheets from being processed" + }, + "use_sheet_names_in_stored_paths": { + "default": true, + "type": "bool", + "help": "if True the stored files path will include 'workbook_name/worksheet_name/...'" + } + } + }, + "console_db": { + "name": "console_db", + "display_name": "Console Database", + "manifest": { + "name": "Console Database", + "author": "Bellingcat", + "type": [ + "database" + ], + "requires_setup": false, + "description": "\nProvides a simple database implementation that outputs archival results and status updates to the console.\n\n### Features\n- Logs the status of archival tasks directly to the console, including:\n - started\n - failed (with error details)\n - aborted\n - done (with optional caching status)\n- Useful for debugging or lightweight setups where no external database is required.\n\n### Setup\nNo additional configuration is required.\n", + "dependencies": { + "python": [ + "loguru" + ] + }, + "entry_point": "", + "version": "1.0", + "configs": {} + }, + "configs": null + }, + "csv_db": { + "name": "csv_db", + "display_name": "CSV Database", + "manifest": { + "name": "CSV Database", + "author": "Bellingcat", + "type": [ + "database" + ], + "requires_setup": false, + "description": "\nHandles exporting archival results to a CSV file.\n\n### Features\n- Saves archival metadata as rows in a CSV file.\n- Automatically creates the CSV file with a header if it does not exist.\n- Appends new metadata entries to the existing file.\n\n### Setup\nRequired config:\n- csv_file: Path to the CSV file where results will be stored (default: \"db.csv\").\n", + "dependencies": { + "python": [ + "loguru" + ] + }, + "entry_point": "csv_db::CSVDb", + "version": "1.0", + "configs": { + "csv_file": { + "default": "db.csv", + "help": "CSV file name to save metadata to" + } + } + }, + "configs": { + "csv_file": { + "default": "db.csv", + "help": "CSV file name to save metadata to" + } + } + }, + "gdrive_storage": { + "name": "gdrive_storage", + "display_name": "Google Drive Storage", + "manifest": { + "name": "Google Drive Storage", + "author": "Dave Mateer", + "type": [ + "storage" + ], + "requires_setup": true, + "description": "\n \n GDriveStorage: A storage module for saving archived content to Google Drive.\n\n Source Documentation: https://davemateer.com/2022/04/28/google-drive-with-python\n\n ### Features\n - Saves media files to Google Drive, organizing them into folders based on the provided path structure.\n - Supports OAuth token-based authentication or service account credentials for API access.\n - Automatically creates folders in Google Drive if they don't exist.\n - Retrieves CDN URLs for stored files, enabling easy sharing and access.\n\n ### Notes\n - Requires setup with either a Google OAuth token or a service account JSON file.\n - Files are uploaded to the specified `root_folder_id` and organized by the `media.key` structure.\n - Automatically handles Google Drive API token refreshes for long-running jobs.\n \n ## Overview\nThis module integrates Google Drive as a storage backend, enabling automatic folder creation and file uploads. It supports authentication via **service accounts** (recommended for automation) or **OAuth tokens** (for user-based authentication).\n\n## Features\n- Saves files to Google Drive, organizing them into structured folders.\n- Supports both **service account** and **OAuth token** authentication.\n- Automatically creates folders if they don't exist.\n- Generates public URLs for easy file sharing.\n\n## Setup Guide\n1. **Enable Google Drive API**\n - Create a Google Cloud project at [Google Cloud Console](https://console.cloud.google.com/)\n - Enable the **Google Drive API**.\n\n2. **Set Up a Google Drive Folder**\n - Create a folder in **Google Drive** and copy its **folder ID** from the URL.\n - Add the **folder ID** to your configuration (`orchestration.yaml`):\n ```yaml\n root_folder_id: \"FOLDER_ID\"\n ```\n\n3. **Authentication Options**\n - **Option 1: Service Account (Recommended)**\n - Create a **service account** in Google Cloud IAM.\n - Download the JSON key file and save it as:\n ```\n secrets/service_account.json\n ```\n - **Share your Drive folder** with the service account\u2019s `client_email` (found in the JSON file).\n \n - **Option 2: OAuth Token (User Authentication)**\n - Create OAuth **Desktop App credentials** in Google Cloud.\n - Save the credentials as:\n ```\n secrets/oauth_credentials.json\n ```\n - Generate an OAuth token by running:\n ```sh\n python scripts/create_update_gdrive_oauth_token.py -c secrets/oauth_credentials.json\n ```\n\n \n Notes on the OAuth token:\n Tokens are refreshed after 1 hour however keep working for 7 days (tbc)\n so as long as the job doesn't last for 7 days then this method of refreshing only once per run will work\n see this link for details on the token:\n https://davemateer.com/2022/04/28/google-drive-with-python#tokens\n \n \n", + "dependencies": { + "python": [ + "loguru", + "googleapiclient", + "google" + ] + }, + "entry_point": "gdrive_storage::GDriveStorage", + "version": "1.0", + "configs": { + "path_generator": { + "default": "url", + "help": "how to store the file in terms of directory structure: 'flat' sets to root; 'url' creates a directory based on the provided URL; 'random' creates a random directory.", + "choices": [ + "flat", + "url", + "random" + ] + }, + "filename_generator": { + "default": "static", + "help": "how to name stored files: 'random' creates a random string; 'static' uses a replicable strategy such as a hash.", + "choices": [ + "random", + "static" + ] + }, + "root_folder_id": { + "required": true, + "help": "root google drive folder ID to use as storage, found in URL: 'https://drive.google.com/drive/folders/FOLDER_ID'" + }, + "oauth_token": { + "default": null, + "help": "JSON filename with Google Drive OAuth token: check auto-archiver repository scripts folder for create_update_gdrive_oauth_token.py. NOTE: storage used will count towards owner of GDrive folder, therefore it is best to use oauth_token_filename over service_account." + }, + "service_account": { + "default": "secrets/service_account.json", + "help": "service account JSON file path, same as used for Google Sheets. NOTE: storage used will count towards the developer account." + } + } + }, + "configs": { + "path_generator": { + "default": "url", + "help": "how to store the file in terms of directory structure: 'flat' sets to root; 'url' creates a directory based on the provided URL; 'random' creates a random directory.", + "choices": [ + "flat", + "url", + "random" + ] + }, + "filename_generator": { + "default": "static", + "help": "how to name stored files: 'random' creates a random string; 'static' uses a replicable strategy such as a hash.", + "choices": [ + "random", + "static" + ] + }, + "root_folder_id": { + "required": true, + "help": "root google drive folder ID to use as storage, found in URL: 'https://drive.google.com/drive/folders/FOLDER_ID'" + }, + "oauth_token": { + "default": null, + "help": "JSON filename with Google Drive OAuth token: check auto-archiver repository scripts folder for create_update_gdrive_oauth_token.py. NOTE: storage used will count towards owner of GDrive folder, therefore it is best to use oauth_token_filename over service_account." + }, + "service_account": { + "default": "secrets/service_account.json", + "help": "service account JSON file path, same as used for Google Sheets. NOTE: storage used will count towards the developer account." + } + } + }, + "atlos_storage": { + "name": "atlos_storage", + "display_name": "Atlos Storage", + "manifest": { + "name": "Atlos Storage", + "author": "Bellingcat", + "type": [ + "storage" + ], + "requires_setup": true, + "description": "\n Stores media files in a [Atlos](https://www.atlos.org/).\n\n ### Features\n - Saves media files to Atlos, organizing them into folders based on the provided path structure.\n\n ### Notes\n - Requires setup with Atlos credentials.\n - Files are uploaded to the specified `root_folder_id` and organized by the `media.key` structure.\n ", + "dependencies": { + "python": [ + "loguru", + "boto3" + ], + "bin": [] + }, + "entry_point": "", + "version": "1.0", + "configs": { + "api_token": { + "default": null, + "help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/", + "required": true, + "type": "str" + }, + "atlos_url": { + "default": "https://platform.atlos.org", + "help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.", + "type": "str" + } + } + }, + "configs": { + "api_token": { + "default": null, + "help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/", + "required": true, + "type": "str" + }, + "atlos_url": { + "default": "https://platform.atlos.org", + "help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.", + "type": "str" + } + } + }, + "s3_storage": { + "name": "s3_storage", + "display_name": "S3 Storage", + "manifest": { + "name": "S3 Storage", + "author": "Bellingcat", + "type": [ + "storage" + ], + "requires_setup": true, + "description": "\n S3Storage: A storage module for saving media files to an S3-compatible object storage.\n\n ### Features\n - Uploads media files to an S3 bucket with customizable configurations.\n - Supports `random_no_duplicate` mode to avoid duplicate uploads by checking existing files based on SHA-256 hashes.\n - Automatically generates unique paths for files when duplicates are found.\n - Configurable endpoint and CDN URL for different S3-compatible providers.\n - Supports both private and public file storage, with public files being readable online.\n\n ### Notes\n - Requires S3 credentials (API key and secret) and a bucket name to function.\n - The `random_no_duplicate` option ensures no duplicate uploads by leveraging hash-based folder structures.\n - Uses `boto3` for interaction with the S3 API.\n - Depends on the `HashEnricher` module for hash calculation.\n ", + "dependencies": { + "python": [ + "hash_enricher", + "boto3", + "loguru" + ] + }, + "entry_point": "", + "version": "1.0", + "configs": { + "path_generator": { + "default": "flat", + "help": "how to store the file in terms of directory structure: 'flat' sets to root; 'url' creates a directory based on the provided URL; 'random' creates a random directory.", + "choices": [ + "flat", + "url", + "random" + ] + }, + "filename_generator": { + "default": "static", + "help": "how to name stored files: 'random' creates a random string; 'static' uses a replicable strategy such as a hash.", + "choices": [ + "random", + "static" + ] + }, + "bucket": { + "default": null, + "help": "S3 bucket name" + }, + "region": { + "default": null, + "help": "S3 region name" + }, + "key": { + "default": null, + "help": "S3 API key" + }, + "secret": { + "default": null, + "help": "S3 API secret" + }, + "random_no_duplicate": { + "default": false, + "type": "bool", + "help": "if set, it will override `path_generator`, `filename_generator` and `folder`. It will check if the file already exists and if so it will not upload it again. Creates a new root folder path `no-dups/`" + }, + "endpoint_url": { + "default": "https://{region}.digitaloceanspaces.com", + "help": "S3 bucket endpoint, {region} are inserted at runtime" + }, + "cdn_url": { + "default": "https://{bucket}.{region}.cdn.digitaloceanspaces.com/{key}", + "help": "S3 CDN url, {bucket}, {region} and {key} are inserted at runtime" + }, + "private": { + "default": false, + "type": "bool", + "help": "if true S3 files will not be readable online" + } + } + }, + "configs": { + "path_generator": { + "default": "flat", + "help": "how to store the file in terms of directory structure: 'flat' sets to root; 'url' creates a directory based on the provided URL; 'random' creates a random directory.", + "choices": [ + "flat", + "url", + "random" + ] + }, + "filename_generator": { + "default": "static", + "help": "how to name stored files: 'random' creates a random string; 'static' uses a replicable strategy such as a hash.", + "choices": [ + "random", + "static" + ] + }, + "bucket": { + "default": null, + "help": "S3 bucket name" + }, + "region": { + "default": null, + "help": "S3 region name" + }, + "key": { + "default": null, + "help": "S3 API key" + }, + "secret": { + "default": null, + "help": "S3 API secret" + }, + "random_no_duplicate": { + "default": false, + "type": "bool", + "help": "if set, it will override `path_generator`, `filename_generator` and `folder`. It will check if the file already exists and if so it will not upload it again. Creates a new root folder path `no-dups/`" + }, + "endpoint_url": { + "default": "https://{region}.digitaloceanspaces.com", + "help": "S3 bucket endpoint, {region} are inserted at runtime" + }, + "cdn_url": { + "default": "https://{bucket}.{region}.cdn.digitaloceanspaces.com/{key}", + "help": "S3 CDN url, {bucket}, {region} and {key} are inserted at runtime" + }, + "private": { + "default": false, + "type": "bool", + "help": "if true S3 files will not be readable online" + } + } + }, + "local_storage": { + "name": "local_storage", + "display_name": "Local Storage", + "manifest": { + "name": "Local Storage", + "author": "Bellingcat", + "type": [ + "storage" + ], + "requires_setup": false, + "description": "\n LocalStorage: A storage module for saving archived content locally on the filesystem.\n\n ### Features\n - Saves archived media files to a specified folder on the local filesystem.\n - Maintains file metadata during storage using `shutil.copy2`.\n - Supports both absolute and relative paths for stored files, configurable via `save_absolute`.\n - Automatically creates directories as needed for storing files.\n\n ### Notes\n - Default storage folder is `./archived`, but this can be changed via the `save_to` configuration.\n - The `save_absolute` option can reveal the file structure in output formats; use with caution.\n ", + "dependencies": { + "python": [ + "loguru" + ] + }, + "entry_point": "", + "version": "1.0", + "configs": { + "path_generator": { + "default": "flat", + "help": "how to store the file in terms of directory structure: 'flat' sets to root; 'url' creates a directory based on the provided URL; 'random' creates a random directory.", + "choices": [ + "flat", + "url", + "random" + ] + }, + "filename_generator": { + "default": "static", + "help": "how to name stored files: 'random' creates a random string; 'static' uses a replicable strategy such as a hash.", + "choices": [ + "random", + "static" + ] + }, + "save_to": { + "default": "./local_archive", + "help": "folder where to save archived content" + }, + "save_absolute": { + "default": false, + "type": "bool", + "help": "whether the path to the stored file is absolute or relative in the output result inc. formatters (WARN: leaks the file structure)" + } + } + }, + "configs": { + "path_generator": { + "default": "flat", + "help": "how to store the file in terms of directory structure: 'flat' sets to root; 'url' creates a directory based on the provided URL; 'random' creates a random directory.", + "choices": [ + "flat", + "url", + "random" + ] + }, + "filename_generator": { + "default": "static", + "help": "how to name stored files: 'random' creates a random string; 'static' uses a replicable strategy such as a hash.", + "choices": [ + "random", + "static" + ] + }, + "save_to": { + "default": "./local_archive", + "help": "folder where to save archived content" + }, + "save_absolute": { + "default": false, + "type": "bool", + "help": "whether the path to the stored file is absolute or relative in the output result inc. formatters (WARN: leaks the file structure)" + } + } + }, + "mute_formatter": { + "name": "mute_formatter", + "display_name": "Mute Formatter", + "manifest": { + "name": "Mute Formatter", + "author": "Bellingcat", + "type": [ + "formatter" + ], + "requires_setup": true, + "description": " Default formatter.\n ", + "dependencies": {}, + "entry_point": "", + "version": "1.0", + "configs": {} + }, + "configs": null + }, + "html_formatter": { + "name": "html_formatter", + "display_name": "HTML Formatter", + "manifest": { + "name": "HTML Formatter", + "author": "Bellingcat", + "type": [ + "formatter" + ], + "requires_setup": false, + "description": " ", + "dependencies": { + "python": [ + "hash_enricher", + "loguru", + "jinja2" + ], + "bin": [ + "" + ] + }, + "entry_point": "", + "version": "1.0", + "configs": { + "detect_thumbnails": { + "default": true, + "help": "if true will group by thumbnails generated by thumbnail enricher by id 'thumbnail_00'", + "type": "bool" + } + } + }, + "configs": { + "detect_thumbnails": { + "default": true, + "help": "if true will group by thumbnails generated by thumbnail enricher by id 'thumbnail_00'", + "type": "bool" + } + } + } + }, + "steps": { + "feeders": [ + "cli_feeder", + "gsheet_feeder", + "atlos_feeder", + "csv_feeder" + ], + "extractors": [ + "wayback_extractor_enricher", + "wacz_extractor_enricher", + "instagram_api_extractor", + "instagram_tbot_extractor", + "generic_extractor", + "twitter_api_extractor", + "instagram_extractor", + "telethon_extractor", + "vk_extractor", + "telegram_extractor" + ], + "enrichers": [ + "wayback_extractor_enricher", + "wacz_extractor_enricher", + "metadata_enricher", + "timestamping_enricher", + "thumbnail_enricher", + "screenshot_enricher", + "meta_enricher", + "pdq_hash_enricher", + "whisper_enricher", + "ssl_enricher", + "hash_enricher" + ], + "databases": [ + "console_db", + "atlos_db", + "api_db", + "csv_db", + "gsheet_db" + ], + "storages": [ + "local_storage", + "gdrive_storage", + "atlos_storage", + "s3_storage" + ], + "formatters": [ + "html_formatter", + "mute_formatter" + ] + }, + "configs": [ + "gsheet_feeder", + "atlos_feeder", + "csv_feeder", + "cli_feeder", + "instagram_api_extractor", + "instagram_tbot_extractor", + "twitter_api_extractor", + "instagram_extractor", + "telethon_extractor", + "vk_extractor", + "generic_extractor", + "wayback_extractor_enricher", + "wacz_extractor_enricher", + "timestamping_enricher", + "screenshot_enricher", + "whisper_enricher", + "thumbnail_enricher", + "ssl_enricher", + "hash_enricher", + "atlos_db", + "api_db", + "gsheet_db", + "csv_db", + "gdrive_storage", + "atlos_storage", + "s3_storage", + "local_storage", + "html_formatter" + ], + "module_types": [ + "feeder", + "extractor", + "enricher", + "database", + "storage", + "formatter" + ], + "empty_config": "# Auto Archiver Configuration\n\n# Steps are the modules that will be run in the order they are defined\nsteps:\n feeders: []\n extractors: []\n enrichers: []\n databases: []\n storages: []\n formatters: []\n\n# Global configuration\n\n# Authentication\n# a dictionary of authentication information that can be used by extractors to login to website. \n# you can use a comma separated list for multiple domains on the same line (common usecase: x.com,twitter.com)\n# Common login 'types' are username/password, cookie, api key/token.\n# There are two special keys for using cookies, they are: cookies_file and cookies_from_browser. \n# Some Examples:\n# facebook.com:\n# username: \"my_username\"\n# password: \"my_password\"\n# or for a site that uses an API key:\n# twitter.com,x.com:\n# api_key\n# api_secret\n# youtube.com:\n# cookie: \"login_cookie=value ; other_cookie=123\" # multiple 'key=value' pairs should be separated by ;\n\nauthentication: {}\n\n# These are the global configurations that are used by the modules\n\nlogging:\n level: INFO\n\n" +} \ No newline at end of file diff --git a/scripts/settings/src/types.d.ts b/scripts/settings/src/types.d.ts new file mode 100644 index 0000000..fdf80fc --- /dev/null +++ b/scripts/settings/src/types.d.ts @@ -0,0 +1,21 @@ +export interface Config { + name: string; + description: string; + type: string?; + default: any; + help: string; + choices: string[]; + required: boolean; +} + +interface Manifest { + description: string; +} + +export interface Module { + name: string; + description: string; + configs: { [key: string]: Config }; + manifest: Manifest; + display_name: string; +} diff --git a/scripts/settings/tsconfig.json b/scripts/settings/tsconfig.json new file mode 100644 index 0000000..3d0a51a --- /dev/null +++ b/scripts/settings/tsconfig.json @@ -0,0 +1,21 @@ +{ + "compilerOptions": { + "target": "ESNext", + "useDefineForClassFields": true, + "lib": ["DOM", "DOM.Iterable", "ESNext"], + "allowJs": false, + "skipLibCheck": true, + "esModuleInterop": false, + "allowSyntheticDefaultImports": true, + "strict": true, + "forceConsistentCasingInFileNames": true, + "module": "ESNext", + "moduleResolution": "Node", + "resolveJsonModule": true, + "isolatedModules": true, + "noEmit": true, + "jsx": "react-jsx" + }, + "include": ["src"], + "references": [{ "path": "./tsconfig.node.json" }] +} diff --git a/scripts/settings/tsconfig.node.json b/scripts/settings/tsconfig.node.json new file mode 100644 index 0000000..9d31e2a --- /dev/null +++ b/scripts/settings/tsconfig.node.json @@ -0,0 +1,9 @@ +{ + "compilerOptions": { + "composite": true, + "module": "ESNext", + "moduleResolution": "Node", + "allowSyntheticDefaultImports": true + }, + "include": ["vite.config.ts"] +} diff --git a/scripts/settings/vite.config.ts b/scripts/settings/vite.config.ts new file mode 100644 index 0000000..a04d8c7 --- /dev/null +++ b/scripts/settings/vite.config.ts @@ -0,0 +1,12 @@ +import { defineConfig } from 'vite'; +import react from '@vitejs/plugin-react'; +import { viteSingleFile } from "vite-plugin-singlefile" + +// https://vite.dev/config/ +export default defineConfig({ + plugins: [react(), viteSingleFile()], + build: { + minify: false, + sourcemap: true, + } +}); diff --git a/src/auto_archiver/__main__.py b/src/auto_archiver/__main__.py index 0023a59..f901d21 100644 --- a/src/auto_archiver/__main__.py +++ b/src/auto_archiver/__main__.py @@ -3,7 +3,7 @@ from auto_archiver.core.orchestrator import ArchivingOrchestrator import sys def main(): - ArchivingOrchestrator().run(sys.argv[1:]) + for _ in ArchivingOrchestrator()._command_line_run(sys.argv[1:]): pass if __name__ == "__main__": main() diff --git a/src/auto_archiver/core/__init__.py b/src/auto_archiver/core/__init__.py index ae4c41c..78d9a3d 100644 --- a/src/auto_archiver/core/__init__.py +++ b/src/auto_archiver/core/__init__.py @@ -3,7 +3,7 @@ """ from .metadata import Metadata from .media import Media -from .module import BaseModule +from .base_module import BaseModule # cannot import ArchivingOrchestrator/Config to avoid circular dep # from .orchestrator import ArchivingOrchestrator diff --git a/src/auto_archiver/core/base_module.py b/src/auto_archiver/core/base_module.py index ece4719..d6e4455 100644 --- a/src/auto_archiver/core/base_module.py +++ b/src/auto_archiver/core/base_module.py @@ -1,13 +1,18 @@ -from urllib.parse import urlparse -from typing import Mapping, Any +from __future__ import annotations + +from typing import Mapping, Any, Type, TYPE_CHECKING from abc import ABC from copy import deepcopy, copy from tempfile import TemporaryDirectory from auto_archiver.utils import url as UrlUtil +from auto_archiver.core.consts import MODULE_TYPES as CONF_MODULE_TYPES from loguru import logger +if TYPE_CHECKING: + from .module import ModuleFactory + class BaseModule(ABC): """ @@ -17,41 +22,24 @@ class BaseModule(ABC): however modules can have a .setup() method to run any setup code (e.g. logging in to a site, spinning up a browser etc.) - See BaseModule.MODULE_TYPES for the types of modules you can create, noting that + See consts.MODULE_TYPES for the types of modules you can create, noting that a subclass can be of multiple types. For example, a module that extracts data from a website and stores it in a database would be both an 'extractor' and a 'database' module. Each module is a python package, and should have a __manifest__.py file in the same directory as the module file. The __manifest__.py specifies the module information - like name, author, version, dependencies etc. See BaseModule._DEFAULT_MANIFEST for the + like name, author, version, dependencies etc. See DEFAULT_MANIFEST for the default manifest structure. """ - MODULE_TYPES = [ - 'feeder', - 'extractor', - 'enricher', - 'database', - 'storage', - 'formatter' - ] - - _DEFAULT_MANIFEST = { - 'name': '', # the display name of the module - 'author': 'Bellingcat', # creator of the module, leave this as Bellingcat or set your own name! - 'type': [], # the type of the module, can be one or more of BaseModule.MODULE_TYPES - 'requires_setup': True, # whether or not this module requires additional setup such as setting API Keys or installing additional softare - 'description': '', # a description of the module - 'dependencies': {}, # external dependencies, e.g. python packages or binaries, in dictionary format - 'entry_point': '', # the entry point for the module, in the format 'module_name::ClassName'. This can be left blank to use the default entry point of module_name::ModuleName - 'version': '1.0', # the version of the module - 'configs': {} # any configuration options this module has, these will be exposed to the user in the config file or via the command line -} + MODULE_TYPES = CONF_MODULE_TYPES + # NOTE: these here are declard as class variables, but they are overridden by the instance variables in the __init__ method config: Mapping[str, Any] authentication: Mapping[str, Mapping[str, str]] name: str + module_factory: ModuleFactory # this is set by the orchestrator prior to archiving tmp_dir: TemporaryDirectory = None @@ -62,14 +50,6 @@ class BaseModule(ABC): def config_setup(self, config: dict): - authentication = config.get('authentication', {}) - # extract out concatenated sites - for key, val in copy(authentication).items(): - if "," in key: - for site in key.split(","): - authentication[site] = val - del authentication[key] - # this is important. Each instance is given its own deepcopied config, so modules cannot # change values to affect other modules config = deepcopy(config) @@ -89,21 +69,28 @@ class BaseModule(ABC): Returns the authentication information for a given site. This is used to authenticate with a site before extracting data. The site should be the domain of the site, e.g. 'twitter.com' - extract_cookies: bool - whether or not to extract cookies from the given browser and return the - cookie jar (disabling can speed up) processing if you don't actually need the cookies jar + :param site: the domain of the site to get authentication information for + :param extract_cookies: whether or not to extract cookies from the given browser/file and return the cookie jar (disabling can speed up processing if you don't actually need the cookies jar). - Currently, the dict can have keys of the following types: - - username: str - the username to use for login - - password: str - the password to use for login - - api_key: str - the API key to use for login - - api_secret: str - the API secret to use for login - - cookie: str - a cookie string to use for login (specific to this site) - - cookies_jar: YoutubeDLCookieJar | http.cookiejar.MozillaCookieJar - a cookie jar compatible with requests (e.g. `requests.get(cookies=cookie_jar)`) + :returns: authdict dict of login information for the given site + + **Global options:**\n + * cookies_from_browser: str - the name of the browser to extract cookies from (e.g. 'chrome', 'firefox' - uses ytdlp under the hood to extract\n + * cookies_file: str - the path to a cookies file to use for login\n + + **Currently, the sites dict can have keys of the following types:**\n + * username: str - the username to use for login\n + * password: str - the password to use for login\n + * api_key: str - the API key to use for login\n + * api_secret: str - the API secret to use for login\n + * cookie: str - a cookie string to use for login (specific to this site)\n + * cookies_file: str - the path to a cookies file to use for login (specific to this site)\n + * cookies_from_browser: str - the name of the browser to extract cookies from (specitic for this site)\n """ # TODO: think about if/how we can deal with sites that have multiple domains (main one is x.com/twitter.com) # for now the user must enter them both, like "x.com,twitter.com" in their config. Maybe we just hard-code? - site = UrlUtil.domain_for_url(site) + site = UrlUtil.domain_for_url(site).lstrip("www.") # add the 'www' version of the site to the list of sites to check authdict = {} @@ -118,8 +105,8 @@ class BaseModule(ABC): for key in self.authentication.keys(): if key in site or site in key: logger.debug(f"Could not find exact authentication information for site '{site}'. \ - did find information for '{key}' which is close, is this what you meant? \ - If so, edit your authentication settings to make sure it exactly matches.") +did find information for '{key}' which is close, is this what you meant? \ +If so, edit your authentication settings to make sure it exactly matches.") def get_ytdlp_cookiejar(args): import yt_dlp @@ -130,16 +117,29 @@ class BaseModule(ABC): ytdlp_opts = getattr(parse_options(args), 'ydl_opts') return yt_dlp.YoutubeDL(ytdlp_opts).cookiejar - # get the cookies jar, prefer the browser cookies than the file - if 'cookies_from_browser' in self.authentication: + get_cookiejar_options = None + + # order of priority: + # 1. cookies_from_browser setting in site config + # 2. cookies_file setting in site config + # 3. cookies_from_browser setting in global config + # 4. cookies_file setting in global config + + if 'cookies_from_browser' in authdict: + get_cookiejar_options = ['--cookies-from-browser', authdict['cookies_from_browser']] + elif 'cookies_file' in authdict: + get_cookiejar_options = ['--cookies', authdict['cookies_file']] + elif 'cookies_from_browser' in self.authentication: authdict['cookies_from_browser'] = self.authentication['cookies_from_browser'] - if extract_cookies: - authdict['cookies_jar'] = get_ytdlp_cookiejar(['--cookies-from-browser', self.authentication['cookies_from_browser']]) + get_cookiejar_options = ['--cookies-from-browser', self.authentication['cookies_from_browser']] elif 'cookies_file' in self.authentication: authdict['cookies_file'] = self.authentication['cookies_file'] - if extract_cookies: - authdict['cookies_jar'] = get_ytdlp_cookiejar(['--cookies', self.authentication['cookies_file']]) + get_cookiejar_options = ['--cookies', self.authentication['cookies_file']] + + if get_cookiejar_options: + authdict['cookies_jar'] = get_ytdlp_cookiejar(get_cookiejar_options) + return authdict def repr(self): diff --git a/src/auto_archiver/core/config.py b/src/auto_archiver/core/config.py index 9bb080f..0282d41 100644 --- a/src/auto_archiver/core/config.py +++ b/src/auto_archiver/core/config.py @@ -7,21 +7,23 @@ flexible setup in various environments. import argparse from ruamel.yaml import YAML, CommentedMap, add_representer +import json from loguru import logger from copy import deepcopy -from .module import BaseModule +from auto_archiver.core.consts import MODULE_TYPES -from typing import Any, List, Type, Tuple _yaml: YAML = YAML() +DEFAULT_CONFIG_FILE = "secrets/orchestration.yaml" + EMPTY_CONFIG = _yaml.load(""" # Auto Archiver Configuration -# Steps are the modules that will be run in the order they are defined -steps:""" + "".join([f"\n {module}s: []" for module in BaseModule.MODULE_TYPES]) + \ +# Steps are the modules that will be run in the order they are defined +steps:""" + "".join([f"\n {module}s: []" for module in MODULE_TYPES]) + \ """ # Global configuration @@ -48,9 +50,61 @@ authentication: {} logging: level: INFO + """) # note: 'logging' is explicitly added above in order to better format the config file + +# Arg Parse Actions/Classes +class AuthenticationJsonParseAction(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + + try: + auth_dict = json.loads(values) + setattr(namespace, self.dest, auth_dict) + except json.JSONDecodeError as e: + raise argparse.ArgumentTypeError(f"Invalid JSON input for argument '{self.dest}': {e}") + + def load_from_file(path): + try: + with open(path, 'r') as f: + try: + auth_dict = json.load(f) + except json.JSONDecodeError: + f.seek(0) + # maybe it's yaml, try that + auth_dict = _yaml.load(f) + if auth_dict.get('authentication'): + auth_dict = auth_dict['authentication'] + auth_dict['load_from_file'] = path + return auth_dict + except: + return None + + if isinstance(auth_dict, dict) and auth_dict.get('from_file'): + auth_dict = load_from_file(auth_dict['from_file']) + elif isinstance(auth_dict, str): + # if it's a string + auth_dict = load_from_file(auth_dict) + + if not isinstance(auth_dict, dict): + raise argparse.ArgumentTypeError("Authentication must be a dictionary of site names and their authentication methods") + global_options = ['cookies_from_browser', 'cookies_file', 'load_from_file'] + for key, auth in auth_dict.items(): + if key in global_options: + continue + if not isinstance(key, str) or not isinstance(auth, dict): + raise argparse.ArgumentTypeError(f"Authentication must be a dictionary of site names and their authentication methods. Valid global configs are {global_options}") + + setattr(namespace, self.dest, auth_dict) + + +class UniqueAppendAction(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + for value in values: + if value not in getattr(namespace, self.dest): + getattr(namespace, self.dest).append(value) + class DefaultValidatingParser(argparse.ArgumentParser): def error(self, message): @@ -81,6 +135,7 @@ class DefaultValidatingParser(argparse.ArgumentParser): return super().parse_known_args(args, namespace) +# Config Utils def to_dot_notation(yaml_conf: CommentedMap | dict) -> dict: dotdict = {} @@ -128,6 +183,11 @@ def merge_dicts(dotdict: dict, yaml_dict: CommentedMap) -> CommentedMap: yaml_subdict[key] = value continue + if key == 'steps': + for module_type, modules in value.items(): + # overwrite the 'steps' from the config file with the ones from the CLI + yaml_subdict[key][module_type] = modules + if is_dict_type(value): update_dict(value, yaml_subdict[key]) elif is_list_type(value): @@ -136,7 +196,6 @@ def merge_dicts(dotdict: dict, yaml_dict: CommentedMap) -> CommentedMap: yaml_subdict[key] = value update_dict(from_dot_notation(dotdict), yaml_dict) - return yaml_dict def read_yaml(yaml_filename: str) -> CommentedMap: @@ -148,8 +207,8 @@ def read_yaml(yaml_filename: str) -> CommentedMap: pass if not config: - config = EMPTY_CONFIG - + config = deepcopy(EMPTY_CONFIG) + return config # TODO: make this tidier/find a way to notify of which keys should not be stored @@ -158,6 +217,14 @@ def read_yaml(yaml_filename: str) -> CommentedMap: def store_yaml(config: CommentedMap, yaml_filename: str) -> None: config_to_save = deepcopy(config) + auth_dict = config_to_save.get("authentication", {}) + if auth_dict and auth_dict.get('load_from_file'): + # remove all other values from the config, don't want to store it in the config file + auth_dict = {"load_from_file": auth_dict["load_from_file"]} + config_to_save.pop('urls', None) with open(yaml_filename, "w", encoding="utf-8") as outf: - _yaml.dump(config_to_save, outf) \ No newline at end of file + _yaml.dump(config_to_save, outf) + +def is_valid_config(config: CommentedMap) -> bool: + return config and config != EMPTY_CONFIG \ No newline at end of file diff --git a/src/auto_archiver/core/consts.py b/src/auto_archiver/core/consts.py new file mode 100644 index 0000000..597ce4e --- /dev/null +++ b/src/auto_archiver/core/consts.py @@ -0,0 +1,25 @@ +class SetupError(ValueError): + pass + +MODULE_TYPES = [ + 'feeder', + 'extractor', + 'enricher', + 'database', + 'storage', + 'formatter' +] + +MANIFEST_FILE = "__manifest__.py" + +DEFAULT_MANIFEST = { + 'name': '', # the display name of the module + 'author': 'Bellingcat', # creator of the module, leave this as Bellingcat or set your own name! + 'type': [], # the type of the module, can be one or more of MODULE_TYPES + 'requires_setup': True, # whether or not this module requires additional setup such as setting API Keys or installing additional software + 'description': '', # a description of the module + 'dependencies': {}, # external dependencies, e.g. python packages or binaries, in dictionary format + 'entry_point': '', # the entry point for the module, in the format 'module_name::ClassName'. This can be left blank to use the default entry point of module_name::ModuleName + 'version': '1.0', # the version of the module + 'configs': {} # any configuration options this module has, these will be exposed to the user in the config file or via the command line +} \ No newline at end of file diff --git a/src/auto_archiver/core/database.py b/src/auto_archiver/core/database.py index 0eb5d81..a6e76e5 100644 --- a/src/auto_archiver/core/database.py +++ b/src/auto_archiver/core/database.py @@ -1,3 +1,8 @@ +""" +Database module for the auto-archiver that defines the interface for implementing database modules +in the media archiving framework. +""" + from __future__ import annotations from abc import abstractmethod from typing import Union @@ -5,6 +10,11 @@ from typing import Union from auto_archiver.core import Metadata, BaseModule class Database(BaseModule): + """ + Base class for implementing database modules in the media archiving framework. + + Subclasses must implement the `fetch` and `done` methods to define platform-specific behavior. + """ def started(self, item: Metadata) -> None: """signals the DB that the given item archival has started""" diff --git a/src/auto_archiver/core/enricher.py b/src/auto_archiver/core/enricher.py index 0e50fa9..a862223 100644 --- a/src/auto_archiver/core/enricher.py +++ b/src/auto_archiver/core/enricher.py @@ -1,5 +1,5 @@ """ -Enrichers are modular components that enhance archived content by adding +Base module for Enrichers – modular components that enhance archived content by adding context, metadata, or additional processing. These add additional information to the context, such as screenshots, hashes, and metadata. @@ -13,7 +13,16 @@ from abc import abstractmethod from auto_archiver.core import Metadata, BaseModule class Enricher(BaseModule): - """Base classes and utilities for enrichers in the Auto-Archiver system.""" + """Base classes and utilities for enrichers in the Auto Archiver system. + + Enricher modules must implement the `enrich` method to define their behavior. + """ @abstractmethod - def enrich(self, to_enrich: Metadata) -> None: pass + def enrich(self, to_enrich: Metadata) -> None: + """ + Enriches a Metadata object with additional information or context. + + Takes the metadata object to enrich as an argument and modifies it in place, returning None. + """ + pass diff --git a/src/auto_archiver/core/extractor.py b/src/auto_archiver/core/extractor.py index 794c06c..f84be98 100644 --- a/src/auto_archiver/core/extractor.py +++ b/src/auto_archiver/core/extractor.py @@ -29,14 +29,24 @@ class Extractor(BaseModule): valid_url: re.Pattern = None def cleanup(self) -> None: - # called when extractors are done, or upon errors, cleanup any resources + """ + Called when extractors are done, or upon errors, cleanup any resources + """ pass def sanitize_url(self, url: str) -> str: - # used to clean unnecessary URL parameters OR unfurl redirect links + """ + Used to clean unnecessary URL parameters OR unfurl redirect links + """ return url def match_link(self, url: str) -> re.Match: + """ + Returns a match object if the given URL matches the valid_url pattern or False/None if not. + + Normally used in the `suitable` method to check if the URL is supported by this extractor. + + """ return self.valid_url.match(url) def suitable(self, url: str) -> bool: @@ -71,7 +81,8 @@ class Extractor(BaseModule): if len(to_filename) > 64: to_filename = to_filename[-64:] to_filename = os.path.join(self.tmp_dir, to_filename) - if verbose: logger.debug(f"downloading {url[0:50]=} {to_filename=}") + if verbose: + logger.debug(f"downloading {url[0:50]=} {to_filename=}") headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36' } @@ -80,8 +91,8 @@ class Extractor(BaseModule): d.raise_for_status() # get mimetype from the response headers - if not Path(to_filename).suffix: - content_type = d.headers.get('Content-Type') + if not mimetypes.guess_type(to_filename)[0]: + content_type = d.headers.get('Content-Type') or self._guess_file_type(url) extension = mimetypes.guess_extension(content_type) if extension: to_filename += extension diff --git a/src/auto_archiver/core/feeder.py b/src/auto_archiver/core/feeder.py index 352cfd9..e8302e6 100644 --- a/src/auto_archiver/core/feeder.py +++ b/src/auto_archiver/core/feeder.py @@ -1,3 +1,7 @@ +""" +The feeder base module defines the interface for implementing feeders in the media archiving framework. +""" + from __future__ import annotations from abc import abstractmethod from auto_archiver.core import Metadata @@ -5,5 +9,17 @@ from auto_archiver.core import BaseModule class Feeder(BaseModule): + """ + Base class for implementing feeders in the media archiving framework. + + Subclasses must implement the `__iter__` method to define platform-specific behavior. + """ + @abstractmethod - def __iter__(self) -> Metadata: return None \ No newline at end of file + def __iter__(self) -> Metadata: + """ + Returns an iterator (use `yield`) over the items to be archived. + + These should be instances of Metadata, typically created with Metadata().set_url(url). + """ + return None \ No newline at end of file diff --git a/src/auto_archiver/core/formatter.py b/src/auto_archiver/core/formatter.py index cf27cb3..3bfc250 100644 --- a/src/auto_archiver/core/formatter.py +++ b/src/auto_archiver/core/formatter.py @@ -1,9 +1,24 @@ +""" +Base module for formatters – modular components that format metadata into media objects for storage. + +The most commonly used formatter is the HTML formatter, which takes metadata and formats it into an HTML file for storage. +""" + from __future__ import annotations from abc import abstractmethod from auto_archiver.core import Metadata, Media, BaseModule class Formatter(BaseModule): + """ + Base class for implementing formatters in the media archiving framework. + + Subclasses must implement the `format` method to define their behavior. + """ @abstractmethod - def format(self, item: Metadata) -> Media: return None \ No newline at end of file + def format(self, item: Metadata) -> Media: + """ + Formats a Metadata object into a user-viewable format (e.g. HTML) and stores it if needed. + """ + return None \ No newline at end of file diff --git a/src/auto_archiver/core/media.py b/src/auto_archiver/core/media.py index b6820ab..ecaef19 100644 --- a/src/auto_archiver/core/media.py +++ b/src/auto_archiver/core/media.py @@ -6,7 +6,7 @@ nested media retrieval, and type validation. from __future__ import annotations import os import traceback -from typing import Any, List +from typing import Any, List, Iterator from dataclasses import dataclass, field from dataclasses_json import dataclass_json, config import mimetypes @@ -21,14 +21,13 @@ class Media: Represents a media file with associated properties and storage details. Attributes: - - filename: The file path of the media. - - key: An optional identifier for the media. + - filename: The file path of the media as saved locally (temporarily, before uploading to the storage). - urls: A list of URLs where the media is stored or accessible. - properties: Additional metadata or transformations for the media. - _mimetype: The media's mimetype (e.g., image/jpeg, video/mp4). """ filename: str - key: str = None + _key: str = None urls: List[str] = field(default_factory=list) properties: dict = field(default_factory=dict) _mimetype: str = None # eg: image/jpeg @@ -47,7 +46,7 @@ class Media: for any_media in self.all_inner_media(include_self=True): s.store(any_media, url, metadata=metadata) - def all_inner_media(self, include_self=False): + def all_inner_media(self, include_self=False) -> Iterator[Media]: """Retrieves all media, including nested media within properties or transformations on original media. This function returns a generator for all the inner media. @@ -67,6 +66,10 @@ class Media: # checks if the media is already stored in the given storage return len(self.urls) > 0 and len(self.urls) == len(in_storage.config["steps"]["storages"]) + @property + def key(self) -> str: + return self._key + def set(self, key: str, value: Any) -> Media: self.properties[key] = value return self diff --git a/src/auto_archiver/core/module.py b/src/auto_archiver/core/module.py index c81e26a..2c6617d 100644 --- a/src/auto_archiver/core/module.py +++ b/src/auto_archiver/core/module.py @@ -6,7 +6,7 @@ by handling user configuration, validating the steps properties, and implementin from __future__ import annotations from dataclasses import dataclass -from typing import List +from typing import List, TYPE_CHECKING import shutil import ast import copy @@ -16,99 +16,116 @@ import os from os.path import join from loguru import logger import auto_archiver -from .base_module import BaseModule +from auto_archiver.core.consts import DEFAULT_MANIFEST, MANIFEST_FILE -_LAZY_LOADED_MODULES = {} - -MANIFEST_FILE = "__manifest__.py" +if TYPE_CHECKING: + from .base_module import BaseModule -def setup_paths(paths: list[str]) -> None: - """ - Sets up the paths for the modules to be loaded from - - This is necessary for the modules to be imported correctly - - """ - for path in paths: - # check path exists, if it doesn't, log a warning - if not os.path.exists(path): - logger.warning(f"Path '{path}' does not exist. Skipping...") - continue +HAS_SETUP_PATHS = False - # see odoo/module/module.py -> initialize_sys_path - if path not in auto_archiver.modules.__path__: - auto_archiver.modules.__path__.append(path) +class ModuleFactory: - # sort based on the length of the path, so that the longest path is last in the list - auto_archiver.modules.__path__ = sorted(auto_archiver.modules.__path__, key=len, reverse=True) + def __init__(self): + self._lazy_modules = {} -def get_module(module_name: str, config: dict) -> BaseModule: - """ - Gets and sets up a module using the provided config - - This will actually load and instantiate the module, and load all its dependencies (i.e. not lazy) - - """ - return get_module_lazy(module_name).load(config) + def setup_paths(self, paths: list[str]) -> None: + """ + Sets up the paths for the modules to be loaded from + + This is necessary for the modules to be imported correctly + + """ + global HAS_SETUP_PATHS -def get_module_lazy(module_name: str, suppress_warnings: bool = False) -> LazyBaseModule: - """ - Lazily loads a module, returning a LazyBaseModule - - This has all the information about the module, but does not load the module itself or its dependencies - - To load an actual module, call .setup() on a lazy module - - """ - if module_name in _LAZY_LOADED_MODULES: - return _LAZY_LOADED_MODULES[module_name] - - available = available_modules(limit_to_modules=[module_name], suppress_warnings=suppress_warnings) - if not available: - raise IndexError(f"Module '{module_name}' not found. Are you sure it's installed/exists?") - return available[0] - -def available_modules(with_manifest: bool=False, limit_to_modules: List[str]= [], suppress_warnings: bool = False) -> List[LazyBaseModule]: - - # search through all valid 'modules' paths. Default is 'modules' in the current directory - - # see odoo/modules/module.py -> get_modules - def is_really_module(module_path): - if os.path.isfile(join(module_path, MANIFEST_FILE)): - return True - - all_modules = [] - - for module_folder in auto_archiver.modules.__path__: - # walk through each module in module_folder and check if it has a valid manifest - try: - possible_modules = os.listdir(module_folder) - except FileNotFoundError: - logger.warning(f"Module folder {module_folder} does not exist") - continue - - for possible_module in possible_modules: - if limit_to_modules and possible_module not in limit_to_modules: + for path in paths: + # check path exists, if it doesn't, log a warning + if not os.path.exists(path): + logger.warning(f"Path '{path}' does not exist. Skipping...") continue - possible_module_path = join(module_folder, possible_module) - if not is_really_module(possible_module_path): + # see odoo/module/module.py -> initialize_sys_path + if path not in auto_archiver.modules.__path__: + if HAS_SETUP_PATHS == True: + logger.warning(f"You are attempting to re-initialise the module paths with: '{path}' for a 2nd time. \ + This could lead to unexpected behaviour. It is recommended to only use a single modules path. \ + If you wish to load modules from different paths then load a 2nd python interpreter (e.g. using multiprocessing).") + auto_archiver.modules.__path__.append(path) + + # sort based on the length of the path, so that the longest path is last in the list + auto_archiver.modules.__path__ = sorted(auto_archiver.modules.__path__, key=len, reverse=True) + + HAS_SETUP_PATHS = True + + def get_module(self, module_name: str, config: dict) -> BaseModule: + """ + Gets and sets up a module using the provided config + + This will actually load and instantiate the module, and load all its dependencies (i.e. not lazy) + + """ + return self.get_module_lazy(module_name).load(config) + + def get_module_lazy(self, module_name: str, suppress_warnings: bool = False) -> LazyBaseModule: + """ + Lazily loads a module, returning a LazyBaseModule + + This has all the information about the module, but does not load the module itself or its dependencies + + To load an actual module, call .setup() on a lazy module + + """ + if module_name in self._lazy_modules: + return self._lazy_modules[module_name] + + available = self.available_modules(limit_to_modules=[module_name], suppress_warnings=suppress_warnings) + if not available: + message = f"Module '{module_name}' not found. Are you sure it's installed/exists?" + if 'archiver' in module_name: + message += f" Did you mean {module_name.replace('archiver', 'extractor')}?" + raise IndexError(message) + return available[0] + + def available_modules(self, limit_to_modules: List[str]= [], suppress_warnings: bool = False) -> List[LazyBaseModule]: + + # search through all valid 'modules' paths. Default is 'modules' in the current directory + + # see odoo/modules/module.py -> get_modules + def is_really_module(module_path): + if os.path.isfile(join(module_path, MANIFEST_FILE)): + return True + + all_modules = [] + + for module_folder in auto_archiver.modules.__path__: + # walk through each module in module_folder and check if it has a valid manifest + try: + possible_modules = os.listdir(module_folder) + except FileNotFoundError: + logger.warning(f"Module folder {module_folder} does not exist") continue - if _LAZY_LOADED_MODULES.get(possible_module): - continue - lazy_module = LazyBaseModule(possible_module, possible_module_path) - _LAZY_LOADED_MODULES[possible_module] = lazy_module + for possible_module in possible_modules: + if limit_to_modules and possible_module not in limit_to_modules: + continue - all_modules.append(lazy_module) - - if not suppress_warnings: - for module in limit_to_modules: - if not any(module == m.name for m in all_modules): - logger.warning(f"Module '{module}' not found. Are you sure it's installed?") + possible_module_path = join(module_folder, possible_module) + if not is_really_module(possible_module_path): + continue + if self._lazy_modules.get(possible_module): + continue + lazy_module = LazyBaseModule(possible_module, possible_module_path, factory=self) - return all_modules + self._lazy_modules[possible_module] = lazy_module + + all_modules.append(lazy_module) + + if not suppress_warnings: + for module in limit_to_modules: + if not any(module == m.name for m in all_modules): + logger.warning(f"Module '{module}' not found. Are you sure it's installed?") + + return all_modules @dataclass class LazyBaseModule: @@ -120,17 +137,22 @@ class LazyBaseModule: """ name: str - type: list description: str path: str + module_factory: ModuleFactory _manifest: dict = None _instance: BaseModule = None _entry_point: str = None - def __init__(self, module_name, path): + def __init__(self, module_name, path, factory: ModuleFactory): self.name = module_name self.path = path + self.module_factory = factory + + @property + def type(self): + return self.manifest['type'] @property def entry_point(self): @@ -161,16 +183,15 @@ class LazyBaseModule: return self._manifest # print(f"Loading manifest for module {module_path}") # load the manifest file - manifest = copy.deepcopy(BaseModule._DEFAULT_MANIFEST) + manifest = copy.deepcopy(DEFAULT_MANIFEST) with open(join(self.path, MANIFEST_FILE)) as f: try: manifest.update(ast.literal_eval(f.read())) except (ValueError, TypeError, SyntaxError, MemoryError, RecursionError) as e: - logger.error(f"Error loading manifest from file {self.path}/{MANIFEST_FILE}: {e}") + raise ValueError(f"Error loading manifest from file {self.path}/{MANIFEST_FILE}: {e}") self._manifest = manifest - self.type = manifest['type'] self._entry_point = manifest['entry_point'] self.description = manifest['description'] self.version = manifest['version'] @@ -189,13 +210,14 @@ class LazyBaseModule: # clear out any empty strings that a user may have erroneously added continue if not check(dep): - logger.error(f"Module '{self.name}' requires external dependency '{dep}' which is not available/setup. Have you installed the required dependencies for the '{self.name}' module? See the README for more information.") + logger.error(f"Module '{self.name}' requires external dependency '{dep}' which is not available/setup. \ + Have you installed the required dependencies for the '{self.name}' module? See the README for more information.") exit(1) def check_python_dep(dep): # first check if it's a module: try: - m = get_module_lazy(dep, suppress_warnings=True) + m = self.module_factory.get_module_lazy(dep, suppress_warnings=True) try: # we must now load this module and set it up with the config m.load(config) @@ -230,19 +252,21 @@ class LazyBaseModule: __import__(f'{qualname}.{file_name}', fromlist=[self.entry_point]) # finally, get the class instance instance: BaseModule = getattr(sys.modules[sub_qualname], class_name)() - if not getattr(instance, 'name', None): - instance.name = self.name - - if not getattr(instance, 'display_name', None): - instance.display_name = self.display_name - - self._instance = instance + # set the name, display name and module factory + instance.name = self.name + instance.display_name = self.display_name + instance.module_factory = self.module_factory + # merge the default config with the user config - default_config = dict((k, v['default']) for k, v in self.configs.items() if v.get('default')) + default_config = dict((k, v['default']) for k, v in self.configs.items() if 'default' in v) + config[self.name] = default_config | config.get(self.name, {}) instance.config_setup(config) instance.setup() + + # save the instance for future easy loading + self._instance = instance return instance def __repr__(self): diff --git a/src/auto_archiver/core/orchestrator.py b/src/auto_archiver/core/orchestrator.py index bb5f9e3..6a95046 100644 --- a/src/auto_archiver/core/orchestrator.py +++ b/src/auto_archiver/core/orchestrator.py @@ -5,88 +5,61 @@ """ from __future__ import annotations -from typing import Generator, Union, List, Type -from urllib.parse import urlparse -from ipaddress import ip_address +from typing import Generator, Union, List, Type, TYPE_CHECKING import argparse import os import sys -import json from tempfile import TemporaryDirectory import traceback +from copy import copy from rich_argparse import RichHelpFormatter - +from loguru import logger +import requests from .metadata import Metadata, Media from auto_archiver.version import __version__ -from .config import _yaml, read_yaml, store_yaml, to_dot_notation, merge_dicts, EMPTY_CONFIG, DefaultValidatingParser -from .module import available_modules, LazyBaseModule, get_module, setup_paths +from .config import read_yaml, store_yaml, to_dot_notation, merge_dicts, is_valid_config, \ + DefaultValidatingParser, UniqueAppendAction, AuthenticationJsonParseAction, DEFAULT_CONFIG_FILE +from .module import ModuleFactory, LazyBaseModule from . import validators, Feeder, Extractor, Database, Storage, Formatter, Enricher -from .module import BaseModule +from .consts import MODULE_TYPES, SetupError +from auto_archiver.utils.url import check_url_or_raise -from loguru import logger - - -DEFAULT_CONFIG_FILE = "orchestration.yaml" - -class JsonParseAction(argparse.Action): - def __call__(self, parser, namespace, values, option_string=None): - try: - setattr(namespace, self.dest, json.loads(values)) - except json.JSONDecodeError as e: - raise argparse.ArgumentTypeError(f"Invalid JSON input for argument '{self.dest}': {e}") - - -class AuthenticationJsonParseAction(JsonParseAction): - def __call__(self, parser, namespace, values, option_string=None): - super().__call__(parser, namespace, values, option_string) - auth_dict = getattr(namespace, self.dest) - if isinstance(auth_dict, str): - # if it's a string - try: - with open(auth_dict, 'r') as f: - try: - auth_dict = json.load(f) - except json.JSONDecodeError: - # maybe it's yaml, try that - auth_dict = _yaml.load(f) - except: - pass - - if not isinstance(auth_dict, dict): - raise argparse.ArgumentTypeError("Authentication must be a dictionary of site names and their authentication methods") - for site, auth in auth_dict.items(): - if not isinstance(site, str) or not isinstance(auth, dict): - raise argparse.ArgumentTypeError("Authentication must be a dictionary of site names and their authentication methods") - setattr(namespace, self.dest, auth_dict) -class UniqueAppendAction(argparse.Action): - def __call__(self, parser, namespace, values, option_string=None): - if not hasattr(namespace, self.dest): - setattr(namespace, self.dest, []) - for value in values: - if value not in getattr(namespace, self.dest): - getattr(namespace, self.dest).append(value) +if TYPE_CHECKING: + from .base_module import BaseModule + from .module import LazyBaseModule class ArchivingOrchestrator: + # instance variables + module_factory: ModuleFactory + setup_finished: bool + logger_id: int + + # instance variables, used for convenience to access modules by step feeders: List[Type[Feeder]] extractors: List[Type[Extractor]] enrichers: List[Type[Enricher]] databases: List[Type[Database]] storages: List[Type[Storage]] formatters: List[Type[Formatter]] - + + def __init__(self): + self.module_factory = ModuleFactory() + self.setup_finished = False + self.logger_id = None + def setup_basic_parser(self): parser = argparse.ArgumentParser( - prog="auto-archiver", - add_help=False, - description=""" + prog="auto-archiver", + add_help=False, + description=""" Auto Archiver is a CLI tool to archive media/metadata from online URLs; it can read URLs from many sources (Google Sheets, Command Line, ...); and write results to many destinations too (CSV, Google Sheets, MongoDB, ...)! """, - epilog="Check the code at https://github.com/bellingcat/auto-archiver", - formatter_class=RichHelpFormatter, + epilog="Check the code at https://github.com/bellingcat/auto-archiver", + formatter_class=RichHelpFormatter, ) parser.add_argument('--help', '-h', action='store_true', dest='help', help='show a full help message and exit') parser.add_argument('--version', action='version', version=__version__) @@ -98,105 +71,136 @@ class ArchivingOrchestrator: self.basic_parser = parser return parser + + def check_steps(self, config): + for module_type in MODULE_TYPES: + if not config['steps'].get(f"{module_type}s", []): + if module_type == 'feeder' or module_type == 'formatter' and config['steps'].get(f"{module_type}"): + raise SetupError(f"It appears you have '{module_type}' set under 'steps' in your configuration file, but as of version 0.13.0 of Auto Archiver, you must use '{module_type}s'. Change this in your configuration file and try again. \ +Here's how that would look: \n\nsteps:\n {module_type}s:\n - [your_{module_type}_name_here]\n {'extractors:...' if module_type == 'feeder' else '...'}\n") + if module_type == 'extractor' and config['steps'].get('archivers'): + raise SetupError(f"As of version 0.13.0 of Auto Archiver, the 'archivers' step name has been changed to 'extractors'. Change this in your configuration file and try again. \ +Here's how that would look: \n\nsteps:\n extractors:\n - [your_extractor_name_here]\n enrichers:...\n") + raise SetupError(f"No {module_type}s were configured. Make sure to set at least one {module_type} in your configuration file or on the command line (using --{module_type}s)") def setup_complete_parser(self, basic_config: dict, yaml_config: dict, unused_args: list[str]) -> None: + + # modules parser to get the overridden 'steps' values + modules_parser = argparse.ArgumentParser( + add_help=False, + ) + self.add_modules_args(modules_parser) + cli_modules, unused_args = modules_parser.parse_known_args(unused_args) + for module_type in MODULE_TYPES: + yaml_config['steps'][f"{module_type}s"] = getattr(cli_modules, f"{module_type}s", []) or yaml_config['steps'].get(f"{module_type}s", []) + parser = DefaultValidatingParser( add_help=False, ) self.add_additional_args(parser) + # merge command line module args (--feeders, --enrichers etc.) and add them to the config + # check what mode we're in # if we have a config file, use that to decide which modules to load # if simple, we'll load just the modules that has requires_setup = False # if full, we'll load all modules # TODO: BUG** - basic_config won't have steps in it, since these args aren't added to 'basic_parser' # but should we add them? Or should we just add them to the 'complete' parser? - if yaml_config != EMPTY_CONFIG: + + if is_valid_config(yaml_config): + self.check_steps(yaml_config) # only load the modules enabled in config # TODO: if some steps are empty (e.g. 'feeders' is empty), should we default to the 'simple' ones? Or only if they are ALL empty? enabled_modules = [] # first loads the modules from the config file, then from the command line - for config in [yaml_config['steps'], basic_config.__dict__]: - for module_type in BaseModule.MODULE_TYPES: - enabled_modules.extend(config.get(f"{module_type}s", [])) + for module_type in MODULE_TYPES: + enabled_modules.extend(yaml_config['steps'].get(f"{module_type}s", [])) # clear out duplicates, but keep the order enabled_modules = list(dict.fromkeys(enabled_modules)) - avail_modules = available_modules(with_manifest=True, limit_to_modules=enabled_modules, suppress_warnings=True) - self.add_module_args(avail_modules, parser) + avail_modules = self.module_factory.available_modules(limit_to_modules=enabled_modules, suppress_warnings=True) + self.add_individual_module_args(avail_modules, parser) elif basic_config.mode == 'simple': - simple_modules = [module for module in available_modules(with_manifest=True) if not module.requires_setup] - self.add_module_args(simple_modules, parser) + simple_modules = [module for module in self.module_factory.available_modules() if not module.requires_setup] + self.add_individual_module_args(simple_modules, parser) - # for simple mode, we use the cli_feeder and any modules that don't require setup - yaml_config['steps']['feeders'] = ['cli_feeder'] - # add them to the config for module in simple_modules: for module_type in module.type: yaml_config['steps'].setdefault(f"{module_type}s", []).append(module.name) else: # load all modules, they're not using the 'simple' mode - self.add_module_args(available_modules(with_manifest=True), parser) + all_modules = self.module_factory.available_modules() + # add all the modules to the steps + for module in all_modules: + for module_type in module.type: + yaml_config['steps'].setdefault(f"{module_type}s", []).append(module.name) + self.add_individual_module_args(all_modules, parser) + parser.set_defaults(**to_dot_notation(yaml_config)) # reload the parser with the new arguments, now that we have them parsed, unknown = parser.parse_known_args(unused_args) - # merge the new config with the old one - self.config = merge_dicts(vars(parsed), yaml_config) + config = merge_dicts(vars(parsed), yaml_config) + + # set up the authentication dict as needed + config = self.setup_authentication(config) + # clean out args from the base_parser that we don't want in the config for key in vars(basic_config): - self.config.pop(key, None) + config.pop(key, None) # setup the logging - self.setup_logging() + self.setup_logging(config) if unknown: logger.warning(f"Ignoring unknown/unused arguments: {unknown}\nPerhaps you don't have this module enabled?") - - if (self.config != yaml_config and basic_config.store) or not os.path.isfile(basic_config.config_file): + + if (config != yaml_config and basic_config.store) or not os.path.isfile(basic_config.config_file): logger.info(f"Storing configuration file to {basic_config.config_file}") - store_yaml(self.config, basic_config.config_file) - - return self.config + store_yaml(config, basic_config.config_file) + + return config + def add_modules_args(self, parser: argparse.ArgumentParser = None): + if not parser: + parser = self.parser + + # Module loading from the command line + for module_type in MODULE_TYPES: + parser.add_argument(f'--{module_type}s', dest=f'{module_type}s', nargs='+', help=f'the {module_type}s to use', default=[], action=UniqueAppendAction) + def add_additional_args(self, parser: argparse.ArgumentParser = None): if not parser: parser = self.parser - - # allow passing URLs directly on the command line - parser.add_argument('urls', nargs='*', default=[], help='URL(s) to archive, either a single URL or a list of urls, should not come from config.yaml') - - parser.add_argument('--feeders', dest='steps.feeders', nargs='+', default=['cli_feeder'], help='the feeders to use', action=UniqueAppendAction) - parser.add_argument('--enrichers', dest='steps.enrichers', nargs='+', help='the enrichers to use', action=UniqueAppendAction) - parser.add_argument('--extractors', dest='steps.extractors', nargs='+', help='the extractors to use', action=UniqueAppendAction) - parser.add_argument('--databases', dest='steps.databases', nargs='+', help='the databases to use', action=UniqueAppendAction) - parser.add_argument('--storages', dest='steps.storages', nargs='+', help='the storages to use', action=UniqueAppendAction) - parser.add_argument('--formatters', dest='steps.formatters', nargs='+', help='the formatter to use', action=UniqueAppendAction) - parser.add_argument('--authentication', dest='authentication', help='A dictionary of sites and their authentication methods \ (token, username etc.) that extractors can use to log into \ a website. If passing this on the command line, use a JSON string. \ - You may also pass a path to a valid JSON/YAML file which will be parsed.',\ + You may also pass a path to a valid JSON/YAML file which will be parsed.', default={}, + nargs="?", action=AuthenticationJsonParseAction) + # logging arguments parser.add_argument('--logging.level', action='store', dest='logging.level', choices=['INFO', 'DEBUG', 'ERROR', 'WARNING'], help='the logging level to use', default='INFO', type=str.upper) parser.add_argument('--logging.file', action='store', dest='logging.file', help='the logging file to write to', default=None) parser.add_argument('--logging.rotation', action='store', dest='logging.rotation', help='the logging rotation to use', default=None) - - def add_module_args(self, modules: list[LazyBaseModule] = None, parser: argparse.ArgumentParser = None) -> None: + def add_individual_module_args(self, modules: list[LazyBaseModule] = None, parser: argparse.ArgumentParser = None) -> None: if not modules: - modules = available_modules(with_manifest=True) - - module: LazyBaseModule + modules = self.module_factory.available_modules() + for module in modules: - + if module.name == 'cli_feeder': + # special case. For the CLI feeder, allow passing URLs directly on the command line without setting --cli_feeder.urls= + parser.add_argument('urls', nargs='*', default=[], help='URL(s) to archive, either a single URL or a list of urls, should not come from config.yaml') + continue + if not module.configs: # this module has no configs, don't show anything in the help # (TODO: do we want to show something about this module though, like a description?) @@ -224,21 +228,35 @@ class ArchivingOrchestrator: arg.should_store = should_store def show_help(self, basic_config: dict): - # for the help message, we want to load *all* possible modules and show the help - # add configs as arg parser arguments - + # for the help message, we want to load manifests from *all* possible modules and show their help/settings + # add configs as arg parser arguments + + self.add_modules_args(self.basic_parser) self.add_additional_args(self.basic_parser) - self.add_module_args(parser=self.basic_parser) + self.add_individual_module_args(parser=self.basic_parser) self.basic_parser.print_help() self.basic_parser.exit() - - def setup_logging(self): + + def setup_logging(self, config): + + logging_config = config['logging'] + + if logging_config.get('enabled', True) is False: + # disabled logging settings, they're set on a higher level + logger.disable('auto_archiver') + return + # setup loguru logging - logger.remove(0) # remove the default logger - logging_config = self.config['logging'] - logger.add(sys.stderr, level=logging_config['level']) - if log_file := logging_config['file']: - logger.add(log_file) if not logging_config['rotation'] else logger.add(log_file, rotation=logging_config['rotation']) + try: + logger.remove(0) # remove the default logger + except ValueError: + pass + + # add other logging info + if self.logger_id is None: # note - need direct comparison to None since need to consider falsy value 0 + self.logger_id = logger.add(sys.stderr, level=logging_config['level']) + if log_file := logging_config['file']: + logger.add(log_file) if not logging_config['rotation'] else logger.add(log_file, rotation=logging_config['rotation']) def install_modules(self, modules_by_type): """ @@ -246,59 +264,38 @@ class ArchivingOrchestrator: orchestrator's attributes (self.feeders, self.extractors etc.). If no modules of a certain type are loaded, the program will exit with an error message. """ - + invalid_modules = [] - for module_type in BaseModule.MODULE_TYPES: + for module_type in MODULE_TYPES: step_items = [] modules_to_load = modules_by_type[f"{module_type}s"] - assert modules_to_load, f"No {module_type}s were configured. Make sure to set at least one {module_type} in your configuration file or on the command line (using --{module_type}s)" + if not modules_to_load: + raise SetupError(f"No {module_type}s were configured. Make sure to set at least one {module_type} in your configuration file or on the command line (using --{module_type}s)") def check_steps_ok(): if not len(step_items): - logger.error(f"NO {module_type.upper()}S LOADED. Please check your configuration and try again.") if len(modules_to_load): - logger.error(f"Tried to load the following modules, but none were available: {modules_to_load}") - exit() + logger.error(f"Unable to load any {module_type}s. Tried the following, but none were available: {modules_to_load}") + raise SetupError(f"NO {module_type.upper()}S LOADED. Please check your configuration and try again.") + if (module_type == 'feeder' or module_type == 'formatter') and len(step_items) > 1: - logger.error(f"Only one {module_type} is allowed, found {len(step_items)} {module_type}s. Please remove one of the following from your configuration file: {modules_to_load}") - exit() + raise SetupError(f"Only one {module_type} is allowed, found {len(step_items)} {module_type}s. Please remove one of the following from your configuration file: {modules_to_load}") for module in modules_to_load: - if module == 'cli_feeder': - # pseudo module, don't load it - urls = self.config['urls'] - if not urls: - logger.error("No URLs provided. Please provide at least one URL via the command line, or set up an alternative feeder. Use --help for more information.") - exit() - # cli_feeder is a pseudo module, it just takes the command line args - def feed(self) -> Generator[Metadata]: - for url in urls: - logger.debug(f"Processing URL: '{url}'") - yield Metadata().set_url(url) - - pseudo_module = type('CLIFeeder', (Feeder,), { - 'name': 'cli_feeder', - 'display_name': 'CLI Feeder', - '__iter__': feed - - })() - - - pseudo_module.__iter__ = feed - step_items.append(pseudo_module) - continue if module in invalid_modules: continue + + loaded_module = None try: - loaded_module: BaseModule = get_module(module, self.config) + loaded_module: BaseModule = self.module_factory.get_module(module, self.config) except (KeyboardInterrupt, Exception) as e: logger.error(f"Error during setup of modules: {e}\n{traceback.format_exc()}") - if module_type == 'extractor' and loaded_module.name == module: + if loaded_module and module_type == 'extractor': loaded_module.cleanup() - exit() + raise e if not loaded_module: invalid_modules.append(module) @@ -308,48 +305,107 @@ class ArchivingOrchestrator: check_steps_ok() setattr(self, f"{module_type}s", step_items) - + def load_config(self, config_file: str) -> dict: if not os.path.exists(config_file) and config_file != DEFAULT_CONFIG_FILE: logger.error(f"The configuration file {config_file} was not found. Make sure the file exists and try again, or run without the --config file to use the default settings.") - exit() + raise FileNotFoundError(f"Configuration file {config_file} not found") return read_yaml(config_file) + + def setup_config(self, args: list) -> dict: + """ + Sets up the configuration file, merging the default config with the user's config + + This function should only ever be run once. + """ - def run(self, args: list) -> None: - self.setup_basic_parser() # parse the known arguments for now (basically, we want the config file) basic_config, unused_args = self.basic_parser.parse_known_args(args) # setup any custom module paths, so they'll show in the help and for arg parsing - setup_paths(basic_config.module_paths) + self.module_factory.setup_paths(basic_config.module_paths) # if help flag was called, then show the help if basic_config.help: self.show_help(basic_config) - + # merge command line --feeder etc. args with what's in the yaml config yaml_config = self.load_config(basic_config.config_file) - self.setup_complete_parser(basic_config, yaml_config, unused_args) + + return self.setup_complete_parser(basic_config, yaml_config, unused_args) + + def check_for_updates(self): + response = requests.get("https://pypi.org/pypi/auto-archiver/json").json() + latest_version = response['info']['version'] + # check version compared to current version + if latest_version != __version__: + if os.environ.get('RUNNING_IN_DOCKER'): + update_cmd = "`docker pull bellingcat/auto-archiver:latest`" + else: + update_cmd = "`pip install --upgrade auto-archiver`" + logger.warning("") + logger.warning("********* IMPORTANT: UPDATE AVAILABLE ********") + logger.warning(f"A new version of auto-archiver is available (v{latest_version}, you have {__version__})") + logger.warning(f"Make sure to update to the latest version using: {update_cmd}") + logger.warning("") + + + def setup(self, args: list): + """ + Function to configure all setup of the orchestrator: setup configs and load modules. + + This method should only ever be called once + """ + + self.check_for_updates() + + if self.setup_finished: + logger.warning("The `setup_config()` function should only ever be run once. \ + If you need to re-run the setup, please re-instantiate a new instance of the orchestrator. \ + For code implementatations, you should call .setup_config() once then you may call .feed() \ + multiple times to archive multiple URLs.") + return + + self.setup_basic_parser() + self.config = self.setup_config(args) logger.info(f"======== Welcome to the AUTO ARCHIVER ({__version__}) ==========") self.install_modules(self.config['steps']) # log out the modules that were loaded - for module_type in BaseModule.MODULE_TYPES: + for module_type in MODULE_TYPES: logger.info(f"{module_type.upper()}S: " + ", ".join(m.display_name for m in getattr(self, f"{module_type}s"))) + + self.setup_finished = True - for _ in self.feed(): - pass + def _command_line_run(self, args: list) -> Generator[Metadata]: + """ + This is the main entry point for the orchestrator, when run from the command line. - def cleanup(self)->None: + :param args: list of arguments to pass to the orchestrator - these are the command line args + + You should not call this method from code implementations. + + This method sets up the configuration, loads the modules, and runs the feed. + If you wish to make code invocations yourself, you should use the 'setup' and 'feed' methods separately. + To test configurations, without loading any modules you can also first call 'setup_configs' + """ + try: + self.setup(args) + return self.feed() + except Exception as e: + logger.error(e) + exit(1) + + def cleanup(self) -> None: logger.info("Cleaning up") for e in self.extractors: e.cleanup() def feed(self) -> Generator[Metadata]: - + url_count = 0 for feeder in self.feeders: for item in feeder: @@ -393,7 +449,6 @@ class ArchivingOrchestrator: m.tmp_dir = None tmp_dir.cleanup() - def archive(self, result: Metadata) -> Union[Metadata, None]: """ Runs the archiving process for a single URL @@ -407,8 +462,8 @@ class ArchivingOrchestrator: original_url = result.get_url().strip() try: - self.assert_valid_url(original_url) - except AssertionError as e: + check_url_or_raise(original_url) + except ValueError as e: logger.error(f"Error archiving URL {original_url}: {e}") raise e @@ -440,13 +495,13 @@ class ArchivingOrchestrator: try: result.merge(a.download(result)) if result.is_success(): break - except Exception as e: + except Exception as e: logger.error(f"ERROR archiver {a.name}: {e}: {traceback.format_exc()}") # 4 - call enrichers to work with archived content for e in self.enrichers: try: e.enrich(result) - except Exception as exc: + except Exception as exc: logger.error(f"ERROR enricher {e.name}: {exc}: {traceback.format_exc()}") # 5 - store all downloaded/generated media @@ -468,30 +523,30 @@ class ArchivingOrchestrator: logger.error(f"ERROR database {d.name}: {e}: {traceback.format_exc()}") return result + - def assert_valid_url(self, url: str) -> bool: + def setup_authentication(self, config: dict) -> dict: """ - Blocks localhost, private, reserved, and link-local IPs and all non-http/https schemes. + Setup authentication for all modules that require it + + Split up strings into multiple sites if they are comma separated """ - assert url.startswith("http://") or url.startswith("https://"), f"Invalid URL scheme" + + authentication = config.get('authentication', {}) + + # extract out concatenated sites + for key, val in copy(authentication).items(): + if "," in key: + for site in key.split(","): + site = site.strip() + authentication[site] = val + del authentication[key] - parsed = urlparse(url) - assert parsed.scheme in ["http", "https"], f"Invalid URL scheme" - assert parsed.hostname, f"Invalid URL hostname" - assert parsed.hostname != "localhost", f"Invalid URL" - - try: # special rules for IP addresses - ip = ip_address(parsed.hostname) - except ValueError: pass - else: - assert ip.is_global, f"Invalid IP used" - assert not ip.is_reserved, f"Invalid IP used" - assert not ip.is_link_local, f"Invalid IP used" - assert not ip.is_private, f"Invalid IP used" - + config['authentication'] = authentication + return config # Helper Properties - + @property def all_modules(self) -> List[Type[BaseModule]]: - return self.feeders + self.extractors + self.enrichers + self.databases + self.storages + self.formatters \ No newline at end of file + return self.feeders + self.extractors + self.enrichers + self.databases + self.storages + self.formatters diff --git a/src/auto_archiver/core/storage.py b/src/auto_archiver/core/storage.py index 5dfa39d..63ccf8d 100644 --- a/src/auto_archiver/core/storage.py +++ b/src/auto_archiver/core/storage.py @@ -1,7 +1,29 @@ +""" +Base module for Storage modules – modular components that store media objects in various locations. + +If you are looking to implement a new storage module, you should subclass the `Storage` class and +implement the `get_cdn_url` and `uploadf` methods. + +Your module **must** also have two config variables 'path_generator' and 'filename_generator' which +determine how the key is generated for the media object. The 'path_generator' and 'filename_generator' +variables can be set to one of the following values: +- 'flat': A flat structure with no subfolders +- 'url': A structure based on the URL of the media object +- 'random': A random structure + +The 'filename_generator' variable can be set to one of the following values: +- 'random': A random string +- 'static': A replicable strategy such as a hash + +If you don't want to use this naming convention, you can override the `set_key` method in your subclass. + +""" + from __future__ import annotations from abc import abstractmethod from typing import IO import os +import platform from loguru import logger from slugify import slugify @@ -10,56 +32,85 @@ from auto_archiver.utils.misc import random_str from auto_archiver.core import Media, BaseModule, Metadata from auto_archiver.modules.hash_enricher.hash_enricher import HashEnricher -from auto_archiver.core.module import get_module + class Storage(BaseModule): + + """ + Base class for implementing storage modules in the media archiving framework. + + Subclasses must implement the `get_cdn_url` and `uploadf` methods to define their behavior. + """ def store(self, media: Media, url: str, metadata: Metadata=None) -> None: if media.is_stored(in_storage=self): logger.debug(f"{media.key} already stored, skipping") return + self.set_key(media, url, metadata) self.upload(media, metadata=metadata) media.add_url(self.get_cdn_url(media)) @abstractmethod - def get_cdn_url(self, media: Media) -> str: pass + def get_cdn_url(self, media: Media) -> str: + """ + Returns the URL of the media object stored in the CDN. + """ + pass @abstractmethod - def uploadf(self, file: IO[bytes], key: str, **kwargs: dict) -> bool: pass + def uploadf(self, file: IO[bytes], key: str, **kwargs: dict) -> bool: + """ + Uploads (or saves) a file to the storage service/location. + + This method should not be called directly, but instead through the 'store' method, + which sets up the media for storage. + """ + pass def upload(self, media: Media, **kwargs) -> bool: + """ + Uploads a media object to the storage service. + + This method should not be called directly, but instead be called through the 'store' method, + which sets up the media for storage. + """ logger.debug(f'[{self.__class__.__name__}] storing file {media.filename} with key {media.key}') with open(media.filename, 'rb') as f: return self.uploadf(f, media, **kwargs) - def set_key(self, media: Media, url, metadata: Metadata) -> None: + def set_key(self, media: Media, url: str, metadata: Metadata) -> None: """takes the media and optionally item info and generates a key""" - if media.key is not None and len(media.key) > 0: return + + if media.key is not None and len(media.key) > 0: + # media key is already set + return + folder = metadata.get_context('folder', '') filename, ext = os.path.splitext(media.filename) # Handle path_generator logic - path_generator = self.config.get("path_generator", "url") + path_generator = self.path_generator if path_generator == "flat": path = "" - filename = slugify(filename) # Ensure filename is slugified elif path_generator == "url": - path = slugify(url) + path = slugify(url)[:70] elif path_generator == "random": - path = self.config.get("random_path", random_str(24), True) + path = random_str(24) else: raise ValueError(f"Invalid path_generator: {path_generator}") # Handle filename_generator logic - filename_generator = self.config.get("filename_generator", "random") + filename_generator = self.filename_generator if filename_generator == "random": filename = random_str(24) elif filename_generator == "static": # load the hash_enricher module - he = get_module(HashEnricher, self.config) + he = self.module_factory.get_module("hash_enricher", self.config) hd = he.calculate_hash(media.filename) filename = hd[:24] else: raise ValueError(f"Invalid filename_generator: {filename_generator}") + + key = os.path.join(folder, path, f"{filename}{ext}") - media.key = os.path.join(folder, path, f"{filename}{ext}") + media._key = key \ No newline at end of file diff --git a/src/auto_archiver/core/validators.py b/src/auto_archiver/core/validators.py index b868ddf..0d3f01f 100644 --- a/src/auto_archiver/core/validators.py +++ b/src/auto_archiver/core/validators.py @@ -1,6 +1,7 @@ # used as validators for config values. Should raise an exception if the value is invalid. from pathlib import Path import argparse +import json def example_validator(value): if "example" not in value: @@ -16,4 +17,7 @@ def positive_number(value): def valid_file(value): if not Path(value).is_file(): raise argparse.ArgumentTypeError(f"File '{value}' does not exist.") - return value \ No newline at end of file + return value + +def json_loader(cli_val): + return json.loads(cli_val) \ No newline at end of file diff --git a/src/auto_archiver/modules/api_db/__manifest__.py b/src/auto_archiver/modules/api_db/__manifest__.py index 8359174..c9ac461 100644 --- a/src/auto_archiver/modules/api_db/__manifest__.py +++ b/src/auto_archiver/modules/api_db/__manifest__.py @@ -1,5 +1,5 @@ { - "name": "Auto-Archiver API Database", + "name": "Auto Archiver API Database", "type": ["database"], "entry_point": "api_db::AAApiDb", "requires_setup": True, @@ -24,9 +24,9 @@ "help": "which group of users have access to the archive in case public=false as author", }, "use_api_cache": { - "default": True, + "default": False, "type": "bool", - "help": "if False then the API database will be queried prior to any archiving operations and stop if the link has already been archived", + "help": "if True then the API database will be queried prior to any archiving operations and stop if the link has already been archived", }, "store_results": { "default": True, @@ -39,7 +39,7 @@ }, }, "description": """ - Provides integration with the Auto-Archiver API for querying and storing archival data. + Provides integration with the Auto Archiver API for querying and storing archival data. ### Features - **API Integration**: Supports querying for existing archives and submitting results. @@ -49,6 +49,6 @@ - **Optional Storage**: Archives results conditionally based on configuration. ### Setup -Requires access to an Auto-Archiver API instance and a valid API token. +Requires access to an Auto Archiver API instance and a valid API token. """, } diff --git a/src/auto_archiver/modules/atlos_db/__init__.py b/src/auto_archiver/modules/atlos_db/__init__.py deleted file mode 100644 index 1552e39..0000000 --- a/src/auto_archiver/modules/atlos_db/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from atlos_db import AtlosDb \ No newline at end of file diff --git a/src/auto_archiver/modules/atlos_db/__manifest__.py b/src/auto_archiver/modules/atlos_db/__manifest__.py deleted file mode 100644 index b9cabf2..0000000 --- a/src/auto_archiver/modules/atlos_db/__manifest__.py +++ /dev/null @@ -1,36 +0,0 @@ -{ - "name": "Atlos Database", - "type": ["database"], - "entry_point": "atlos_db::AtlosDb", - "requires_setup": True, - "dependencies": - {"python": ["loguru", - ""], - "bin": [""]}, - "configs": { - "api_token": { - "default": None, - "help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/", - }, - "atlos_url": { - "default": "https://platform.atlos.org", - "help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.", - "type": "str" - }, - }, - "description": """ -Handles integration with the Atlos platform for managing archival results. - -### Features -- Outputs archival results to the Atlos API for storage and tracking. -- Updates failure status with error details when archiving fails. -- Processes and formats metadata, including ISO formatting for datetime fields. -- Skips processing for items without an Atlos ID. - -### Setup -Required configs: -- atlos_url: Base URL for the Atlos API. -- api_token: Authentication token for API access. -""" -, -} diff --git a/src/auto_archiver/modules/atlos_db/atlos_db.py b/src/auto_archiver/modules/atlos_db/atlos_db.py deleted file mode 100644 index baa9fef..0000000 --- a/src/auto_archiver/modules/atlos_db/atlos_db.py +++ /dev/null @@ -1,66 +0,0 @@ -from typing import Union - -import requests -from loguru import logger - -from auto_archiver.core import Database -from auto_archiver.core import Metadata - - -class AtlosDb(Database): - """ - Outputs results to Atlos - """ - - def failed(self, item: Metadata, reason: str) -> None: - """Update DB accordingly for failure""" - # If the item has no Atlos ID, there's nothing for us to do - if not item.metadata.get("atlos_id"): - logger.info(f"Item {item.get_url()} has no Atlos ID, skipping") - return - - requests.post( - f"{self.atlos_url}/api/v2/source_material/metadata/{item.metadata['atlos_id']}/auto_archiver", - headers={"Authorization": f"Bearer {self.api_token}"}, - json={"metadata": {"processed": True, "status": "error", "error": reason}}, - ).raise_for_status() - logger.info( - f"Stored failure for {item.get_url()} (ID {item.metadata['atlos_id']}) on Atlos: {reason}" - ) - - def fetch(self, item: Metadata) -> Union[Metadata, bool]: - """check and fetch if the given item has been archived already, each - database should handle its own caching, and configuration mechanisms""" - return False - - def _process_metadata(self, item: Metadata) -> dict: - """Process metadata for storage on Atlos. Will convert any datetime - objects to ISO format.""" - - return { - k: v.isoformat() if hasattr(v, "isoformat") else v - for k, v in item.metadata.items() - } - - def done(self, item: Metadata, cached: bool = False) -> None: - """archival result ready - should be saved to DB""" - - if not item.metadata.get("atlos_id"): - logger.info(f"Item {item.get_url()} has no Atlos ID, skipping") - return - - requests.post( - f"{self.atlos_url}/api/v2/source_material/metadata/{item.metadata['atlos_id']}/auto_archiver", - headers={"Authorization": f"Bearer {self.api_token}"}, - json={ - "metadata": dict( - processed=True, - status="success", - results=self._process_metadata(item), - ) - }, - ).raise_for_status() - - logger.info( - f"Stored success for {item.get_url()} (ID {item.metadata['atlos_id']}) on Atlos" - ) diff --git a/src/auto_archiver/modules/atlos_db/base_configs.py b/src/auto_archiver/modules/atlos_db/base_configs.py deleted file mode 100644 index f672f82..0000000 --- a/src/auto_archiver/modules/atlos_db/base_configs.py +++ /dev/null @@ -1,13 +0,0 @@ -def get_atlos_config_options(): - return { - "api_token": { - "default": None, - "help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/", - "type": str - }, - "atlos_url": { - "default": "https://platform.atlos.org", - "help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.", - "type": str - }, - } \ No newline at end of file diff --git a/src/auto_archiver/modules/atlos_feeder/__init__.py b/src/auto_archiver/modules/atlos_feeder/__init__.py deleted file mode 100644 index 67b243a..0000000 --- a/src/auto_archiver/modules/atlos_feeder/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .atlos_feeder import AtlosFeeder \ No newline at end of file diff --git a/src/auto_archiver/modules/atlos_feeder/__manifest__.py b/src/auto_archiver/modules/atlos_feeder/__manifest__.py deleted file mode 100644 index d59f420..0000000 --- a/src/auto_archiver/modules/atlos_feeder/__manifest__.py +++ /dev/null @@ -1,34 +0,0 @@ -{ - "name": "Atlos Feeder", - "type": ["feeder"], - "requires_setup": True, - "dependencies": { - "python": ["loguru", "requests"], - }, - "configs": { - "api_token": { - "type": "str", - "required": True, - "help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/", - }, - "atlos_url": { - "default": "https://platform.atlos.org", - "help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.", - "type": "str" - }, - }, - "description": """ - AtlosFeeder: A feeder module that integrates with the Atlos API to fetch source material URLs for archival. - - ### Features - - Connects to the Atlos API to retrieve a list of source material URLs. - - Filters source materials based on visibility, processing status, and metadata. - - Converts filtered source materials into `Metadata` objects with the relevant `atlos_id` and URL. - - Iterates through paginated results using a cursor for efficient API interaction. - - ### Notes - - Requires an Atlos API endpoint and a valid API token for authentication. - - Ensures only unprocessed, visible, and ready-to-archive URLs are returned. - - Handles pagination transparently when retrieving data from the Atlos API. - """ -} diff --git a/src/auto_archiver/modules/atlos_feeder/atlos_feeder.py b/src/auto_archiver/modules/atlos_feeder/atlos_feeder.py deleted file mode 100644 index 8c8f9cb..0000000 --- a/src/auto_archiver/modules/atlos_feeder/atlos_feeder.py +++ /dev/null @@ -1,42 +0,0 @@ -import requests -from loguru import logger - -from auto_archiver.core import Feeder -from auto_archiver.core import Metadata - - -class AtlosFeeder(Feeder): - - def __iter__(self) -> Metadata: - # Get all the urls from the Atlos API - count = 0 - cursor = None - while True: - response = requests.get( - f"{self.atlos_url}/api/v2/source_material", - headers={"Authorization": f"Bearer {self.api_token}"}, - params={"cursor": cursor}, - ) - data = response.json() - response.raise_for_status() - cursor = data["next"] - - for item in data["results"]: - if ( - item["source_url"] not in [None, ""] - and ( - item["metadata"] - .get("auto_archiver", {}) - .get("processed", False) - != True - ) - and item["visibility"] == "visible" - and item["status"] not in ["processing", "pending"] - ): - yield Metadata().set_url(item["source_url"]).set( - "atlos_id", item["id"] - ) - count += 1 - - if len(data["results"]) == 0 or cursor is None: - break diff --git a/src/auto_archiver/modules/atlos_feeder_db_storage/__init__.py b/src/auto_archiver/modules/atlos_feeder_db_storage/__init__.py new file mode 100644 index 0000000..8d62823 --- /dev/null +++ b/src/auto_archiver/modules/atlos_feeder_db_storage/__init__.py @@ -0,0 +1 @@ +from .atlos_feeder_db_storage import AtlosFeederDbStorage \ No newline at end of file diff --git a/src/auto_archiver/modules/atlos_feeder_db_storage/__manifest__.py b/src/auto_archiver/modules/atlos_feeder_db_storage/__manifest__.py new file mode 100644 index 0000000..2ea8f8f --- /dev/null +++ b/src/auto_archiver/modules/atlos_feeder_db_storage/__manifest__.py @@ -0,0 +1,46 @@ +{ + "name": "Atlos Feeder Database Storage", + "type": ["feeder", "database", "storage"], +"entry_point": "atlos_feeder_db_storage::AtlosFeederDbStorage", + "requires_setup": True, + "dependencies": { + "python": ["loguru", "requests"], + }, + "configs": { + "api_token": { + "type": "str", + "required": True, + "help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/", + }, + "atlos_url": { + "default": "https://platform.atlos.org", + "help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.", + "type": "str" + }, + }, + "description": """ + A module that integrates with the Atlos API to fetch source material URLs for archival, uplaod extracted media, + + [Atlos](https://www.atlos.org/) is a visual investigation and archiving platform designed for investigative research, journalism, and open-source intelligence (OSINT). + It helps users organize, analyze, and store media from various sources, making it easier to track and investigate digital evidence. + + To get started create a new project and obtain an API token from the settings page. You can group event's into Atlos's 'incidents'. + Here you can add 'source material' by URLn and the Atlos feeder will fetch these URLs for archival. + + You can use Atlos only as a 'feeder', however you can also implement the 'database' and 'storage' features to store the media files in Atlos which is recommended. + The Auto Archiver will retain the Atlos ID for each item, ensuring that the media and database outputs are uplaoded back into the relevant media item. + + + ### Features + - Connects to the Atlos API to retrieve a list of source material URLs. + - Iterates through the URLs from all source material items which are unprocessed, visible, and ready to archive. + - If the storage option is selected, it will store the media files alongside the original source material item in Atlos. + - Is the database option is selected it will output the results to the media item, as well as updating failure status with error details when archiving fails. + - Skips Storege/ database upload for items without an Atlos ID - restricting that you must use the Atlos feeder so that it has the Atlos ID to store the results with. + + ### Notes + - Requires an Atlos account with a project and a valid API token for authentication. + - Ensures only unprocessed, visible, and ready-to-archive URLs are returned. + - Feches any media items within an Atlos project, regardless of separation into incidents. + """ +} diff --git a/src/auto_archiver/modules/atlos_feeder_db_storage/atlos_feeder_db_storage.py b/src/auto_archiver/modules/atlos_feeder_db_storage/atlos_feeder_db_storage.py new file mode 100644 index 0000000..87b4f82 --- /dev/null +++ b/src/auto_archiver/modules/atlos_feeder_db_storage/atlos_feeder_db_storage.py @@ -0,0 +1,153 @@ +import hashlib +import os +from typing import IO, Iterator, Optional, Union + +import requests +from loguru import logger + +from auto_archiver.core import Database, Feeder, Media, Metadata, Storage +from auto_archiver.utils import calculate_file_hash + + +class AtlosFeederDbStorage(Feeder, Database, Storage): + + def setup(self) -> requests.Session: + """create and return a persistent session.""" + self.session = requests.Session() + + def _get(self, endpoint: str, params: Optional[dict] = None) -> dict: + """Wrapper for GET requests to the Atlos API.""" + url = f"{self.atlos_url}{endpoint}" + response = self.session.get( + url, headers={"Authorization": f"Bearer {self.api_token}"}, params=params + ) + response.raise_for_status() + return response.json() + + def _post( + self, + endpoint: str, + json: Optional[dict] = None, + params: Optional[dict] = None, + files: Optional[dict] = None, + ) -> dict: + """Wrapper for POST requests to the Atlos API.""" + url = f"{self.atlos_url}{endpoint}" + response = self.session.post( + url, + headers={"Authorization": f"Bearer {self.api_token}"}, + json=json, + params=params, + files=files, + ) + response.raise_for_status() + return response.json() + + # ! Atlos Module - Feeder Methods + + def __iter__(self) -> Iterator[Metadata]: + """Iterate over unprocessed, visible source materials from Atlos.""" + cursor = None + while True: + data = self._get("/api/v2/source_material", params={"cursor": cursor}) + cursor = data.get("next") + results = data.get("results", []) + for item in results: + if ( + item.get("source_url") not in [None, ""] + and not item.get("metadata", {}).get("auto_archiver", {}).get("processed", False) + and item.get("visibility") == "visible" + and item.get("status") not in ["processing", "pending"] + ): + yield Metadata().set_url(item["source_url"]).set("atlos_id", item["id"]) + if not results or cursor is None: + break + + # ! Atlos Module - Database Methods + + def failed(self, item: Metadata, reason: str) -> None: + """Mark an item as failed in Atlos, if the ID exists.""" + atlos_id = item.metadata.get("atlos_id") + if not atlos_id: + logger.info(f"Item {item.get_url()} has no Atlos ID, skipping") + return + self._post( + f"/api/v2/source_material/metadata/{atlos_id}/auto_archiver", + json={"metadata": {"processed": True, "status": "error", "error": reason}}, + ) + logger.info(f"Stored failure for {item.get_url()} (ID {atlos_id}) on Atlos: {reason}") + + def fetch(self, item: Metadata) -> Union[Metadata, bool]: + """check and fetch if the given item has been archived already, each + database should handle its own caching, and configuration mechanisms""" + return False + + def _process_metadata(self, item: Metadata) -> dict: + """Process metadata for storage on Atlos. Will convert any datetime + objects to ISO format.""" + return { + k: v.isoformat() if hasattr(v, "isoformat") else v + for k, v in item.metadata.items() + } + + def done(self, item: Metadata, cached: bool = False) -> None: + """Mark an item as successfully archived in Atlos.""" + atlos_id = item.metadata.get("atlos_id") + if not atlos_id: + logger.info(f"Item {item.get_url()} has no Atlos ID, skipping") + return + self._post( + f"/api/v2/source_material/metadata/{atlos_id}/auto_archiver", + json={ + "metadata": { + "processed": True, + "status": "success", + "results": self._process_metadata(item), + } + }, + ) + logger.info(f"Stored success for {item.get_url()} (ID {atlos_id}) on Atlos") + + # ! Atlos Module - Storage Methods + + def get_cdn_url(self, _media: Media) -> str: + """Return the base Atlos URL as the CDN URL.""" + return self.atlos_url + + def upload(self, media: Media, metadata: Optional[Metadata] = None, **_kwargs) -> bool: + """Upload a media file to Atlos if it has not been uploaded already.""" + if metadata is None: + logger.error(f"No metadata provided for {media.filename}") + return False + + atlos_id = metadata.get("atlos_id") + if not atlos_id: + logger.error(f"No Atlos ID found in metadata; can't store {media.filename} in Atlos.") + return False + + media_hash = calculate_file_hash(media.filename, hash_algo=hashlib.sha256, chunksize=4096) + + # Check whether the media has already been uploaded + source_material = self._get(f"/api/v2/source_material/{atlos_id}")["result"] + existing_media = [ + artifact.get("file_hash_sha256") + for artifact in source_material.get("artifacts", []) + ] + if media_hash in existing_media: + logger.info(f"{media.filename} with SHA256 {media_hash} already uploaded to Atlos") + return True + + # Upload the media to the Atlos API + with open(media.filename, "rb") as file_obj: + self._post( + f"/api/v2/source_material/upload/{atlos_id}", + params={"title": media.properties}, + files={"file": (os.path.basename(media.filename), file_obj)}, + ) + logger.info(f"Uploaded {media.filename} to Atlos with ID {atlos_id} and title {media.key}") + return True + + def uploadf(self, file: IO[bytes], key: str, **kwargs: dict) -> bool: + """Upload a file-like object; not implemented.""" + pass + diff --git a/src/auto_archiver/modules/atlos_storage/atlos_storage.py b/src/auto_archiver/modules/atlos_storage/atlos_storage.py deleted file mode 100644 index f8eef68..0000000 --- a/src/auto_archiver/modules/atlos_storage/atlos_storage.py +++ /dev/null @@ -1,66 +0,0 @@ -import hashlib -import os -from typing import IO, Optional - -import requests -from loguru import logger - -from auto_archiver.core import Media, Metadata -from auto_archiver.core import Storage - - -class AtlosStorage(Storage): - - def get_cdn_url(self, _media: Media) -> str: - # It's not always possible to provide an exact URL, because it's - # possible that the media once uploaded could have been copied to - # another project. - return self.atlos_url - - def _hash(self, media: Media) -> str: - # Hash the media file using sha-256. We don't use the existing auto archiver - # hash because there's no guarantee that the configuerer is using sha-256, which - # is how Atlos hashes files. - - sha256 = hashlib.sha256() - with open(media.filename, "rb") as f: - while True: - buf = f.read(4096) - if not buf: break - sha256.update(buf) - return sha256.hexdigest() - - def upload(self, media: Media, metadata: Optional[Metadata]=None, **_kwargs) -> bool: - atlos_id = metadata.get("atlos_id") - if atlos_id is None: - logger.error(f"No Atlos ID found in metadata; can't store {media.filename} on Atlos") - return False - - media_hash = self._hash(media) - - # Check whether the media has already been uploaded - source_material = requests.get( - f"{self.atlos_url}/api/v2/source_material/{atlos_id}", - headers={"Authorization": f"Bearer {self.api_token}"}, - ).json()["result"] - existing_media = [x["file_hash_sha256"] for x in source_material.get("artifacts", [])] - if media_hash in existing_media: - logger.info(f"{media.filename} with SHA256 {media_hash} already uploaded to Atlos") - return True - - # Upload the media to the Atlos API - requests.post( - f"{self.atlos_url}/api/v2/source_material/upload/{atlos_id}", - headers={"Authorization": f"Bearer {self.api_token}"}, - params={ - "title": media.properties - }, - files={"file": (os.path.basename(media.filename), open(media.filename, "rb"))}, - ).raise_for_status() - - logger.info(f"Uploaded {media.filename} to Atlos with ID {atlos_id} and title {media.key}") - - return True - - # must be implemented even if unused - def uploadf(self, file: IO[bytes], key: str, **kwargs: dict) -> bool: pass diff --git a/src/auto_archiver/modules/cli_feeder/__manifest__.py b/src/auto_archiver/modules/cli_feeder/__manifest__.py new file mode 100644 index 0000000..609aa3e --- /dev/null +++ b/src/auto_archiver/modules/cli_feeder/__manifest__.py @@ -0,0 +1,23 @@ +{ + 'name': 'Command Line Feeder', + 'type': ['feeder'], + 'entry_point': 'cli_feeder::CLIFeeder', + 'requires_setup': False, + 'description': 'Feeds URLs to orchestrator from the command line', + 'configs': { + 'urls': { + 'default': None, + 'help': 'URL(s) to archive, either a single URL or a list of urls, should not come from config.yaml', + }, + }, + 'description': """ +The Command Line Feeder is the default enabled feeder for the Auto Archiver. It allows you to pass URLs directly to the orchestrator from the command line +without the need to specify any additional configuration or command line arguments: + +`auto-archiver --feeder cli_feeder -- "https://example.com/1/,https://example.com/2/"` + +You can pass multiple URLs by separating them with a space. The URLs will be processed in the order they are provided. + +`auto-archiver --feeder cli_feeder -- https://example.com/1/ https://example.com/2/` +""", +} \ No newline at end of file diff --git a/src/auto_archiver/modules/cli_feeder/cli_feeder.py b/src/auto_archiver/modules/cli_feeder/cli_feeder.py new file mode 100644 index 0000000..fac584d --- /dev/null +++ b/src/auto_archiver/modules/cli_feeder/cli_feeder.py @@ -0,0 +1,20 @@ +from loguru import logger + +from auto_archiver.core.feeder import Feeder +from auto_archiver.core.metadata import Metadata + +class CLIFeeder(Feeder): + + def setup(self) -> None: + self.urls = self.config['urls'] + if not self.urls: + raise ValueError("No URLs provided. Please provide at least one URL via the command line, or set up an alternative feeder. Use --help for more information.") + + def __iter__(self) -> Metadata: + urls = self.config['urls'] + for url in urls: + logger.debug(f"Processing {url}") + m = Metadata().set_url(url) + yield m + + logger.success(f"Processed {len(urls)} URL(s)") \ No newline at end of file diff --git a/src/auto_archiver/modules/console_db/console_db.py b/src/auto_archiver/modules/console_db/console_db.py index 48609b0..b26a605 100644 --- a/src/auto_archiver/modules/console_db/console_db.py +++ b/src/auto_archiver/modules/console_db/console_db.py @@ -10,7 +10,7 @@ class ConsoleDb(Database): """ def started(self, item: Metadata) -> None: - logger.warning(f"STARTED {item}") + logger.info(f"STARTED {item}") def failed(self, item: Metadata, reason:str) -> None: logger.error(f"FAILED {item}: {reason}") diff --git a/src/auto_archiver/modules/csv_db/__manifest__.py b/src/auto_archiver/modules/csv_db/__manifest__.py index 507ce14..d9733b2 100644 --- a/src/auto_archiver/modules/csv_db/__manifest__.py +++ b/src/auto_archiver/modules/csv_db/__manifest__.py @@ -6,7 +6,7 @@ }, 'entry_point': 'csv_db::CSVDb', "configs": { - "csv_file": {"default": "db.csv", "help": "CSV file name"} + "csv_file": {"default": "db.csv", "help": "CSV file name to save metadata to"}, }, "description": """ Handles exporting archival results to a CSV file. diff --git a/src/auto_archiver/modules/gdrive_storage/__manifest__.py b/src/auto_archiver/modules/gdrive_storage/__manifest__.py index 632e52b..73784b8 100644 --- a/src/auto_archiver/modules/gdrive_storage/__manifest__.py +++ b/src/auto_archiver/modules/gdrive_storage/__manifest__.py @@ -32,7 +32,6 @@ GDriveStorage: A storage module for saving archived content to Google Drive. - Author: Dave Mateer, (And maintained by: ) Source Documentation: https://davemateer.com/2022/04/28/google-drive-with-python ### Features diff --git a/src/auto_archiver/modules/generic_extractor/__manifest__.py b/src/auto_archiver/modules/generic_extractor/__manifest__.py index caa3ae1..1d3b365 100644 --- a/src/auto_archiver/modules/generic_extractor/__manifest__.py +++ b/src/auto_archiver/modules/generic_extractor/__manifest__.py @@ -28,6 +28,13 @@ the broader archiving framework. metadata objects. Some dropins are included in this generic_archiver by default, but custom dropins can be created to handle additional websites and passed to the archiver via the command line using the `--dropins` option (TODO!). + +### Auto-Updates + +The Generic Extractor will also automatically check for updates to `yt-dlp` (every 5 days by default). +This can be configured using the `ytdlp_update_interval` setting (or disabled by setting it to -1). +If you are having issues with the extractor, you can review the version of `yt-dlp` being used with `yt-dlp --version`. + """, "configs": { "subtitles": {"default": True, "help": "download subtitles if available", "type": "bool"}, @@ -64,5 +71,10 @@ via the command line using the `--dropins` option (TODO!). "default": "inf", "help": "Use to limit the number of videos to download when a channel or long page is being extracted. 'inf' means no limit.", }, + "ytdlp_update_interval": { + "default": 5, + "help": "How often to check for yt-dlp updates (days). If positive, will check and update yt-dlp every [num] days. Set it to -1 to disable, or 0 to always update on every run.", + "type": "int", + }, }, } diff --git a/src/auto_archiver/modules/generic_extractor/bluesky.py b/src/auto_archiver/modules/generic_extractor/bluesky.py index f2086b0..5eef520 100644 --- a/src/auto_archiver/modules/generic_extractor/bluesky.py +++ b/src/auto_archiver/modules/generic_extractor/bluesky.py @@ -39,11 +39,11 @@ class Bluesky(GenericDropin): for image_media in image_medias: url = media_url.format(image_media['image']['ref']['$link'], post['author']['did']) image_media = archiver.download_from_url(url) - media.append(image_media) + media.append(Media(image_media)) for video_media in video_medias: url = media_url.format(video_media['ref']['$link'], post['author']['did']) video_media = archiver.download_from_url(url) - media.append(video_media) + media.append(Media(video_media)) return media diff --git a/src/auto_archiver/modules/generic_extractor/facebook.py b/src/auto_archiver/modules/generic_extractor/facebook.py index 352d44e..fed8e09 100644 --- a/src/auto_archiver/modules/generic_extractor/facebook.py +++ b/src/auto_archiver/modules/generic_extractor/facebook.py @@ -8,7 +8,8 @@ class Facebook(GenericDropin): url.replace('://m.facebook.com/', '://www.facebook.com/'), video_id) webpage = ie_instance._download_webpage(url, ie_instance._match_valid_url(url).group('id')) - post_data = ie_instance._extract_from_url.extract_metadata(webpage) + # TODO: fix once https://github.com/yt-dlp/yt-dlp/pull/12275 is merged + post_data = ie_instance._extract_metadata(webpage) return post_data def create_metadata(self, post: dict, ie_instance, archiver, url): diff --git a/src/auto_archiver/modules/generic_extractor/generic_extractor.py b/src/auto_archiver/modules/generic_extractor/generic_extractor.py index 86e978f..5acce46 100644 --- a/src/auto_archiver/modules/generic_extractor/generic_extractor.py +++ b/src/auto_archiver/modules/generic_extractor/generic_extractor.py @@ -1,7 +1,11 @@ -import datetime, os, yt_dlp, pysubs2 +import datetime, os import importlib -from typing import Type +import subprocess +from typing import Generator, Type + +import yt_dlp from yt_dlp.extractor.common import InfoExtractor +import pysubs2 from loguru import logger @@ -11,7 +15,45 @@ from auto_archiver.core import Metadata, Media class GenericExtractor(Extractor): _dropins = {} - def suitable_extractors(self, url: str) -> list[str]: + def setup(self): + # check for file .ytdlp-update in the secrets folder + if self.ytdlp_update_interval < 0: + return + + use_secrets = os.path.exists('secrets') + path = os.path.join('secrets' if use_secrets else '', '.ytdlp-update') + next_update_check = None + if os.path.exists(path): + with open(path, "r") as f: + next_update_check = datetime.datetime.fromisoformat(f.read()) + + if not next_update_check or next_update_check < datetime.datetime.now(): + self.update_ytdlp() + + next_update_check = datetime.datetime.now() + datetime.timedelta(days=self.ytdlp_update_interval) + with open(path, "w") as f: + f.write(next_update_check.isoformat()) + + def update_ytdlp(self): + logger.info("Checking and updating yt-dlp...") + logger.info(f"Tip: change the 'ytdlp_update_interval' setting to control how often yt-dlp is updated. Set to -1 to disable or 0 to enable on every run. Current setting: {self.ytdlp_update_interval}") + from importlib.metadata import version as get_version + old_version = get_version("yt-dlp") + try: + # try and update with pip (this works inside poetry environment and in a normal virtualenv) + result = subprocess.run(["pip", "install", "--upgrade", "yt-dlp"], check=True, capture_output=True) + + if "Successfully installed yt-dlp" in result.stdout.decode(): + new_version = importlib.metadata.version("yt-dlp") + logger.info(f"yt-dlp successfully (from {old_version} to {new_version})") + importlib.reload(yt_dlp) + else: + logger.info("yt-dlp already up to date") + + except Exception as e: + logger.error(f"Error updating yt-dlp: {e}") + + def suitable_extractors(self, url: str) -> Generator[str, None, None]: """ Returns a list of valid extractors for the given URL""" for info_extractor in yt_dlp.YoutubeDL()._ies.values(): @@ -86,7 +128,7 @@ class GenericExtractor(Extractor): # keep both 'title' and 'fulltitle', but prefer 'title', falling back to 'fulltitle' if it doesn't exist result.set_title(video_data.pop('title', video_data.pop('fulltitle', ""))) result.set_url(url) - + if "description" in video_data: result.set_content(video_data["description"]) # extract comments if enabled if self.comments: result.set("comments", [{ @@ -116,7 +158,7 @@ class GenericExtractor(Extractor): def get_metadata_for_post(self, info_extractor: Type[InfoExtractor], url: str, ydl: yt_dlp.YoutubeDL) -> Metadata: """ - Calls into the ytdlp InfoExtract subclass to use the prive _extract_post method to get the post metadata. + Calls into the ytdlp InfoExtract subclass to use the private _extract_post method to get the post metadata. """ ie_instance = info_extractor(downloader=ydl) @@ -266,6 +308,11 @@ class GenericExtractor(Extractor): def download(self, item: Metadata) -> Metadata: url = item.get_url() + #TODO: this is a temporary hack until this issue is closed: https://github.com/yt-dlp/yt-dlp/issues/11025 + if url.startswith("https://ya.ru"): + url = url.replace("https://ya.ru", "https://yandex.ru") + item.set("replaced_url", url) + ydl_options = {'outtmpl': os.path.join(self.tmp_dir, f'%(id)s.%(ext)s'), 'quiet': False, 'noplaylist': not self.allow_playlist , @@ -275,7 +322,8 @@ class GenericExtractor(Extractor): # set up auth auth = self.auth_for_site(url, extract_cookies=False) - # order of importance: username/pasword -> api_key -> cookie -> cookie_from_browser -> cookies_file + + # order of importance: username/pasword -> api_key -> cookie -> cookies_from_browser -> cookies_file if auth: if 'username' in auth and 'password' in auth: logger.debug(f'Using provided auth username and password for {url}') @@ -284,12 +332,12 @@ class GenericExtractor(Extractor): elif 'cookie' in auth: logger.debug(f'Using provided auth cookie for {url}') yt_dlp.utils.std_headers['cookie'] = auth['cookie'] - elif 'cookie_from_browser' in auth: - logger.debug(f'Using extracted cookies from browser {self.cookies_from_browser} for {url}') + elif 'cookies_from_browser' in auth: + logger.debug(f'Using extracted cookies from browser {auth["cookies_from_browser"]} for {url}') ydl_options['cookiesfrombrowser'] = auth['cookies_from_browser'] elif 'cookies_file' in auth: - logger.debug(f'Using cookies from file {self.cookie_file} for {url}') - ydl_options['cookiesfile'] = auth['cookies_file'] + logger.debug(f'Using cookies from file {auth["cookies_file"]} for {url}') + ydl_options['cookiefile'] = auth['cookies_file'] ydl = yt_dlp.YoutubeDL(ydl_options) # allsubtitles and subtitleslangs not working as expected, so default lang is always "en" diff --git a/src/auto_archiver/modules/gsheet_db/__init__.py b/src/auto_archiver/modules/gsheet_db/__init__.py deleted file mode 100644 index 01fdee6..0000000 --- a/src/auto_archiver/modules/gsheet_db/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .gsheet_db import GsheetsDb \ No newline at end of file diff --git a/src/auto_archiver/modules/gsheet_db/__manifest__.py b/src/auto_archiver/modules/gsheet_db/__manifest__.py deleted file mode 100644 index cf95245..0000000 --- a/src/auto_archiver/modules/gsheet_db/__manifest__.py +++ /dev/null @@ -1,38 +0,0 @@ -{ - "name": "Google Sheets Database", - "type": ["database"], - "entry_point": "gsheet_db::GsheetsDb", - "requires_setup": True, - "dependencies": { - "python": ["loguru", "gspread", "slugify"], - }, - "configs": { - "allow_worksheets": { - "default": set(), - "help": "(CSV) only worksheets whose name is included in allow are included (overrides worksheet_block), leave empty so all are allowed", - }, - "block_worksheets": { - "default": set(), - "help": "(CSV) explicitly block some worksheets from being processed", - }, - "use_sheet_names_in_stored_paths": { - "default": True, - "type": "bool", - "help": "if True the stored files path will include 'workbook_name/worksheet_name/...'", - } - }, - "description": """ - GsheetsDatabase: - Handles integration with Google Sheets for tracking archival tasks. - -### Features -- Updates a Google Sheet with the status of the archived URLs, including in progress, success or failure, and method used. -- Saves metadata such as title, text, timestamp, hashes, screenshots, and media URLs to designated columns. -- Formats media-specific metadata, such as thumbnails and PDQ hashes for the sheet. -- Skips redundant updates for empty or invalid data fields. - -### Notes -- Currently works only with metadata provided by GsheetFeeder. -- Requires configuration of a linked Google Sheet and appropriate API credentials. - """ -} diff --git a/src/auto_archiver/modules/gsheet_db/gsheet_db.py b/src/auto_archiver/modules/gsheet_db/gsheet_db.py deleted file mode 100644 index c19f2ae..0000000 --- a/src/auto_archiver/modules/gsheet_db/gsheet_db.py +++ /dev/null @@ -1,114 +0,0 @@ -from typing import Union, Tuple -from urllib.parse import quote - -from loguru import logger - -from auto_archiver.core import Database -from auto_archiver.core import Metadata, Media -from auto_archiver.modules.gsheet_feeder import GWorksheet -from auto_archiver.utils.misc import get_current_timestamp - - -class GsheetsDb(Database): - """ - NB: only works if GsheetFeeder is used. - could be updated in the future to support non-GsheetFeeder metadata - """ - - def started(self, item: Metadata) -> None: - logger.warning(f"STARTED {item}") - gw, row = self._retrieve_gsheet(item) - gw.set_cell(row, "status", "Archive in progress") - - def failed(self, item: Metadata, reason: str) -> None: - logger.error(f"FAILED {item}") - self._safe_status_update(item, f"Archive failed {reason}") - - def aborted(self, item: Metadata) -> None: - logger.warning(f"ABORTED {item}") - self._safe_status_update(item, "") - - def fetch(self, item: Metadata) -> Union[Metadata, bool]: - """check if the given item has been archived already""" - return False - - def done(self, item: Metadata, cached: bool = False) -> None: - """archival result ready - should be saved to DB""" - logger.success(f"DONE {item.get_url()}") - gw, row = self._retrieve_gsheet(item) - # self._safe_status_update(item, 'done') - - cell_updates = [] - row_values = gw.get_row(row) - - def batch_if_valid(col, val, final_value=None): - final_value = final_value or val - try: - if val and gw.col_exists(col) and gw.get_cell(row_values, col) == "": - cell_updates.append((row, col, final_value)) - except Exception as e: - logger.error(f"Unable to batch {col}={final_value} due to {e}") - - status_message = item.status - if cached: - status_message = f"[cached] {status_message}" - cell_updates.append((row, "status", status_message)) - - media: Media = item.get_final_media() - if hasattr(media, "urls"): - batch_if_valid("archive", "\n".join(media.urls)) - batch_if_valid("date", True, get_current_timestamp()) - batch_if_valid("title", item.get_title()) - batch_if_valid("text", item.get("content", "")) - batch_if_valid("timestamp", item.get_timestamp()) - if media: - batch_if_valid("hash", media.get("hash", "not-calculated")) - - # merge all pdq hashes into a single string, if present - pdq_hashes = [] - all_media = item.get_all_media() - for m in all_media: - if pdq := m.get("pdq_hash"): - pdq_hashes.append(pdq) - if len(pdq_hashes): - batch_if_valid("pdq_hash", ",".join(pdq_hashes)) - - if (screenshot := item.get_media_by_id("screenshot")) and hasattr( - screenshot, "urls" - ): - batch_if_valid("screenshot", "\n".join(screenshot.urls)) - - if thumbnail := item.get_first_image("thumbnail"): - if hasattr(thumbnail, "urls"): - batch_if_valid("thumbnail", f'=IMAGE("{thumbnail.urls[0]}")') - - if browsertrix := item.get_media_by_id("browsertrix"): - batch_if_valid("wacz", "\n".join(browsertrix.urls)) - batch_if_valid( - "replaywebpage", - "\n".join( - [ - f"https://replayweb.page/?source={quote(wacz)}#view=pages&url={quote(item.get_url())}" - for wacz in browsertrix.urls - ] - ), - ) - - gw.batch_set_cell(cell_updates) - - def _safe_status_update(self, item: Metadata, new_status: str) -> None: - try: - gw, row = self._retrieve_gsheet(item) - gw.set_cell(row, "status", new_status) - except Exception as e: - logger.debug(f"Unable to update sheet: {e}") - - def _retrieve_gsheet(self, item: Metadata) -> Tuple[GWorksheet, int]: - - if gsheet := item.get_context("gsheet"): - gw: GWorksheet = gsheet.get("worksheet") - row: int = gsheet.get("row") - elif self.sheet_id: - logger.error(f"Unable to retrieve Gsheet for {item.get_url()}, GsheetDB must be used alongside GsheetFeeder.") - - return gw, row diff --git a/src/auto_archiver/modules/gsheet_feeder/__init__.py b/src/auto_archiver/modules/gsheet_feeder/__init__.py deleted file mode 100644 index bb4230a..0000000 --- a/src/auto_archiver/modules/gsheet_feeder/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .gworksheet import GWorksheet -from .gsheet_feeder import GsheetsFeeder \ No newline at end of file diff --git a/src/auto_archiver/modules/gsheet_feeder/__manifest__.py b/src/auto_archiver/modules/gsheet_feeder/__manifest__.py deleted file mode 100644 index 7b74072..0000000 --- a/src/auto_archiver/modules/gsheet_feeder/__manifest__.py +++ /dev/null @@ -1,71 +0,0 @@ -{ - "name": "Google Sheets Feeder", - "type": ["feeder"], - "entry_point": "gsheet_feeder::GsheetsFeeder", - "requires_setup": True, - "dependencies": { - "python": ["loguru", "gspread", "slugify"], - }, - "configs": { - "sheet": {"default": None, "help": "name of the sheet to archive"}, - "sheet_id": { - "default": None, - "help": "(alternative to sheet name) the id of the sheet to archive", - }, - "header": {"default": 1, "help": "index of the header row (starts at 1)", "type": "int"}, - "service_account": { - "default": "secrets/service_account.json", - "help": "service account JSON file path", - }, - "columns": { - "default": { - "url": "link", - "status": "archive status", - "folder": "destination folder", - "archive": "archive location", - "date": "archive date", - "thumbnail": "thumbnail", - "timestamp": "upload timestamp", - "title": "upload title", - "text": "text content", - "screenshot": "screenshot", - "hash": "hash", - "pdq_hash": "perceptual hashes", - "wacz": "wacz", - "replaywebpage": "replaywebpage", - }, - "help": "names of columns in the google sheet (stringified JSON object)", - "type": "auto_archiver.utils.json_loader", - }, - "allow_worksheets": { - "default": set(), - "help": "(CSV) only worksheets whose name is included in allow are included (overrides worksheet_block), leave empty so all are allowed", - }, - "block_worksheets": { - "default": set(), - "help": "(CSV) explicitly block some worksheets from being processed", - }, - "use_sheet_names_in_stored_paths": { - "default": True, - "help": "if True the stored files path will include 'workbook_name/worksheet_name/...'", - "type": "bool", - }, - }, - "description": """ - GsheetsFeeder - A Google Sheets-based feeder for the Auto Archiver. - - This reads data from Google Sheets and filters rows based on user-defined rules. - The filtered rows are processed into `Metadata` objects. - - ### Features - - Validates the sheet structure and filters rows based on input configurations. - - Processes only worksheets allowed by the `allow_worksheets` and `block_worksheets` configurations. - - Ensures only rows with valid URLs and unprocessed statuses are included for archival. - - Supports organizing stored files into folder paths based on sheet and worksheet names. - - ### Notes - - Requires a Google Service Account JSON file for authentication. Suggested location is `secrets/gsheets_service_account.json`. - - Create the sheet using the template provided in the docs. - """, -} diff --git a/src/auto_archiver/modules/gsheet_feeder/gsheet_feeder.py b/src/auto_archiver/modules/gsheet_feeder/gsheet_feeder.py deleted file mode 100644 index 8612d02..0000000 --- a/src/auto_archiver/modules/gsheet_feeder/gsheet_feeder.py +++ /dev/null @@ -1,96 +0,0 @@ -""" -GsheetsFeeder: A Google Sheets-based feeder for the Auto Archiver. - -This reads data from Google Sheets and filters rows based on user-defined rules. -The filtered rows are processed into `Metadata` objects. - -### Key properties -- validates the sheet's structure and filters rows based on input configurations. -- Ensures only rows with valid URLs and unprocessed statuses are included. -""" -import os -import gspread - -from loguru import logger -from slugify import slugify - -from auto_archiver.core import Feeder -from auto_archiver.core import Metadata -from . import GWorksheet - - -class GsheetsFeeder(Feeder): - - def setup(self) -> None: - self.gsheets_client = gspread.service_account(filename=self.service_account) - # TODO mv to validators - assert self.sheet or self.sheet_id, ( - "You need to define either a 'sheet' name or a 'sheet_id' in your manifest." - ) - - def open_sheet(self): - if self.sheet: - return self.gsheets_client.open(self.sheet) - else: # self.sheet_id - return self.gsheets_client.open_by_key(self.sheet_id) - - def __iter__(self) -> Metadata: - sh = self.open_sheet() - for ii, worksheet in enumerate(sh.worksheets()): - if not self.should_process_sheet(worksheet.title): - logger.debug(f"SKIPPED worksheet '{worksheet.title}' due to allow/block rules") - continue - logger.info(f'Opening worksheet {ii=}: {worksheet.title=} header={self.header}') - gw = GWorksheet(worksheet, header_row=self.header, columns=self.columns) - if len(missing_cols := self.missing_required_columns(gw)): - logger.warning(f"SKIPPED worksheet '{worksheet.title}' due to missing required column(s) for {missing_cols}") - continue - - # process and yield metadata here: - yield from self._process_rows(gw) - logger.success(f'Finished worksheet {worksheet.title}') - - def _process_rows(self, gw: GWorksheet): - for row in range(1 + self.header, gw.count_rows() + 1): - url = gw.get_cell(row, 'url').strip() - if not len(url): continue - original_status = gw.get_cell(row, 'status') - status = gw.get_cell(row, 'status', fresh=original_status in ['', None]) - # TODO: custom status parser(?) aka should_retry_from_status - if status not in ['', None]: continue - - # All checks done - archival process starts here - m = Metadata().set_url(url) - self._set_context(m, gw, row) - yield m - - def _set_context(self, m: Metadata, gw: GWorksheet, row: int) -> Metadata: - # TODO: Check folder value not being recognised - m.set_context("gsheet", {"row": row, "worksheet": gw}) - - if gw.get_cell_or_default(row, 'folder', "") is None: - folder = '' - else: - folder = slugify(gw.get_cell_or_default(row, 'folder', "").strip()) - if len(folder): - if self.use_sheet_names_in_stored_paths: - m.set_context("folder", os.path.join(folder, slugify(self.sheet), slugify(gw.wks.title))) - else: - m.set_context("folder", folder) - - - def should_process_sheet(self, sheet_name: str) -> bool: - if len(self.allow_worksheets) and sheet_name not in self.allow_worksheets: - # ALLOW rules exist AND sheet name not explicitly allowed - return False - if len(self.block_worksheets) and sheet_name in self.block_worksheets: - # BLOCK rules exist AND sheet name is blocked - return False - return True - - def missing_required_columns(self, gw: GWorksheet) -> list: - missing = [] - for required_col in ['url', 'status']: - if not gw.col_exists(required_col): - missing.append(required_col) - return missing diff --git a/src/auto_archiver/modules/gsheet_feeder_db/__init__.py b/src/auto_archiver/modules/gsheet_feeder_db/__init__.py new file mode 100644 index 0000000..2e9ac02 --- /dev/null +++ b/src/auto_archiver/modules/gsheet_feeder_db/__init__.py @@ -0,0 +1,2 @@ +from .gworksheet import GWorksheet +from .gsheet_feeder_db import GsheetsFeederDB \ No newline at end of file diff --git a/src/auto_archiver/modules/gsheet_feeder_db/__manifest__.py b/src/auto_archiver/modules/gsheet_feeder_db/__manifest__.py new file mode 100644 index 0000000..9dd6b87 --- /dev/null +++ b/src/auto_archiver/modules/gsheet_feeder_db/__manifest__.py @@ -0,0 +1,94 @@ +{ + "name": "Google Sheets Feeder Database", + "type": ["feeder", "database"], + "entry_point": "gsheet_feeder_db::GsheetsFeederDB", + "requires_setup": True, + "dependencies": { + "python": ["loguru", "gspread", "slugify"], + }, + "configs": { + "sheet": {"default": None, "help": "name of the sheet to archive"}, + "sheet_id": { + "default": None, + "help": "the id of the sheet to archive (alternative to 'sheet' config)", + }, + "header": {"default": 1, + "type": "int", + "help": "index of the header row (starts at 1)", "type": "int"}, + "service_account": { + "default": "secrets/service_account.json", + "help": "service account JSON file path. Learn how to create one: https://gspread.readthedocs.io/en/latest/oauth2.html", + "required": True, + }, + "columns": { + "default": { + "url": "link", + "status": "archive status", + "folder": "destination folder", + "archive": "archive location", + "date": "archive date", + "thumbnail": "thumbnail", + "timestamp": "upload timestamp", + "title": "upload title", + "text": "text content", + "screenshot": "screenshot", + "hash": "hash", + "pdq_hash": "perceptual hashes", + "wacz": "wacz", + "replaywebpage": "replaywebpage", + }, + "help": "Custom names for the columns in your Google sheet. If you don't want to use the default column names, change them with this setting", + "type": "json_loader", + }, + "allow_worksheets": { + "default": set(), + "help": "A list of worksheet names that should be processed (overrides worksheet_block), leave empty so all are allowed", + }, + "block_worksheets": { + "default": set(), + "help": "A list of worksheet names for worksheets that should be explicitly blocked from being processed", + }, + "use_sheet_names_in_stored_paths": { + "default": True, + "help": "if True the stored files path will include 'workbook_name/worksheet_name/...'", + "type": "bool", + }, + "allow_worksheets": { + "default": set(), + "help": "(CSV) only worksheets whose name is included in allow are included (overrides worksheet_block), leave empty so all are allowed", + }, + "block_worksheets": { + "default": set(), + "help": "(CSV) explicitly block some worksheets from being processed", + }, + "use_sheet_names_in_stored_paths": { + "default": True, + "type": "bool", + "help": "if True the stored files path will include 'workbook_name/worksheet_name/...'", + } + }, + "description": """ + GsheetsFeederDatabase + A Google Sheets-based feeder and optional database for the Auto Archiver. + + This reads data from Google Sheets and filters rows based on user-defined rules. + The filtered rows are processed into `Metadata` objects. + + ### Features + - Validates the sheet structure and filters rows based on input configurations. + - Processes only worksheets allowed by the `allow_worksheets` and `block_worksheets` configurations. + - Ensures only rows with valid URLs and unprocessed statuses are included for archival. + - Supports organizing stored files into folder paths based on sheet and worksheet names. + - If the database is enabled, this updates the Google Sheet with the status of the archived URLs, including in progress, success or failure, and method used. + - Saves metadata such as title, text, timestamp, hashes, screenshots, and media URLs to designated columns. + - Formats media-specific metadata, such as thumbnails and PDQ hashes for the sheet. + - Skips redundant updates for empty or invalid data fields. + + ### Setup + - Requires a Google Service Account JSON file for authentication, which should be stored in `secrets/gsheets_service_account.json`. + To set up a service account, follow the instructions [here](https://gspread.readthedocs.io/en/latest/oauth2.html). + - Define the `sheet` or `sheet_id` configuration to specify the sheet to archive. + - Customize the column names in your Google sheet using the `columns` configuration. + - The Google Sheet can be used soley as a feeder or as a feeder and database, but note you can't currently feed into the database from an alternate feeder. + """, +} diff --git a/src/auto_archiver/modules/gsheet_feeder_db/gsheet_feeder_db.py b/src/auto_archiver/modules/gsheet_feeder_db/gsheet_feeder_db.py new file mode 100644 index 0000000..406eeb4 --- /dev/null +++ b/src/auto_archiver/modules/gsheet_feeder_db/gsheet_feeder_db.py @@ -0,0 +1,196 @@ +""" +GsheetsFeeder: A Google Sheets-based feeder for the Auto Archiver. + +This reads data from Google Sheets and filters rows based on user-defined rules. +The filtered rows are processed into `Metadata` objects. + +### Key properties +- validates the sheet's structure and filters rows based on input configurations. +- Ensures only rows with valid URLs and unprocessed statuses are included. +""" +import os +from typing import Tuple, Union +from urllib.parse import quote + +import gspread +from loguru import logger +from slugify import slugify + +from auto_archiver.core import Feeder, Database, Media +from auto_archiver.core import Metadata +from auto_archiver.modules.gsheet_feeder_db import GWorksheet +from auto_archiver.utils.misc import calculate_file_hash, get_current_timestamp + + +class GsheetsFeederDB(Feeder, Database): + + def setup(self) -> None: + self.gsheets_client = gspread.service_account(filename=self.service_account) + # TODO mv to validators + if not self.sheet and not self.sheet_id: + raise ValueError("You need to define either a 'sheet' name or a 'sheet_id' in your manifest.") + + def open_sheet(self): + if self.sheet: + return self.gsheets_client.open(self.sheet) + else: # self.sheet_id + return self.gsheets_client.open_by_key(self.sheet_id) + + def __iter__(self) -> Metadata: + sh = self.open_sheet() + for ii, worksheet in enumerate(sh.worksheets()): + if not self.should_process_sheet(worksheet.title): + logger.debug(f"SKIPPED worksheet '{worksheet.title}' due to allow/block rules") + continue + logger.info(f'Opening worksheet {ii=}: {worksheet.title=} header={self.header}') + gw = GWorksheet(worksheet, header_row=self.header, columns=self.columns) + if len(missing_cols := self.missing_required_columns(gw)): + logger.warning(f"SKIPPED worksheet '{worksheet.title}' due to missing required column(s) for {missing_cols}") + continue + + # process and yield metadata here: + yield from self._process_rows(gw) + logger.success(f'Finished worksheet {worksheet.title}') + + def _process_rows(self, gw: GWorksheet): + for row in range(1 + self.header, gw.count_rows() + 1): + url = gw.get_cell(row, 'url').strip() + if not len(url): continue + original_status = gw.get_cell(row, 'status') + status = gw.get_cell(row, 'status', fresh=original_status in ['', None]) + # TODO: custom status parser(?) aka should_retry_from_status + if status not in ['', None]: continue + + # All checks done - archival process starts here + m = Metadata().set_url(url) + self._set_context(m, gw, row) + yield m + + def _set_context(self, m: Metadata, gw: GWorksheet, row: int) -> Metadata: + # TODO: Check folder value not being recognised + m.set_context("gsheet", {"row": row, "worksheet": gw}) + + if gw.get_cell_or_default(row, 'folder', "") is None: + folder = '' + else: + folder = slugify(gw.get_cell_or_default(row, 'folder', "").strip()) + if len(folder): + if self.use_sheet_names_in_stored_paths: + m.set_context("folder", os.path.join(folder, slugify(self.sheet), slugify(gw.wks.title))) + else: + m.set_context("folder", folder) + + def should_process_sheet(self, sheet_name: str) -> bool: + if len(self.allow_worksheets) and sheet_name not in self.allow_worksheets: + # ALLOW rules exist AND sheet name not explicitly allowed + return False + if len(self.block_worksheets) and sheet_name in self.block_worksheets: + # BLOCK rules exist AND sheet name is blocked + return False + return True + + def missing_required_columns(self, gw: GWorksheet) -> list: + missing = [] + for required_col in ['url', 'status']: + if not gw.col_exists(required_col): + missing.append(required_col) + return missing + + + def started(self, item: Metadata) -> None: + logger.warning(f"STARTED {item}") + gw, row = self._retrieve_gsheet(item) + gw.set_cell(row, "status", "Archive in progress") + + def failed(self, item: Metadata, reason: str) -> None: + logger.error(f"FAILED {item}") + self._safe_status_update(item, f"Archive failed {reason}") + + def aborted(self, item: Metadata) -> None: + logger.warning(f"ABORTED {item}") + self._safe_status_update(item, "") + + def fetch(self, item: Metadata) -> Union[Metadata, bool]: + """check if the given item has been archived already""" + return False + + def done(self, item: Metadata, cached: bool = False) -> None: + """archival result ready - should be saved to DB""" + logger.success(f"DONE {item.get_url()}") + gw, row = self._retrieve_gsheet(item) + # self._safe_status_update(item, 'done') + + cell_updates = [] + row_values = gw.get_row(row) + + def batch_if_valid(col, val, final_value=None): + final_value = final_value or val + try: + if val and gw.col_exists(col) and gw.get_cell(row_values, col) == "": + cell_updates.append((row, col, final_value)) + except Exception as e: + logger.error(f"Unable to batch {col}={final_value} due to {e}") + + status_message = item.status + if cached: + status_message = f"[cached] {status_message}" + cell_updates.append((row, "status", status_message)) + + media: Media = item.get_final_media() + if hasattr(media, "urls"): + batch_if_valid("archive", "\n".join(media.urls)) + batch_if_valid("date", True, get_current_timestamp()) + batch_if_valid("title", item.get_title()) + batch_if_valid("text", item.get("content", "")) + batch_if_valid("timestamp", item.get_timestamp()) + if media: + batch_if_valid("hash", media.get("hash", "not-calculated")) + + # merge all pdq hashes into a single string, if present + pdq_hashes = [] + all_media = item.get_all_media() + for m in all_media: + if pdq := m.get("pdq_hash"): + pdq_hashes.append(pdq) + if len(pdq_hashes): + batch_if_valid("pdq_hash", ",".join(pdq_hashes)) + + if (screenshot := item.get_media_by_id("screenshot")) and hasattr( + screenshot, "urls" + ): + batch_if_valid("screenshot", "\n".join(screenshot.urls)) + + if thumbnail := item.get_first_image("thumbnail"): + if hasattr(thumbnail, "urls"): + batch_if_valid("thumbnail", f'=IMAGE("{thumbnail.urls[0]}")') + + if browsertrix := item.get_media_by_id("browsertrix"): + batch_if_valid("wacz", "\n".join(browsertrix.urls)) + batch_if_valid( + "replaywebpage", + "\n".join( + [ + f"https://replayweb.page/?source={quote(wacz)}#view=pages&url={quote(item.get_url())}" + for wacz in browsertrix.urls + ] + ), + ) + + gw.batch_set_cell(cell_updates) + + def _safe_status_update(self, item: Metadata, new_status: str) -> None: + try: + gw, row = self._retrieve_gsheet(item) + gw.set_cell(row, "status", new_status) + except Exception as e: + logger.debug(f"Unable to update sheet: {e}") + + def _retrieve_gsheet(self, item: Metadata) -> Tuple[GWorksheet, int]: + + if gsheet := item.get_context("gsheet"): + gw: GWorksheet = gsheet.get("worksheet") + row: int = gsheet.get("row") + elif self.sheet_id: + logger.error(f"Unable to retrieve Gsheet for {item.get_url()}, GsheetDB must be used alongside GsheetFeeder.") + + return gw, row diff --git a/src/auto_archiver/modules/gsheet_feeder/gworksheet.py b/src/auto_archiver/modules/gsheet_feeder_db/gworksheet.py similarity index 99% rename from src/auto_archiver/modules/gsheet_feeder/gworksheet.py rename to src/auto_archiver/modules/gsheet_feeder_db/gworksheet.py index 3044780..ba2d691 100644 --- a/src/auto_archiver/modules/gsheet_feeder/gworksheet.py +++ b/src/auto_archiver/modules/gsheet_feeder_db/gworksheet.py @@ -17,6 +17,7 @@ class GWorksheet: 'thumbnail': 'thumbnail', 'timestamp': 'upload timestamp', 'title': 'upload title', + 'text': 'text content', 'screenshot': 'screenshot', 'hash': 'hash', 'pdq_hash': 'perceptual hashes', diff --git a/src/auto_archiver/modules/html_formatter/__manifest__.py b/src/auto_archiver/modules/html_formatter/__manifest__.py index ec19cf8..6e51c7a 100644 --- a/src/auto_archiver/modules/html_formatter/__manifest__.py +++ b/src/auto_archiver/modules/html_formatter/__manifest__.py @@ -7,7 +7,9 @@ "bin": [""] }, "configs": { - "detect_thumbnails": {"default": True, "help": "if true will group by thumbnails generated by thumbnail enricher by id 'thumbnail_00'"} + "detect_thumbnails": {"default": True, + "help": "if true will group by thumbnails generated by thumbnail enricher by id 'thumbnail_00'", + "type": "bool"}, }, "description": """ """, } diff --git a/src/auto_archiver/modules/html_formatter/html_formatter.py b/src/auto_archiver/modules/html_formatter/html_formatter.py index 3691735..deb4b44 100644 --- a/src/auto_archiver/modules/html_formatter/html_formatter.py +++ b/src/auto_archiver/modules/html_formatter/html_formatter.py @@ -9,9 +9,7 @@ import base64 from auto_archiver.version import __version__ from auto_archiver.core import Metadata, Media from auto_archiver.core import Formatter -from auto_archiver.modules.hash_enricher import HashEnricher from auto_archiver.utils.misc import random_str -from auto_archiver.core.module import get_module class HtmlFormatter(Formatter): environment: Environment = None @@ -51,7 +49,7 @@ class HtmlFormatter(Formatter): final_media = Media(filename=html_path, _mimetype="text/html") # get the already instantiated hash_enricher module - he = get_module('hash_enricher', self.config) + he = self.module_factory.get_module('hash_enricher', self.config) if len(hd := he.calculate_hash(final_media.filename)): final_media.set("hash", f"{he.algorithm}:{hd}") diff --git a/src/auto_archiver/modules/html_formatter/templates/html_template.html b/src/auto_archiver/modules/html_formatter/templates/html_template.html index 8bdf5ef..62d6b0b 100644 --- a/src/auto_archiver/modules/html_formatter/templates/html_template.html +++ b/src/auto_archiver/modules/html_formatter/templates/html_template.html @@ -200,7 +200,7 @@ el.innerHTML = decodeCertificate(certificate); let cyberChefUrl = - `https://gchq.github.io/CyberChef/#recipe=Parse_X.509_certificate('PEM')&input=${btoa(certificate)}`; + `https://gchq.github.io/CyberChef/#recipe=Parse_X.509_certificate('PEM')&input=${btoa(certificate).replace(/=+$/, '')}`; // create a new anchor with this url and append after the code let a = document.createElement("a"); a.href = cyberChefUrl; diff --git a/src/auto_archiver/modules/instagram_extractor/__manifest__.py b/src/auto_archiver/modules/instagram_extractor/__manifest__.py index 05cae19..a66389f 100644 --- a/src/auto_archiver/modules/instagram_extractor/__manifest__.py +++ b/src/auto_archiver/modules/instagram_extractor/__manifest__.py @@ -10,25 +10,30 @@ "requires_setup": True, "configs": { "username": {"required": True, - "help": "a valid Instagram username"}, + "help": "A valid Instagram username."}, "password": { "required": True, - "help": "the corresponding Instagram account password", + "help": "The corresponding Instagram account password.", }, "download_folder": { "default": "instaloader", - "help": "name of a folder to temporarily download content to", + "help": "Name of a folder to temporarily download content to.", }, "session_file": { "default": "secrets/instaloader.session", - "help": "path to the instagram session which saves session credentials", + "help": "Path to the instagram session file which saves session credentials. If one doesn't exist this gives the path to store a new one.", }, # TODO: fine-grain # "download_stories": {"default": True, "help": "if the link is to a user profile: whether to get stories information"}, }, "description": """ - Uses the [Instaloader library](https://instaloader.github.io/as-module.html) to download content from Instagram. This class handles both individual posts - and user profiles, downloading as much information as possible, including images, videos, text, stories, + Uses the [Instaloader library](https://instaloader.github.io/as-module.html) to download content from Instagram. + + > ⚠️ **Warning** + > This module is not actively maintained due to known issues with blocking. + > Prioritise usage of the [Instagram Tbot Extractor](./instagram_tbot_extractor.md) and [Instagram API Extractor](./instagram_api_extractor.md) + + This class handles both individual posts and user profiles, downloading as much information as possible, including images, videos, text, stories, highlights, and tagged posts. Authentication is required via username/password or a session file. diff --git a/src/auto_archiver/modules/instagram_extractor/instagram_extractor.py b/src/auto_archiver/modules/instagram_extractor/instagram_extractor.py index 0af2c32..7e195ad 100644 --- a/src/auto_archiver/modules/instagram_extractor/instagram_extractor.py +++ b/src/auto_archiver/modules/instagram_extractor/instagram_extractor.py @@ -3,7 +3,7 @@ highlights, and tagged posts. Authentication is required via username/password or a session file. """ -import re, os, shutil, traceback +import re, os, shutil import instaloader from loguru import logger @@ -15,10 +15,9 @@ class InstagramExtractor(Extractor): """ Uses Instaloader to download either a post (inc images, videos, text) or as much as possible from a profile (posts, stories, highlights, ...) """ + # NB: post regex should be tested before profile - valid_url = re.compile(r"(?:(?:http|https):\/\/)?(?:www.)?(?:instagram.com|instagr.am|instagr.com)\/") - # https://regex101.com/r/MGPquX/1 post_pattern = re.compile(r"{valid_url}(?:p|reel)\/(\w+)".format(valid_url=valid_url)) # https://regex101.com/r/6Wbsxa/1 @@ -28,19 +27,22 @@ class InstagramExtractor(Extractor): def setup(self) -> None: self.insta = instaloader.Instaloader( - download_geotags=True, download_comments=True, compress_json=False, dirname_pattern=self.download_folder, filename_pattern="{date_utc}_UTC_{target}__{typename}" + download_geotags=True, + download_comments=True, + compress_json=False, + dirname_pattern=self.download_folder, + filename_pattern="{date_utc}_UTC_{target}__{typename}" ) try: self.insta.load_session_from_file(self.username, self.session_file) except Exception as e: - logger.error(f"Unable to login from session file: {e}\n{traceback.format_exc()}") try: - self.insta.login(self.username, config.instagram_self.password) - # TODO: wait for this issue to be fixed https://github.com/instaloader/instaloader/issues/1758 + logger.debug(f"Session file failed", exc_info=True) + logger.info("No valid session file found - Attempting login with use and password.") + self.insta.login(self.username, self.password) self.insta.save_session_to_file(self.session_file) - except Exception as e2: - logger.error(f"Unable to finish login (retrying from file): {e2}\n{traceback.format_exc()}") - + except Exception as e: + logger.error(f"Failed to setup Instagram Extractor with Instagrapi. {e}") def download(self, item: Metadata) -> Metadata: diff --git a/src/auto_archiver/modules/instagram_tbot_extractor/instagram_tbot_extractor.py b/src/auto_archiver/modules/instagram_tbot_extractor/instagram_tbot_extractor.py index d4b7a8e..1416da9 100644 --- a/src/auto_archiver/modules/instagram_tbot_extractor/instagram_tbot_extractor.py +++ b/src/auto_archiver/modules/instagram_tbot_extractor/instagram_tbot_extractor.py @@ -77,13 +77,14 @@ class InstagramTbotExtractor(Extractor): chat, since_id = self._send_url_to_bot(url) message = self._process_messages(chat, since_id, tmp_dir, result) + # This may be outdated and replaced by the below message, but keeping until confirmed if "You must enter a URL to a post" in message: logger.debug(f"invalid link {url=} for {self.name}: {message}") return False - # # TODO: It currently returns this as a success - is that intentional? - # if "Media not found or unavailable" in message: - # logger.debug(f"invalid link {url=} for {self.name}: {message}") - # return False + + if "Media not found or unavailable" in message: + logger.debug(f"No media found for link {url=} for {self.name}: {message}") + return False if message: result.set_content(message).set_title(message[:128]) @@ -103,7 +104,7 @@ class InstagramTbotExtractor(Extractor): message = "" time.sleep(3) # media is added before text by the bot so it can be used as a stop-logic mechanism - while attempts < (self.timeout - 3) and (not message or not len(seen_media)): + while attempts < max(self.timeout - 3, 3) and (not message or not len(seen_media)): attempts += 1 time.sleep(1) for post in self.client.iter_messages(chat, min_id=since_id): diff --git a/src/auto_archiver/modules/local_storage/__manifest__.py b/src/auto_archiver/modules/local_storage/__manifest__.py index 6d9cf53..8ad6381 100644 --- a/src/auto_archiver/modules/local_storage/__manifest__.py +++ b/src/auto_archiver/modules/local_storage/__manifest__.py @@ -17,7 +17,9 @@ "choices": ["random", "static"], }, "save_to": {"default": "./local_archive", "help": "folder where to save archived content"}, - "save_absolute": {"default": False, "help": "whether the path to the stored file is absolute or relative in the output result inc. formatters (WARN: leaks the file structure)"}, + "save_absolute": {"default": False, + "type": "bool", + "help": "whether the path to the stored file is absolute or relative in the output result inc. formatters (WARN: leaks the file structure)"}, }, "description": """ LocalStorage: A storage module for saving archived content locally on the filesystem. diff --git a/src/auto_archiver/modules/local_storage/local_storage.py b/src/auto_archiver/modules/local_storage/local_storage.py index b995577..2b1a101 100644 --- a/src/auto_archiver/modules/local_storage/local_storage.py +++ b/src/auto_archiver/modules/local_storage/local_storage.py @@ -6,25 +6,42 @@ from loguru import logger from auto_archiver.core import Media from auto_archiver.core import Storage - +from auto_archiver.core.consts import SetupError class LocalStorage(Storage): + + def setup(self) -> None: + if len(self.save_to) > 200: + raise SetupError(f"Your save_to path is too long, this will cause issues saving files on your computer. Please use a shorter path.") + def get_cdn_url(self, media: Media) -> str: - # TODO: is this viable with Storage.configs on path/filename? - dest = os.path.join(self.save_to, media.key) + dest = media.key + if self.save_absolute: dest = os.path.abspath(dest) return dest + def set_key(self, media, url, metadata): + # clarify we want to save the file to the save_to folder + + old_folder = metadata.get('folder', '') + metadata.set_context('folder', os.path.join(self.save_to, metadata.get('folder', ''))) + super().set_key(media, url, metadata) + # don't impact other storages that might want a different 'folder' set + metadata.set_context('folder', old_folder) + def upload(self, media: Media, **kwargs) -> bool: # override parent so that we can use shutil.copy2 and keep metadata - dest = os.path.join(self.save_to, media.key) + dest = media.key + os.makedirs(os.path.dirname(dest), exist_ok=True) logger.debug(f'[{self.__class__.__name__}] storing file {media.filename} with key {media.key} to {dest}') + res = shutil.copy2(media.filename, dest) logger.info(res) return True # must be implemented even if unused - def uploadf(self, file: IO[bytes], key: str, **kwargs: dict) -> bool: pass + def uploadf(self, file: IO[bytes], key: str, **kwargs: dict) -> bool: + pass \ No newline at end of file diff --git a/src/auto_archiver/modules/s3_storage/s3_storage.py b/src/auto_archiver/modules/s3_storage/s3_storage.py index 6590ac9..183a944 100644 --- a/src/auto_archiver/modules/s3_storage/s3_storage.py +++ b/src/auto_archiver/modules/s3_storage/s3_storage.py @@ -42,7 +42,7 @@ class S3Storage(Storage): logger.warning(f"Unable to get mimetype for {media.key=}, error: {e}") self.s3.upload_fileobj(file, Bucket=self.bucket, Key=media.key, ExtraArgs=extra_args) return True - + def is_upload_needed(self, media: Media) -> bool: if self.random_no_duplicate: # checks if a folder with the hash already exists, if so it skips the upload @@ -50,13 +50,13 @@ class S3Storage(Storage): path = os.path.join(NO_DUPLICATES_FOLDER, hd[:24]) if existing_key:=self.file_in_folder(path): - media.key = existing_key + media._key = existing_key media.set("previously archived", True) logger.debug(f"skipping upload of {media.filename} because it already exists in {media.key}") return False _, ext = os.path.splitext(media.key) - media.key = os.path.join(path, f"{random_str(24)}{ext}") + media._key = os.path.join(path, f"{random_str(24)}{ext}") return True def file_in_folder(self, path:str) -> str: @@ -66,5 +66,4 @@ class S3Storage(Storage): resp = self.s3.list_objects(Bucket=self.bucket, Prefix=path, Delimiter='/', MaxKeys=1) if 'Contents' in resp: return resp['Contents'][0]['Key'] - return False - + return False \ No newline at end of file diff --git a/src/auto_archiver/modules/screenshot_enricher/__manifest__.py b/src/auto_archiver/modules/screenshot_enricher/__manifest__.py index 52842c9..92c0883 100644 --- a/src/auto_archiver/modules/screenshot_enricher/__manifest__.py +++ b/src/auto_archiver/modules/screenshot_enricher/__manifest__.py @@ -4,16 +4,27 @@ "requires_setup": True, "dependencies": { "python": ["loguru", "selenium"], - "bin": ["chromedriver"] }, "configs": { - "width": {"default": 1280, "help": "width of the screenshots"}, - "height": {"default": 720, "help": "height of the screenshots"}, - "timeout": {"default": 60, "help": "timeout for taking the screenshot"}, - "sleep_before_screenshot": {"default": 4, "help": "seconds to wait for the pages to load before taking screenshot"}, + "width": {"default": 1280, + "type": "int", + "help": "width of the screenshots"}, + "height": {"default": 1024, + "type": "int", + "help": "height of the screenshots"}, + "timeout": {"default": 60, + "type": "int", + "help": "timeout for taking the screenshot"}, + "sleep_before_screenshot": {"default": 4, + "type": "int", + "help": "seconds to wait for the pages to load before taking screenshot"}, "http_proxy": {"default": "", "help": "http proxy to use for the webdriver, eg http://proxy-user:password@proxy-ip:port"}, - "save_to_pdf": {"default": False, "help": "save the page as pdf along with the screenshot. PDF saving options can be adjusted with the 'print_options' parameter"}, - "print_options": {"default": {}, "help": "options to pass to the pdf printer"} + "save_to_pdf": {"default": False, + "type": "bool", + "help": "save the page as pdf along with the screenshot. PDF saving options can be adjusted with the 'print_options' parameter"}, + "print_options": {"default": {}, + "help": "options to pass to the pdf printer, in JSON format. See https://www.selenium.dev/documentation/webdriver/interactions/print_page/ for more information", + "type": "json_loader"}, }, "description": """ Captures screenshots and optionally saves web pages as PDFs using a WebDriver. diff --git a/src/auto_archiver/modules/screenshot_enricher/screenshot_enricher.py b/src/auto_archiver/modules/screenshot_enricher/screenshot_enricher.py index e1da99d..832d0f8 100644 --- a/src/auto_archiver/modules/screenshot_enricher/screenshot_enricher.py +++ b/src/auto_archiver/modules/screenshot_enricher/screenshot_enricher.py @@ -11,6 +11,10 @@ from auto_archiver.core import Media, Metadata class ScreenshotEnricher(Enricher): + def __init__(self, webdriver_factory=None): + super().__init__() + self.webdriver_factory = webdriver_factory or Webdriver + def enrich(self, to_enrich: Metadata) -> None: url = to_enrich.get_url() @@ -20,7 +24,8 @@ class ScreenshotEnricher(Enricher): logger.debug(f"Enriching screenshot for {url=}") auth = self.auth_for_site(url) - with Webdriver(self.width, self.height, self.timeout, facebook_accept_cookies='facebook.com' in url, + with self.webdriver_factory( + self.width, self.height, self.timeout, facebook_accept_cookies='facebook.com' in url, http_proxy=self.http_proxy, print_options=self.print_options, auth=auth) as driver: try: driver.get(url) @@ -38,3 +43,4 @@ class ScreenshotEnricher(Enricher): logger.info("TimeoutException loading page for screenshot") except Exception as e: logger.error(f"Got error while loading webdriver for screenshot enricher: {e}") + diff --git a/src/auto_archiver/modules/ssl_enricher/__manifest__.py b/src/auto_archiver/modules/ssl_enricher/__manifest__.py index 9028f14..097cd21 100644 --- a/src/auto_archiver/modules/ssl_enricher/__manifest__.py +++ b/src/auto_archiver/modules/ssl_enricher/__manifest__.py @@ -7,7 +7,9 @@ }, 'entry_point': 'ssl_enricher::SSLEnricher', "configs": { - "skip_when_nothing_archived": {"default": True, "help": "if true, will skip enriching when no media is archived"}, + "skip_when_nothing_archived": {"default": True, + "type": 'bool', + "help": "if true, will skip enriching when no media is archived"}, }, "description": """ Retrieves SSL certificate information for a domain and stores it as a file. diff --git a/src/auto_archiver/modules/telegram_extractor/__manifest__.py b/src/auto_archiver/modules/telegram_extractor/__manifest__.py index cb0ee1e..0b94582 100644 --- a/src/auto_archiver/modules/telegram_extractor/__manifest__.py +++ b/src/auto_archiver/modules/telegram_extractor/__manifest__.py @@ -20,5 +20,6 @@ - Processes HTML content of messages to retrieve embedded media. - Sets structured metadata, including timestamps, content, and media details. - Does not require user authentication for Telegram. + """, } diff --git a/src/auto_archiver/modules/telethon_extractor/__manifest__.py b/src/auto_archiver/modules/telethon_extractor/__manifest__.py index 6b37654..5e58203 100644 --- a/src/auto_archiver/modules/telethon_extractor/__manifest__.py +++ b/src/auto_archiver/modules/telethon_extractor/__manifest__.py @@ -1,5 +1,5 @@ { - "name": "telethon_extractor", + "name": "Telethon Extractor", "type": ["extractor"], "requires_setup": True, "dependencies": { @@ -14,11 +14,13 @@ "api_hash": {"default": None, "help": "telegram API_HASH value, go to https://my.telegram.org/apps"}, "bot_token": {"default": None, "help": "optional, but allows access to more content such as large videos, talk to @botfather"}, "session_file": {"default": "secrets/anon", "help": "optional, records the telegram login session for future usage, '.session' will be appended to the provided value."}, - "join_channels": {"default": True, "help": "disables the initial setup with channel_invites config, useful if you have a lot and get stuck"}, + "join_channels": {"default": True, + "type": "bool", + "help": "disables the initial setup with channel_invites config, useful if you have a lot and get stuck"}, "channel_invites": { "default": {}, "help": "(JSON string) private channel invite links (format: t.me/joinchat/HASH OR t.me/+HASH) and (optional but important to avoid hanging for minutes on startup) channel id (format: CHANNEL_ID taken from a post url like https://t.me/c/CHANNEL_ID/1), the telegram account will join any new channels on setup", - "type": "auto_archiver.utils.json_loader", + "type": "json_loader", } }, "description": """ @@ -40,5 +42,9 @@ To use the `TelethonExtractor`, you must configure the following: - **Bot Token**: Optional, allows access to additional content (e.g., large videos) but limits private channel archiving. - **Channel Invites**: Optional, specify a JSON string of invite links to join channels during setup. +### First Time Login +The first time you run, you will be prompted to do a authentication with the phone number associated, alternatively you can put your `anon.session` in the root. + + """ } \ No newline at end of file diff --git a/src/auto_archiver/modules/thumbnail_enricher/__manifest__.py b/src/auto_archiver/modules/thumbnail_enricher/__manifest__.py index e47397f..1bd23b5 100644 --- a/src/auto_archiver/modules/thumbnail_enricher/__manifest__.py +++ b/src/auto_archiver/modules/thumbnail_enricher/__manifest__.py @@ -7,8 +7,12 @@ "bin": ["ffmpeg"] }, "configs": { - "thumbnails_per_minute": {"default": 60, "help": "how many thumbnails to generate per minute of video, can be limited by max_thumbnails"}, - "max_thumbnails": {"default": 16, "help": "limit the number of thumbnails to generate per video, 0 means no limit"}, + "thumbnails_per_minute": {"default": 60, + "type": "int", + "help": "how many thumbnails to generate per minute of video, can be limited by max_thumbnails"}, + "max_thumbnails": {"default": 16, + "type": "int", + "help": "limit the number of thumbnails to generate per video, 0 means no limit"}, }, "description": """ Generates thumbnails for video files to provide visual previews. diff --git a/src/auto_archiver/modules/thumbnail_enricher/thumbnail_enricher.py b/src/auto_archiver/modules/thumbnail_enricher/thumbnail_enricher.py index e0ac937..8178cd8 100644 --- a/src/auto_archiver/modules/thumbnail_enricher/thumbnail_enricher.py +++ b/src/auto_archiver/modules/thumbnail_enricher/thumbnail_enricher.py @@ -42,7 +42,7 @@ class ThumbnailEnricher(Enricher): logger.error(f"error getting duration of video {m.filename}: {e}") return - num_thumbs = int(min(max(1, duration * self.thumbnails_per_minute), self.max_thumbnails)) + num_thumbs = int(min(max(1, (duration / 60) * self.thumbnails_per_minute), self.max_thumbnails)) timestamps = [duration / (num_thumbs + 1) * i for i in range(1, num_thumbs + 1)] thumbnails_media = [] diff --git a/src/auto_archiver/modules/tiktok_tikwm_extractor/__init__.py b/src/auto_archiver/modules/tiktok_tikwm_extractor/__init__.py new file mode 100644 index 0000000..25a20f5 --- /dev/null +++ b/src/auto_archiver/modules/tiktok_tikwm_extractor/__init__.py @@ -0,0 +1 @@ +from .tiktok_tikwm_extractor import TiktokTikwmExtractor \ No newline at end of file diff --git a/src/auto_archiver/modules/tiktok_tikwm_extractor/__manifest__.py b/src/auto_archiver/modules/tiktok_tikwm_extractor/__manifest__.py new file mode 100644 index 0000000..56d8e3e --- /dev/null +++ b/src/auto_archiver/modules/tiktok_tikwm_extractor/__manifest__.py @@ -0,0 +1,23 @@ +{ + "name": "Tiktok Tikwm Extractor", + "type": ["extractor"], + "requires_setup": False, + "dependencies": { + "python": ["loguru", "requests"], + "bin": [] + }, + "description": """ + Uses an unofficial TikTok video download platform's API to download videos: https://tikwm.com/ + + This extractor complements the generic_extractor which can already get TikTok videos, but this one can extract special videos like those marked as sensitive. + + ### Features + - Downloads the video and, if possible, also the video cover. + - Stores extra metadata about the post like author information, and more as returned by tikwm.com. + + ### Notes + - If tikwm.com is down, this extractor will not work. + - If tikwm.com changes their API, this extractor may break. + - If no video is found, this extractor will consider the extraction failed. + """ +} diff --git a/src/auto_archiver/modules/tiktok_tikwm_extractor/tiktok_tikwm_extractor.py b/src/auto_archiver/modules/tiktok_tikwm_extractor/tiktok_tikwm_extractor.py new file mode 100644 index 0000000..8b07775 --- /dev/null +++ b/src/auto_archiver/modules/tiktok_tikwm_extractor/tiktok_tikwm_extractor.py @@ -0,0 +1,75 @@ +import re +import requests +from loguru import logger +from datetime import datetime, timezone +from yt_dlp.extractor.tiktok import TikTokIE + +from auto_archiver.core import Extractor +from auto_archiver.core import Metadata, Media + + +class TiktokTikwmExtractor(Extractor): + """ + Extractor for TikTok that uses an unofficial API and can capture content that requires a login, like sensitive content. + """ + TIKWM_ENDPOINT = "https://www.tikwm.com/api/?url={url}" + + def download(self, item: Metadata) -> Metadata: + url = item.get_url() + + if not re.match(TikTokIE._VALID_URL, url): + return False + + endpoint = TiktokTikwmExtractor.TIKWM_ENDPOINT.format(url=url) + + r = requests.get(endpoint) + if r.status_code != 200: + logger.error(f"unexpected status code '{r.status_code}' from tikwm.com for {url=}:") + return False + + try: + json_response = r.json() + except ValueError: + logger.error(f"failed to parse JSON response from tikwm.com for {url=}") + return False + + if not json_response.get('msg') == 'success' or not (api_data := json_response.get('data', {})): + logger.error(f"failed to get a valid response from tikwm.com for {url=}: {json_response}") + return False + + # tries to get the non-watermarked version first + video_url = api_data.pop("play", api_data.pop("wmplay", None)) + if not video_url: + logger.error(f"no valid video URL found in response from tikwm.com for {url=}") + return False + + # prepare result, start by downloading video + result = Metadata() + + # get the cover if possible + cover_url = api_data.pop("origin_cover", api_data.pop("cover", api_data.pop("ai_dynamic_cover", None))) + if cover_url and (cover_downloaded := self.download_from_url(cover_url)): + result.add_media(Media(cover_downloaded)) + + # get the video or fail + video_downloaded = self.download_from_url(video_url, f"vid_{api_data.get('id', '')}") + if not video_downloaded: + logger.error(f"failed to download video from {video_url}") + return False + video_media = Media(video_downloaded) + if duration := api_data.pop("duration", None): + video_media.set("duration", duration) + result.add_media(video_media) + + # add remaining metadata + result.set_title(api_data.pop("title", "")) + + if created_at := api_data.pop("create_time", None): + result.set_timestamp(datetime.fromtimestamp(created_at, tz=timezone.utc)) + + if (author := api_data.pop("author", None)): + result.set("author", author) + + result.set("api_data", api_data) + + return result.success("tikwm") diff --git a/src/auto_archiver/modules/wacz_enricher/__init__.py b/src/auto_archiver/modules/wacz_enricher/__init__.py deleted file mode 100644 index 686b8d8..0000000 --- a/src/auto_archiver/modules/wacz_enricher/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .wacz_enricher import WaczExtractorEnricher diff --git a/src/auto_archiver/modules/wacz_extractor_enricher/__init__.py b/src/auto_archiver/modules/wacz_extractor_enricher/__init__.py new file mode 100644 index 0000000..b9a53e3 --- /dev/null +++ b/src/auto_archiver/modules/wacz_extractor_enricher/__init__.py @@ -0,0 +1 @@ +from .wacz_extractor_enricher import WaczExtractorEnricher diff --git a/src/auto_archiver/modules/wacz_enricher/__manifest__.py b/src/auto_archiver/modules/wacz_extractor_enricher/__manifest__.py similarity index 59% rename from src/auto_archiver/modules/wacz_enricher/__manifest__.py rename to src/auto_archiver/modules/wacz_extractor_enricher/__manifest__.py index 46ce05e..9b373b9 100644 --- a/src/auto_archiver/modules/wacz_enricher/__manifest__.py +++ b/src/auto_archiver/modules/wacz_extractor_enricher/__manifest__.py @@ -1,7 +1,7 @@ { - "name": "WACZ Enricher", - "type": ["enricher", "archiver"], - "entry_point": "wacz_enricher::WaczExtractorEnricher", + "name": "WACZ Enricher (and Extractor)", + "type": ["enricher", "extractor"], + "entry_point": "wacz_extractor_enricher::WaczExtractorEnricher", "requires_setup": True, "dependencies": { "python": [ @@ -17,11 +17,19 @@ "configs": { "profile": {"default": None, "help": "browsertrix-profile (for profile generation see https://github.com/webrecorder/browsertrix-crawler#creating-and-using-browser-profiles)."}, "docker_commands": {"default": None, "help":"if a custom docker invocation is needed"}, - "timeout": {"default": 120, "help": "timeout for WACZ generation in seconds"}, - "extract_media": {"default": False, "help": "If enabled all the images/videos/audio present in the WACZ archive will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched."}, - "extract_screenshot": {"default": True, "help": "If enabled the screenshot captured by browsertrix will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched."}, + "timeout": {"default": 120, + "type": "int", + "help": "timeout for WACZ generation in seconds", "type": "int"}, + "extract_media": {"default": False, + "type": 'bool', + "help": "If enabled all the images/videos/audio present in the WACZ archive will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched." + }, + "extract_screenshot": {"default": True, + "type": 'bool', + "help": "If enabled the screenshot captured by browsertrix will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched." + }, "socks_proxy_host": {"default": None, "help": "SOCKS proxy host for browsertrix-crawler, use in combination with socks_proxy_port. eg: user:password@host"}, - "socks_proxy_port": {"default": None, "help": "SOCKS proxy port for browsertrix-crawler, use in combination with socks_proxy_host. eg 1234"}, + "socks_proxy_port": {"default": None, "type":"int", "help": "SOCKS proxy port for browsertrix-crawler, use in combination with socks_proxy_host. eg 1234"}, "proxy_server": {"default": None, "help": "SOCKS server proxy URL, in development"}, }, "description": """ diff --git a/src/auto_archiver/modules/wacz_enricher/wacz_enricher.py b/src/auto_archiver/modules/wacz_extractor_enricher/wacz_extractor_enricher.py similarity index 99% rename from src/auto_archiver/modules/wacz_enricher/wacz_enricher.py rename to src/auto_archiver/modules/wacz_extractor_enricher/wacz_extractor_enricher.py index c324c62..ff7314a 100644 --- a/src/auto_archiver/modules/wacz_enricher/wacz_enricher.py +++ b/src/auto_archiver/modules/wacz_extractor_enricher/wacz_extractor_enricher.py @@ -221,4 +221,4 @@ class WaczExtractorEnricher(Enricher, Extractor): to_enrich.add_media(m, warc_fn) counter += 1 seen_urls.add(record_url) - logger.info(f"WACZ extract_media/extract_screenshot finished, found {counter} relevant media file(s)") + logger.info(f"WACZ extract_media/extract_screenshot finished, found {counter} relevant media file(s)") \ No newline at end of file diff --git a/src/auto_archiver/modules/wayback_extractor_enricher/__manifest__.py b/src/auto_archiver/modules/wayback_extractor_enricher/__manifest__.py index baecc14..62a7e8a 100644 --- a/src/auto_archiver/modules/wayback_extractor_enricher/__manifest__.py +++ b/src/auto_archiver/modules/wayback_extractor_enricher/__manifest__.py @@ -1,6 +1,6 @@ { - "name": "Wayback Machine Enricher", - "type": ["enricher", "archiver"], + "name": "Wayback Machine Enricher (and Extractor)", + "type": ["enricher", "extractor"], "entry_point": "wayback_extractor_enricher::WaybackExtractorEnricher", "requires_setup": True, "dependencies": { @@ -9,6 +9,7 @@ "configs": { "timeout": { "default": 15, + "type": "int", "help": "seconds to wait for successful archive confirmation from wayback, if more than this passes the result contains the job_id so the status can later be checked manually.", }, "if_not_archived_within": { diff --git a/src/auto_archiver/modules/whisper_enricher/__manifest__.py b/src/auto_archiver/modules/whisper_enricher/__manifest__.py index 98e743e..0e09d03 100644 --- a/src/auto_archiver/modules/whisper_enricher/__manifest__.py +++ b/src/auto_archiver/modules/whisper_enricher/__manifest__.py @@ -10,8 +10,12 @@ "help": "WhisperApi api endpoint, eg: https://whisperbox-api.com/api/v1, a deployment of https://github.com/bellingcat/whisperbox-transcribe."}, "api_key": {"required": True, "help": "WhisperApi api key for authentication"}, - "include_srt": {"default": False, "help": "Whether to include a subtitle SRT (SubRip Subtitle file) for the video (can be used in video players)."}, - "timeout": {"default": 90, "help": "How many seconds to wait at most for a successful job completion."}, + "include_srt": {"default": False, + "type": "bool", + "help": "Whether to include a subtitle SRT (SubRip Subtitle file) for the video (can be used in video players)."}, + "timeout": {"default": 90, + "type": "int", + "help": "How many seconds to wait at most for a successful job completion."}, "action": {"default": "translate", "help": "which Whisper operation to execute", "choices": ["transcribe", "translate", "language_detection"]}, diff --git a/src/auto_archiver/modules/whisper_enricher/whisper_enricher.py b/src/auto_archiver/modules/whisper_enricher/whisper_enricher.py index 89579f9..d63d2ed 100644 --- a/src/auto_archiver/modules/whisper_enricher/whisper_enricher.py +++ b/src/auto_archiver/modules/whisper_enricher/whisper_enricher.py @@ -4,7 +4,6 @@ from loguru import logger from auto_archiver.core import Enricher from auto_archiver.core import Metadata, Media -from auto_archiver.core.module import get_module class WhisperEnricher(Enricher): """ @@ -15,7 +14,7 @@ class WhisperEnricher(Enricher): def setup(self) -> None: self.stores = self.config['steps']['storages'] - self.s3 = get_module("s3_storage", self.config) + self.s3 = self.module_factory.get_module("s3_storage", self.config) if not "s3_storage" in self.stores: logger.error("WhisperEnricher: To use the WhisperEnricher you need to use S3Storage so files are accessible publicly to the whisper service being called.") return @@ -29,8 +28,7 @@ class WhisperEnricher(Enricher): job_results = {} for i, m in enumerate(to_enrich.media): if m.is_video() or m.is_audio(): - # TODO: this used to pass all storage items to store now - # Now only passing S3, the rest will get added later in the usual order (?) + # Only storing S3, the rest will get added later in the usual order (?) m.store(url=url, metadata=to_enrich, storages=[self.s3]) try: job_id = self.submit_job(m) diff --git a/src/auto_archiver/utils/__init__.py b/src/auto_archiver/utils/__init__.py index ed2d3bb..46ca191 100644 --- a/src/auto_archiver/utils/__init__.py +++ b/src/auto_archiver/utils/__init__.py @@ -2,7 +2,6 @@ # we need to explicitly expose the available imports here from .misc import * from .webdriver import Webdriver -from .atlos import get_atlos_config_options # handy utils from ytdlp from yt_dlp.utils import (clean_html, traverse_obj, strip_or_none, url_or_none) \ No newline at end of file diff --git a/src/auto_archiver/utils/atlos.py b/src/auto_archiver/utils/atlos.py deleted file mode 100644 index c47c711..0000000 --- a/src/auto_archiver/utils/atlos.py +++ /dev/null @@ -1,13 +0,0 @@ -def get_atlos_config_options(): - return { - "api_token": { - "default": None, - "help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/", - "cli_set": lambda cli_val, _: cli_val - }, - "atlos_url": { - "default": "https://platform.atlos.org", - "help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.", - "cli_set": lambda cli_val, _: cli_val - }, - } \ No newline at end of file diff --git a/src/auto_archiver/utils/gsheet.py b/src/auto_archiver/utils/gsheet.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/auto_archiver/utils/misc.py b/src/auto_archiver/utils/misc.py index cd03b49..49ef0b5 100644 --- a/src/auto_archiver/utils/misc.py +++ b/src/auto_archiver/utils/misc.py @@ -1,9 +1,11 @@ -import os +import hashlib import json +import os import uuid from datetime import datetime, timezone +from dateutil.parser import parse as parse_dt + import requests -import hashlib from loguru import logger @@ -46,7 +48,7 @@ def dump_payload(p): def update_nested_dict(dictionary, update_dict): - # takes 2 dicts and overwrites the first with the second only on the changed balues + # takes 2 dicts and overwrites the first with the second only on the changed values for key, value in update_dict.items(): if key in dictionary and isinstance(value, dict) and isinstance(dictionary[key], dict): update_nested_dict(dictionary[key], value) @@ -59,10 +61,6 @@ def random_str(length: int = 32) -> str: return str(uuid.uuid4()).replace("-", "")[:length] -def json_loader(cli_val): - return json.loads(cli_val) - - def calculate_file_hash(filename: str, hash_algo = hashlib.sha256, chunksize: int = 16000000) -> str: hash = hash_algo() with open(filename, "rb") as f: @@ -72,26 +70,34 @@ def calculate_file_hash(filename: str, hash_algo = hashlib.sha256, chunksize: in hash.update(buf) return hash.hexdigest() -def get_current_datetime_iso() -> str: - return datetime.now(timezone.utc).replace(tzinfo=timezone.utc).isoformat() +def get_datetime_from_str(dt_str: str, fmt: str | None = None, dayfirst=True) -> datetime | None: + """ parse a datetime string with option of passing a specific format -def get_datetime_from_str(dt_str: str, fmt: str | None = None) -> datetime | None: - # parse a datetime string with option of passing a specific format + Args: + dt_str: the datetime string to parse + fmt: the python date format of the datetime string, if None, dateutil.parser.parse is used + dayfirst: Use this to signify between date formats which put the day first, vs the month first: + e.g. DD/MM/YYYY vs MM/DD/YYYY + """ try: - return datetime.strptime(dt_str, fmt) if fmt else datetime.fromisoformat(dt_str) + return datetime.strptime(dt_str, fmt) if fmt else parse_dt(dt_str, dayfirst=dayfirst) except ValueError as e: logger.error(f"Unable to parse datestring {dt_str}: {e}") return None -def get_timestamp(ts, utc=True, iso=True) -> str | datetime | None: - # Consistent parsing of timestamps - # If utc=True, the timezone is set to UTC, - # if iso=True, the output is an iso string +def get_timestamp(ts, utc=True, iso=True, dayfirst=True) -> str | datetime | None: + """ Consistent parsing of timestamps. + Args: + If utc=True, the timezone is set to UTC, + if iso=True, the output is an iso string + Use dayfirst to signify between date formats which put the date vs month first: + e.g. DD/MM/YYYY vs MM/DD/YYYY + """ if not ts: return try: - if isinstance(ts, str): ts = datetime.fromisoformat(ts) + if isinstance(ts, str): ts = parse_dt(ts, dayfirst=dayfirst) if isinstance(ts, (int, float)): ts = datetime.fromtimestamp(ts) if utc: ts = ts.replace(tzinfo=timezone.utc) if iso: return ts.isoformat() @@ -100,5 +106,6 @@ def get_timestamp(ts, utc=True, iso=True) -> str | datetime | None: logger.error(f"Unable to parse timestamp {ts}: {e}") return None + def get_current_timestamp() -> str: - return get_timestamp(datetime.now()) \ No newline at end of file + return get_timestamp(datetime.now()) diff --git a/src/auto_archiver/utils/url.py b/src/auto_archiver/utils/url.py index 40884da..061f4aa 100644 --- a/src/auto_archiver/utils/url.py +++ b/src/auto_archiver/utils/url.py @@ -1,5 +1,6 @@ import re from urllib.parse import urlparse, urlunparse +from ipaddress import ip_address AUTHWALL_URLS = [ @@ -7,6 +8,43 @@ AUTHWALL_URLS = [ re.compile(r"https:\/\/www\.instagram\.com"), # instagram ] + +def check_url_or_raise(url: str) -> bool | ValueError: + """ + Blocks localhost, private, reserved, and link-local IPs and all non-http/https schemes. + """ + + + if not (url.startswith("http://") or url.startswith("https://")): + raise ValueError(f"Invalid URL scheme for url {url}") + + parsed = urlparse(url) + if not parsed.hostname: + raise ValueError(f"Invalid URL hostname for url {url}") + + if parsed.hostname == "localhost": + raise ValueError(f"Localhost URLs cannot be parsed for security reasons (for url {url})") + + if parsed.scheme not in ["http", "https"]: + raise ValueError(f"Invalid URL scheme, only http and https supported (for url {url})") + + try: # special rules for IP addresses + ip = ip_address(parsed.hostname) + except ValueError: + pass + + else: + if not ip.is_global: + raise ValueError(f"IP address {ip} is not globally reachable") + if ip.is_reserved: + raise ValueError(f"Reserved IP address {ip} used") + if ip.is_link_local: + raise ValueError(f"Link-local IP address {ip} used") + if ip.is_private: + raise ValueError(f"Private IP address {ip} used") + + return True + def domain_for_url(url: str) -> str: """ SECURITY: parse the domain using urllib to avoid any potential security issues diff --git a/src/auto_archiver/utils/webdriver.py b/src/auto_archiver/utils/webdriver.py index db26d04..cb4e2a9 100644 --- a/src/auto_archiver/utils/webdriver.py +++ b/src/auto_archiver/utils/webdriver.py @@ -1,18 +1,23 @@ """ This Webdriver class acts as a context manager for the selenium webdriver. """ from __future__ import annotations -from selenium import webdriver -from selenium.common.exceptions import TimeoutException -from selenium.webdriver.common.proxy import Proxy, ProxyType -from selenium.webdriver.common.print_page_options import PrintOptions -from loguru import logger -from selenium.webdriver.common.by import By +import os import time #import domain_for_url from urllib.parse import urlparse, urlunparse from http.cookiejar import MozillaCookieJar +from selenium import webdriver +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support import expected_conditions as EC +from selenium.common import exceptions as selenium_exceptions +from selenium.webdriver.common.print_page_options import PrintOptions +from selenium.webdriver.common.by import By + +from loguru import logger + + class CookieSettingDriver(webdriver.Firefox): facebook_accept_cookies: bool @@ -20,6 +25,10 @@ class CookieSettingDriver(webdriver.Firefox): cookiejar: MozillaCookieJar def __init__(self, cookies, cookiejar, facebook_accept_cookies, *args, **kwargs): + if os.environ.get('RUNNING_IN_DOCKER'): + # Selenium doesn't support linux-aarch64 driver, we need to set this manually + kwargs['service'] = webdriver.FirefoxService(executable_path='/usr/local/bin/geckodriver') + super(CookieSettingDriver, self).__init__(*args, **kwargs) self.cookies = cookies self.cookiejar = cookiejar @@ -64,14 +73,29 @@ class CookieSettingDriver(webdriver.Firefox): time.sleep(2) except Exception as e: logger.warning(f'Failed on fb accept cookies.', e) + + # now get the actual URL super(CookieSettingDriver, self).get(url) if self.facebook_accept_cookies: # try and click the 'close' button on the 'login' window to close it - close_button = self.find_element(By.XPATH, "//div[@role='dialog']//div[@aria-label='Close']") - if close_button: - close_button.click() + try: + xpath = "//div[@role='dialog']//div[@aria-label='Close']" + WebDriverWait(self, 5).until(EC.element_to_be_clickable((By.XPATH, xpath))).click() + except selenium_exceptions.NoSuchElementException: + logger.warning("Unable to find the 'close' button on the facebook login window") + pass + else: + + # for all other sites, try and use some common button text to reject/accept cookies + for text in ["Refuse non-essential cookies", "Decline optional cookies", "Reject additional cookies", "Reject all", "Accept all cookies"]: + try: + xpath = f"//*[contains(translate(text(), 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz'), '{text.lower()}')]" + WebDriverWait(self, 5).until(EC.element_to_be_clickable((By.XPATH, xpath))).click() + break + except selenium_exceptions.WebDriverException: + pass class Webdriver: @@ -90,7 +114,6 @@ class Webdriver: setattr(self.print_options, k, v) def __enter__(self) -> webdriver: - options = webdriver.FirefoxOptions() options.add_argument("--headless") options.add_argument(f'--proxy-server={self.http_proxy}') @@ -105,7 +128,7 @@ class Webdriver: self.driver.set_window_size(self.width, self.height) self.driver.set_page_load_timeout(self.timeout_seconds) self.driver.print_options = self.print_options - except TimeoutException as e: + except selenium_exceptions.TimeoutException as e: logger.error(f"failed to get new webdriver, possibly due to insufficient system resources or timeout settings: {e}") return self.driver diff --git a/tests/conftest.py b/tests/conftest.py index 8675fbc..a94abcd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,12 +3,14 @@ pytest conftest file, for shared fixtures and configuration """ import os import pickle +from datetime import datetime, timezone from tempfile import TemporaryDirectory from typing import Dict, Tuple import hashlib + import pytest from auto_archiver.core.metadata import Metadata -from auto_archiver.core.module import get_module, _LAZY_LOADED_MODULES +from auto_archiver.core.module import ModuleFactory # Test names inserted into this list will be run last. This is useful for expensive/costly tests # that you only want to run if everything else succeeds (e.g. API calls). The order here is important @@ -20,19 +22,19 @@ TESTS_TO_RUN_LAST = ['test_twitter_api_archiver'] def setup_module(request): def _setup_module(module_name, config={}): + module_factory = ModuleFactory() + if isinstance(module_name, type): # get the module name: # if the class does not have a .name, use the name of the parent folder module_name = module_name.__module__.rsplit(".",2)[-2] - m = get_module(module_name, {module_name: config}) - + m = module_factory.get_module(module_name, {module_name: config}) # add the tmp_dir to the module tmp_dir = TemporaryDirectory() m.tmp_dir = tmp_dir.name - + def cleanup(): - _LAZY_LOADED_MODULES.pop(module_name) tmp_dir.cleanup() request.addfinalizer(cleanup) @@ -122,10 +124,36 @@ def pytest_runtest_setup(item): def unpickle(): """ Returns a helper function that unpickles a file - ** gets the file from the test_files directory: tests/data/test_files ** + ** gets the file from the test_files directory: tests/data/ ** """ def _unpickle(path): - test_data_dir = os.path.join(os.path.dirname(__file__), "data", "test_files") - with open(os.path.join(test_data_dir, path), "rb") as f: + with open(os.path.join("tests/data", path), "rb") as f: return pickle.load(f) - return _unpickle \ No newline at end of file + return _unpickle + + +@pytest.fixture +def mock_binary_dependencies(mocker): + mock_shutil_which = mocker.patch("shutil.which") + # Mock all binary dependencies as available + mock_shutil_which.return_value = "/usr/bin/fake_binary" + return mock_shutil_which + + +@pytest.fixture +def sample_datetime(): + return datetime(2023, 1, 1, 12, 0, tzinfo=timezone.utc) + + +@pytest.fixture(autouse=True) +def mock_sleep(mocker): + """Globally mock time.sleep to avoid delays.""" + return mocker.patch("time.sleep") + + +@pytest.fixture +def metadata(): + metadata = Metadata() + metadata.set("_processed_at", "2021-01-01T00:00:00") + metadata.set_url("https://example.com") + return metadata \ No newline at end of file diff --git a/tests/data/metadata_enricher_exif.pickle b/tests/data/metadata_enricher_exif.pickle new file mode 100644 index 0000000..5607a9b Binary files /dev/null and b/tests/data/metadata_enricher_exif.pickle differ diff --git a/tests/data/metadata_enricher_ytshort_expected.pickle b/tests/data/metadata_enricher_ytshort_expected.pickle new file mode 100644 index 0000000..7e54268 Binary files /dev/null and b/tests/data/metadata_enricher_ytshort_expected.pickle differ diff --git a/tests/data/metadata_enricher_ytshort_input.pickle b/tests/data/metadata_enricher_ytshort_input.pickle new file mode 100644 index 0000000..2495c46 Binary files /dev/null and b/tests/data/metadata_enricher_ytshort_input.pickle differ diff --git a/tests/data/test_modules/example_module/__manifest__.py b/tests/data/test_modules/example_module/__manifest__.py index f2ebdbf..e3a26bb 100644 --- a/tests/data/test_modules/example_module/__manifest__.py +++ b/tests/data/test_modules/example_module/__manifest__.py @@ -1,11 +1,29 @@ { + # Display Name of your module "name": "Example Module", + # The author of your module (optional) + "author": "John Doe", + # Optional version number, for your own versioning purposes + "version": 2.0, + # The type of the module, must be one (or more) of the built in module types "type": ["extractor", "feeder", "formatter", "storage", "enricher", "database"], + # a boolean indicating whether or not a module requires additional user setup before it can be used + # for example: adding API keys, installing additional software etc. "requires_setup": False, - "dependencies": {"python": ["loguru"] - }, + # a dictionary of dependencies for this module, that must be installed before the module is loaded. + # Can be python dependencies (external packages, or other auto-archiver modules), or you can + # provide external bin dependencies (e.g. ffmpeg, docker etc.) + "dependencies": { + "python": ["loguru"], + "bin": ["bash"], + }, + # configurations that this module takes. These are argparse-compliant dicationaries, that are + # used to create command line arguments when the programme is run. + # The full name of the config option will become: `module_name.config_name` "configs": { "csv_file": {"default": "db.csv", "help": "CSV file name"}, "required_field": {"required": True, "help": "required field in the CSV file"}, }, + # A description of the module, used for documentation + "description": "This is an example module", } \ No newline at end of file diff --git a/tests/data/test_service_account.json b/tests/data/test_service_account.json new file mode 100644 index 0000000..5aae894 --- /dev/null +++ b/tests/data/test_service_account.json @@ -0,0 +1,14 @@ +{ + "type": "service_account", + "project_id": "some-project-id", + "private_key_id": "some-private-key-id", + "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDPlcaFJgt7HzoC\n4z0b18PzI2R5c892mLnNwRO8DOKid5INt6z5RAWKDPdnIyHjRBx74qNZl6768pia\nztQNgnud7mKcmvOvGrpUbFx2BdAw8xTyAlRVMalOBhUS9RKvjP5WgSwR5EKwfvzy\nrGioC6ml/segz5EchSaIzgASwB17ir0w6IrymBxUeNelfzCGJpCRhqG5nG+eEjct\nUYU0QIyihRD1Lq0f3Z3D0xfTLLZ630iFBj/Wr0BCJHkl6hdVuGhnyn4S98sMX1Bd\ntaJF/lWi4jdt7SoXD3+FWv66kHPpFfINMpReuB9u0ogfYkORgiRBOMhYBkGGQjUG\nOnBTxEc3AgMBAAECgf9bKiK8DdSz0ALzQbRLhgj2B9485jHI49wjgINOyceZ23uS\nQYXaO+DFLcgLqBkVSGanuHMpU0+qCpeM0v9yXSTIW8RguWMnFd8ID/yLRktxfQa1\n1FAQh+NlF4/gnuUoM8N/FYSy6R5grfaxwU8Qfg66IQXUB52OezSVu5lxNO4G5Rwv\nJ2e/+XYBUv/H26BnQSmjFCzbJkdbtrOeThpaLwLexKcollvoHKGyus0jpWg4C9Ez\n9EJaE+on4nd+cM1Vd+dWaHXoZ9Db9IvxPBqFJE8fynap7RDBeZK678OuCvQntrp4\nrTsE9hW8073Jhl/LbhfbDC0lhFR0JUHygVGE01ECgYEA+g+ddpGGY90yhhM76bTr\nkU6WwislMmfS0WDdLPemNgzLwCtkC2vsQgzg/egxqkVF5dJ9upiFhVgpYxY7ap9U\nSGFemb6T1ASl/1yeNhd0yc4PZFsJ29k+kNgSIlJYm9KDCIMqS1wPoXvFQhbMitOf\n/gLCPugxl67c+qg6nfuODTkCgYEA1IPngESOJnV8oa2WReWrO6+u6xb/OhqdmBzI\n5yq1z3f5gb98XESZR/rCH2vAOmHIJPn3XdZHsznOuxhZwGr1oztiRIurLmBlxQoL\n7tq0jDOUVSD2yeyQwKt5LaBH94P598FiauGxXM4raREWKtcNBGoOX1u1+kEBsoL4\ntf10Z+8CgYEA3QFkB+ECR8y91KW3NAzEjj5JG/8J9wyv1IGpuQ5/hhG1Gni/CSEv\nRAkh6QaIrpZe+ooYuQwIJhwPKBYEGW4MDZSRCYzYFnCtTY5L/j6o55sJG4cipX3R\nwC5XiKIC0mUxjhpvDP+miPBdHNYNnT0AkH1btEF/YzIW+Coq9GnZ2HECgYAOOpax\ne+WYpZ0mphy9qVcBtA2eJ/gGx+ltWeAJuk5aCcpm6Y9GDkHFFAETYX+JaSqhbysk\n2UgLs/8nf8XioEa6GyvFMyTPAh1OSBHseDBGgt2XpZFgi7pVbCW87FJlPCzsbcJN\nLbdWY2d8rWwyihuRBBjaQaW5j8ixTxuf88xreQKBgQCST4Fr8C5CkpakTA+KOost\nLOlziUBm0534mTg7dTcOE1H1+gxtqpXlXcJylpGz1lUXRlHCIutN5iPJcN5cxFES\nsP7wBd7BhficsMKDiWPm9XbP2zXVZu0ldUxA1mONMsS1P4p7i3Dh4uzrRDmSkTUL\njUpppYDumg3oM7wSJ6sTQA==\n-----END PRIVATE KEY-----", + "client_email": "some-email", + "client_id": "some-client-email", + "auth_uri": "https://example.com/o/oauth2/auth", + "token_uri": "https://oauth2.example.com/token", + "auth_provider_x509_cert_url": "https://www.example.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.example.com/robot/v1/metadata/x509/some-email", + "universe_domain": "example.com" + } + \ No newline at end of file diff --git a/tests/databases/test_api_db.py b/tests/databases/test_api_db.py new file mode 100644 index 0000000..5d1ea84 --- /dev/null +++ b/tests/databases/test_api_db.py @@ -0,0 +1,59 @@ +import pytest + +from auto_archiver.core import Metadata +from auto_archiver.modules.api_db import AAApiDb + + +@pytest.fixture +def api_db(setup_module): + configs: dict = { + "api_endpoint": "https://api.example.com", + "api_token": "test-token", + "public": False, + "author_id": "Someone", + "group_id": "123", + "use_api_cache": True, + "store_results": True, + "tags": "[]", + } + return setup_module(AAApiDb, configs) + + +def test_fetch_no_cache(api_db, metadata): + # Test fetch + api_db.use_api_cache = False + assert api_db.fetch(metadata) is None + + +def test_fetch_fail_status(api_db, metadata, mocker): + # Test response fail in fetch method + mock_get = mocker.patch("auto_archiver.modules.api_db.api_db.requests.get") + mock_get.return_value.status_code = 400 + mock_get.return_value.json.return_value = {} + mock_error = mocker.patch("loguru.logger.error") + assert api_db.fetch(metadata) is False + mock_error.assert_called_once_with("AA API FAIL (400): {}") + + +def test_fetch(api_db, metadata, mocker): + # Test successful fetch method + mock_get = mocker.patch("auto_archiver.modules.api_db.api_db.requests.get") + mock_datetime = mocker.patch("auto_archiver.core.metadata.datetime.datetime") + mock_datetime.now.return_value = "2021-01-01T00:00:00" + mock_get.return_value.status_code = 200 + mock_get.return_value.json.return_value = [{"result": {}}, {"result": + {'media': [], 'metadata': {'_processed_at': '2021-01-01T00:00:00', 'url': 'https://example.com'}, + 'status': 'no archiver'}}] + assert api_db.fetch(metadata) == metadata + + +def test_done_success(api_db, metadata, mocker): + mock_post = mocker.patch("auto_archiver.modules.api_db.api_db.requests.post") + mock_post.return_value.status_code = 201 + api_db.done(metadata) + mock_post.assert_called_once() + mock_post.assert_called_once_with("https://api.example.com/interop/submit-archive", + json={'author_id': 'Someone', 'url': 'https://example.com', + 'public': False, 'group_id': '123', 'tags': ['[', ']'], 'result': '{"status": "no archiver", "metadata": {"_processed_at": "2021-01-01T00:00:00", "url": "https://example.com"}, "media": []}'}, + headers={'Authorization': 'Bearer test-token'}) + diff --git a/tests/databases/test_atlos_db.py b/tests/databases/test_atlos_db.py new file mode 100644 index 0000000..a73f1df --- /dev/null +++ b/tests/databases/test_atlos_db.py @@ -0,0 +1,109 @@ +import pytest +from datetime import datetime + +from auto_archiver.core import Metadata +from auto_archiver.modules.atlos_feeder_db_storage import AtlosFeederDbStorage as AtlosDb + + +class FakeAPIResponse: + """Simulate a response object.""" + + def __init__(self, data: dict, raise_error: bool = False) -> None: + self._data = data + self.raise_error = raise_error + + def json(self) -> dict: + return self._data + + def raise_for_status(self) -> None: + if self.raise_error: + raise Exception("HTTP error") + + +@pytest.fixture +def atlos_db(setup_module, mocker) -> AtlosDb: + """Fixture for AtlosDb.""" + configs: dict = { + "api_token": "abc123", + "atlos_url": "https://platform.atlos.org", + } + mocker.patch("requests.Session") + atlos_feeder = setup_module("atlos_feeder_db_storage", configs) + fake_session = mocker.MagicMock() + # Configure the default response to have no results so that __iter__ terminates + fake_session.get.return_value = FakeAPIResponse({"next": None, "results": []}) + atlos_feeder.session = fake_session + return atlos_feeder + + +def test_failed_no_atlos_id(atlos_db, metadata, mocker): + """Test failed() skips posting when no atlos_id present.""" + post_mock = mocker.patch("requests.post") + atlos_db.failed(metadata, "failure reason") + post_mock.assert_not_called() + + +def test_failed_with_atlos_id(atlos_db, metadata, mocker): + """Test failed() posts failure when atlos_id is present.""" + metadata.set("atlos_id", 42) + fake_resp = FakeAPIResponse({}, raise_error=False) + post_mock = mocker.patch.object(atlos_db, "_post", return_value=fake_resp) + atlos_db.failed(metadata, "failure reason") + expected_endpoint = f"/api/v2/source_material/metadata/42/auto_archiver" + expected_json = { + "metadata": {"processed": True, "status": "error", "error": "failure reason"} + } + post_mock.assert_called_once_with(expected_endpoint, json=expected_json) + + +def test_failed_http_error(atlos_db, metadata, mocker): + """Test failed() raises exception on HTTP error.""" + metadata.set("atlos_id", 42) + # Patch _post to raise an exception instead of returning a fake response. + mocker.patch.object(atlos_db, "_post", side_effect=Exception("HTTP error")) + with pytest.raises(Exception, match="HTTP error"): + atlos_db.failed(metadata, "failure reason") + + +def test_fetch_returns_false(atlos_db): + """Test fetch() always returns False.""" + item = Metadata() + assert atlos_db.fetch(item) is False + + +def test_done_no_atlos_id(atlos_db, mocker): + """Test done() skips posting when no atlos_id present.""" + item = Metadata().set_url("http://example.com") + post_mock = mocker.patch("requests.post") + atlos_db.done(item) + post_mock.assert_not_called() + + +def test_done_with_atlos_id(atlos_db, metadata, mocker): + """Test done() posts success when atlos_id is present.""" + metadata.set("atlos_id", 99) + now = datetime.now() + metadata.set("timestamp", now) + fake_resp = FakeAPIResponse({}, raise_error=False) + post_mock = mocker.patch.object(atlos_db, "_post", return_value=fake_resp) + atlos_db.done(metadata) + expected_endpoint = f"/api/v2/source_material/metadata/99/auto_archiver" + expected_results = metadata.metadata.copy() + expected_results["timestamp"] = now.isoformat() + expected_json = { + "metadata": { + "processed": True, + "status": "success", + "results": expected_results, + } + } + post_mock.assert_called_once_with(expected_endpoint, json=expected_json) + + +def test_done_http_error(atlos_db, metadata, mocker): + """Test done() raises an exception on HTTP error.""" + metadata.set("atlos_id", 123) + # Patch _post to raise an exception. + mocker.patch.object(atlos_db, "_post", side_effect=Exception("HTTP error")) + with pytest.raises(Exception, match="HTTP error"): + atlos_db.done(metadata) diff --git a/tests/databases/test_gsheet_db.py b/tests/databases/test_gsheet_db.py index 18a22f1..2f1202d 100644 --- a/tests/databases/test_gsheet_db.py +++ b/tests/databases/test_gsheet_db.py @@ -1,16 +1,13 @@ from datetime import datetime, timezone -from unittest.mock import MagicMock, patch - import pytest from auto_archiver.core import Metadata, Media -from auto_archiver.modules.gsheet_db import GsheetsDb -from auto_archiver.modules.gsheet_feeder import GWorksheet +from auto_archiver.modules.gsheet_feeder_db import GsheetsFeederDB, GWorksheet @pytest.fixture -def mock_gworksheet(): - mock_gworksheet = MagicMock(spec=GWorksheet) +def mock_gworksheet(mocker): + mock_gworksheet = mocker.MagicMock(spec=GWorksheet) mock_gworksheet.col_exists.return_value = True mock_gworksheet.get_cell.return_value = "" mock_gworksheet.get_row.return_value = {} @@ -18,14 +15,14 @@ def mock_gworksheet(): @pytest.fixture -def mock_metadata(): - metadata: Metadata = MagicMock(spec=Metadata) +def mock_metadata(mocker): + metadata: Metadata = mocker.MagicMock(spec=Metadata) metadata.get_url.return_value = "http://example.com" metadata.status = "done" metadata.get_title.return_value = "Example Title" metadata.get.return_value = "Example Content" metadata.get_timestamp.return_value = "2025-01-01T00:00:00" - metadata.get_final_media.return_value = MagicMock(spec=Media) + metadata.get_final_media.return_value = mocker.MagicMock(spec=Media) metadata.get_all_media.return_value = [] metadata.get_media_by_id.return_value = None metadata.get_first_image.return_value = None @@ -47,21 +44,28 @@ def metadata(): @pytest.fixture -def mock_media(): +def mock_media(mocker): """Fixture for a mock Media object.""" - mock_media = MagicMock(spec=Media) + mock_media = mocker.MagicMock(spec=Media) mock_media.urls = ["http://example.com/media"] mock_media.get.return_value = "not-calculated" return mock_media @pytest.fixture -def gsheets_db(mock_gworksheet, setup_module): - db = setup_module("gsheet_db", { - "allow_worksheets": "set()", - "block_worksheets": "set()", - "use_sheet_names_in_stored_paths": "True", - }) - db._retrieve_gsheet = MagicMock(return_value=(mock_gworksheet, 1)) +def gsheets_db(mock_gworksheet, setup_module, mocker): + mocker.patch("gspread.service_account") + config: dict = { + "sheet": "testsheet", + "sheet_id": None, + "header": 1, + "service_account": "test/service_account.json", + "columns": {'url': 'link', 'status': 'archive status', 'folder': 'destination folder', 'archive': 'archive location', 'date': 'archive date', 'thumbnail': 'thumbnail', 'timestamp': 'upload timestamp', 'title': 'upload title', 'text': 'text content', 'screenshot': 'screenshot', 'hash': 'hash', 'pdq_hash': 'perceptual hashes', 'wacz': 'wacz', 'replaywebpage': 'replaywebpage'}, + "allow_worksheets": set(), + "block_worksheets": set(), + "use_sheet_names_in_stored_paths": True, + } + db = setup_module("gsheet_feeder_db", config) + db._retrieve_gsheet = mocker.MagicMock(return_value=(mock_gworksheet, 1)) return db @@ -109,27 +113,26 @@ def test_aborted(gsheets_db, mock_metadata, mock_gworksheet): mock_gworksheet.set_cell.assert_called_once_with(1, 'status', '') -def test_done(gsheets_db, metadata, mock_gworksheet, expected_calls): - with patch("auto_archiver.modules.gsheet_db.gsheet_db.get_current_timestamp", return_value='2025-02-01T00:00:00+00:00'): - gsheets_db.done(metadata) +def test_done(gsheets_db, metadata, mock_gworksheet, expected_calls, mocker): + mocker.patch("auto_archiver.modules.gsheet_feeder_db.gsheet_feeder_db.get_current_timestamp", return_value='2025-02-01T00:00:00+00:00') + gsheets_db.done(metadata) mock_gworksheet.batch_set_cell.assert_called_once_with(expected_calls) -def test_done_cached(gsheets_db, metadata, mock_gworksheet): - with patch("auto_archiver.modules.gsheet_db.gsheet_db.get_current_timestamp", return_value='2025-02-01T00:00:00+00:00'): - gsheets_db.done(metadata, cached=True) +def test_done_cached(gsheets_db, metadata, mock_gworksheet, mocker): + mocker.patch("auto_archiver.modules.gsheet_feeder_db.gsheet_feeder_db.get_current_timestamp", return_value='2025-02-01T00:00:00+00:00') + gsheets_db.done(metadata, cached=True) # Verify the status message includes "[cached]" call_args = mock_gworksheet.batch_set_cell.call_args[0][0] assert any(call[2].startswith("[cached]") for call in call_args) -def test_done_missing_media(gsheets_db, metadata, mock_gworksheet): +def test_done_missing_media(gsheets_db, metadata, mock_gworksheet, mocker): # clear media from metadata metadata.media = [] - with patch("auto_archiver.modules.gsheet_db.gsheet_db.get_current_timestamp", - return_value='2025-02-01T00:00:00+00:00'): - gsheets_db.done(metadata) + mocker.patch("auto_archiver.modules.gsheet_feeder_db.gsheet_feeder_db.get_current_timestamp", return_value='2025-02-01T00:00:00+00:00') + gsheets_db.done(metadata) # Verify nothing media-related gets updated call_args = mock_gworksheet.batch_set_cell.call_args[0][0] media_fields = {'archive', 'screenshot', 'thumbnail', 'wacz', 'replaywebpage'} diff --git a/tests/enrichers/test_hash_enricher.py b/tests/enrichers/test_hash_enricher.py index 4b61fc2..c2fe67a 100644 --- a/tests/enrichers/test_hash_enricher.py +++ b/tests/enrichers/test_hash_enricher.py @@ -2,7 +2,7 @@ import pytest from auto_archiver.modules.hash_enricher import HashEnricher from auto_archiver.core import Metadata, Media -from auto_archiver.core.module import get_module_lazy +from auto_archiver.core.module import ModuleFactory @pytest.mark.parametrize("algorithm, filename, expected_hash", [ ("SHA-256", "tests/data/testfile_1.txt", "1b4f0e9851971998e732078544c96b36c3d01cedf7caa332359d6f1d83567014"), @@ -22,7 +22,7 @@ def test_default_config_values(setup_module): def test_config(): # test default config - c = get_module_lazy('hash_enricher').configs + c = ModuleFactory().get_module_lazy('hash_enricher').configs assert c["algorithm"]["default"] == "SHA-256" assert c["chunksize"]["default"] == 16000000 assert c["algorithm"]["choices"] == ["SHA-256", "SHA3-512"] diff --git a/tests/enrichers/test_meta_enricher.py b/tests/enrichers/test_meta_enricher.py index a09aaa9..476e25b 100644 --- a/tests/enrichers/test_meta_enricher.py +++ b/tests/enrichers/test_meta_enricher.py @@ -1,6 +1,5 @@ import datetime from datetime import datetime, timedelta, timezone -from unittest.mock import MagicMock, patch import pytest @@ -9,29 +8,21 @@ from auto_archiver.modules.meta_enricher import MetaEnricher @pytest.fixture -def mock_metadata(): +def mock_metadata(mocker): """Creates a mock Metadata object.""" - mock: Metadata = MagicMock(spec=Metadata) + mock: Metadata = mocker.MagicMock(spec=Metadata) mock.get_url.return_value = "https://example.com" mock.is_empty.return_value = False # Default to not empty mock.get_all_media.return_value = [] return mock @pytest.fixture -def mock_media(): +def mock_media(mocker): """Creates a mock Media object.""" - mock: Media = MagicMock(spec=Media) + mock: Media = mocker.MagicMock(spec=Media) mock.filename = "mock_file.txt" return mock -@pytest.fixture -def metadata(): - m = Metadata() - m.set_url("https://example.com") - m.set_title("Test Title") - m.set_content("Test Content") - return m - @pytest.fixture(autouse=True) def meta_enricher(setup_module): @@ -90,14 +81,14 @@ def test_enrich_file_sizes_no_media(meta_enricher, metadata): assert metadata.get("total_size") == "0.0 bytes" -def test_enrich_archive_duration(meta_enricher, metadata): +def test_enrich_archive_duration(meta_enricher, metadata, mocker): # Set fixed "processed at" time in the past processed_at = datetime.now(timezone.utc) - timedelta(minutes=10, seconds=30) metadata.set("_processed_at", processed_at) # patch datetime - with patch("datetime.datetime") as mock_datetime: - mock_now = datetime.now(timezone.utc) - mock_datetime.now.return_value = mock_now - meta_enricher.enrich_archive_duration(metadata) + mock_datetime = mocker.patch("datetime.datetime") + mock_now = datetime.now(timezone.utc) + mock_datetime.now.return_value = mock_now + meta_enricher.enrich_archive_duration(metadata) assert metadata.get("archive_duration_seconds") == 630 \ No newline at end of file diff --git a/tests/enrichers/test_metadata_enricher.py b/tests/enrichers/test_metadata_enricher.py new file mode 100644 index 0000000..888837d --- /dev/null +++ b/tests/enrichers/test_metadata_enricher.py @@ -0,0 +1,88 @@ + +import pytest + +from auto_archiver.core import Media + + +@pytest.fixture +def mock_media(mocker): + """Creates a mock Media object.""" + mock: Media = mocker.MagicMock(spec=Media) + mock.filename = "mock_file.txt" + return mock + + +@pytest.fixture +def enricher(setup_module, mock_binary_dependencies): + return setup_module("metadata_enricher", {}) + + +@pytest.mark.parametrize( + "output,expected", + [ + ("Key1: Value1\nKey2: Value2", {"Key1": "Value1", "Key2": "Value2"}), + ("InvalidLine", {}), + ("", {}), + ], +) +def test_get_metadata(enricher, output, expected, mocker): + mock_run = mocker.patch("subprocess.run") + mock_run.return_value.stdout = output + mock_run.return_value.stderr = "" + mock_run.return_value.returncode = 0 + + result = enricher.get_metadata("test.jpg") + assert result == expected + mock_run.assert_called_once_with( + ["exiftool", "test.jpg"], capture_output=True, text=True + ) + + +def test_get_metadata_exiftool_not_found(enricher, mocker): + mock_run = mocker.patch("subprocess.run") + mock_run.side_effect = FileNotFoundError + result = enricher.get_metadata("test.jpg") + assert result == {} + + +def test_enrich_sets_metadata(enricher, mocker): + media1 = mocker.Mock(filename="img1.jpg") + media2 = mocker.Mock(filename="img2.jpg") + metadata = mocker.Mock() + metadata.media = [media1, media2] + enricher.get_metadata = lambda f: {"key": "value"} if f == "img1.jpg" else {} + + enricher.enrich(metadata) + + media1.set.assert_called_once_with("metadata", {"key": "value"}) + media2.set.assert_not_called() + assert metadata.media == [media1, media2] + + +def test_enrich_empty_media(enricher, mocker): + metadata = mocker.Mock() + metadata.media = [] + # Should not raise errors + enricher.enrich(metadata) + + +def test_get_metadata_error_handling(enricher, mocker): + mocker.patch("subprocess.run", side_effect=Exception("Test error")) + mock_log = mocker.patch("loguru.logger.error") + result = enricher.get_metadata("test.jpg") + assert result == {} + assert "Error occurred: " in mock_log.call_args[0][0] + + +def test_metadata_pickle(enricher, unpickle, mocker): + mock_run = mocker.patch("subprocess.run") + # Uses pickled values + mock_run.return_value = unpickle("metadata_enricher_exif.pickle") + metadata = unpickle("metadata_enricher_ytshort_input.pickle") + expected = unpickle("metadata_enricher_ytshort_expected.pickle") + enricher.enrich(metadata) + expected_media = expected.media + actual_media = metadata.media + assert len(expected_media) == len(actual_media) + assert actual_media[0].properties.get("metadata") == expected_media[0].properties.get("metadata") + diff --git a/tests/enrichers/test_pdq_hash_enricher.py b/tests/enrichers/test_pdq_hash_enricher.py new file mode 100644 index 0000000..61fa778 --- /dev/null +++ b/tests/enrichers/test_pdq_hash_enricher.py @@ -0,0 +1,76 @@ +import pytest +from PIL import UnidentifiedImageError + +from auto_archiver.core import Metadata, Media +from auto_archiver.modules.pdq_hash_enricher import PdqHashEnricher + + +@pytest.fixture +def enricher(setup_module): + return setup_module("pdq_hash_enricher", {}) + + +@pytest.fixture +def metadata_with_images(): + m = Metadata() + m.set_url("https://example.com") + m.add_media(Media(filename="image1.jpg", _key="image1")) + m.add_media(Media(filename="image2.jpg", _key="image2")) + return m + + +def test_successful_enrich(metadata_with_images, mocker): + mocker.patch("pdqhash.compute", return_value=([1, 0, 1, 0] * 64, 100)) + mocker.patch("PIL.Image.open") + mocker.patch.object(Media, "is_image", return_value=True) + enricher = PdqHashEnricher() + enricher.enrich(metadata_with_images) + + # Ensure the hash is set for image media + for media in metadata_with_images.media: + assert media.get("pdq_hash") is not None + + +def test_enrich_skip_non_image(metadata_with_images, mocker): + mocker.patch.object(Media, "is_image", return_value=False) + mock_pdq = mocker.patch("pdqhash.compute") + + enricher = PdqHashEnricher() + enricher.enrich(metadata_with_images) + mock_pdq.assert_not_called() + + +def test_enrich_handles_corrupted_image(metadata_with_images, mocker): + mocker.patch("PIL.Image.open", side_effect=UnidentifiedImageError("Corrupted image")) + mock_pdq = mocker.patch("pdqhash.compute") + mock_logger = mocker.patch("loguru.logger.error") + enricher = PdqHashEnricher() + enricher.enrich(metadata_with_images) + + assert mock_logger.call_count == len(metadata_with_images.media) + mock_pdq.assert_not_called() + + +@pytest.mark.parametrize( + "media_id, should_have_hash", + [ + ("screenshot", False), + ("warc-file-123", False), + ("regular-image", True), + ] +) +def test_enrich_excludes_by_filetype(media_id, should_have_hash, mocker): + metadata = Metadata() + metadata.set_url("https://example.com") + metadata.add_media(Media(filename="image.jpg").set("id", media_id)) + + mocker.patch("pdqhash.compute", return_value=([1, 0, 1, 0] * 64, 100)) + mocker.patch("PIL.Image.open") + mocker.patch.object(Media, "is_image", return_value=True) + + enricher = PdqHashEnricher() + enricher.enrich(metadata) + + media_item = metadata.media[0] + assert (media_item.get("pdq_hash") is not None) == should_have_hash + diff --git a/tests/enrichers/test_screenshot_enricher.py b/tests/enrichers/test_screenshot_enricher.py new file mode 100644 index 0000000..25ca51d --- /dev/null +++ b/tests/enrichers/test_screenshot_enricher.py @@ -0,0 +1,195 @@ +import base64 + +import pytest +from selenium.common.exceptions import TimeoutException + +from auto_archiver.core import Metadata, Media +from auto_archiver.modules.screenshot_enricher import ScreenshotEnricher + + +@pytest.fixture +def mock_selenium_env(mocker): + """Patches Selenium calls and driver checks in one place.""" + + # Patch external dependencies + mock_which = mocker.patch("shutil.which") + mock_driver_class = mocker.patch("auto_archiver.utils.webdriver.CookieSettingDriver") + mock_binary_paths = mocker.patch("selenium.webdriver.common.selenium_manager.SeleniumManager.binary_paths") + mock_is_file = mocker.patch("pathlib.Path.is_file", return_value=True) + mock_popen = mocker.patch("subprocess.Popen") + mock_is_connectable = mocker.patch("selenium.webdriver.common.service.Service.is_connectable", return_value=True) + mock_firefox_options = mocker.patch("selenium.webdriver.FirefoxOptions") + # Define side effect for `shutil.which` + def mock_which_side_effect(dep): + return "/mock/geckodriver" if dep == "geckodriver" else None + mock_which.side_effect = mock_which_side_effect + + # Mock binary paths + mock_binary_paths.return_value = { + "driver_path": "/mock/driver", + "browser_path": "/mock/browser", + } + # Mock `subprocess.Popen` + mock_proc = mocker.MagicMock() + mock_proc.poll.return_value = None + mock_popen.return_value = mock_proc + # Mock `CookieSettingDriver` + mock_driver = mocker.MagicMock() + mock_driver_class.return_value = mock_driver + # Mock `FirefoxOptions` + mock_options_instance = mocker.MagicMock() + mock_firefox_options.return_value = mock_options_instance + yield mock_driver, mock_driver_class, mock_options_instance + + +@pytest.fixture +def common_patches(tmp_path, mocker): + """Patches common utilities used across multiple tests.""" + mocker.patch("auto_archiver.utils.url.is_auth_wall", return_value=False) + mocker.patch("os.path.join", return_value=str(tmp_path / "test.png")) + mocker.patch("time.sleep") + yield + + +@pytest.fixture +def screenshot_enricher(setup_module, mock_binary_dependencies) -> ScreenshotEnricher: + configs: dict = { + "width": 1280, + "height": 720, + "timeout": 60, + "sleep_before_screenshot": 4, + "http_proxy": "", + "save_to_pdf": "False", + "print_options": {}, + } + return setup_module("screenshot_enricher", configs) + + +@pytest.fixture +def metadata_with_video(): + m = Metadata() + m.set_url("https://example.com") + m.add_media(Media(filename="video.mp4").set("id", "video1")) + return m + + +def test_enrich_adds_screenshot( + screenshot_enricher, + metadata_with_video, + mock_selenium_env, + common_patches, + tmp_path, +): + mock_driver, mock_driver_class, mock_options_instance = mock_selenium_env + screenshot_enricher.enrich(metadata_with_video) + mock_driver_class.assert_called_once_with( + cookies=None, + cookiejar=None, + facebook_accept_cookies=False, + options=mock_options_instance, + ) + # Verify the actual calls on the returned mock_driver + mock_driver.get.assert_called_once_with("https://example.com") + mock_driver.save_screenshot.assert_called_once_with(str(tmp_path / "test.png")) + # Check that the media was added (2 = original video + screenshot) + assert len(metadata_with_video.media) == 2 + assert metadata_with_video.media[1].properties.get("id") == "screenshot" + + +@pytest.mark.parametrize( + "url,is_auth", + [ + ("https://example.com", False), + ("https://private.com", True), + ], +) +def test_enrich_auth_wall( + screenshot_enricher, + metadata_with_video, + mock_selenium_env, + common_patches, + url, + is_auth, + mocker +): + # Testing with and without is_auth_wall + mock_driver, mock_driver_class, _ = mock_selenium_env + mocker.patch("auto_archiver.utils.url.is_auth_wall", return_value=is_auth) + metadata_with_video.set_url(url) + screenshot_enricher.enrich(metadata_with_video) + + if is_auth: + mock_driver.get.assert_not_called() + assert len(metadata_with_video.media) == 1 + assert metadata_with_video.media[0].properties.get("id") == "video1" + else: + mock_driver.get.assert_called_once_with(url) + assert len(metadata_with_video.media) == 2 + assert metadata_with_video.media[1].properties.get("id") == "screenshot" + + +def test_handle_timeout_exception( + screenshot_enricher, metadata_with_video, mock_selenium_env, mocker +): + mock_driver, mock_driver_class, mock_options_instance = mock_selenium_env + + mock_driver.get.side_effect = TimeoutException + mock_log = mocker.patch("loguru.logger.info") + screenshot_enricher.enrich(metadata_with_video) + mock_log.assert_called_once_with("TimeoutException loading page for screenshot") + assert len(metadata_with_video.media) == 1 + + +def test_handle_general_exception( + screenshot_enricher, metadata_with_video, mock_selenium_env, mocker +): + """Test proper handling of unexpected general exceptions""" + mock_driver, mock_driver_class, mock_options_instance = mock_selenium_env + # Simulate a generic exception when save_screenshot is called + mock_driver.get.return_value = None + mock_driver.save_screenshot.side_effect = Exception("Unexpected Error") + + mock_log = mocker.patch("loguru.logger.error") + screenshot_enricher.enrich(metadata_with_video) + # Verify that the exception was logged with the log + mock_log.assert_called_once_with( + "Got error while loading webdriver for screenshot enricher: Unexpected Error" + ) + # And no new media was added due to the error + assert len(metadata_with_video.media) == 1 + + +def test_pdf_creation(mocker, screenshot_enricher, metadata_with_video, mock_selenium_env): + """Test PDF creation when save_to_pdf is enabled""" + mock_driver, mock_driver_class, mock_options_instance = mock_selenium_env + # Override the save_to_pdf option + screenshot_enricher.save_to_pdf = True + # Mock the print_page method to return base64-encoded content + mock_driver.print_page.return_value = base64.b64encode(b"fake_pdf_content").decode("utf-8") + # Patch functions with mocker + mock_os_path_join = mocker.patch("os.path.join", side_effect=lambda *args: f"{args[-1]}") + mock_random_str = mocker.patch( + "auto_archiver.modules.screenshot_enricher.screenshot_enricher.random_str", + return_value="fixed123", + ) + mock_open = mocker.patch("builtins.open", new_callable=mocker.mock_open) + mock_log_error = mocker.patch("loguru.logger.error") + + screenshot_enricher.enrich(metadata_with_video) + # Verify screenshot and PDF creation + mock_driver.save_screenshot.assert_called_once() + mock_driver.print_page.assert_called_once_with(mock_driver.print_options) + # Check that PDF file was opened and written + mock_open.assert_any_call("pdf_fixed123.pdf", "wb") + + # Ensure both screenshot and PDF were added as media + assert len(metadata_with_video.media) == 3 + assert metadata_with_video.media[1].properties.get("id") == "screenshot" + assert metadata_with_video.media[2].properties.get("id") == "pdf" + + +@pytest.fixture(autouse=True) +def cleanup_files(tmp_path): + yield + for file in tmp_path.iterdir(): + file.unlink() diff --git a/tests/enrichers/test_ssl_enricher.py b/tests/enrichers/test_ssl_enricher.py new file mode 100644 index 0000000..eb7ba6b --- /dev/null +++ b/tests/enrichers/test_ssl_enricher.py @@ -0,0 +1,54 @@ +import ssl +import pytest + +from auto_archiver.core import Metadata, Media + + +@pytest.fixture +def enricher(setup_module): + configs: dict = { + "skip_when_nothing_archived": "True", + } + return setup_module("ssl_enricher", configs) + + +@pytest.fixture +def metadata(): + m = Metadata() + m.set_url("https://example.com") + m.add_media(Media("tests/data/testfile_1.txt")) + m.add_media(Media("tests/data/testfile_2.txt")) + return m + + +def test_http_raises(metadata, enricher): + metadata.set_url("http://example.com") + with pytest.raises(AssertionError) as exc_info: + enricher.enrich(metadata) + assert "Invalid URL scheme" in str(exc_info.value) + + +def test_empty_metadata(metadata, enricher): + metadata.media = [] + assert enricher.enrich(metadata) is None + + +def test_ssl_enrich(metadata, enricher, mocker): + mocker.patch("ssl.get_server_certificate", return_value="TEST_CERT") + mock_file = mocker.patch("builtins.open", mocker.mock_open()) + media_len_before = len(metadata.media) + enricher.enrich(metadata) + + ssl.get_server_certificate.assert_called_once_with(("example.com", 443)) + mock_file.assert_called_once_with(f"{enricher.tmp_dir}/example-com.pem", "w") + mock_file().write.assert_called_once_with("TEST_CERT") + assert len(metadata.media) == media_len_before + 1 + # Ensure the certificate is added to metadata + assert any(media.filename.endswith("example-com.pem") for media in metadata.media) + + +def test_ssl_error_handling(enricher, metadata, mocker): + mocker.patch("ssl.get_server_certificate", side_effect=ssl.SSLError("SSL error")) + with pytest.raises(ssl.SSLError, match="SSL error"): + enricher.enrich(metadata) + diff --git a/tests/enrichers/test_thumbnail_enricher.py b/tests/enrichers/test_thumbnail_enricher.py new file mode 100644 index 0000000..effc25e --- /dev/null +++ b/tests/enrichers/test_thumbnail_enricher.py @@ -0,0 +1,148 @@ +import pytest +from auto_archiver.core import Metadata, Media +from auto_archiver.modules.thumbnail_enricher import ThumbnailEnricher + + +@pytest.fixture +def thumbnail_enricher(setup_module, mock_binary_dependencies) -> ThumbnailEnricher: + config: dict = { + "thumbnails_per_minute": 60, + "max_thumbnails": 4, + } + return setup_module("thumbnail_enricher", config) + + +@pytest.fixture +def metadata_with_video(): + m = Metadata() + m.set_url("https://example.com") + m.add_media(Media(filename="video.mp4").set("id", "video1")) + return m + + +@pytest.fixture +def mock_ffmpeg_environment(mocker): + # Mocking all the ffmpeg calls in one place + mock_ffmpeg_input = mocker.patch("ffmpeg.input") + mock_makedirs = mocker.patch("os.makedirs") + mocker.patch.object(Media, "is_video", return_value=True), + mock_probe = mocker.patch( + "ffmpeg.probe", + return_value={ + "streams": [ + {"codec_type": "video", "duration": "120"} + ] # Default 2-minute duration, but can override in tests + }, + ) + mock_output = mocker.MagicMock() + mock_ffmpeg_input.return_value.filter.return_value.output.return_value = ( + mock_output + ) + + return { + "mock_ffmpeg_input": mock_ffmpeg_input, + "mock_makedirs": mock_makedirs, + "mock_output": mock_output, + "mock_probe": mock_probe, + } + + +@pytest.mark.parametrize("thumbnails_per_minute, max_thumbnails, expected_count", [ + (10, 5, 5), # Capped at max_thumbnails + (1, 10, 2), # Less than max_thumbnails + (60, 7, 7), # Matches exactly +]) +def test_enrich_thumbnail_limits( + thumbnail_enricher, metadata_with_video, mock_ffmpeg_environment, + thumbnails_per_minute, max_thumbnails, expected_count +): + thumbnail_enricher.thumbnails_per_minute = thumbnails_per_minute + thumbnail_enricher.max_thumbnails = max_thumbnails + + thumbnail_enricher.enrich(metadata_with_video) + + assert mock_ffmpeg_environment["mock_output"].run.call_count == expected_count + thumbnails = metadata_with_video.media[0].get("thumbnails") + assert len(thumbnails) == expected_count + +def test_enrich_handles_probe_failure(thumbnail_enricher, metadata_with_video, mocker): + + mocker.patch("ffmpeg.probe", side_effect=Exception("Probe error")) + mocker.patch("os.makedirs") + mock_logger = mocker.patch("loguru.logger.error") + mocker.patch.object(Media, "is_video", return_value=True) + + thumbnail_enricher.enrich(metadata_with_video) + # Ensure error was logged + mock_logger.assert_called_with( + f"error getting duration of video video.mp4: Probe error" + ) + # Ensure no thumbnails were created + thumbnails = metadata_with_video.media[0].get("thumbnails") + assert thumbnails is None + + +def test_enrich_skips_non_video_files(thumbnail_enricher, metadata_with_video, mocker): + mocker.patch.object(Media, "is_video", return_value=False) + mock_ffmpeg = mocker.patch("ffmpeg.input") + thumbnail_enricher.enrich(metadata_with_video) + mock_ffmpeg.assert_not_called() + + +@pytest.mark.parametrize("thumbnails_per_minute,max_thumbnails,expected_count", [ + (60, 5, 5), # caught by max + (60, 20, 10), # caught by t/min + (0, 20, 1), # test min caught (1) + (11, 20, 1), # test min caught (1) + (12, 20, 2), # test caught by t/min +]) +def test_enrich_handles_short_video( + thumbnail_enricher, metadata_with_video, mock_ffmpeg_environment, thumbnails_per_minute, max_thumbnails, expected_count, mocker +): + # override mock duration + fake_duration = 10 + mocker.patch( + "ffmpeg.probe", + return_value={ "streams": [{"codec_type": "video", "duration": str(fake_duration)}]}, + ) + thumbnail_enricher.thumbnails_per_minute = thumbnails_per_minute + thumbnail_enricher.max_thumbnails = max_thumbnails + + thumbnail_enricher.enrich(metadata_with_video) + assert mock_ffmpeg_environment["mock_output"].run.call_count == expected_count + thumbnails = metadata_with_video.media[0].get("thumbnails") + assert len(thumbnails) == expected_count + + +def test_uses_existing_duration( + thumbnail_enricher, metadata_with_video, mock_ffmpeg_environment +): + metadata_with_video.media[0].set("duration", 60) + thumbnail_enricher.enrich(metadata_with_video) + mock_ffmpeg_environment["mock_probe"].assert_not_called() + assert mock_ffmpeg_environment["mock_output"].run.call_count == 4 + + +def test_enrich_metadata_structure(thumbnail_enricher, metadata_with_video, mock_ffmpeg_environment, mocker): + fake_duration = 120 + mocker.patch("ffmpeg.probe", return_value={'streams': [{'codec_type': 'video', 'duration': str(fake_duration)}]}) + thumbnail_enricher.thumbnails_per_minute = 2 + thumbnail_enricher.max_thumbnails = 4 + + thumbnail_enricher.enrich(metadata_with_video) + + media_item = metadata_with_video.media[0] + thumbnails = media_item.get("thumbnails") + + # Assert normal metadata + assert media_item.get("id") == "video1" + assert media_item.get("duration") == fake_duration + # Evenly spaced timestamps + expected_timestamps = ["24.000s", "48.000s", "72.000s", "96.000s"] + assert thumbnails is not None + assert len(thumbnails) == 4 + + for index, thumbnail in enumerate(thumbnails): + assert thumbnail.filename is not None + assert thumbnail.properties.get("id") == f"thumbnail_{index}" + assert thumbnail.properties.get("timestamp") == expected_timestamps[index] diff --git a/tests/enrichers/test_wacz_enricher.py b/tests/enrichers/test_wacz_enricher.py new file mode 100644 index 0000000..ceab83b --- /dev/null +++ b/tests/enrichers/test_wacz_enricher.py @@ -0,0 +1,112 @@ +import os +from zipfile import ZipFile + +import pytest + +from auto_archiver.core import Metadata, Media + + +@pytest.fixture +def wacz_enricher(setup_module, mock_binary_dependencies): + configs: dict = { + "profile": None, + "docker_commands": None, + "timeout": 120, + "extract_media": False, + "extract_screenshot": True, + "socks_proxy_host": None, + "socks_proxy_port": None, + "proxy_server": None, + } + wacz = setup_module("wacz_extractor_enricher", configs) + return wacz + + +def test_setup_without_docker(wacz_enricher, mocker): + mocker.patch.dict(os.environ, {"RUNNING_IN_DOCKER": "1"}, clear=True) + wacz_enricher.setup() + assert not wacz_enricher.docker_in_docker + + +def test_setup_with_docker(wacz_enricher, mocker): + mocker.patch.dict(os.environ, {"WACZ_ENABLE_DOCKER": "1"}, clear=True) + wacz_enricher.setup() + assert wacz_enricher.use_docker + + +def test_already_ran(wacz_enricher, metadata, mocker): + metadata.add_media(Media("test.wacz"), id="browsertrix") + mock_log = mocker.patch("loguru.logger.info") + assert wacz_enricher.enrich(metadata) is True + assert "WACZ enricher had already been executed" in mock_log.call_args[0][0] + + +def test_basic_call_execution(wacz_enricher, mocker): + mock_run = mocker.patch("subprocess.run") + mock_run.return_value = mocker.Mock(returncode=0) + metadata = Metadata().set_url("https://example.com") + wacz_enricher.enrich(metadata) + assert mock_run.called + # Checks that the url is passed to the cmd + assert "--url https://example.com" in " ".join(mock_run.call_args[0][0]) + + +def test_download_success(wacz_enricher, mocker) -> None: + """Test download returns metadata on successful enrichment.""" + basic_metadata = Metadata().set_url("https://example.com") + mocker.patch.object(wacz_enricher, "enrich", return_value=True) + result = wacz_enricher.download(basic_metadata) + assert result is not None + assert isinstance(result, Metadata) + assert result.status == "wacz: success" + + +def test_enrich_already_executed(wacz_enricher, mocker) -> None: + """Test enrich if already executed.""" + mock_log = mocker.patch("loguru.logger.info") + metadata = Metadata().set_url("https://example.com") + media = Media(filename="some_file.wacz") + metadata.add_media(media, id="browsertrix") + result = wacz_enricher.enrich(metadata) + assert result is True + assert "WACZ enricher had already been executed:" in mock_log.call_args[0][0] + + +def test_enrich_subprocess_exception(wacz_enricher, mocker, tmp_path) -> None: + """Test enrich returns False when subprocess fails.""" + wacz_enricher.tmp_dir = str(tmp_path) + wacz_enricher.extract_media = False + wacz_enricher.extract_screenshot = True + mocker.patch("auto_archiver.utils.misc.random_str", return_value="TESTCOL") + mocker.patch("subprocess.run", side_effect=Exception("fail")) + basic_metadata = Metadata().set_url("https://example.com") + result = wacz_enricher.enrich(basic_metadata) + assert result is False + + +def test_extract_media(wacz_enricher, metadata, tmp_path, mocker) -> None: + """Test extract_media_from_wacz extracts screenshot media.""" + wacz_enricher.tmp_dir = str(tmp_path) + + # Create a *real* zip file so ZipFile won't fail. + wacz_file = tmp_path / "dummy.wacz" + with ZipFile(wacz_file, "w") as zf: + zf.writestr("dummy.txt", "test content") + + mocker.patch("os.listdir", return_value=[]) + warc_data = ( + b"WARC/1.0\r\n" + b"WARC-Type: resource\r\n" + b"Content-Type: image/png\r\n" + b"WARC-Target-URI: http://example.com/image.png\r\n" + b"Content-Length: 12\r\n" + b"\r\n" + b"image-bytes" + b"\r\n\r\nWARC/1.0\r\n\r\n" + ) + mock_file = mocker.mock_open(read_data=warc_data) + mocker.patch("builtins.open", mock_file) + metadata.add_media(Media("something.wacz"), "browsertrix") + wacz_enricher.extract_media_from_wacz(metadata, str(wacz_file)) + assert len(metadata.media) == 2 + assert metadata.media[1].properties.get("id") == "browsertrix-screenshot" diff --git a/tests/enrichers/test_wayback_enricher.py b/tests/enrichers/test_wayback_enricher.py new file mode 100644 index 0000000..5406e39 --- /dev/null +++ b/tests/enrichers/test_wayback_enricher.py @@ -0,0 +1,168 @@ +import json +import requests +import pytest +from auto_archiver.modules.wayback_extractor_enricher import WaybackExtractorEnricher +from auto_archiver.core import Metadata + + +@pytest.fixture +def mock_is_auth_wall(mocker): + """Fixture to mock is_auth_wall behavior.""" + def _mock_is_auth_wall(return_value: bool): + return mocker.patch("auto_archiver.utils.url.is_auth_wall", return_value=return_value) + return _mock_is_auth_wall + +@pytest.fixture +def mock_post_success(mocker): + """Fixture to mock POST requests with a successful response.""" + def _mock_post(json_data: dict = None, status_code: int = 200): + json_data = {"job_id": "job123"} if json_data is None else json_data + resp = mocker.Mock(status_code=status_code) + resp.json.return_value = json_data + return mocker.patch("requests.post", return_value=resp) + return _mock_post + +@pytest.fixture +def mock_get_success(mocker): + """Fixture to mock GET requests returning a completed archive status.""" + def _mock_get(json_data: dict = None, status_code: int = 200): + json_data = json_data or { + "status": "success", + "timestamp": "20250101010101", + "original_url": "https://example.com" + } + resp = mocker.Mock(status_code=status_code) + resp.json.return_value = json_data + return mocker.patch("requests.get", return_value=resp) + return _mock_get + +@pytest.fixture +def wayback_extractor_enricher(setup_module) -> WaybackExtractorEnricher: + configs: dict = { + "timeout": 5, + "if_not_archived_within": None, + "key": "somekey", + "secret": "secret", + "proxy_http": None, + "proxy_https": None, + } + return setup_module("wayback_extractor_enricher", configs) + + +def test_download_success( + wayback_extractor_enricher, + mock_is_auth_wall, + mock_post_success, + mock_get_success +): + mock_is_auth_wall(False) + mock_post_success() + mock_get_success() + # Basic metadata to allow merge + metadata = Metadata().set_url("https://example.com") + result = wayback_extractor_enricher.download(metadata) + assert result.get("wayback") == "https://web.archive.org/web/20250101010101/https://example.com" + +def test_enrich_auth_wall(wayback_extractor_enricher, metadata, mock_is_auth_wall): + mock_is_auth_wall(True) + result = wayback_extractor_enricher.enrich(metadata) + assert result is None + +def test_enrich_already_enriched(wayback_extractor_enricher, metadata): + metadata.set("wayback", "existing") + result = wayback_extractor_enricher.enrich(metadata) + assert result is True + +def test_enrich_post_failure( + wayback_extractor_enricher, + metadata, + mock_is_auth_wall, + mock_post_success +): + mock_is_auth_wall(False) + mock_post_success(json_data={"error": "server error"}, status_code=500) + result = wayback_extractor_enricher.enrich(metadata) + assert result is False + assert "Internet archive failed with status of 500" in metadata.get("wayback") + +def test_enrich_post_json_decode_error( + wayback_extractor_enricher, + metadata, + mock_is_auth_wall, + mocker +): + mock_is_auth_wall(False) + resp = mocker.Mock(status_code=200) + resp.json.side_effect = json.decoder.JSONDecodeError("msg", "doc", 0) + resp.text = "invalid json" + mocker.patch("requests.post", return_value=resp) + assert wayback_extractor_enricher.enrich(metadata) is False + +def test_enrich_no_job_id( + wayback_extractor_enricher, + metadata, + mock_is_auth_wall, + mock_post_success +): + mock_is_auth_wall(False) + mock_post_success(json_data={}) + assert wayback_extractor_enricher.enrich(metadata) is False + +def test_enrich_get_success( + wayback_extractor_enricher, + metadata, + mock_is_auth_wall, + mock_post_success, + mock_get_success +): + mock_is_auth_wall(False) + mock_post_success() + mock_get_success() + assert wayback_extractor_enricher.enrich(metadata) is True + assert metadata.get("wayback") == "https://web.archive.org/web/20250101010101/https://example.com" + assert metadata.get("check wayback") == "https://web.archive.org/web/*/https://example.com" + +def test_enrich_get_failure( + wayback_extractor_enricher, + metadata, + mock_is_auth_wall, + mock_post_success, + mock_get_success +): + mock_is_auth_wall(False) + mock_post_success() + mock_get_success(json_data={"status": "failed"}, status_code=400) + assert wayback_extractor_enricher.enrich(metadata) is False + +def test_enrich_get_request_exception( + wayback_extractor_enricher, + metadata, + mock_is_auth_wall, + mock_post_success, + mocker +): + mock_is_auth_wall(False) + mock_post_success() + mocker.patch("requests.get", side_effect=requests.exceptions.RequestException("error")) + mocker.patch("time.sleep", return_value=None) + # check it still enriches the job_id information + assert wayback_extractor_enricher.enrich(metadata) is True + assert metadata.get("wayback").get("job_id") == "job123" + +def test_enrich_get_json_decode_error( + wayback_extractor_enricher, + metadata, + mock_is_auth_wall, + mock_post_success, + mocker +): + mock_is_auth_wall(False) + mock_post_success() + resp = mocker.Mock() + resp.json.side_effect = json.decoder.JSONDecodeError("msg", "doc", 0) + resp.text = "invalid json" + mocker.patch("requests.get", return_value=resp) + mocker.patch("time.sleep", return_value=None) + # check it still enriches the job_id information + assert wayback_extractor_enricher.enrich(metadata) is True + assert metadata.get("wayback").get("job_id") == "job123" diff --git a/tests/enrichers/test_whisper_enricher.py b/tests/enrichers/test_whisper_enricher.py new file mode 100644 index 0000000..ee1844a --- /dev/null +++ b/tests/enrichers/test_whisper_enricher.py @@ -0,0 +1,133 @@ +import pytest + +from auto_archiver.core import Metadata, Media +from auto_archiver.modules.s3_storage import S3Storage +from auto_archiver.modules.whisper_enricher import WhisperEnricher + +TEST_S3_URL = "http://cdn.example.com/test.mp4" + + +@pytest.fixture +def enricher(mocker): + """Fixture with mocked S3 and API dependencies""" + config = { + "api_endpoint": "http://testapi", + "api_key": "whisper-key", + "include_srt": False, + "timeout": 5, + "action": "translate", + "steps": {"storages": ["s3_storage"]} + } + mock_s3 = mocker.MagicMock(spec=S3Storage) + mock_s3.get_cdn_url.return_value = TEST_S3_URL + instance = WhisperEnricher() + instance.name = "whisper_enricher" + instance.display_name = "Whisper Enricher" + instance.config_setup({instance.name: config}) + # bypassing the setup method and mocking S3 setup + instance.stores = config['steps']['storages'] + instance.s3 = mock_s3 + yield instance, mock_s3 + + +@pytest.fixture +def metadata(): + metadata = Metadata() + metadata.set_url("http://test.url") + metadata.set_title("test title") + return metadata + + +@pytest.fixture +def mock_requests(mocker): + mock_requests = mocker.patch("auto_archiver.modules.whisper_enricher.whisper_enricher.requests") + mock_response = mocker.MagicMock() + mock_response.status_code = 201 + mock_response.json.return_value = {"id": "job123"} + mock_requests.post.return_value = mock_response + yield mock_requests + + +def test_successful_job_submission(enricher, metadata, mock_requests, mocker): + """Test successful media processing with S3 configured""" + whisper, mock_s3 = enricher + # Configure mock S3 URL to match test expectation + mock_s3.get_cdn_url.return_value = TEST_S3_URL + + # Create test media with matching CDN URL + m = Media("test.mp4") + m.mimetype = "video/mp4" + m.add_url(mock_s3.get_cdn_url.return_value) + metadata.media = [m] + + # Mock the complete API interaction chain + mock_status_response = mocker.MagicMock() + mock_status_response.status_code = 200 + mock_status_response.json.return_value = { + "status": "success", + "meta": {} + } + mock_artifacts_response = mocker.MagicMock() + mock_artifacts_response.status_code = 200 + mock_artifacts_response.json.return_value = [{ + "data": [{"start": 0, "end": 5, "text": "test transcript"}] + }] + # Set up mock response sequence + mock_requests.get.side_effect = [ + mock_status_response, # First call: status check + mock_artifacts_response # Second call: artifacts check + ] + + # Run enrichment (without opening file) + whisper.enrich(metadata) + # Check API interactions + mock_requests.post.assert_called_once_with( + "http://testapi/jobs", + json={"url": "http://cdn.example.com/test.mp4", "type": "translate"}, + headers={"Authorization": "Bearer whisper-key"} + ) + # Verify job status checks + assert mock_requests.get.call_count == 2 + assert "artifact_0_text" in metadata.media[0].get("whisper_model") + assert metadata.media[0].get("whisper_model") == {'artifact_0_text': 'test transcript', + 'job_artifacts_check': 'http://testapi/jobs/job123/artifacts', + 'job_id': 'job123', + 'job_status_check': 'http://testapi/jobs/job123'} + + +def test_submit_job(enricher, mocker): + """Test job submission method""" + whisper, _ = enricher + m = Media("test.mp4") + m.add_url(TEST_S3_URL) + mock_requests = mocker.patch("auto_archiver.modules.whisper_enricher.whisper_enricher.requests") + mock_response = mocker.MagicMock() + mock_response.status_code = 201 + mock_response.json.return_value = {"id": "job123"} + mock_requests.post.return_value = mock_response + job_id = whisper.submit_job(m) + assert job_id == "job123" + + +def test_submit_raises_status(enricher, mocker): + whisper, _ = enricher + m = Media("test.mp4") + m.add_url(TEST_S3_URL) + mock_requests = mocker.patch("auto_archiver.modules.whisper_enricher.whisper_enricher.requests") + mock_response = mocker.MagicMock() + mock_response.status_code = 400 + mock_response.json.return_value = {"id": "job123"} + mock_requests.post.return_value = mock_response + with pytest.raises(AssertionError) as exc_info: + whisper.submit_job(m) + assert str(exc_info.value) == "calling the whisper api http://testapi returned a non-success code: 400" + + +# @pytest.mark.parametrize("test_url, status", ["http://cdn.example.com/test.mp4",]) +def test_submit_job_fails(enricher): + """Test assertion fails with non-S3 URL""" + whisper, mock_s3 = enricher + m = Media("test.mp4") + m.add_url("http://cdn.wrongurl.com/test.mp4") + with pytest.raises(AssertionError): + whisper.submit_job(m) diff --git a/tests/extractors/test_generic_extractor.py b/tests/extractors/test_generic_extractor.py index c70a51f..33f35b7 100644 --- a/tests/extractors/test_generic_extractor.py +++ b/tests/extractors/test_generic_extractor.py @@ -9,6 +9,7 @@ import pytest from auto_archiver.modules.generic_extractor.generic_extractor import GenericExtractor from .test_extractor_base import TestExtractorBase +CI=os.getenv("GITHUB_ACTIONS", '') == 'true' class TestGenericExtractor(TestExtractorBase): """Tests Generic Extractor """ @@ -67,7 +68,7 @@ class TestGenericExtractor(TestExtractorBase): "twitter.com/bellingcat/status/123", "https://www.youtube.com/watch?v=1" ]) - def test_download_nonexistend_media(self, make_item, url): + def test_download_nonexistent_media(self, make_item, url): """ Test to make sure that the extractor doesn't break on non-existend posts/media @@ -77,10 +78,11 @@ class TestGenericExtractor(TestExtractorBase): result = self.extractor.download(item) assert not result - + @pytest.mark.skipif(CI, reason="Currently no way to authenticate when on CI. Youtube (yt-dlp) doesn't support logging in with username/password.") @pytest.mark.download def test_youtube_download(self, make_item): # url https://www.youtube.com/watch?v=5qap5aO4i9A + item = make_item("https://www.youtube.com/watch?v=J---aiyznGQ") result = self.extractor.download(item) assert result.get_url() == "https://www.youtube.com/watch?v=J---aiyznGQ" @@ -114,6 +116,7 @@ class TestGenericExtractor(TestExtractorBase): result = self.extractor.download(item) assert result is not False + @pytest.mark.skipif(CI, reason="Truth social blocks GH actions.") @pytest.mark.download def test_truthsocial_download_video(self, make_item): item = make_item("https://truthsocial.com/@DaynaTrueman/posts/110602446619561579") @@ -121,18 +124,21 @@ class TestGenericExtractor(TestExtractorBase): assert len(result.media) == 1 assert result is not False + @pytest.mark.skipif(CI, reason="Truth social blocks GH actions.") @pytest.mark.download def test_truthsocial_download_no_media(self, make_item): item = make_item("https://truthsocial.com/@bbcnewa/posts/109598702184774628") result = self.extractor.download(item) assert result is not False + @pytest.mark.skipif(CI, reason="Truth social blocks GH actions.") @pytest.mark.download def test_truthsocial_download_poll(self, make_item): item = make_item("https://truthsocial.com/@CNN_US/posts/113724326568555098") result = self.extractor.download(item) assert result is not False + @pytest.mark.skipif(CI, reason="Truth social blocks GH actions.") @pytest.mark.download def test_truthsocial_download_single_image(self, make_item): item = make_item("https://truthsocial.com/@mariabartiromo/posts/113861116433335006") @@ -140,6 +146,7 @@ class TestGenericExtractor(TestExtractorBase): assert len(result.media) == 1 assert result is not False + @pytest.mark.skipif(CI, reason="Truth social blocks GH actions.") @pytest.mark.download def test_truthsocial_download_multiple_images(self, make_item): item = make_item("https://truthsocial.com/@trrth/posts/113861302149349135") diff --git a/tests/extractors/test_instagram_api_extractor.py b/tests/extractors/test_instagram_api_extractor.py index c119e3f..7eba8e9 100644 --- a/tests/extractors/test_instagram_api_extractor.py +++ b/tests/extractors/test_instagram_api_extractor.py @@ -1,15 +1,12 @@ from datetime import datetime -from typing import Type import pytest -from unittest.mock import patch, MagicMock from auto_archiver.core import Metadata from auto_archiver.modules.instagram_api_extractor.instagram_api_extractor import InstagramAPIExtractor from .test_extractor_base import TestExtractorBase - @pytest.fixture def mock_user_response(): return { @@ -115,74 +112,74 @@ class TestInstagramAPIExtractor(TestExtractorBase): # test gets text (metadata title) pass - def test_download_profile_basic(self, metadata, mock_user_response): + def test_download_profile_basic(self, metadata, mock_user_response, mocker): """Test basic profile download without full_profile""" - with patch.object(self.extractor, 'call_api') as mock_call, \ - patch.object(self.extractor, 'download_from_url') as mock_download: - # Mock API responses - mock_call.return_value = mock_user_response - mock_download.return_value = "profile.jpg" + mock_call = mocker.patch.object(self.extractor, 'call_api') + mock_download = mocker.patch.object(self.extractor, 'download_from_url') + # Mock API responses + mock_call.return_value = mock_user_response + mock_download.return_value = "profile.jpg" - result = self.extractor.download_profile(metadata, "test_user") - assert result.status == "insta profile: success" - assert result.get_title() == "Test User" - assert result.get("data") == self.extractor.cleanup_dict(mock_user_response["user"]) - # Verify profile picture download - mock_call.assert_called_once_with("v2/user/by/username", {"username": "test_user"}) - mock_download.assert_called_once_with("http://example.com/profile.jpg") - assert len(result.media) == 1 - assert result.media[0].filename == "profile.jpg" + result = self.extractor.download_profile(metadata, "test_user") + assert result.status == "insta profile: success" + assert result.get_title() == "Test User" + assert result.get("data") == self.extractor.cleanup_dict(mock_user_response["user"]) + # Verify profile picture download + mock_call.assert_called_once_with("v2/user/by/username", {"username": "test_user"}) + mock_download.assert_called_once_with("http://example.com/profile.jpg") + assert len(result.media) == 1 + assert result.media[0].filename == "profile.jpg" - def test_download_profile_full(self, metadata, mock_user_response, mock_story_response): + def test_download_profile_full(self, metadata, mock_user_response, mock_story_response, mocker): """Test full profile download with stories/posts""" - with patch.object(self.extractor, 'call_api') as mock_call, \ - patch.object(self.extractor, 'download_all_posts') as mock_posts, \ - patch.object(self.extractor, 'download_all_highlights') as mock_highlights, \ - patch.object(self.extractor, 'download_all_tagged') as mock_tagged, \ - patch.object(self.extractor, '_download_stories_reusable') as mock_stories: + mock_call = mocker.patch.object(self.extractor, 'call_api') + mock_posts = mocker.patch.object(self.extractor, 'download_all_posts') + mock_highlights = mocker.patch.object(self.extractor, 'download_all_highlights') + mock_tagged = mocker.patch.object(self.extractor, 'download_all_tagged') + mock_stories = mocker.patch.object(self.extractor, '_download_stories_reusable') - self.extractor.full_profile = True - mock_call.side_effect = [ - mock_user_response, - mock_story_response - ] - mock_highlights.return_value = None - mock_stories.return_value = mock_story_response - mock_posts.return_value = None - mock_tagged.return_value = None + self.extractor.full_profile = True + mock_call.side_effect = [ + mock_user_response, + mock_story_response + ] + mock_highlights.return_value = None + mock_stories.return_value = mock_story_response + mock_posts.return_value = None + mock_tagged.return_value = None - result = self.extractor.download_profile(metadata, "test_user") - assert result.get("#stories") == len(mock_story_response) - mock_posts.assert_called_once_with(result, "123") - assert "errors" not in result.metadata + result = self.extractor.download_profile(metadata, "test_user") + assert result.get("#stories") == len(mock_story_response) + mock_posts.assert_called_once_with(result, "123") + assert "errors" not in result.metadata - def test_download_profile_not_found(self, metadata): + def test_download_profile_not_found(self, metadata, mocker): """Test profile not found error""" - with patch.object(self.extractor, 'call_api') as mock_call: - mock_call.return_value = {"user": None} - with pytest.raises(AssertionError) as exc_info: - self.extractor.download_profile(metadata, "invalid_user") - assert "User invalid_user not found" in str(exc_info.value) + mock_call = mocker.patch.object(self.extractor, 'call_api') + mock_call.return_value = {"user": None} + with pytest.raises(AssertionError) as exc_info: + self.extractor.download_profile(metadata, "invalid_user") + assert "User invalid_user not found" in str(exc_info.value) - def test_download_profile_error_handling(self, metadata, mock_user_response): + def test_download_profile_error_handling(self, metadata, mock_user_response, mocker): """Test error handling in full profile mode""" - with (patch.object(self.extractor, 'call_api') as mock_call, \ - patch.object(self.extractor, 'download_all_highlights') as mock_highlights, \ - patch.object(self.extractor, 'download_all_tagged') as mock_tagged, \ - patch.object(self.extractor, '_download_stories_reusable') as stories_tagged, \ - patch.object(self.extractor, 'download_all_posts') as mock_posts - ): - self.extractor.full_profile = True - mock_call.side_effect = [ - mock_user_response, - Exception("Stories API failed"), - Exception("Posts API failed") - ] - mock_highlights.return_value = None - mock_tagged.return_value = None - stories_tagged.return_value = None - mock_posts.return_value = None - result = self.extractor.download_profile(metadata, "test_user") + mock_call = mocker.patch.object(self.extractor, 'call_api') + mock_highlights = mocker.patch.object(self.extractor, 'download_all_highlights') + mock_tagged = mocker.patch.object(self.extractor, 'download_all_tagged') + stories_tagged = mocker.patch.object(self.extractor, '_download_stories_reusable') + mock_posts = mocker.patch.object(self.extractor, 'download_all_posts') - assert result.is_success() - assert "Error downloading stories for test_user" in result.metadata["errors"] \ No newline at end of file + self.extractor.full_profile = True + mock_call.side_effect = [ + mock_user_response, + Exception("Stories API failed"), + Exception("Posts API failed") + ] + mock_highlights.return_value = None + mock_tagged.return_value = None + stories_tagged.return_value = None + mock_posts.return_value = None + result = self.extractor.download_profile(metadata, "test_user") + + assert result.is_success() + assert "Error downloading stories for test_user" in result.metadata["errors"] \ No newline at end of file diff --git a/tests/extractors/test_instagram_extractor.py b/tests/extractors/test_instagram_extractor.py index 7efe1b1..647cab4 100644 --- a/tests/extractors/test_instagram_extractor.py +++ b/tests/extractors/test_instagram_extractor.py @@ -1,21 +1,36 @@ import pytest from auto_archiver.modules.instagram_extractor import InstagramExtractor -from .test_extractor_base import TestExtractorBase -class TestInstagramExtractor(TestExtractorBase): + +@pytest.fixture +def instagram_extractor(setup_module, mocker): extractor_module: str = 'instagram_extractor' - config: dict = {} + config: dict = { + "username": "user_name", + "password": "password123", + "download_folder": "instaloader", + "session_file": "secrets/instaloader.session", + } + fake_loader = mocker.MagicMock() + fake_loader.load_session_from_file.return_value = None + fake_loader.login.return_value = None + fake_loader.save_session_to_file.return_value = None + mocker.patch("instaloader.Instaloader", return_value=fake_loader,) + return setup_module(extractor_module, config) - @pytest.mark.parametrize("url", [ - "https://www.instagram.com/p/", - "https://www.instagram.com/p/1234567890/", - "https://www.instagram.com/reel/1234567890/", - "https://www.instagram.com/username/", - "https://www.instagram.com/username/stories/", - "https://www.instagram.com/username/highlights/", - ]) - def test_regex_matches(self, url): - # post - assert InstagramExtractor.valid_url.match(url) + +@pytest.mark.parametrize("url", [ + "https://www.instagram.com/p/", + "https://www.instagram.com/p/1234567890/", + "https://www.instagram.com/reel/1234567890/", + "https://www.instagram.com/username/", + "https://www.instagram.com/username/stories/", + "https://www.instagram.com/username/highlights/", +]) +def test_regex_matches(url: str, instagram_extractor: InstagramExtractor) -> None: + """ + Ensure that the valid_url regex matches all provided Instagram URLs. + """ + assert instagram_extractor.valid_url.match(url) \ No newline at end of file diff --git a/tests/extractors/test_instagram_tbot_extractor.py b/tests/extractors/test_instagram_tbot_extractor.py index d7a1e53..f274728 100644 --- a/tests/extractors/test_instagram_tbot_extractor.py +++ b/tests/extractors/test_instagram_tbot_extractor.py @@ -1,94 +1,108 @@ import os -from typing import Type -from unittest.mock import patch, MagicMock import pytest from auto_archiver.core import Metadata -from auto_archiver.core.extractor import Extractor from auto_archiver.modules.instagram_tbot_extractor import InstagramTbotExtractor from tests.extractors.test_extractor_base import TestExtractorBase -TESTFILES = os.path.join(os.path.dirname(__file__), "testfiles") - @pytest.fixture -def session_file(tmpdir): - """Fixture to create a test session file.""" - session_file = os.path.join(tmpdir, "test_session.session") - with open(session_file, "w") as f: - f.write("mock_session_data") - return session_file.replace(".session", "") +def patch_extractor_methods(request, setup_module, mocker): + mocker.patch.object(InstagramTbotExtractor, '_prepare_session_file', return_value=None) + mocker.patch.object(InstagramTbotExtractor, '_initialize_telegram_client', return_value=None) + yield -@pytest.fixture(autouse=True) -def patch_extractor_methods(request, setup_module): - with patch.object(InstagramTbotExtractor, '_prepare_session_file', return_value=None), \ - patch.object(InstagramTbotExtractor, '_initialize_telegram_client', return_value=None): - if hasattr(request, 'cls') and hasattr(request.cls, 'config'): - request.cls.extractor = setup_module("instagram_tbot_extractor", request.cls.config) - - yield - @pytest.fixture def metadata_sample(): m = Metadata() m.set_title("Test Title") - m.set_timestamp("2021-01-01T00:00:00Z") + m.set_timestamp("2021-01-01T00:00:00") m.set_url("https://www.instagram.com/p/1234567890") return m -class TestInstagramTbotExtractor: +@pytest.fixture +def mock_telegram_client(mocker): + """Fixture to mock TelegramClient interactions.""" + mock_client = mocker.patch("auto_archiver.modules.instagram_tbot_extractor.client") + instance = mocker.MagicMock() + mock_client.return_value = instance + return instance + +@pytest.fixture +def extractor(setup_module, patch_extractor_methods, mocker): extractor_module = "instagram_tbot_extractor" - extractor: InstagramTbotExtractor config = { "api_id": 12345, "api_hash": "test_api_hash", "session_file": "test_session", + "timeout": 4 + } + extractor = setup_module(extractor_module, config) + extractor.client = mocker.MagicMock() + extractor.session_file = "test_session" + return extractor + + +def test_non_instagram_url(extractor, metadata_sample): + metadata_sample.set_url("https://www.youtube.com") + assert extractor.download(metadata_sample) is False + + +def test_download_success(extractor, metadata_sample, mocker): + mocker.patch.object(extractor, "_send_url_to_bot", return_value=(mocker.MagicMock(), 101)) + mocker.patch.object(extractor, "_process_messages", return_value="Sample Instagram post caption") + result = extractor.download(metadata_sample) + assert result.is_success() + assert result.status == "insta-via-bot: success" + assert result.metadata.get("title") == "Sample Instagram post caption" + + +def test_download_invalid(extractor, metadata_sample, mocker): + mocker.patch.object(extractor, "_send_url_to_bot", return_value=(mocker.MagicMock(), 101)) + mocker.patch.object(extractor, "_process_messages", return_value="You must enter a URL to a post") + assert extractor.download(metadata_sample) is False + + +@pytest.mark.skip(reason="Requires authentication.") +class TestInstagramTbotExtractorReal(TestExtractorBase): + # To run these tests set the TELEGRAM_API_ID and TELEGRAM_API_HASH environment variables, and ensure the session file exists. + # Note these are true at this point in time, but changes to source media could be reason for failure. + extractor_module = "instagram_tbot_extractor" + extractor: InstagramTbotExtractor + config = { + "api_id": os.environ.get("TELEGRAM_API_ID"), + "api_hash": os.environ.get("TELEGRAM_API_HASH"), + "session_file": "secrets/anon-insta", } - @pytest.fixture - def mock_telegram_client(self): - """Fixture to mock TelegramClient interactions.""" - with patch("auto_archiver.modules.instagram_tbot_extractor._initialize_telegram_client") as mock_client: - instance = MagicMock() - mock_client.return_value = instance - yield instance - - def test_extractor_is_initialized(self): - assert self.extractor is not None - - - @patch("time.sleep") - @pytest.mark.parametrize("url, expected_status, bot_responses", [ - ("https://www.instagram.com/p/C4QgLbrIKXG", "insta-via-bot: success", [MagicMock(id=101, media=None, message="Are you new to Bellingcat? - The way we share our investigations is different. 💭\nWe want you to read our story but also learn ou")]), - ("https://www.instagram.com/reel/DEVLK8qoIbg/", "insta-via-bot: success", [MagicMock(id=101, media=None, message="Our volunteer community is at the centre of many incredible Bellingcat investigations and tools. Stephanie Ladel is one such vol")]), - # todo tbot not working for stories :( - ("https://www.instagram.com/stories/bellingcatofficial/3556336382743057476/", False, [MagicMock(id=101, media=None, message="Media not found or unavailable")]), - ("https://www.youtube.com/watch?v=ymCMy8OffHM", False, []), - ("https://www.instagram.com/p/INVALID", False, [MagicMock(id=101, media=None, message="You must enter a URL to a post")]), + @pytest.mark.parametrize("url, expected_status, message, len_media", [ + ("https://www.instagram.com/p/C4QgLbrIKXG", "insta-via-bot: success", + "Are you new to Bellingcat? - The way we share our investigations is different. 💭\nWe want you to read our story but also learn ou", + 6), + ("https://www.instagram.com/reel/DEVLK8qoIbg/", "insta-via-bot: success", + "Our volunteer community is at the centre of many incredible Bellingcat investigations and tools. Stephanie Ladel is one such vol", + 3), + # instagram tbot not working (potentially intermittently?) for stories - replace with a live story to retest + # ("https://www.instagram.com/stories/bellingcatofficial/3556336382743057476/", False, "Media not found or unavailable"), + # Seems to be working intermittently for highlights + # ("https://www.instagram.com/stories/highlights/17868810693068139/", "insta-via-bot: success", None, 50), + # Marking invalid url as success + ("https://www.instagram.com/p/INVALID", "insta-via-bot: success", "Media not found or unavailable", 0), + ("https://www.youtube.com/watch?v=ymCMy8OffHM", False, None, 0), ]) - def test_download(self, mock_sleep, url, expected_status, bot_responses, metadata_sample): + def test_download(self, url, expected_status, message, len_media, metadata_sample): """Test the `download()` method with various Instagram URLs.""" metadata_sample.set_url(url) - self.extractor.client = MagicMock() + result = self.extractor.download(metadata_sample) - pass - # TODO fully mock or use as authenticated test - # if expected_status: - # assert result.is_success() - # assert result.status == expected_status - # assert result.metadata.get("title") in [msg.message[:128] for msg in bot_responses if msg.message] - # else: - # assert result is False - - - - - # Test story -# Test expired story -# Test requires login/ access (?) -# Test post -# Test multiple images? \ No newline at end of file + if expected_status: + assert result.is_success() + assert result.status == expected_status + assert result.metadata.get("title") == message + assert len(result.media) == len_media + else: + assert result is False diff --git a/tests/extractors/test_tiktok_tikwm_extractor.py b/tests/extractors/test_tiktok_tikwm_extractor.py new file mode 100644 index 0000000..e8ad8df --- /dev/null +++ b/tests/extractors/test_tiktok_tikwm_extractor.py @@ -0,0 +1,154 @@ +from datetime import datetime, timezone +import time +import pytest + +from auto_archiver.modules.tiktok_tikwm_extractor.tiktok_tikwm_extractor import TiktokTikwmExtractor +from .test_extractor_base import TestExtractorBase + + +class TestTiktokTikwmExtractor(TestExtractorBase): + """ + Test suite for TestTiktokTikwmExtractor. + """ + + extractor_module = "tiktok_tikwm_extractor" + extractor: TiktokTikwmExtractor + + config = {} + + VALID_EXAMPLE_URL = "https://www.tiktok.com/@example/video/1234" + + @staticmethod + def get_mockers(mocker): + mock_get = mocker.patch("auto_archiver.modules.tiktok_tikwm_extractor.tiktok_tikwm_extractor.requests.get") + mock_logger = mocker.patch("auto_archiver.modules.tiktok_tikwm_extractor.tiktok_tikwm_extractor.logger") + return mock_get, mock_logger + + @pytest.mark.parametrize("url,valid_url", [ + ("https://bellingcat.com", False), + ("https://youtube.com", False), + ("https://tiktok.co/", False), + ("https://tiktok.com/", False), + ("https://www.tiktok.com/", False), + ("https://api.cool.tiktok.com/", False), + (VALID_EXAMPLE_URL, True), + ("https://www.tiktok.com/@bbcnews/video/7478038212070411542", True), + ("https://www.tiktok.com/@ggs68taiwan.official/video/7441821351142362375", True), + ]) + def test_valid_urls(self, mocker, make_item, url, valid_url): + mock_get, mock_logger = self.get_mockers(mocker) + if valid_url: + mock_get.return_value.status_code = 404 + assert self.extractor.download(make_item(url)) == False + assert mock_get.call_count == int(valid_url) + assert mock_logger.error.call_count == int(valid_url) + + def test_invalid_json_responses(self, mocker, make_item): + mock_get, mock_logger = self.get_mockers(mocker) + mock_get.return_value.status_code = 200 + mock_get.return_value.json.side_effect = ValueError + assert self.extractor.download(make_item(self.VALID_EXAMPLE_URL)) == False + mock_get.assert_called_once() + mock_get.return_value.json.assert_called_once() + mock_logger.error.assert_called_once() + assert mock_logger.error.call_args[0][0].startswith("failed to parse JSON response") + + mock_get.return_value.json.side_effect = Exception + with pytest.raises(Exception): + self.extractor.download(make_item(self.VALID_EXAMPLE_URL)) + mock_get.assert_called() + assert mock_get.call_count == 2 + assert mock_get.return_value.json.call_count == 2 + + @pytest.mark.parametrize("response", [ + ({"msg": "failure"}), + ({"msg": "success"}), + ]) + def test_unsuccessful_responses(self, mocker, make_item, response): + mock_get, mock_logger = self.get_mockers(mocker) + mock_get.return_value.status_code = 200 + mock_get.return_value.json.return_value = response + assert self.extractor.download(make_item(self.VALID_EXAMPLE_URL)) == False + mock_get.assert_called_once() + mock_get.return_value.json.assert_called_once() + mock_logger.error.assert_called_once() + assert mock_logger.error.call_args[0][0].startswith("failed to get a valid response") + + @pytest.mark.parametrize("response,has_vid", [ + ({"data": {"id": 123}}, False), + ({"data": {"wmplay": "url"}}, True), + ({"data": {"play": "url"}}, True), + ]) + def test_correct_extraction(self, mocker, make_item, response, has_vid): + mock_get, mock_logger = self.get_mockers(mocker) + mock_get.return_value.status_code = 200 + mock_get.return_value.json.return_value = {"msg": "success", **response} + + result = self.extractor.download(make_item(self.VALID_EXAMPLE_URL)) + if not has_vid: + assert result == False + else: + assert result.is_success() + assert len(result.media) == 1 + mock_get.assert_called() + assert mock_get.call_count == 1 + int(has_vid) + mock_get.return_value.json.assert_called_once() + if not has_vid: + mock_logger.error.assert_called_once() + assert mock_logger.error.call_args[0][0].startswith("no valid video URL found") + else: + mock_logger.error.assert_not_called() + + def test_correct_extraction(self, mocker, make_item): + mock_get, _ = self.get_mockers(mocker) + mock_get.return_value.status_code = 200 + mock_get.return_value.json.return_value = {"msg": "success", "data": { + "wmplay": "url", + "origin_cover": "cover.jpg", + "title": "Title", + "id": 123, + "duration": 60, + "create_time": 1736301699, + "author": "Author", + "other": "data" + }} + + result = self.extractor.download(make_item(self.VALID_EXAMPLE_URL)) + assert result.is_success() + assert len(result.media) == 2 + assert result.get_title() == "Title" + assert result.get("author") == "Author" + assert result.get("api_data") == {"other": "data", "id": 123} + assert result.media[1].get("duration") == 60 + assert result.get("timestamp") == datetime.fromtimestamp(1736301699, tz=timezone.utc) + + @pytest.mark.download + def test_download_video(self, make_item): + url = "https://www.tiktok.com/@bbcnews/video/7478038212070411542" + + result = self.extractor.download(make_item(url)) + assert result.is_success() + assert len(result.media) == 2 + assert result.get_title() == "The A23a iceberg is one of the world's oldest and it's so big you can see it from space. #Iceberg #A23a #Antarctica #Ice #ClimateChange #DavidAttenborough #Ocean #Sea #SouthGeorgia #BBCNews " + assert result.get("author").get("unique_id") == "bbcnews" + assert result.get("api_data").get("id") == '7478038212070411542' + assert result.media[1].get("duration") == 59 + assert result.get("timestamp") == datetime.fromtimestamp(1741122000, tz=timezone.utc) + + @pytest.mark.download + def test_download_sensitive_video(self, make_item, mock_sleep): + # sleep is needed because of the rate limit + mock_sleep.stop() + time.sleep(1.1) + mock_sleep.start() + + url = "https://www.tiktok.com/@ggs68taiwan.official/video/7441821351142362375" + + result = self.extractor.download(make_item(url)) + assert result.is_success() + assert len(result.media) == 2 + assert result.get_title() == "Căng nhất lúc này #ggs68 #ggs68taiwan #taiwan #dailoan #tiktoknews" + assert result.get("author").get("id") == "7197400619475649562" + assert result.get("api_data").get("id") == '7441821351142362375' + assert result.media[1].get("duration") == 34 + assert result.get("timestamp") == datetime.fromtimestamp(1732684060, tz=timezone.utc) diff --git a/tests/extractors/test_twitter_api_extractor.py b/tests/extractors/test_twitter_api_extractor.py index d9a8eb0..26394ac 100644 --- a/tests/extractors/test_twitter_api_extractor.py +++ b/tests/extractors/test_twitter_api_extractor.py @@ -23,7 +23,6 @@ class TestTwitterApiExtractor(TestExtractorBase): } @pytest.mark.parametrize("url, expected", [ - ("https://t.co/yl3oOJatFp", "https://www.bellingcat.com/category/resources/"), # t.co URL ("https://x.com/bellingcat/status/1874097816571961839", "https://x.com/bellingcat/status/1874097816571961839"), # x.com urls unchanged ("https://twitter.com/bellingcat/status/1874097816571961839", "https://twitter.com/bellingcat/status/1874097816571961839"), # twitter urls unchanged ("https://twitter.com/bellingcat/status/1874097816571961839?s=20&t=3d0g4ZQis7dCbSDg-mE7-w", "https://twitter.com/bellingcat/status/1874097816571961839?s=20&t=3d0g4ZQis7dCbSDg-mE7-w"), # don't strip params from twitter urls (changed Jan 2025) @@ -32,7 +31,11 @@ class TestTwitterApiExtractor(TestExtractorBase): ]) def test_sanitize_url(self, url, expected): assert expected == self.extractor.sanitize_url(url) - + + @pytest.mark.download + def test_sanitize_url_download(self): + assert "https://www.bellingcat.com/category/resources/" == self.extractor.sanitize_url("https://t.co/yl3oOJatFp") + @pytest.mark.parametrize("url, exptected_username, exptected_tweetid", [ ("https://twitter.com/bellingcat/status/1874097816571961839", "bellingcat", "1874097816571961839"), ("https://x.com/bellingcat/status/1874097816571961839", "bellingcat", "1874097816571961839"), diff --git a/tests/extractors/test_vk_extractor.py b/tests/extractors/test_vk_extractor.py new file mode 100644 index 0000000..80eb9dd --- /dev/null +++ b/tests/extractors/test_vk_extractor.py @@ -0,0 +1,76 @@ +import pytest + +from auto_archiver.core import Metadata +from auto_archiver.modules.vk_extractor import VkExtractor + + +@pytest.fixture +def mock_vk_scraper(mocker): + """Fixture to mock VkScraper.""" + return mocker.patch("auto_archiver.modules.vk_extractor.vk_extractor.VkScraper") + +@pytest.fixture +def vk_extractor(setup_module, mock_vk_scraper) -> VkExtractor: + """Fixture to initialize VkExtractor with mocked VkScraper.""" + extractor_module = "vk_extractor" + configs = { + "username": "name", + "password": "password123", + "session_file": "secrets/vk_config.v2.json", + } + vk = setup_module(extractor_module, configs) + vk.vks = mock_vk_scraper.return_value + return vk + + +def test_netloc(vk_extractor, metadata): + # metadata url set as: "https://example.com/" + assert vk_extractor.download(metadata) is False + + +def test_vk_url_but_scrape_returns_empty(vk_extractor, metadata): + metadata.set_url("https://vk.com/valid-wall") + vk_extractor.vks.scrape.return_value = [] + assert vk_extractor.download(metadata) is False + assert metadata.netloc == "vk.com" + vk_extractor.vks.scrape.assert_called_once_with(metadata.get_url()) + + +def test_successful_scrape_and_download(vk_extractor, metadata, mocker): + mock_scrapes = [ + {"text": "Post Title", "datetime": "2023-01-01T00:00:00", "id": 1}, + {"text": "Another Post", "datetime": "2023-01-02T00:00:00", "id": 2} + ] + mock_filenames = ["image1.jpg", "image2.png"] + vk_extractor.vks.scrape.return_value = mock_scrapes + vk_extractor.vks.download_media.return_value = mock_filenames + metadata.set_url("https://vk.com/valid-wall") + result = vk_extractor.download(metadata) + # Test metadata + assert result.is_success() + assert result.status == "vk: success" + assert result.get_title() == "Post Title" + assert result.get_timestamp() == "2023-01-01T00:00:00+00:00" + assert "Another Post" in result.metadata["content"] + # Test Media objects + assert len(result.media) == 2 + assert result.media[0].filename == "image1.jpg" + assert result.media[1].filename == "image2.png" + vk_extractor.vks.download_media.assert_called_once_with( + mock_scrapes, vk_extractor.tmp_dir + ) + + +def test_adds_first_title_and_timestamp(vk_extractor): + metadata = Metadata().set_url("https://vk.com/no-metadata") + metadata.set_url("https://vk.com/no-metadata") + mock_scrapes = [{"text": "value", "datetime": "2023-01-01T00:00:00"}, + {"text": "value2", "datetime": "2023-01-02T00:00:00"}] + vk_extractor.vks.scrape.return_value = mock_scrapes + vk_extractor.vks.download_media.return_value = [] + result = vk_extractor.download(metadata) + + assert result.get_title() == "value" + # formatted timestamp + assert result.get_timestamp() == "2023-01-01T00:00:00+00:00" + assert result.is_success() \ No newline at end of file diff --git a/tests/feeders/test_atlos_feeder.py b/tests/feeders/test_atlos_feeder.py new file mode 100644 index 0000000..1ef9fab --- /dev/null +++ b/tests/feeders/test_atlos_feeder.py @@ -0,0 +1,109 @@ +import pytest +from auto_archiver.modules.atlos_feeder_db_storage import AtlosFeederDbStorage as AtlosFeeder + + +class FakeAPIResponse: + """Simulate a response object.""" + + def __init__(self, data: dict, raise_error: bool = False) -> None: + self._data = data + self.raise_error = raise_error + + def json(self) -> dict: + return self._data + + def raise_for_status(self) -> None: + if self.raise_error: + raise Exception("HTTP error") + + +@pytest.fixture +def atlos_feeder(setup_module, mocker) -> AtlosFeeder: + """Fixture for AtlosFeeder.""" + configs: dict = { + "api_token": "abc123", + "atlos_url": "https://platform.atlos.org", + } + mocker.patch("requests.Session") + atlos_feeder = setup_module("atlos_feeder_db_storage", configs) + fake_session = mocker.MagicMock() + # Configure the default response to have no results so that __iter__ terminates + fake_session.get.return_value = FakeAPIResponse({"next": None, "results": []}) + atlos_feeder.session = fake_session + return atlos_feeder + + +@pytest.fixture +def mock_atlos_api(atlos_feeder): + """Fixture to update the atlos_feeder.session.get side_effect.""" + def _mock_responses(responses): + atlos_feeder.session.get.side_effect = [FakeAPIResponse(data) for data in responses] + return _mock_responses + + +def test_atlos_feeder_iter_yields_valid_metadata(atlos_feeder, mock_atlos_api): + """Test valid items are yielded and invalid ones ignored.""" + mock_atlos_api([ + { + "next": None, + "results": [ + {"source_url": "http://example.com", "id": 1, + "metadata": {"auto_archiver": {"processed": False}}, + "visibility": "visible", "status": "complete"}, + {"source_url": "", "id": 2, + "metadata": {"auto_archiver": {"processed": False}}, + "visibility": "visible", "status": "complete"}, + {"source_url": "http://example.org", "id": 3, + "metadata": {"auto_archiver": {"processed": True}}, + "visibility": "visible", "status": "complete"}, + ], + } + ]) + + items = list(atlos_feeder) + assert len(items) == 1 + assert items[0].get_url() == "http://example.com" + assert items[0].get("atlos_id") == 1 + + +def test_atlos_feeder_multiple_pages(atlos_feeder, mock_atlos_api): + """Test iteration over multiple pages with valid items.""" + mock_atlos_api([ + { + "next": "cursor2", + "results": [ + {"source_url": "http://example1.com", "id": 10, + "metadata": {"auto_archiver": {"processed": False}}, + "visibility": "visible", "status": "complete"}, + ], + }, + { + "next": None, + "results": [ + {"source_url": "http://example2.com", "id": 20, + "metadata": {"auto_archiver": {"processed": False}}, + "visibility": "visible", "status": "complete"}, + ], + }, + ]) + + items = list(atlos_feeder) + assert len(items) == 2 + assert items[0].get_url() == "http://example1.com" + assert items[0].get("atlos_id") == 10 + assert items[1].get_url() == "http://example2.com" + assert items[1].get("atlos_id") == 20 + + +def test_atlos_feeder_no_results(atlos_feeder, mock_atlos_api): + """Test iteration stops when no results are returned.""" + mock_atlos_api([{"next": None, "results": []}]) + assert list(atlos_feeder) == [] + + +def test_atlos_feeder_http_error(atlos_feeder, mocker): + """Test raises an exception on HTTP error.""" + fake_response = FakeAPIResponse({"next": None, "results": []}, raise_error=True) + atlos_feeder.session.get.side_effect = [fake_response] + with pytest.raises(Exception, match="HTTP error"): + list(atlos_feeder) diff --git a/tests/feeders/test_gsheet_feeder.py b/tests/feeders/test_gsheet_feeder.py index b86e329..9ca81b0 100644 --- a/tests/feeders/test_gsheet_feeder.py +++ b/tests/feeders/test_gsheet_feeder.py @@ -2,27 +2,23 @@ from typing import Type import gspread import pytest -from unittest.mock import patch, MagicMock -from auto_archiver.modules.gsheet_feeder import GsheetsFeeder +from auto_archiver.modules.gsheet_feeder_db import GsheetsFeederDB from auto_archiver.core import Metadata, Feeder -def test_setup_without_sheet_and_sheet_id(setup_module): +def test_setup_without_sheet_and_sheet_id(setup_module, mocker): # Ensure setup() raises AssertionError if neither sheet nor sheet_id is set. - with patch("gspread.service_account"): - with pytest.raises(AssertionError): - setup_module( - "gsheet_feeder", - {"service_account": "dummy.json", "sheet": None, "sheet_id": None}, - ) + mocker.patch("gspread.service_account") + with pytest.raises(ValueError): + setup_module( + "gsheet_feeder_db", + {"service_account": "dummy.json", "sheet": None, "sheet_id": None}, + ) @pytest.fixture -def gsheet_feeder(setup_module) -> GsheetsFeeder: - with patch("gspread.service_account"): - feeder = setup_module( - "gsheet_feeder", - { +def gsheet_feeder(setup_module, mocker) -> GsheetsFeederDB: + config: dict = { "service_account": "dummy.json", "sheet": "test-auto-archiver", "sheet_id": None, @@ -46,9 +42,13 @@ def gsheet_feeder(setup_module) -> GsheetsFeeder: "allow_worksheets": set(), "block_worksheets": set(), "use_sheet_names_in_stored_paths": True, - }, - ) - feeder.gsheets_client = MagicMock() + } + mocker.patch("gspread.service_account") + feeder = setup_module( + "gsheet_feeder_db", + config + ) + feeder.gsheets_client = mocker.MagicMock() return feeder @@ -90,7 +90,7 @@ class MockWorksheet: return matching.get(col_name, default) -def test__process_rows(gsheet_feeder: GsheetsFeeder): +def test__process_rows(gsheet_feeder: GsheetsFeederDB): testworksheet = MockWorksheet() metadata_items = list(gsheet_feeder._process_rows(testworksheet)) assert len(metadata_items) == 3 @@ -98,7 +98,7 @@ def test__process_rows(gsheet_feeder: GsheetsFeeder): assert metadata_items[0].get("url") == "http://example.com" -def test__set_metadata(gsheet_feeder: GsheetsFeeder): +def test__set_metadata(gsheet_feeder: GsheetsFeederDB): worksheet = MockWorksheet() metadata = Metadata() gsheet_feeder._set_context(metadata, worksheet, 1) @@ -106,12 +106,12 @@ def test__set_metadata(gsheet_feeder: GsheetsFeeder): @pytest.mark.skip(reason="Not recognising folder column") -def test__set_metadata_with_folder_pickled(gsheet_feeder: GsheetsFeeder, worksheet): +def test__set_metadata_with_folder_pickled(gsheet_feeder: GsheetsFeederDB, worksheet): gsheet_feeder._set_context(worksheet, 7) assert Metadata.get_context("gsheet") == {"row": 1, "worksheet": worksheet} -def test__set_metadata_with_folder(gsheet_feeder: GsheetsFeeder): +def test__set_metadata_with_folder(gsheet_feeder: GsheetsFeederDB): testworksheet = MockWorksheet() metadata = Metadata() testworksheet.wks.title = "TestSheet" @@ -129,56 +129,56 @@ def test__set_metadata_with_folder(gsheet_feeder: GsheetsFeeder): ], ) def test_open_sheet_with_name_or_id( - setup_module, sheet, sheet_id, expected_method, expected_arg, description + setup_module, sheet, sheet_id, expected_method, expected_arg, description, mocker ): """Ensure open_sheet() correctly opens by name or ID based on configuration.""" - with patch("gspread.service_account") as mock_service_account: - mock_client = MagicMock() - mock_service_account.return_value = mock_client - mock_client.open.return_value = "MockSheet" - mock_client.open_by_key.return_value = "MockSheet" + mock_service_account = mocker.patch("gspread.service_account") + mock_client = mocker.MagicMock() + mock_service_account.return_value = mock_client + mock_client.open.return_value = "MockSheet" + mock_client.open_by_key.return_value = "MockSheet" - # Setup module with parameterized values - feeder = setup_module( - "gsheet_feeder", - {"service_account": "dummy.json", "sheet": sheet, "sheet_id": sheet_id}, - ) - sheet_result = feeder.open_sheet() - # Validate the correct method was called - getattr(mock_client, expected_method).assert_called_once_with( - expected_arg - ), f"Failed: {description}" - assert sheet_result == "MockSheet", f"Failed: {description}" + # Setup module with parameterized values + feeder = setup_module( + "gsheet_feeder_db", + {"service_account": "dummy.json", "sheet": sheet, "sheet_id": sheet_id}, + ) + sheet_result = feeder.open_sheet() + # Validate the correct method was called + getattr(mock_client, expected_method).assert_called_once_with( + expected_arg + ), f"Failed: {description}" + assert sheet_result == "MockSheet", f"Failed: {description}" @pytest.mark.usefixtures("setup_module") -def test_open_sheet_with_sheet_id(setup_module): +def test_open_sheet_with_sheet_id(setup_module, mocker): """Ensure open_sheet() correctly opens a sheet by ID.""" - with patch("gspread.service_account") as mock_service_account: - mock_client = MagicMock() - mock_service_account.return_value = mock_client - mock_client.open_by_key.return_value = "MockSheet" - feeder = setup_module( - "gsheet_feeder", - {"service_account": "dummy.json", "sheet": None, "sheet_id": "ABC123"}, - ) - sheet = feeder.open_sheet() - mock_client.open_by_key.assert_called_once_with("ABC123") - assert sheet == "MockSheet" + mock_service_account = mocker.patch("gspread.service_account") + mock_client = mocker.MagicMock() + mock_service_account.return_value = mock_client + mock_client.open_by_key.return_value = "MockSheet" + feeder = setup_module( + "gsheet_feeder_db", + {"service_account": "dummy.json", "sheet": None, "sheet_id": "ABC123"}, + ) + sheet = feeder.open_sheet() + mock_client.open_by_key.assert_called_once_with("ABC123") + assert sheet == "MockSheet" -def test_should_process_sheet(setup_module): - with patch("gspread.service_account"): - gdb = setup_module( - "gsheet_feeder", - { - "service_account": "dummy.json", - "sheet": "TestSheet", - "sheet_id": None, - "allow_worksheets": {"TestSheet", "Sheet2"}, - "block_worksheets": {"Sheet3"}, - }, - ) +def test_should_process_sheet(setup_module, mocker): + mocker.patch("gspread.service_account") + gdb = setup_module( + "gsheet_feeder_db", + { + "service_account": "dummy.json", + "sheet": "TestSheet", + "sheet_id": None, + "allow_worksheets": {"TestSheet", "Sheet2"}, + "block_worksheets": {"Sheet3"}, + }, + ) assert gdb.should_process_sheet("TestSheet") == True assert gdb.should_process_sheet("Sheet3") == False # False if allow_worksheets is set @@ -187,10 +187,10 @@ def test_should_process_sheet(setup_module): @pytest.mark.skip(reason="Requires a real connection") class TestGSheetsFeederReal: - """Testing GSheetsFeeder class""" + """Testing GsheetsFeeder class""" - module_name: str = "gsheet_feeder" - feeder: GsheetsFeeder + module_name: str = "gsheet_feeder_db" + feeder: GsheetsFeederDB # You must follow the setup process explain in the docs for this to work config: dict = { "service_account": "secrets/service_account.json", diff --git a/tests/feeders/test_gworksheet.py b/tests/feeders/test_gworksheet.py index e6f5cc6..b6a0b5c 100644 --- a/tests/feeders/test_gworksheet.py +++ b/tests/feeders/test_gworksheet.py @@ -1,13 +1,13 @@ +# Note this isn't a feeder, but contained as utility of the gsheet feeder module import pytest -from unittest.mock import MagicMock -from auto_archiver.modules.gsheet_feeder import GWorksheet +from auto_archiver.modules.gsheet_feeder_db import GWorksheet class TestGWorksheet: @pytest.fixture - def mock_worksheet(self): - mock_ws = MagicMock() + def mock_worksheet(self, mocker): + mock_ws = mocker.MagicMock() mock_ws.get_values.return_value = [ ["Link", "Archive Status", "Archive Location", "Archive Date"], ["url1", "archived", "filepath1", "2023-01-01"], @@ -136,8 +136,8 @@ class TestGWorksheet: assert gworksheet.to_a1(row, col) == expected # Test empty worksheet - def test_empty_worksheet_initialization(self): - mock_ws = MagicMock() + def test_empty_worksheet_initialization(self, mocker): + mock_ws = mocker.MagicMock() mock_ws.get_values.return_value = [] g = GWorksheet(mock_ws) assert g.headers == [] diff --git a/tests/storages/test_S3_storage.py b/tests/storages/test_S3_storage.py index 2a5d026..abf9763 100644 --- a/tests/storages/test_S3_storage.py +++ b/tests/storages/test_S3_storage.py @@ -1,6 +1,5 @@ from typing import Type import pytest -from unittest.mock import MagicMock, patch from auto_archiver.core import Media from auto_archiver.modules.s3_storage import S3Storage @@ -11,7 +10,6 @@ class TestS3Storage: """ module_name: str = "s3_storage" storage: Type[S3Storage] - s3: MagicMock config: dict = { "path_generator": "flat", "filename_generator": "static", @@ -25,100 +23,83 @@ class TestS3Storage: "private": False, } - @patch('boto3.client') @pytest.fixture(autouse=True) - def setup_storage(self, setup_module): + def setup_storage(self, setup_module, mocker): + self.s3 = S3Storage() self.storage = setup_module(self.module_name, self.config) def test_client_initialization(self): """Test that S3 client is initialized with correct parameters""" + assert self.storage.s3 is not None assert self.storage.s3.meta.region_name == 'test-region' def test_get_cdn_url_generation(self): """Test CDN URL formatting """ media = Media("test.txt") - media.key = "path/to/file.txt" + media._key = "path/to/file.txt" url = self.storage.get_cdn_url(media) assert url == "https://cdn.example.com/path/to/file.txt" - media.key = "another/path.jpg" + media._key = "another/path.jpg" assert self.storage.get_cdn_url(media) == "https://cdn.example.com/another/path.jpg" - def test_uploadf_sets_acl_public(self): + def test_uploadf_sets_acl_public(self, mocker): media = Media("test.txt") - mock_file = MagicMock() - with patch.object(self.storage.s3, 'upload_fileobj') as mock_s3_upload, \ - patch.object(self.storage, 'is_upload_needed', return_value=True): - self.storage.uploadf(mock_file, media) - mock_s3_upload.assert_called_once_with( - mock_file, - Bucket='test-bucket', - Key=media.key, - ExtraArgs={'ACL': 'public-read', 'ContentType': 'text/plain'} - ) + mock_file = mocker.MagicMock() + mock_s3_upload = mocker.patch.object(self.storage.s3, 'upload_fileobj') + mocker.patch.object(self.storage, 'is_upload_needed', return_value=True) + self.storage.uploadf(mock_file, media) + mock_s3_upload.assert_called_once_with( + mock_file, + Bucket='test-bucket', + Key=media.key, + ExtraArgs={'ACL': 'public-read', 'ContentType': 'text/plain'} + ) - def test_upload_decision_logic(self): + def test_upload_decision_logic(self, mocker): """Test is_upload_needed under different conditions""" media = Media("test.txt") - # Test default state (random_no_duplicate=False) assert self.storage.is_upload_needed(media) is True - # Set duplicate checking config to true: - self.storage.random_no_duplicate = True - with patch('auto_archiver.modules.s3_storage.s3_storage.calculate_file_hash') as mock_calc_hash, \ - patch.object(self.storage, 'file_in_folder') as mock_file_in_folder: - mock_calc_hash.return_value = 'beepboop123beepboop123beepboop123' - mock_file_in_folder.return_value = 'existing_key.txt' - # Test duplicate result - assert self.storage.is_upload_needed(media) is False - assert media.key == 'existing_key.txt' - mock_file_in_folder.assert_called_with( - # (first 24 chars of hash) - 'no-dups/beepboop123beepboop123be' - ) + mock_calc_hash = mocker.patch('auto_archiver.modules.s3_storage.s3_storage.calculate_file_hash', return_value='beepboop123beepboop123beepboop123') + mock_file_in_folder = mocker.patch.object(self.storage, 'file_in_folder', return_value='existing_key.txt') + assert self.storage.is_upload_needed(media) is False + assert media.key == 'existing_key.txt' + mock_file_in_folder.assert_called_with('no-dups/beepboop123beepboop123be') - - @patch.object(S3Storage, 'file_in_folder') - def test_skips_upload_when_duplicate_exists(self, mock_file_in_folder): + def test_skips_upload_when_duplicate_exists(self, mocker): """Test that upload skips when file_in_folder finds existing object""" self.storage.random_no_duplicate = True - mock_file_in_folder.return_value = "existing_folder/existing_file.txt" - # Create test media with calculated hash + mock_file_in_folder = mocker.patch.object(S3Storage, 'file_in_folder', return_value="existing_folder/existing_file.txt") media = Media("test.txt") - media.key = "original_path.txt" - with patch('auto_archiver.modules.s3_storage.s3_storage.calculate_file_hash') as mock_calculate_hash: - mock_calculate_hash.return_value = "beepboop123beepboop123beepboop123" - # Verify upload - assert self.storage.is_upload_needed(media) is False - assert media.key == "existing_folder/existing_file.txt" - assert media.get("previously archived") is True - with patch.object(self.storage.s3, 'upload_fileobj') as mock_upload: - result = self.storage.uploadf(None, media) - mock_upload.assert_not_called() - assert result is True + media._key = "original_path.txt" + mock_calculate_hash = mocker.patch('auto_archiver.modules.s3_storage.s3_storage.calculate_file_hash', return_value="beepboop123beepboop123beepboop123") + assert self.storage.is_upload_needed(media) is False + assert media.key == "existing_folder/existing_file.txt" + assert media.get("previously archived") is True + mock_upload = mocker.patch.object(self.storage.s3, 'upload_fileobj') + result = self.storage.uploadf(None, media) + mock_upload.assert_not_called() + assert result is True - @patch.object(S3Storage, 'is_upload_needed') - def test_uploads_with_correct_parameters(self, mock_upload_needed): + def test_uploads_with_correct_parameters(self, mocker): media = Media("test.txt") - media.key = "original_key.txt" - mock_upload_needed.return_value = True + media._key = "original_key.txt" + mocker.patch.object(S3Storage, 'is_upload_needed', return_value=True) media.mimetype = 'image/png' - mock_file = MagicMock() + mock_file = mocker.MagicMock() + mock_upload = mocker.patch.object(self.storage.s3, 'upload_fileobj') + self.storage.uploadf(mock_file, media) + mock_upload.assert_called_once_with( + mock_file, + Bucket='test-bucket', + Key='original_key.txt', + ExtraArgs={ + 'ACL': 'public-read', + 'ContentType': 'image/png' + } + ) - with patch.object(self.storage.s3, 'upload_fileobj') as mock_upload: - self.storage.uploadf(mock_file, media) - # verify call occured with these params - mock_upload.assert_called_once_with( - mock_file, - Bucket='test-bucket', - Key='original_key.txt', - ExtraArgs={ - 'ACL': 'public-read', - 'ContentType': 'image/png' - } - ) - - def test_file_in_folder_exists(self): - with patch.object(self.storage.s3, 'list_objects') as mock_list_objects: - mock_list_objects.return_value = {'Contents': [{'Key': 'path/to/file.txt'}]} - assert self.storage.file_in_folder('path/to/') == 'path/to/file.txt' \ No newline at end of file + def test_file_in_folder_exists(self, mocker): + mock_list_objects = mocker.patch.object(self.storage.s3, 'list_objects', return_value={'Contents': [{'Key': 'path/to/file.txt'}]}) + assert self.storage.file_in_folder('path/to/') == 'path/to/file.txt' diff --git a/tests/storages/test_atlos_storage.py b/tests/storages/test_atlos_storage.py new file mode 100644 index 0000000..d33ff2f --- /dev/null +++ b/tests/storages/test_atlos_storage.py @@ -0,0 +1,112 @@ +import os +import hashlib +import pytest +from auto_archiver.core import Media, Metadata +from auto_archiver.modules.atlos_feeder_db_storage import AtlosFeederDbStorage as AtlosStorage + + +class FakeAPIResponse: + """Simulate a response object.""" + + def __init__(self, data: dict, raise_error: bool = False) -> None: + self._data = data + self.raise_error = raise_error + + def json(self) -> dict: + return self._data + + def raise_for_status(self) -> None: + if self.raise_error: + raise Exception("HTTP error") + + +@pytest.fixture +def atlos_storage(setup_module, mocker) -> AtlosStorage: + """Fixture for AtlosStorage.""" + configs: dict = { + "api_token": "abc123", + "atlos_url": "https://platform.atlos.org", + } + mocker.patch("requests.Session") + atlos_feeder = setup_module("atlos_feeder_db_storage", configs) + mock_session = mocker.MagicMock() + # Configure the default response to have no results so that __iter__ terminates + mock_session.get.return_value = FakeAPIResponse({"next": None, "results": []}) + atlos_feeder.session = mock_session + return atlos_feeder + + +@pytest.fixture +def media(tmp_path) -> Media: + """Fixture for Media.""" + content = b"media content" + file_path = tmp_path / "media.txt" + file_path.write_bytes(content) + media = Media(filename=str(file_path)) + media.properties = {"something": "Title"} + media._key = "key" + return media + + +def test_get_cdn_url(atlos_storage: AtlosStorage) -> None: + """Test get_cdn_url returns the configured atlos_url.""" + media = Media(filename="dummy.mp4") + url = atlos_storage.get_cdn_url(media) + assert url == atlos_storage.atlos_url + + +def test_upload_no_atlos_id(tmp_path, atlos_storage: AtlosStorage, media: Media, mocker) -> None: + """Test upload() returns False when metadata lacks atlos_id.""" + metadata = Metadata() # atlos_id not set + post_mock = mocker.patch("requests.post") + result = atlos_storage.upload(media, metadata) + assert result is False + post_mock.assert_not_called() + + +def test_upload_already_uploaded(atlos_storage: AtlosStorage, metadata: Metadata, media: Media, mocker) -> None: + """Test upload() returns True if media hash already exists.""" + content = b"media content" + metadata.set("atlos_id", 101) + media_hash = hashlib.sha256(content).hexdigest() + fake_get_response = {"result": {"artifacts": [{"file_hash_sha256": media_hash}]}} + get_mock = mocker.patch.object(atlos_storage, "_get", return_value=fake_get_response) + post_mock = mocker.patch.object(atlos_storage, "_post") + result = atlos_storage.upload(media, metadata) + assert result is True + get_mock.assert_called_once() + post_mock.assert_not_called() + + +def test_upload_not_uploaded(tmp_path, atlos_storage: AtlosStorage, metadata: Metadata, media: Media, mocker) -> None: + """Test upload() uploads media when not already present.""" + metadata.set("atlos_id", 202) + fake_get_response = {"result": {"artifacts": [{"file_hash_sha256": "different_hash"}]}} + get_mock = mocker.patch.object(atlos_storage, "_get", return_value=fake_get_response) + fake_post_response = {"result": "uploaded"} + post_mock = mocker.patch.object(atlos_storage, "_post", return_value=fake_post_response) + result = atlos_storage.upload(media, metadata) + assert result is True + + get_mock.assert_called_once() + post_mock.assert_called_once() + expected_endpoint = f"/api/v2/source_material/upload/202" + call_args = post_mock.call_args[0] + assert call_args[0] == expected_endpoint + call_kwargs = post_mock.call_args[1] + expected_headers = {"Authorization": f"Bearer {atlos_storage.api_token}"} + expected_params = {"title": media.properties} + assert call_kwargs["params"] == expected_params + file_tuple = call_kwargs["files"]["file"] + assert file_tuple[0] == os.path.basename(media.filename) + + +def test_upload_post_http_error(tmp_path, atlos_storage: AtlosStorage, metadata: Metadata, media: Media, mocker) -> None: + """Test upload() propagates HTTP error during POST.""" + metadata.set("atlos_id", 303) + fake_get_response = {"result": {"artifacts": []}} + mocker.patch.object(atlos_storage, "_get", return_value=fake_get_response) + mocker.patch.object(atlos_storage, "_post", side_effect=Exception("HTTP error")) + with pytest.raises(Exception, match="HTTP error"): + atlos_storage.upload(media, metadata) + diff --git a/tests/storages/test_gdrive_storage.py b/tests/storages/test_gdrive_storage.py index aba0a25..d87f5e8 100644 --- a/tests/storages/test_gdrive_storage.py +++ b/tests/storages/test_gdrive_storage.py @@ -1,44 +1,57 @@ from typing import Type import pytest -from unittest.mock import MagicMock, patch +from oauth2client import service_account + from auto_archiver.core import Media from auto_archiver.modules.gdrive_storage import GDriveStorage from auto_archiver.core.metadata import Metadata from tests.storages.test_storage_base import TestStorageBase -class TestGDriveStorage: - """ - Test suite for GDriveStorage. - """ - +@pytest.fixture +def gdrive_storage(setup_module, mocker): module_name: str = "gdrive_storage" - storage: Type[GDriveStorage] + storage: GDriveStorage config: dict = {'path_generator': 'url', 'filename_generator': 'static', 'root_folder_id': "fake_root_folder_id", 'oauth_token': None, 'service_account': 'fake_service_account.json' } - - @pytest.fixture(autouse=True) - def gdrive(self, setup_module): - with patch('google.oauth2.service_account.Credentials.from_service_account_file') as mock_creds: - self.storage = setup_module(self.module_name, self.config) - - def test_initialize_fails_with_non_existent_creds(self): - """ - Test that the Google Drive service raises a FileNotFoundError when the service account file does not exist. - """ - # Act and Assert - with pytest.raises(FileNotFoundError) as exc_info: - self.storage.setup() - assert "No such file or directory" in str(exc_info.value) + mocker.patch('google.oauth2.service_account.Credentials.from_service_account_file') + return setup_module(module_name, config) - def test_path_parts(self): - media = Media(filename="test.jpg") - media.key = "folder1/folder2/test.jpg" +def test_initialize_fails_with_non_existent_creds(setup_module): + """Test that the Google Drive service raises a FileNotFoundError when the service account file does not exist. + (and isn't mocked) + """ + config: dict = {'path_generator': 'url', + 'filename_generator': 'static', + 'root_folder_id': "fake_root_folder_id", + 'oauth_token': None, + 'service_account': 'fake_service_account.json' + } + with pytest.raises(FileNotFoundError) as exc_info: + setup_module("gdrive_storage", config) + assert "No such file or directory" in str(exc_info.value) + + +def test_get_id_from_parent_and_name(gdrive_storage, mocker): + """Test _get_id_from_parent_and_name returns correct id from an API result.""" + fake_list = mocker.MagicMock() + fake_list.execute.return_value = {"files": [{"id": "123", "name": "testname"}]} + fake_service = mocker.MagicMock() + # mock the files.list return value + fake_service.files.return_value.list.return_value = fake_list + gdrive_storage.service = fake_service + result = gdrive_storage._get_id_from_parent_and_name("parent", "mock", retries=1, use_mime_type=False) + assert result == "123" + +def test_path_parts(): + media = Media(filename="test.jpg") + media._key = "folder1/folder2/test.jpg" + @pytest.mark.skip(reason="Requires real credentials") diff --git a/tests/storages/test_local_storage.py b/tests/storages/test_local_storage.py new file mode 100644 index 0000000..c3581df --- /dev/null +++ b/tests/storages/test_local_storage.py @@ -0,0 +1,60 @@ + +import os +from pathlib import Path + +import pytest + +from auto_archiver.core import Media, Metadata +from auto_archiver.modules.local_storage import LocalStorage +from auto_archiver.core.consts import SetupError + +@pytest.fixture +def local_storage(setup_module, tmp_path) -> LocalStorage: + save_to = tmp_path / "local_archive" + save_to.mkdir() + configs: dict = { + "path_generator": "flat", + "filename_generator": "static", + "save_to": str(save_to), + "save_absolute": False, + } + return setup_module("local_storage", configs) + +@pytest.fixture +def sample_media(tmp_path) -> Media: + """Fixture creating a Media object with temporary source file""" + src_file = tmp_path / "source.txt" + src_file.write_text("test content") + return Media(filename=str(src_file)) + +def test_too_long_save_path(setup_module): + with pytest.raises(SetupError): + setup_module("local_storage", {"save_to": "long"*100}) + +def test_get_cdn_url_relative(local_storage): + local_storage.filename_generator = "random" + media = Media(filename="dummy.txt") + local_storage.set_key(media, "https://example.com", Metadata()) + expected = os.path.join(local_storage.save_to, media.key) + assert local_storage.get_cdn_url(media) == expected + +def test_get_cdn_url_absolute(local_storage): + local_storage.filename_generator = "random" + + media = Media(filename="dummy.txt") + local_storage.save_absolute = True + local_storage.set_key(media, "https://example.com", Metadata()) + expected = os.path.abspath(os.path.join(local_storage.save_to, media.key)) + assert local_storage.get_cdn_url(media) == expected + +def test_upload_file_contents_and_metadata(local_storage, sample_media): + local_storage.store(sample_media, "https://example.com", Metadata()) + dest = os.path.join(local_storage.save_to, sample_media.key) + assert Path(sample_media.filename).read_text() == Path(dest).read_text() + +def test_upload_nonexistent_source(local_storage): + media = Media(_key="missing.txt", filename="nonexistent.txt") + with pytest.raises(FileNotFoundError): + local_storage.upload(media) + + diff --git a/tests/storages/test_storage_base.py b/tests/storages/test_storage_base.py index 7578acd..53dfbd7 100644 --- a/tests/storages/test_storage_base.py +++ b/tests/storages/test_storage_base.py @@ -2,9 +2,9 @@ from typing import Type import pytest -from auto_archiver.core.metadata import Metadata +from auto_archiver.core.metadata import Metadata, Media from auto_archiver.core.storage import Storage - +from auto_archiver.core.module import ModuleFactory class TestStorageBase(object): @@ -20,3 +20,76 @@ class TestStorageBase(object): self.storage: Type[Storage] = setup_module( self.module_name, self.config ) + + +class TestBaseStorage(Storage): + + name = "test_storage" + + def get_cdn_url(self, media): + return "cdn_url" + + def uploadf(self, file, key, **kwargs): + return True + +@pytest.fixture +def dummy_file(tmp_path): + # create dummy.txt file + dummy_file = tmp_path / "dummy.txt" + dummy_file.write_text("test content") + return str(dummy_file) + +@pytest.fixture +def storage_base(): + def _storage_base(config): + storage_base = TestBaseStorage() + storage_base.config_setup({TestBaseStorage.name : config}) + storage_base.module_factory = ModuleFactory() + return storage_base + + return _storage_base + +@pytest.mark.parametrize( + "path_generator, filename_generator, url, expected_key", + [ + ("flat", "static", "https://example.com/file/", "folder/6ae8a75555209fd6c44157c0.txt"), + ("flat", "random", "https://example.com/file/", "folder/pretend-random.txt"), + ("url", "static", "https://example.com/file/", "folder/https-example-com-file/6ae8a75555209fd6c44157c0.txt"), + ("url", "random", "https://example.com/file/", "folder/https-example-com-file/pretend-random.txt"), + ("random", "static", "https://example.com/file/", "folder/pretend-random/6ae8a75555209fd6c44157c0.txt"), + ("random", "random", "https://example.com/file/", "folder/pretend-random/pretend-random.txt"), + + ], +) +def test_storage_name_generation(storage_base, path_generator, filename_generator, url, + expected_key, mocker, tmp_path, dummy_file): + mock_random = mocker.patch("auto_archiver.core.storage.random_str") + mock_random.return_value = "pretend-random" + + config: dict = { + "path_generator": path_generator, + "filename_generator": filename_generator, + } + storage: Storage = storage_base(config) + assert storage.path_generator == path_generator + assert storage.filename_generator == filename_generator + + metadata = Metadata() + metadata.set_context("folder", "folder") + media = Media(filename=dummy_file) + storage.set_key(media, url, metadata) + print(media.key) + assert media.key == expected_key + + +def test_really_long_name(storage_base, dummy_file): + config: dict = { + "path_generator": "url", + "filename_generator": "static", + } + storage: Storage = storage_base(config) + + url = f"https://example.com/{'file'*100}" + media = Media(filename=dummy_file) + storage.set_key(media, url, Metadata()) + assert media.key == f"https-example-com-{'file'*13}/6ae8a75555209fd6c44157c0.txt" \ No newline at end of file diff --git a/tests/test_implementation.py b/tests/test_implementation.py index 7e33651..51e9d79 100644 --- a/tests/test_implementation.py +++ b/tests/test_implementation.py @@ -6,7 +6,9 @@ from auto_archiver.__main__ import main @pytest.fixture def orchestration_file_path(tmp_path): - return (tmp_path / "example_orch.yaml").as_posix() + folder = tmp_path / "secrets" + folder.mkdir(exist_ok=True) + return (folder / "example_orch.yaml").as_posix() @pytest.fixture def orchestration_file(orchestration_file_path): @@ -28,6 +30,7 @@ def autoarchiver(tmp_path, monkeypatch, request): logger.add(sys.stderr) request.addfinalizer(cleanup) + (tmp_path / "secrets").mkdir(exist_ok=True) # change dir to tmp_path monkeypatch.chdir(tmp_path) @@ -60,3 +63,16 @@ def test_run_auto_archiver_empty_file(caplog, autoarchiver, orchestration_file): # should treat an empty file as if there is no file at all assert " No URLs provided. Please provide at least one URL via the com" in caplog.text + +def test_call_autoarchiver_main(caplog, monkeypatch, tmp_path): + from auto_archiver.__main__ import main + + # monkey patch to change the current working directory, so that we don't use the user's real config file + monkeypatch.chdir(tmp_path) + (tmp_path / "secrets").mkdir(exist_ok=True) + with monkeypatch.context() as m: + m.setattr(sys, "argv", ["auto-archiver"]) + with pytest.raises(SystemExit): + main() + + assert "No URLs provided. Please provide at least one" in caplog.text \ No newline at end of file diff --git a/tests/test_metadata.py b/tests/test_metadata.py index b07e107..e1f7797 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -162,4 +162,25 @@ def test_get_context(): def test_choose_most_complete(): - pass \ No newline at end of file + m_more = Metadata() + m_more.set_title("Title 1") + m_more.set_content("Content 1") + m_more.set_url("https://example.com") + + m_less = Metadata() + m_less.set_title("Title 2") + m_less.set_content("Content 2") + m_less.set_url("https://example.com") + m_less.set_context("key", "value") + + res = Metadata.choose_most_complete([m_more, m_less]) + assert res.metadata.get("title") == "Title 1" + +def test_choose_most_complete_from_pickles(unpickle): + # test most complete from pickles before and after an enricher has run + # Only compares length of media, not the actual media + m_before_enriching = unpickle("metadata_enricher_ytshort_input.pickle") + m_after_enriching = unpickle("metadata_enricher_ytshort_expected.pickle") + # Iterates `for r in results[1:]:` + res = Metadata.choose_most_complete([Metadata(), m_after_enriching, m_before_enriching]) + assert res.media == m_after_enriching.media diff --git a/tests/test_modules.py b/tests/test_modules.py index 854edb5..7a2b14d 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -1,24 +1,18 @@ import sys import pytest -from auto_archiver.core.module import get_module_lazy, BaseModule, LazyBaseModule, _LAZY_LOADED_MODULES +from auto_archiver.core.module import ModuleFactory, LazyBaseModule +from auto_archiver.core.base_module import BaseModule @pytest.fixture def example_module(): import auto_archiver + module_factory = ModuleFactory() + previous_path = auto_archiver.modules.__path__ auto_archiver.modules.__path__.append("tests/data/test_modules/") - module = get_module_lazy("example_module") - yield module - # cleanup - try: - del module._manifest - except AttributeError: - pass - del _LAZY_LOADED_MODULES["example_module"] - sys.modules.pop("auto_archiver.modules.example_module.example_module", None) - auto_archiver.modules.__path__ = previous_path + return module_factory.get_module_lazy("example_module") def test_get_module_lazy(example_module): assert example_module.name == "example_module" @@ -46,12 +40,14 @@ def test_module_dependency_check_loads_module(example_module): # monkey patch the manifest to include a nonexistnet dependency example_module.manifest["dependencies"]["python"] = ["hash_enricher"] + module_factory = example_module.module_factory + loaded_module = example_module.load({}) assert loaded_module is not None # check the dependency is loaded - assert _LAZY_LOADED_MODULES["hash_enricher"] is not None - assert _LAZY_LOADED_MODULES["hash_enricher"]._instance is not None + assert module_factory._lazy_modules["hash_enricher"] is not None + assert module_factory._lazy_modules["hash_enricher"]._instance is not None def test_load_module(example_module): @@ -69,7 +65,7 @@ def test_load_module(example_module): @pytest.mark.parametrize("module_name", ["local_storage", "generic_extractor", "html_formatter", "csv_db"]) def test_load_modules(module_name): # test that specific modules can be loaded - module = get_module_lazy(module_name) + module = ModuleFactory().get_module_lazy(module_name) assert module is not None assert isinstance(module, LazyBaseModule) assert module.name == module_name @@ -86,7 +82,7 @@ def test_load_modules(module_name): @pytest.mark.parametrize("module_name", ["local_storage", "generic_extractor", "html_formatter", "csv_db"]) def test_lazy_base_module(module_name): - lazy_module = get_module_lazy(module_name) + lazy_module = ModuleFactory().get_module_lazy(module_name) assert lazy_module is not None assert isinstance(lazy_module, LazyBaseModule) diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py index 5ba57d0..72f4949 100644 --- a/tests/test_orchestrator.py +++ b/tests/test_orchestrator.py @@ -4,7 +4,7 @@ from argparse import ArgumentParser, ArgumentTypeError from auto_archiver.core.orchestrator import ArchivingOrchestrator from auto_archiver.version import __version__ from auto_archiver.core.config import read_yaml, store_yaml -from auto_archiver.core.module import _LAZY_LOADED_MODULES +from auto_archiver.core import Metadata TEST_ORCHESTRATION = "tests/data/test_orchestration.yaml" TEST_MODULES = "tests/data/test_modules/" @@ -17,22 +17,7 @@ def test_args(): @pytest.fixture def orchestrator(): - yield ArchivingOrchestrator() - # hack - the loguru logger starts with one logger, but if orchestrator has run before - # it'll remove the default logger, add it back in: - - from loguru import logger - - if not logger._core.handlers.get(0): - logger._core.handlers_count = 0 - logger.add(sys.stderr) - # and remove the custom logger - if logger._core.handlers.get(1): - logger.remove(1) - - # delete out any loaded modules - _LAZY_LOADED_MODULES.clear() - + return ArchivingOrchestrator() @pytest.fixture def basic_parser(orchestrator) -> ArgumentParser: @@ -75,18 +60,36 @@ def test_help(orchestrator, basic_parser, capsys): orchestrator.show_help(args) assert exit_error.value.code == 0 - assert "Usage: auto-archiver [--help] [--version] [--config CONFIG_FILE]" in capsys.readouterr().out + + logs = capsys.readouterr().out + assert "Usage: auto-archiver [--help] [--version] [--config CONFIG_FILE]" in logs + + # basic config options + assert "--version" in logs + + # setting modules options + assert "--feeders" in logs + assert "--extractors" in logs + + # authentication options + assert "--authentication" in logs + + # logging options + assert "--logging.level" in logs + + # individual module configs + assert "--gsheet_feeder_db.sheet_id" in logs def test_add_custom_modules_path(orchestrator, test_args): - orchestrator.run(test_args) + orchestrator.setup_config(test_args) import auto_archiver assert "tests/data/test_modules/" in auto_archiver.modules.__path__ def test_add_custom_modules_path_invalid(orchestrator, caplog, test_args): - orchestrator.run(test_args + # we still need to load the real path to get the example_module + orchestrator.setup_config(test_args + # we still need to load the real path to get the example_module ["--module_paths", "tests/data/invalid_test_modules/"]) assert caplog.records[0].message == "Path 'tests/data/invalid_test_modules/' does not exist. Skipping..." @@ -97,7 +100,7 @@ def test_check_required_values(orchestrator, caplog, test_args): test_args = test_args[:-2] with pytest.raises(SystemExit) as exit_error: - orchestrator.run(test_args) + config = orchestrator.setup_config(test_args) assert caplog.records[1].message == "the following arguments are required: --example_module.required_field" @@ -111,24 +114,72 @@ def test_get_required_values_from_config(orchestrator, test_args, tmp_path): store_yaml(test_yaml, tmp_file) # run the orchestrator - orchestrator.run(["--config", tmp_file, "--module_paths", TEST_MODULES]) - assert orchestrator.config is not None + config = orchestrator.setup_config(["--config", tmp_file, "--module_paths", TEST_MODULES]) + assert config is not None def test_load_authentication_string(orchestrator, test_args): - orchestrator.run(test_args + ["--authentication", '{"facebook.com": {"username": "my_username", "password": "my_password"}}']) - assert orchestrator.config['authentication'] == {"facebook.com": {"username": "my_username", "password": "my_password"}} + config = orchestrator.setup_config(test_args + ["--authentication", '{"facebook.com": {"username": "my_username", "password": "my_password"}}']) + assert config['authentication'] == {"facebook.com": {"username": "my_username", "password": "my_password"}} def test_load_authentication_string_concat_site(orchestrator, test_args): - orchestrator.run(test_args + ["--authentication", '{"x.com,twitter.com": {"api_key": "my_key"}}']) - assert orchestrator.config['authentication'] == {"x.com": {"api_key": "my_key"}, + config = orchestrator.setup_config(test_args + ["--authentication", '{"x.com,twitter.com": {"api_key": "my_key"}}']) + assert config['authentication'] == {"x.com": {"api_key": "my_key"}, "twitter.com": {"api_key": "my_key"}} def test_load_invalid_authentication_string(orchestrator, test_args): with pytest.raises(ArgumentTypeError): - orchestrator.run(test_args + ["--authentication", "{\''invalid_json"]) + orchestrator.setup_config(test_args + ["--authentication", "{\''invalid_json"]) def test_load_authentication_invalid_dict(orchestrator, test_args): with pytest.raises(ArgumentTypeError): - orchestrator.run(test_args + ["--authentication", "[true, false]"]) \ No newline at end of file + orchestrator.setup_config(test_args + ["--authentication", "[true, false]"]) + +def test_load_modules_from_commandline(orchestrator, test_args): + args = test_args + ["--feeders", "example_module", "--extractors", "example_module", "--databases", "example_module", "--enrichers", "example_module", "--formatters", "example_module"] + + orchestrator.setup(args) + + assert len(orchestrator.feeders) == 1 + assert len(orchestrator.extractors) == 1 + assert len(orchestrator.databases) == 1 + assert len(orchestrator.enrichers) == 1 + assert len(orchestrator.formatters) == 1 + + assert orchestrator.feeders[0].name == "example_module" + assert orchestrator.extractors[0].name == "example_module" + assert orchestrator.databases[0].name == "example_module" + assert orchestrator.enrichers[0].name == "example_module" + assert orchestrator.formatters[0].name == "example_module" + +def test_load_settings_for_module_from_commandline(orchestrator, test_args): + args = test_args + ["--feeders", "gsheet_feeder_db", "--gsheet_feeder_db.sheet_id", "123", "--gsheet_feeder_db.service_account", "tests/data/test_service_account.json"] + + orchestrator.setup(args) + + assert len(orchestrator.feeders) == 1 + assert orchestrator.feeders[0].name == "gsheet_feeder_db" + assert orchestrator.config['gsheet_feeder_db']['sheet_id'] == "123" + + +def test_multiple_orchestrator(test_args): + + o1_args = test_args + ["--feeders", "gsheet_feeder_db", "--gsheet_feeder_db.service_account", "tests/data/test_service_account.json"] + o1 = ArchivingOrchestrator() + + with pytest.raises(ValueError) as exit_error: + # this should fail because the gsheet_feeder_db requires a sheet_id / sheet + o1.setup(o1_args) + + + + o2_args = test_args + ["--feeders", "example_module"] + o2 = ArchivingOrchestrator() + o2.setup(o2_args) + + assert o2.feeders[0].name == "example_module" + + output: Metadata = list(o2.feed()) + assert len(output) == 1 + assert output[0].get_url() == "https://example.com" \ No newline at end of file diff --git a/tests/utils/test_misc.py b/tests/utils/test_misc.py new file mode 100644 index 0000000..0023077 --- /dev/null +++ b/tests/utils/test_misc.py @@ -0,0 +1,144 @@ +import hashlib +import json +from datetime import datetime, timezone + +import pytest + +from auto_archiver.utils.misc import ( + mkdir_if_not_exists, + expand_url, + getattr_or, + DateTimeEncoder, + dump_payload, + get_datetime_from_str, + update_nested_dict, + calculate_file_hash, + random_str, + get_timestamp +) + + +@pytest.fixture +def sample_file(tmp_path): + file_path = tmp_path / "test.txt" + file_path.write_text("test content") + return file_path + + +class TestDirectoryUtils: + def test_mkdir_creates_new_directory(self, tmp_path): + new_dir = tmp_path / "new_folder" + mkdir_if_not_exists(new_dir) + assert new_dir.exists() + assert new_dir.is_dir() + + def test_mkdir_exists_quietly(self, tmp_path): + existing_dir = tmp_path / "existing" + existing_dir.mkdir() + mkdir_if_not_exists(existing_dir) + assert existing_dir.exists() + +class TestURLExpansion: + @pytest.mark.parametrize("input_url,expected", [ + ("https://example.com", "https://example.com"), + ("https://t.co/test", "https://expanded.url") + ]) + def test_expand_url(self, input_url, expected, mocker): + mock_response = mocker.Mock() + mock_response.url = "https://expanded.url" + mocker.patch('requests.get', return_value=mock_response) + result = expand_url(input_url) + assert result == expected + + def test_expand_url_handles_errors(self, caplog, mocker): + mocker.patch('requests.get', side_effect=Exception("Connection error")) + url = "https://t.co/error" + result = expand_url(url) + assert result == url + assert f"Failed to expand url {url}" in caplog.text + +class TestAttributeHandling: + class Sample: + exists = "value" + none = None + + @pytest.mark.parametrize("obj,attr,default,expected", [ + (Sample(), "exists", "default", "value"), + (Sample(), "none", "default", "default"), + (Sample(), "missing", "default", "default"), + (None, "anything", "fallback", "fallback"), + ]) + def test_getattr_or(self, obj, attr, default, expected): + # Test gets attribute or returns a default value + assert getattr_or(obj, attr, default) == expected + +class TestDateTimeHandling: + def test_datetime_encoder(self, sample_datetime): + result = json.dumps({"dt": sample_datetime}, cls=DateTimeEncoder) + loaded = json.loads(result) + assert loaded["dt"] == str(sample_datetime) + + def test_dump_payload(self, sample_datetime): + payload = {"timestamp": sample_datetime} + result = dump_payload(payload) + assert str(sample_datetime) in result + + @pytest.mark.parametrize("dt_str,fmt,expected", [ + ("2023-01-01 12:00:00+00:00", None, datetime(2023, 1, 1, 12, 0, tzinfo=timezone.utc)), + ("20230101 120000", "%Y%m%d %H%M%S", datetime(2023, 1, 1, 12, 0)), + ("invalid", None, None), + ]) + def test_datetime_from_string(self, dt_str, fmt, expected): + result = get_datetime_from_str(dt_str, fmt) + if expected is None: + assert result is None + else: + assert result == expected.replace(tzinfo=result.tzinfo) + +class TestDictUtils: + @pytest.mark.parametrize("original,update,expected", [ + ({"a": 1}, {"b": 2}, {"a": 1, "b": 2}), + ({"nested": {"a": 1}}, {"nested": {"b": 2}}, {"nested": {"a": 1, "b": 2}}), + ({"a": {"b": {"c": 1}}}, {"a": {"b": {"c": 2}}}, {"a": {"b": {"c": 2}}}), + ]) + def test_update_nested_dict(self, original, update, expected): + update_nested_dict(original, update) + assert original == expected + +class TestHashingUtils: + def test_file_hashing(self, sample_file): + expected = hashlib.sha256(b"test content").hexdigest() + assert calculate_file_hash(str(sample_file)) == expected + + def test_large_file_hashing(self, tmp_path): + file_path = tmp_path / "large.bin" + content = b"0" * 16_000_000 * 2 # 32MB + file_path.write_bytes(content) + + expected = hashlib.sha256(content).hexdigest() + assert calculate_file_hash(str(file_path)) == expected + +class TestMiscUtils: + def test_random_str_length(self): + for length in [8, 16, 32]: + assert len(random_str(length)) == length + + def test_random_str_raises_too_long(self): + with pytest.raises(AssertionError) as exc_info: + random_str(64) + assert "length must be less than 32 as UUID4 is used" == str(exc_info.value) + + def test_random_str_uniqueness(self): + assert random_str() != random_str() + + @pytest.mark.parametrize("ts_input,utc,iso,expected_type", [ + (datetime.now(), True, True, str), + ("2023-01-01T12:00:00+00:00", False, False, datetime), + (1672574400, True, True, str), + ]) + def test_timestamp_parsing(self, ts_input, utc, iso, expected_type): + result = get_timestamp(ts_input, utc=utc, iso=iso) + assert isinstance(result, expected_type) + + def test_invalid_timestamp_returns_none(self): + assert get_timestamp("invalid-date") is None \ No newline at end of file