diff --git a/.github/workflows/docker-publish.yaml b/.github/workflows/docker-publish.yaml index 4d232e2..2031ad6 100644 --- a/.github/workflows/docker-publish.yaml +++ b/.github/workflows/docker-publish.yaml @@ -11,7 +11,7 @@ on: env: # Use docker.io for Docker Hub if empty - REGISTRY: ghcr.io + REGISTRY: docker.io # github.repository as / IMAGE_NAME: ${{ github.repository }} @@ -45,10 +45,12 @@ jobs: images: bellingcat/auto-archiver - name: Build and push Docker image - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v6 with: context: . platforms: linux/amd64,linux/arm64 push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:cache + cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:cache,mode=max diff --git a/.gitignore b/.gitignore index 701de43..f31bc6c 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,5 @@ dist* docs/_build/ docs/source/autoapi/ docs/source/modules/autogen/ +scripts/settings_page.html +.vite diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 434f805..6dc9fe5 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -9,6 +9,7 @@ build: os: ubuntu-22.04 tools: python: "3.10" + nodejs: "22" jobs: post_install: - pip install poetry @@ -17,6 +18,11 @@ build: # See https://github.com/readthedocs/readthedocs.org/pull/11152/ - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH poetry install --with docs + # generate the config editor page. Schema then HTML + - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH poetry run python scripts/generate_settings_schema.py + # install node dependencies and build the settings + - cd scripts/settings && npm install && npm run build && yes | cp dist/index.html ../../docs/source/installation/settings_base.html && cd ../.. + sphinx: configuration: docs/source/conf.py diff --git a/Dockerfile b/Dockerfile index cbcfdd4..68aed42 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,13 +7,24 @@ ENV RUNNING_IN_DOCKER=1 \ PYTHONFAULTHANDLER=1 \ PATH="/root/.local/bin:$PATH" + +ARG TARGETARCH + # Installing system dependencies RUN add-apt-repository ppa:mozillateam/ppa && \ apt-get update && \ apt-get install -y --no-install-recommends gcc ffmpeg fonts-noto exiftool && \ apt-get install -y --no-install-recommends firefox-esr && \ - ln -s /usr/bin/firefox-esr /usr/bin/firefox && \ - wget https://github.com/mozilla/geckodriver/releases/download/v0.35.0/geckodriver-v0.35.0-linux64.tar.gz && \ + ln -s /usr/bin/firefox-esr /usr/bin/firefox + +ARG GECKODRIVER_VERSION=0.36.0 + +RUN if [ $(uname -m) = "aarch64" ]; then \ + GECKODRIVER_ARCH=linux-aarch64; \ + else \ + GECKODRIVER_ARCH=linux64; \ + fi && \ + wget https://github.com/mozilla/geckodriver/releases/download/v${GECKODRIVER_VERSION}/geckodriver-v${GECKODRIVER_VERSION}-${GECKODRIVER_ARCH}.tar.gz && \ tar -xvzf geckodriver* -C /usr/local/bin && \ chmod +x /usr/local/bin/geckodriver && \ rm geckodriver-v* && \ diff --git a/docs/source/development/developer_guidelines.md b/docs/source/development/developer_guidelines.md index e72193a..0014d8f 100644 --- a/docs/source/development/developer_guidelines.md +++ b/docs/source/development/developer_guidelines.md @@ -31,4 +31,5 @@ docker_development testing docs release +settings_page ``` \ No newline at end of file diff --git a/docs/source/development/release.md b/docs/source/development/release.md index 403dcb9..694af78 100644 --- a/docs/source/development/release.md +++ b/docs/source/development/release.md @@ -13,3 +13,8 @@ manual release to docker hub * `docker image tag auto-archiver bellingcat/auto-archiver:latest` * `docker push bellingcat/auto-archiver` + + +### Building the Settings Page + +The Settings page is built as part of the python-publish workflow and packaged within the app. \ No newline at end of file diff --git a/docs/source/development/settings_page.md b/docs/source/development/settings_page.md new file mode 100644 index 0000000..41271b9 --- /dev/null +++ b/docs/source/development/settings_page.md @@ -0,0 +1,31 @@ +# Configuration Editor + +The [configuration editor](../installation/config_editor.md), is an easy-to-use UI for users to edit their auto-archiver settings. + +The single-file app is built using React and vite. To get started developing the package, follow these steps: + +1. Make sure you have Node v22 installed. + +```{note} Tip: if you don't have node installed: + +Use `nvm` to manage your node installations. Use: +`curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash` to install `nvm` and then `nvm i 22` to install Node v22 +``` + +2. Generate the `schema.json` file for the currently installed modules using `python scripts/generate_settings_schema.py` +3. Go to the settings folder `cd scripts/settings/` and build your environment with `npm i` +4. Run a development version of the page with `npm run dev` and then open localhost:5173. +5. Build a release version of the page with `npm run build` + +A release version creates a single-file app called `dist/index.html`. This file should be copied to `docs/source/installation/settings_base.html` so that it can be integrated into the sphinx docs. + +```{note} + +The single-file app dist/index.html does not include any `` or `` tags as it is designed to be built into a RTD docs page. Edit `index.html` in the settings folder if you wish to modify the built page. +``` + +## Readthedocs Integration + +The configuration editor is built as part of the RTD deployment (see `.readthedocs.yaml` file). This command is run every time RTD is built: + +`cd scripts/settings && npm install && npm run build && yes | cp dist/index.html ../../docs/source/installation/settings_base.html && cd ../..` \ No newline at end of file diff --git a/docs/source/how_to/authentication_how_to.md b/docs/source/how_to/authentication_how_to.md index 8994271..0e842fb 100644 --- a/docs/source/how_to/authentication_how_to.md +++ b/docs/source/how_to/authentication_how_to.md @@ -46,7 +46,7 @@ First, we need to install an extension in our browser to export the cookies for **2. Export the cookies** -```{note} See the note [here](../installation/authentication.md#recommendations-for-authentication) on why you shouldn't use your own personal account for achiving. +```{note} See the note [here](../installation/authentication.md#recommendations-for-authentication) on why you shouldn't use your own personal account for archiving. ``` Once the extension is installed in your preferred browser, login to Twitter in this browser, and then activate the extension and export the cookies. You can choose to export all your cookies for your browser, or just cookies for this specific site. In the image below, we're only exporting cookies for Twitter/x.com: diff --git a/docs/source/how_to/gsheets_setup.md b/docs/source/how_to/gsheets_setup.md index 20cedd5..ade8024 100644 --- a/docs/source/how_to/gsheets_setup.md +++ b/docs/source/how_to/gsheets_setup.md @@ -8,18 +8,25 @@ This guide explains how to set up Google Sheets to process URLs automatically an ### 1. Setting up your Google Sheet -Any Google sheet must have at least *one* column, with the name 'link' (you can change this name afterwards). This is the column with the URLs that you want the Auto Archiver to archive. Your sheet can have many other columns that the Auto Archiver can use, and you can also include any other columns for your own personal use. +Any Google sheet must have at least *one* column, with the name 'link' (you can change this name afterwards). This is the column with the URLs that you want the Auto Archiver to archive. +Your sheet can have many other columns that the Auto Archiver can use, and you can also include any additional columns for your own personal use. The order of the columns does not matter, the naming just needs to be correctly assigned to its corresponding value in the configuration file. -We recommend copying [this template Google Sheet](https://docs.google.com/spreadsheets/d/1NJZo_XZUBKTI1Ghlgi4nTPVvCfb0HXAs6j5tNGas72k/edit?usp=sharing) as a starting point for your project. +We recommend copying [this template Google Sheet](https://docs.google.com/spreadsheets/d/1NJZo_XZUBKTI1Ghlgi4nTPVvCfb0HXAs6j5tNGas72k/edit?usp=sharing) as a starting point for your project, as this matches the default column names. Here's an overview of all the columns, and what a complete sheet would look like. -Inputs: +**Inputs:** -* **Link** *(required)*: the URL of the post to archive +These are processed by the Gsheet Feeder and passed to the Auto Archiver. + +* **Link** *(required)*: the URL of the post that is to be archived * **Destination folder**: custom folder for archived file (regardless of storage) -Outputs: +**Outputs:** + +These are updated by the Gsheet DB module during the archiving process. +Note the required columns are only required if you are using the Gsheet DB module as well as the feeder. + * **Archive status** *(required)*: Status of archive operation * **Archive location**: URL of archived post * **Archive date**: Date archived @@ -33,9 +40,11 @@ Outputs: * **WACZ**: Link to a WACZ web archive of post * **ReplayWebpage**: Link to a ReplayWebpage viewer of the WACZ archive -For example, this is a spreadsheet configured with all of the columns for the auto archiver and a few URLs to archive. (Note that the column names are not case sensitive.) +For example, this is a spreadsheet configured with all of the columns for the auto archiver and a few URLs to archive. +In this example the Ghseet Feeder and Gsheet DB are being used, and the archive is in progress. +(Note that the column names are not case sensitive.) -![A screenshot of a Google Spreadsheet with column headers defined as above, and several Youtube and Twitter URLs in the "Link" column](../demo-before.png) +![A screenshot of a Google Spreadsheet with column headers defined as above, and several Youtube and Twitter URLs in the "Link" column](../../demo-before.png) We'll change the name of the 'Destination Folder' column in step 3. @@ -51,43 +60,47 @@ Once you've downloaded the file, save it to `secrets/service_account.json` Now that you've set up your Google sheet, and you've set up the service account so Auto Archiver can access the sheet, the final step is to set your configuration. -First, make sure you have `gsheet_feeder` set in the `steps.feeders` section of your config. If you wish to store the results of the archiving process back in your Google sheet, make sure to also set the `ghseet_db` settig in the `steps.databases` section. Here's how this might look: +First, make sure you have `gsheet_feeder_db` set in the `steps.feeders` section of your config. If you wish to store the results of the archiving process back in your Google sheet, make sure to also set the `ghseet_db` settig in the `steps.databases` section. Here's how this might look: ```{code} yaml steps: feeders: - - gsheet_feeder + - gsheet_feeder_db ... databases: - - gsheet_db # optional, if you also want to store the results in the Google sheet + - gsheet_feeder_db # optional, if you also want to store the results in the Google sheet and tract the status of active archivals. ... ``` -Next, set up the `gsheet_feeder` configuration settings in the 'Configurations' part of the config `orchestration.yaml` file. Open up he file, and set the `gsheet_feeder.sheet` setting or the `gsheet_feeder.sheet_id` setting. The `sheet` should be the name of your sheet, as it shows in the top left of the sheet. For example, the sheet [here](https://docs.google.com/spreadsheets/d/1NJZo_XZUBKTI1Ghlgi4nTPVvCfb0HXAs6j5tNGas72k/edit?gid=0#gid=0) is called 'Public Auto Archiver template'. +Next, set up the `gsheet_feeder_db` configuration settings in the 'Configurations' part of the config `orchestration.yaml` file. Open up the file, and set the `gsheet_feeder_db.sheet` setting or the `gsheet_feeder_db.sheet_id` setting. The `sheet` should be the name of your sheet, as it shows in the top left of the sheet. +For example, the sheet [here](https://docs.google.com/spreadsheets/d/1NJZo_XZUBKTI1Ghlgi4nTPVvCfb0HXAs6j5tNGas72k/edit?gid=0#gid=0) is called 'Public Auto Archiver template'. Here's how this might look: ```{code} yaml ... -gsheet_feeder: +gsheet_feeder_db: sheet: 'My Awesome Sheet' ... ``` You can also pass these settings directly on the command line without having to edit the file, here'a an example of how to do that (using docker): -`docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver:dockerize --gsheet_feeder.sheet "Auto archive test 2023-2"`. +`docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver:dockerize --gsheet_feeder_db.sheet "My Awesome Sheet 2"`. Here, the sheet name has been overridden/specified in the command line invocation. ### 3a. (Optional) Changing the column names -In step 1, we said we would change the name of the 'Destination Folder'. Perhaps you don't like this name, or already have a sheet with a different name. In our example here, we want to name this column 'Save Folder'. To do this, we need to edit the `ghseet_feeder.column` setting in the configuration file. For more information on this setting, see the [Gsheet Feeder docs](../modules/autogen/feeder/gsheet_feeder.md#configuration-options). We will first copy the default settings from the Gsheet Feeder docs for the 'column' settings, and then edit the 'Destination Folder' section to rename it 'Save Folder'. Our final configuration section looks like: +In step 1, we said we would change the name of the 'Destination Folder'. Perhaps you don't like this name, or already have a sheet with a different name. In our example here, we want to name this column 'Save Folder'. To do this, we need to edit the `ghseet_feeder_db.column` setting in the configuration file. +For more information on this setting, see the [Gsheet Feeder Database docs](../modules/autogen/feeder/gsheet_feeder_db.md#configuration-options). We will first copy the default settings from the Gsheet Feeder docs for the 'column' settings, and then edit the 'Destination Folder' section to rename it 'Save Folder'. Our final configuration section looks like: ```{code} yaml ... -gsheet_feeder: +gsheet_feeder_db: sheet: 'My Awesome Sheet' + header: 1 + service_account: secrets/service_account.json columns: url: link status: archive status @@ -103,20 +116,44 @@ gsheet_feeder: pdq_hash: perceptual hashes wacz: wacz replaywebpage: replaywebpage + ``` +## 4. Running the Auto Archiver +### Feeding the URLs to the Auto Archiver -## Viewing the Results after archiving +The URLs to be archived should be added to the Google Sheet, and optionally a folder value. Leave all the other configured columns empty (but you may add additional columns for your own use, as long as they don't conflict with the column names mapped in the configuration file). +The Auto Archiver will archive any URLs which have an empty 'status' column -With the `ghseet_db` installed, once you start running the Auto Archiver, it will updates the "Archive status" column. +### Viewing the Results after archiving -![A screenshot of a Google Spreadsheet with column headers defined as above, and several Youtube and Twitter URLs in the "Link" column. The auto archiver has added "archive in progress" to one of the status columns.](../demo-progress.png) +With the `ghseet_feeder_db` installed, once you start running the Auto Archiver, it will update the "Archive status" column. +The status will be set to "Archive in progress" once the archival starts. If the archival is stopped during a run, either manually or because an error is raised the status value should be cleared. + +![A screenshot of a Google Spreadsheet with column headers defined as above, and several Youtube and Twitter URLs in the "Link" column. The auto archiver has added "archive in progress" to one of the status columns.](../../demo-progress.png) The links are downloaded and archived, and the spreadsheet is updated to the following: -![A screenshot of a Google Spreadsheet with videos archived and metadata added per the description of the columns above.](../demo-after.png) +![A screenshot of a Google Spreadsheet with videos archived and metadata added per the description of the columns above.](../../demo-after.png) -Note that the first row is skipped, as it is assumed to be a header row (`--gsheet_feeder.header=1` and you can change it if you use more rows above). Rows with an empty URL column, or a non-empty archive column are also skipped. All sheets in the document will be checked. +Note that the first row is skipped, as it is assumed to be a header row (`--gsheet_feeder_db.header=1` and you can change it if you use more rows above). Rows with an empty URL column, or a non-empty archive column are also skipped. All sheets in the document will be checked. The "archive location" link contains the path of the archived file, in local storage, S3, or in Google Drive. -![The archive result for a link in the demo sheet.](../demo-archive.png) +![The archive result for a link in the demo sheet.](../../demo-archive.png) + +### Troubleshooting + +**Hanging Archival in progress status** + +Occasionally system crashes or other unexpected events can cause the Auto Archiver to exit without cleaning up the status value. +If you are sure that all archival processes have stopped but you still see "Archive in progress" in the status column, you can manually clear the status column to allow the Auto Archiver to retry that archival on the next run. + +**Nothing archived status** + +Sometimes this means the tool is genuinely unable to extract the content at this point in time, but sometimes it can be resolved with different configurations. +Try: + - Turning on additional 'extractor' types in the configuration file (this can appear as 'no archiver' in the status column). + - Changing credentials or refreshing session files for extractors which require them + - Check if the extractors can accept any additional configurations such as adding a cookie file. + + diff --git a/docs/source/how_to/new_config_format.md b/docs/source/how_to/new_config_format.md index 6c12276..5cef3c8 100644 --- a/docs/source/how_to/new_config_format.md +++ b/docs/source/how_to/new_config_format.md @@ -1,11 +1,11 @@ -# Upgrading to v0.13 +# Upgrading from v0.12 ```{note} This how-to is only relevant for people who used Auto Archiver before February 2025 (versions prior to 0.13). If you are new to Auto Archiver, then you are already using the latest configuration format and this how-to is not relevant for you. ``` -Version 0.13 of Auto Archiver has breaking changes in the configuration format, which means earlier configuration formats will not work without slight modifications. +Versions 0.13+ of Auto Archiver has breaking changes in the configuration format, which means earlier configuration formats will not work without slight modifications. ## How do I know if I need to update my configuration format? @@ -22,15 +22,13 @@ your configuration file or on the command line (using --feeders) ```{code} yaml steps: - feeder: gsheet_feeder + feeder: cli_feeder ... ``` -## Updating your configuration file +The next two sections outline the two methods you have for updating your file. -To update your configuration file, you can either: - -### 1. Manually edit the configuration file and change the values. +## 1. Manually edit the configuration file and change the values. This is recommended if you want to keep all your old settings. Follow the steps below to change the relevant settings: @@ -75,28 +73,49 @@ The names of the actual modules have also changed, so for any extractor modules - `wayback_archiver_enricher` → `wayback_extractor_enricher` - `vk_archiver` → `vk_extractor` -Additionally, the `youtube_archiver` has been renamed to `generic_extractor` as it is considered the default/fallback extractor. Read more about the [generic extractor](../modules/autogen/extractor/generic_extractor.md). + +#### c) Module Renaming + + +The `youtube_archiver` has been renamed to `generic_extractor` as it is considered the default/fallback extractor. Read more about the [generic extractor](../modules/autogen/extractor/generic_extractor.md). + +The `atlos` modules have been merged into one, as have the `gsheets` feeder and database. + +- `atlos_feeder` → `atlos_feeder_db_storage` +- `atlos_storage` → `atlos_feeder_db_storage` +- `atlos_db` → `atlos_feeder_db_storage` +- `gsheet_feeder` → `gsheet_feeder_db` +- `gsheet_db` → `gsheet_feeder_db` + Example: ```{code} yaml steps: + feeders: + - gsheet_feeder_db # formerly gsheet_feeder ... - archivers: - - telethon_archiver - - youtube_archiver - - vk_archiver - -# renaming 'archiver' to 'extractor', and renaming the youtube_archiver the above config will become: -steps: + extractors: # formerly 'archivers' + - telethon_extractor # formerly telethon_archiver + - generic_extractor # formerly youtube_archiver + - vk_extractor # formerly vk_archiver + databases: + - gsheet_feeder_db # formerly gsheet_db ... - extractors: - - telethon_extractor - - vk_extractor - - generic_extractor ``` -#### c) Redundant / Obsolete Modules +```{note} + +Don't forget to also rename the configuration settings. For example: + +```{code} yaml +gsheet_feeder_db: # formerly gsheet_feeder + service_account: secrets/service_account.json + sheet: My Google Sheet +... +``` + +#### d) Redundant / Obsolete Modules With v0.13 of Auto Archiver, the following modules have been removed and their features have been built in to the generic_extractor. You should remove them from the 'steps' section of your configuration file: @@ -104,7 +123,7 @@ With v0.13 of Auto Archiver, the following modules have been removed and their f * `tiktok_archiver` - use the `generic_extractor` to extract TikTok videos. -### 2. Auto-generate a new config, then copy over your settings. +## 2. Auto-generate a new config, then copy over your settings. Using this method, you can have Auto Archiver auto-generate a configuration file for you, then you can copy over the desired settings from your old config file. This is probably the easiest method and quickest to setup, but it may require some trial and error as you copy over your settings. diff --git a/docs/source/installation/config_editor.md b/docs/source/installation/config_editor.md new file mode 100644 index 0000000..a23ebce --- /dev/null +++ b/docs/source/installation/config_editor.md @@ -0,0 +1,5 @@ +# Configuration Editor + +```{raw} html +:file: settings.html +``` \ No newline at end of file diff --git a/docs/source/installation/settings.html b/docs/source/installation/settings.html new file mode 100644 index 0000000..915ee8d --- /dev/null +++ b/docs/source/installation/settings.html @@ -0,0 +1,48685 @@ + + +
diff --git a/docs/source/installation/setup.md b/docs/source/installation/setup.md index 8d1a6f5..e5c96a6 100644 --- a/docs/source/installation/setup.md +++ b/docs/source/installation/setup.md @@ -6,6 +6,7 @@ installation.md configurations.md +config_editor.md authentication.md requirements.md config_cheatsheet.md diff --git a/scripts/generate_settings_schema.py b/scripts/generate_settings_schema.py new file mode 100644 index 0000000..16cb22f --- /dev/null +++ b/scripts/generate_settings_schema.py @@ -0,0 +1,52 @@ +import json +import os +import io + +from ruamel.yaml import YAML + +from auto_archiver.core.module import ModuleFactory +from auto_archiver.core.consts import MODULE_TYPES +from auto_archiver.core.config import EMPTY_CONFIG + +class SchemaEncoder(json.JSONEncoder): + def default(self, obj): + if isinstance(obj, set): + return list(obj) + return json.JSONEncoder.default(self, obj) + +# Get available modules +module_factory = ModuleFactory() +available_modules = module_factory.available_modules() + +modules_by_type = {} +# Categorize modules by type +for module in available_modules: + for type in module.manifest.get('type', []): + modules_by_type.setdefault(type, []).append(module) + +all_modules_ordered_by_type = sorted(available_modules, key=lambda x: (MODULE_TYPES.index(x.type[0]), not x.requires_setup)) + +yaml: YAML = YAML() + +config_string = io.BytesIO() +yaml.dump(EMPTY_CONFIG, config_string) +config_string = config_string.getvalue().decode('utf-8') +output_schema = { + 'modules': dict((module.name, + { + 'name': module.name, + 'display_name': module.display_name, + 'manifest': module.manifest, + 'configs': module.configs or None + } + ) for module in all_modules_ordered_by_type), + 'steps': dict((f"{module_type}s", [module.name for module in modules_by_type[module_type]]) for module_type in MODULE_TYPES), + 'configs': [m.name for m in all_modules_ordered_by_type if m.configs], + 'module_types': MODULE_TYPES, + 'empty_config': config_string +} + +current_file_dir = os.path.dirname(os.path.abspath(__file__)) +output_file = os.path.join(current_file_dir, 'settings/src/schema.json') +with open(output_file, 'w') as file: + json.dump(output_schema, file, indent=4, cls=SchemaEncoder) \ No newline at end of file diff --git a/scripts/settings/.gitignore b/scripts/settings/.gitignore new file mode 100644 index 0000000..a547bf3 --- /dev/null +++ b/scripts/settings/.gitignore @@ -0,0 +1,24 @@ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* +lerna-debug.log* + +node_modules +dist +dist-ssr +*.local + +# Editor directories and files +.vscode/* +!.vscode/extensions.json +.idea +.DS_Store +*.suo +*.ntvs* +*.njsproj +*.sln +*.sw? diff --git a/scripts/settings/index.html b/scripts/settings/index.html new file mode 100644 index 0000000..22ff169 --- /dev/null +++ b/scripts/settings/index.html @@ -0,0 +1,3 @@ + +
+ diff --git a/scripts/settings/package-lock.json b/scripts/settings/package-lock.json new file mode 100644 index 0000000..cd40c14 --- /dev/null +++ b/scripts/settings/package-lock.json @@ -0,0 +1,3743 @@ +{ + "name": "material-ui-vite-ts", + "version": "5.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "material-ui-vite-ts", + "version": "5.0.0", + "dependencies": { + "@dnd-kit/core": "^6.3.1", + "@dnd-kit/sortable": "^10.0.0", + "@emotion/react": "latest", + "@emotion/styled": "latest", + "@mui/icons-material": "latest", + "@mui/material": "latest", + "react": "19.0.0", + "react-dom": "19.0.0", + "react-markdown": "^10.0.0", + "yaml": "^2.7.0" + }, + "devDependencies": { + "@types/react": "latest", + "@types/react-dom": "latest", + "@vitejs/plugin-react": "latest", + "typescript": "latest", + "vite": "latest", + "vite-plugin-singlefile": "^2.1.0" + } + }, + "node_modules/@ampproject/remapping": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@ampproject/remapping/-/remapping-2.3.0.tgz", + "integrity": "sha512-30iZtAPgz+LTIYoeivqYo853f02jBYSd5uGnGpkFV0M3xOt9aN73erkgYAmZU43x4VfqcnLxW9Kpg3R5LC4YYw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/code-frame": { + "version": "7.26.2", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.26.2.tgz", + "integrity": "sha512-RJlIHRueQgwWitWgF8OdFYGZX328Ax5BCemNGlqHfplnRT9ESi8JkFlvaVYbS+UubVY6dpv87Fs2u5M29iNFVQ==", + "license": "MIT", + "dependencies": { + "@babel/helper-validator-identifier": "^7.25.9", + "js-tokens": "^4.0.0", + "picocolors": "^1.0.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/compat-data": { + "version": "7.26.8", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.26.8.tgz", + "integrity": "sha512-oH5UPLMWR3L2wEFLnFJ1TZXqHufiTKAiLfqw5zkhS4dKXLJ10yVztfil/twG8EDTA4F/tvVNw9nOl4ZMslB8rQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/core": { + "version": "7.26.9", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.26.9.tgz", + "integrity": "sha512-lWBYIrF7qK5+GjY5Uy+/hEgp8OJWOD/rpy74GplYRhEauvbHDeFB8t5hPOZxCZ0Oxf4Cc36tK51/l3ymJysrKw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@ampproject/remapping": "^2.2.0", + "@babel/code-frame": "^7.26.2", + "@babel/generator": "^7.26.9", + "@babel/helper-compilation-targets": "^7.26.5", + "@babel/helper-module-transforms": "^7.26.0", + "@babel/helpers": "^7.26.9", + "@babel/parser": "^7.26.9", + "@babel/template": "^7.26.9", + "@babel/traverse": "^7.26.9", + "@babel/types": "^7.26.9", + "convert-source-map": "^2.0.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.2", + "json5": "^2.2.3", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/babel" + } + }, + "node_modules/@babel/core/node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@babel/generator": { + "version": "7.26.9", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.26.9.tgz", + "integrity": "sha512-kEWdzjOAUMW4hAyrzJ0ZaTOu9OmpyDIQicIh0zg0EEcEkYXZb2TjtBhnHi2ViX7PKwZqF4xwqfAm299/QMP3lg==", + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.26.9", + "@babel/types": "^7.26.9", + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.25", + "jsesc": "^3.0.2" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-compilation-targets": { + "version": "7.26.5", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.26.5.tgz", + "integrity": "sha512-IXuyn5EkouFJscIDuFF5EsiSolseme1s0CZB+QxVugqJLYmKdxI1VfIBOst0SUu4rnk2Z7kqTwmoO1lp3HIfnA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/compat-data": "^7.26.5", + "@babel/helper-validator-option": "^7.25.9", + "browserslist": "^4.24.0", + "lru-cache": "^5.1.1", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-imports": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.25.9.tgz", + "integrity": "sha512-tnUA4RsrmflIM6W6RFTLFSXITtl0wKjgpnLgXyowocVPrbYrLUXSBXDgTs8BlbmIzIdlBySRQjINYs2BAkiLtw==", + "license": "MIT", + "dependencies": { + "@babel/traverse": "^7.25.9", + "@babel/types": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-transforms": { + "version": "7.26.0", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.26.0.tgz", + "integrity": "sha512-xO+xu6B5K2czEnQye6BHA7DolFFmS3LB7stHZFaOLb1pAwO1HWLS8fXA+eh0A2yIvltPVmx3eNNDBJA2SLHXFw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-module-imports": "^7.25.9", + "@babel/helper-validator-identifier": "^7.25.9", + "@babel/traverse": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-plugin-utils": { + "version": "7.26.5", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.26.5.tgz", + "integrity": "sha512-RS+jZcRdZdRFzMyr+wcsaqOmld1/EqTghfaBGQQd/WnRdzdlvSZ//kF7U8VQTxf1ynZ4cjUcYgjVGx13ewNPMg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.25.9.tgz", + "integrity": "sha512-4A/SCr/2KLd5jrtOMFzaKjVtAei3+2r/NChoBNoZ3EyP/+GlhoaEGoWOZUmFmoITP7zOJyHIMm+DYRd8o3PvHA==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.25.9.tgz", + "integrity": "sha512-Ed61U6XJc3CVRfkERJWDz4dJwKe7iLmmJsbOGu9wSloNSFttHV0I8g6UAgb7qnK5ly5bGLPd4oXZlxCdANBOWQ==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-option": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.25.9.tgz", + "integrity": "sha512-e/zv1co8pp55dNdEcCynfj9X7nyUKUXoUEwfXqaZt0omVOmDe9oOTdKStH4GmAw6zxMFs50ZayuMfHDKlO7Tfw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helpers": { + "version": "7.26.9", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.26.9.tgz", + "integrity": "sha512-Mz/4+y8udxBKdmzt/UjPACs4G3j5SshJJEFFKxlCGPydG4JAHXxjWjAwjd09tf6oINvl1VfMJo+nB7H2YKQ0dA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/template": "^7.26.9", + "@babel/types": "^7.26.9" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.26.9", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.26.9.tgz", + "integrity": "sha512-81NWa1njQblgZbQHxWHpxxCzNsa3ZwvFqpUg7P+NNUU6f3UU2jBEg4OlF/J6rl8+PQGh1q6/zWScd001YwcA5A==", + "license": "MIT", + "dependencies": { + "@babel/types": "^7.26.9" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/plugin-transform-react-jsx-self": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-self/-/plugin-transform-react-jsx-self-7.25.9.tgz", + "integrity": "sha512-y8quW6p0WHkEhmErnfe58r7x0A70uKphQm8Sp8cV7tjNQwK56sNVK0M73LK3WuYmsuyrftut4xAkjjgU0twaMg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-react-jsx-source": { + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-source/-/plugin-transform-react-jsx-source-7.25.9.tgz", + "integrity": "sha512-+iqjT8xmXhhYv4/uiYd8FNQsraMFZIfxVSqxxVSZP0WbbSAWvBXAul0m/zu+7Vv4O/3WtApy9pmaTMiumEZgfg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/runtime": { + "version": "7.26.9", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.26.9.tgz", + "integrity": "sha512-aA63XwOkcl4xxQa3HjPMqOP6LiK0ZDv3mUPYEFXkpHbaFjtGggE1A61FjFzJnB+p7/oy2gA8E+rcBNl/zC1tMg==", + "license": "MIT", + "dependencies": { + "regenerator-runtime": "^0.14.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/template": { + "version": "7.26.9", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.26.9.tgz", + "integrity": "sha512-qyRplbeIpNZhmzOysF/wFMuP9sctmh2cFzRAZOn1YapxBsE1i9bJIY586R/WBLfLcmcBlM8ROBiQURnnNy+zfA==", + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.26.2", + "@babel/parser": "^7.26.9", + "@babel/types": "^7.26.9" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/traverse": { + "version": "7.26.9", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.26.9.tgz", + "integrity": "sha512-ZYW7L+pL8ahU5fXmNbPF+iZFHCv5scFak7MZ9bwaRPLUhHh7QQEMjZUg0HevihoqCM5iSYHN61EyCoZvqC+bxg==", + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.26.2", + "@babel/generator": "^7.26.9", + "@babel/parser": "^7.26.9", + "@babel/template": "^7.26.9", + "@babel/types": "^7.26.9", + "debug": "^4.3.1", + "globals": "^11.1.0" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/types": { + "version": "7.26.9", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.26.9.tgz", + "integrity": "sha512-Y3IR1cRnOxOCDvMmNiym7XpXQ93iGDDPHx+Zj+NM+rg0fBaShfQLkg+hKPaZCEvg5N/LeCo4+Rj/i3FuJsIQaw==", + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.25.9", + "@babel/helper-validator-identifier": "^7.25.9" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@dnd-kit/accessibility": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/@dnd-kit/accessibility/-/accessibility-3.1.1.tgz", + "integrity": "sha512-2P+YgaXF+gRsIihwwY1gCsQSYnu9Zyj2py8kY5fFvUM1qm2WA2u639R6YNVfU4GWr+ZM5mqEsfHZZLoRONbemw==", + "license": "MIT", + "dependencies": { + "tslib": "^2.0.0" + }, + "peerDependencies": { + "react": ">=16.8.0" + } + }, + "node_modules/@dnd-kit/core": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/@dnd-kit/core/-/core-6.3.1.tgz", + "integrity": "sha512-xkGBRQQab4RLwgXxoqETICr6S5JlogafbhNsidmrkVv2YRs5MLwpjoF2qpiGjQt8S9AoxtIV603s0GIUpY5eYQ==", + "license": "MIT", + "dependencies": { + "@dnd-kit/accessibility": "^3.1.1", + "@dnd-kit/utilities": "^3.2.2", + "tslib": "^2.0.0" + }, + "peerDependencies": { + "react": ">=16.8.0", + "react-dom": ">=16.8.0" + } + }, + "node_modules/@dnd-kit/sortable": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/@dnd-kit/sortable/-/sortable-10.0.0.tgz", + "integrity": "sha512-+xqhmIIzvAYMGfBYYnbKuNicfSsk4RksY2XdmJhT+HAC01nix6fHCztU68jooFiMUB01Ky3F0FyOvhG/BZrWkg==", + "license": "MIT", + "dependencies": { + "@dnd-kit/utilities": "^3.2.2", + "tslib": "^2.0.0" + }, + "peerDependencies": { + "@dnd-kit/core": "^6.3.0", + "react": ">=16.8.0" + } + }, + "node_modules/@dnd-kit/utilities": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/@dnd-kit/utilities/-/utilities-3.2.2.tgz", + "integrity": "sha512-+MKAJEOfaBe5SmV6t34p80MMKhjvUz0vRrvVJbPT0WElzaOJ/1xs+D+KDv+tD/NE5ujfrChEcshd4fLn0wpiqg==", + "license": "MIT", + "dependencies": { + "tslib": "^2.0.0" + }, + "peerDependencies": { + "react": ">=16.8.0" + } + }, + "node_modules/@emotion/babel-plugin": { + "version": "11.13.5", + "resolved": "https://registry.npmjs.org/@emotion/babel-plugin/-/babel-plugin-11.13.5.tgz", + "integrity": "sha512-pxHCpT2ex+0q+HH91/zsdHkw/lXd468DIN2zvfvLtPKLLMo6gQj7oLObq8PhkrxOZb/gGCq03S3Z7PDhS8pduQ==", + "license": "MIT", + "dependencies": { + "@babel/helper-module-imports": "^7.16.7", + "@babel/runtime": "^7.18.3", + "@emotion/hash": "^0.9.2", + "@emotion/memoize": "^0.9.0", + "@emotion/serialize": "^1.3.3", + "babel-plugin-macros": "^3.1.0", + "convert-source-map": "^1.5.0", + "escape-string-regexp": "^4.0.0", + "find-root": "^1.1.0", + "source-map": "^0.5.7", + "stylis": "4.2.0" + } + }, + "node_modules/@emotion/cache": { + "version": "11.14.0", + "resolved": "https://registry.npmjs.org/@emotion/cache/-/cache-11.14.0.tgz", + "integrity": "sha512-L/B1lc/TViYk4DcpGxtAVbx0ZyiKM5ktoIyafGkH6zg/tj+mA+NE//aPYKG0k8kCHSHVJrpLpcAlOBEXQ3SavA==", + "license": "MIT", + "dependencies": { + "@emotion/memoize": "^0.9.0", + "@emotion/sheet": "^1.4.0", + "@emotion/utils": "^1.4.2", + "@emotion/weak-memoize": "^0.4.0", + "stylis": "4.2.0" + } + }, + "node_modules/@emotion/hash": { + "version": "0.9.2", + "resolved": "https://registry.npmjs.org/@emotion/hash/-/hash-0.9.2.tgz", + "integrity": "sha512-MyqliTZGuOm3+5ZRSaaBGP3USLw6+EGykkwZns2EPC5g8jJ4z9OrdZY9apkl3+UP9+sdz76YYkwCKP5gh8iY3g==", + "license": "MIT" + }, + "node_modules/@emotion/is-prop-valid": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@emotion/is-prop-valid/-/is-prop-valid-1.3.1.tgz", + "integrity": "sha512-/ACwoqx7XQi9knQs/G0qKvv5teDMhD7bXYns9N/wM8ah8iNb8jZ2uNO0YOgiq2o2poIvVtJS2YALasQuMSQ7Kw==", + "license": "MIT", + "dependencies": { + "@emotion/memoize": "^0.9.0" + } + }, + "node_modules/@emotion/memoize": { + "version": "0.9.0", + "resolved": "https://registry.npmjs.org/@emotion/memoize/-/memoize-0.9.0.tgz", + "integrity": "sha512-30FAj7/EoJ5mwVPOWhAyCX+FPfMDrVecJAM+Iw9NRoSl4BBAQeqj4cApHHUXOVvIPgLVDsCFoz/hGD+5QQD1GQ==", + "license": "MIT" + }, + "node_modules/@emotion/react": { + "version": "11.14.0", + "resolved": "https://registry.npmjs.org/@emotion/react/-/react-11.14.0.tgz", + "integrity": "sha512-O000MLDBDdk/EohJPFUqvnp4qnHeYkVP5B0xEG0D/L7cOKP9kefu2DXn8dj74cQfsEzUqh+sr1RzFqiL1o+PpA==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.18.3", + "@emotion/babel-plugin": "^11.13.5", + "@emotion/cache": "^11.14.0", + "@emotion/serialize": "^1.3.3", + "@emotion/use-insertion-effect-with-fallbacks": "^1.2.0", + "@emotion/utils": "^1.4.2", + "@emotion/weak-memoize": "^0.4.0", + "hoist-non-react-statics": "^3.3.1" + }, + "peerDependencies": { + "react": ">=16.8.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@emotion/serialize": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/@emotion/serialize/-/serialize-1.3.3.tgz", + "integrity": "sha512-EISGqt7sSNWHGI76hC7x1CksiXPahbxEOrC5RjmFRJTqLyEK9/9hZvBbiYn70dw4wuwMKiEMCUlR6ZXTSWQqxA==", + "license": "MIT", + "dependencies": { + "@emotion/hash": "^0.9.2", + "@emotion/memoize": "^0.9.0", + "@emotion/unitless": "^0.10.0", + "@emotion/utils": "^1.4.2", + "csstype": "^3.0.2" + } + }, + "node_modules/@emotion/sheet": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@emotion/sheet/-/sheet-1.4.0.tgz", + "integrity": "sha512-fTBW9/8r2w3dXWYM4HCB1Rdp8NLibOw2+XELH5m5+AkWiL/KqYX6dc0kKYlaYyKjrQ6ds33MCdMPEwgs2z1rqg==", + "license": "MIT" + }, + "node_modules/@emotion/styled": { + "version": "11.14.0", + "resolved": "https://registry.npmjs.org/@emotion/styled/-/styled-11.14.0.tgz", + "integrity": "sha512-XxfOnXFffatap2IyCeJyNov3kiDQWoR08gPUQxvbL7fxKryGBKUZUkG6Hz48DZwVrJSVh9sJboyV1Ds4OW6SgA==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.18.3", + "@emotion/babel-plugin": "^11.13.5", + "@emotion/is-prop-valid": "^1.3.0", + "@emotion/serialize": "^1.3.3", + "@emotion/use-insertion-effect-with-fallbacks": "^1.2.0", + "@emotion/utils": "^1.4.2" + }, + "peerDependencies": { + "@emotion/react": "^11.0.0-rc.0", + "react": ">=16.8.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@emotion/unitless": { + "version": "0.10.0", + "resolved": "https://registry.npmjs.org/@emotion/unitless/-/unitless-0.10.0.tgz", + "integrity": "sha512-dFoMUuQA20zvtVTuxZww6OHoJYgrzfKM1t52mVySDJnMSEa08ruEvdYQbhvyu6soU+NeLVd3yKfTfT0NeV6qGg==", + "license": "MIT" + }, + "node_modules/@emotion/use-insertion-effect-with-fallbacks": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@emotion/use-insertion-effect-with-fallbacks/-/use-insertion-effect-with-fallbacks-1.2.0.tgz", + "integrity": "sha512-yJMtVdH59sxi/aVJBpk9FQq+OR8ll5GT8oWd57UpeaKEVGab41JWaCFA7FRLoMLloOZF/c/wsPoe+bfGmRKgDg==", + "license": "MIT", + "peerDependencies": { + "react": ">=16.8.0" + } + }, + "node_modules/@emotion/utils": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/@emotion/utils/-/utils-1.4.2.tgz", + "integrity": "sha512-3vLclRofFziIa3J2wDh9jjbkUz9qk5Vi3IZ/FSTKViB0k+ef0fPV7dYrUIugbgupYDx7v9ud/SjrtEP8Y4xLoA==", + "license": "MIT" + }, + "node_modules/@emotion/weak-memoize": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/@emotion/weak-memoize/-/weak-memoize-0.4.0.tgz", + "integrity": "sha512-snKqtPW01tN0ui7yu9rGv69aJXr/a/Ywvl11sUjNtEcRc+ng/mQriFL0wLXMef74iHa/EkftbDzU9F8iFbH+zg==", + "license": "MIT" + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.25.0.tgz", + "integrity": "sha512-O7vun9Sf8DFjH2UtqK8Ku3LkquL9SZL8OLY1T5NZkA34+wG3OQF7cl4Ql8vdNzM6fzBbYfLaiRLIOZ+2FOCgBQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.25.0.tgz", + "integrity": "sha512-PTyWCYYiU0+1eJKmw21lWtC+d08JDZPQ5g+kFyxP0V+es6VPPSUhM6zk8iImp2jbV6GwjX4pap0JFbUQN65X1g==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.25.0.tgz", + "integrity": "sha512-grvv8WncGjDSyUBjN9yHXNt+cq0snxXbDxy5pJtzMKGmmpPxeAmAhWxXI+01lU5rwZomDgD3kJwulEnhTRUd6g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.25.0.tgz", + "integrity": "sha512-m/ix7SfKG5buCnxasr52+LI78SQ+wgdENi9CqyCXwjVR2X4Jkz+BpC3le3AoBPYTC9NHklwngVXvbJ9/Akhrfg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.25.0.tgz", + "integrity": "sha512-mVwdUb5SRkPayVadIOI78K7aAnPamoeFR2bT5nszFUZ9P8UpK4ratOdYbZZXYSqPKMHfS1wdHCJk1P1EZpRdvw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.25.0.tgz", + "integrity": "sha512-DgDaYsPWFTS4S3nWpFcMn/33ZZwAAeAFKNHNa1QN0rI4pUjgqf0f7ONmXf6d22tqTY+H9FNdgeaAa+YIFUn2Rg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.25.0.tgz", + "integrity": "sha512-VN4ocxy6dxefN1MepBx/iD1dH5K8qNtNe227I0mnTRjry8tj5MRk4zprLEdG8WPyAPb93/e4pSgi1SoHdgOa4w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.25.0.tgz", + "integrity": "sha512-mrSgt7lCh07FY+hDD1TxiTyIHyttn6vnjesnPoVDNmDfOmggTLXRv8Id5fNZey1gl/V2dyVK1VXXqVsQIiAk+A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.25.0.tgz", + "integrity": "sha512-vkB3IYj2IDo3g9xX7HqhPYxVkNQe8qTK55fraQyTzTX/fxaDtXiEnavv9geOsonh2Fd2RMB+i5cbhu2zMNWJwg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.0.tgz", + "integrity": "sha512-9QAQjTWNDM/Vk2bgBl17yWuZxZNQIF0OUUuPZRKoDtqF2k4EtYbpyiG5/Dk7nqeK6kIJWPYldkOcBqjXjrUlmg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.25.0.tgz", + "integrity": "sha512-43ET5bHbphBegyeqLb7I1eYn2P/JYGNmzzdidq/w0T8E2SsYL1U6un2NFROFRg1JZLTzdCoRomg8Rvf9M6W6Gg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.25.0.tgz", + "integrity": "sha512-fC95c/xyNFueMhClxJmeRIj2yrSMdDfmqJnyOY4ZqsALkDrrKJfIg5NTMSzVBr5YW1jf+l7/cndBfP3MSDpoHw==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.25.0.tgz", + "integrity": "sha512-nkAMFju7KDW73T1DdH7glcyIptm95a7Le8irTQNO/qtkoyypZAnjchQgooFUDQhNAy4iu08N79W4T4pMBwhPwQ==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.25.0.tgz", + "integrity": "sha512-NhyOejdhRGS8Iwv+KKR2zTq2PpysF9XqY+Zk77vQHqNbo/PwZCzB5/h7VGuREZm1fixhs4Q/qWRSi5zmAiO4Fw==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.25.0.tgz", + "integrity": "sha512-5S/rbP5OY+GHLC5qXp1y/Mx//e92L1YDqkiBbO9TQOvuFXM+iDqUNG5XopAnXoRH3FjIUDkeGcY1cgNvnXp/kA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.25.0.tgz", + "integrity": "sha512-XM2BFsEBz0Fw37V0zU4CXfcfuACMrppsMFKdYY2WuTS3yi8O1nFOhil/xhKTmE1nPmVyvQJjJivgDT+xh8pXJA==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.25.0.tgz", + "integrity": "sha512-9yl91rHw/cpwMCNytUDxwj2XjFpxML0y9HAOH9pNVQDpQrBxHy01Dx+vaMu0N1CKa/RzBD2hB4u//nfc+Sd3Cw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.25.0.tgz", + "integrity": "sha512-RuG4PSMPFfrkH6UwCAqBzauBWTygTvb1nxWasEJooGSJ/NwRw7b2HOwyRTQIU97Hq37l3npXoZGYMy3b3xYvPw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.25.0.tgz", + "integrity": "sha512-jl+qisSB5jk01N5f7sPCsBENCOlPiS/xptD5yxOx2oqQfyourJwIKLRA2yqWdifj3owQZCL2sn6o08dBzZGQzA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.25.0.tgz", + "integrity": "sha512-21sUNbq2r84YE+SJDfaQRvdgznTD8Xc0oc3p3iW/a1EVWeNj/SdUCbm5U0itZPQYRuRTW20fPMWMpcrciH2EJw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.25.0.tgz", + "integrity": "sha512-2gwwriSMPcCFRlPlKx3zLQhfN/2WjJ2NSlg5TKLQOJdV0mSxIcYNTMhk3H3ulL/cak+Xj0lY1Ym9ysDV1igceg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.25.0.tgz", + "integrity": "sha512-bxI7ThgLzPrPz484/S9jLlvUAHYMzy6I0XiU1ZMeAEOBcS0VePBFxh1JjTQt3Xiat5b6Oh4x7UC7IwKQKIJRIg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.25.0.tgz", + "integrity": "sha512-ZUAc2YK6JW89xTbXvftxdnYy3m4iHIkDtK3CLce8wg8M2L+YZhIvO1DKpxrd0Yr59AeNNkTiic9YLf6FTtXWMw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.25.0.tgz", + "integrity": "sha512-eSNxISBu8XweVEWG31/JzjkIGbGIJN/TrRoiSVZwZ6pkC6VX4Im/WV2cz559/TXLcYbcrDN8JtKgd9DJVIo8GA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.25.0.tgz", + "integrity": "sha512-ZENoHJBxA20C2zFzh6AI4fT6RraMzjYw4xKWemRTRmRVtN9c5DcH9r/f2ihEkMjOW5eGgrwCslG/+Y/3bL+DHQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.8", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.8.tgz", + "integrity": "sha512-imAbBGkb+ebQyxKgzv5Hu2nmROxoDOXHh80evxdoXNOrvAnVx7zimzc1Oo5h9RlfV4vPXaE2iM5pOFbvOCClWA==", + "license": "MIT", + "dependencies": { + "@jridgewell/set-array": "^1.2.1", + "@jridgewell/sourcemap-codec": "^1.4.10", + "@jridgewell/trace-mapping": "^0.3.24" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/set-array": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.2.1.tgz", + "integrity": "sha512-R8gLRTZeyp03ymzP/6Lil/28tGeGEzhx1q2k703KGWRAI1VdvPIXdG70VJc2pAMw3NA6JKL5hhFu1sJX0Mnn/A==", + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.0.tgz", + "integrity": "sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ==", + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.25", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.25.tgz", + "integrity": "sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ==", + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@mui/core-downloads-tracker": { + "version": "6.4.6", + "resolved": "https://registry.npmjs.org/@mui/core-downloads-tracker/-/core-downloads-tracker-6.4.6.tgz", + "integrity": "sha512-rho5Q4IscbrVmK9rCrLTJmjLjfH6m/NcqKr/mchvck0EIXlyYUB9+Z0oVmkt/+Mben43LMRYBH8q/Uzxj/c4Vw==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mui-org" + } + }, + "node_modules/@mui/icons-material": { + "version": "6.4.6", + "resolved": "https://registry.npmjs.org/@mui/icons-material/-/icons-material-6.4.6.tgz", + "integrity": "sha512-rGJBvIQQbQAlyKYljHQ8wAQS/K2/uYwvemcpygnAmCizmCI4zSF9HQPuiG8Ql4YLZ6V/uKjA3WHIYmF/8sV+pQ==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.26.0" + }, + "engines": { + "node": ">=14.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mui-org" + }, + "peerDependencies": { + "@mui/material": "^6.4.6", + "@types/react": "^17.0.0 || ^18.0.0 || ^19.0.0", + "react": "^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@mui/material": { + "version": "6.4.6", + "resolved": "https://registry.npmjs.org/@mui/material/-/material-6.4.6.tgz", + "integrity": "sha512-6UyAju+DBOdMogfYmLiT3Nu7RgliorimNBny1pN/acOjc+THNFVE7hlxLyn3RDONoZJNDi/8vO4AQQr6dLAXqA==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.26.0", + "@mui/core-downloads-tracker": "^6.4.6", + "@mui/system": "^6.4.6", + "@mui/types": "^7.2.21", + "@mui/utils": "^6.4.6", + "@popperjs/core": "^2.11.8", + "@types/react-transition-group": "^4.4.12", + "clsx": "^2.1.1", + "csstype": "^3.1.3", + "prop-types": "^15.8.1", + "react-is": "^19.0.0", + "react-transition-group": "^4.4.5" + }, + "engines": { + "node": ">=14.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mui-org" + }, + "peerDependencies": { + "@emotion/react": "^11.5.0", + "@emotion/styled": "^11.3.0", + "@mui/material-pigment-css": "^6.4.6", + "@types/react": "^17.0.0 || ^18.0.0 || ^19.0.0", + "react": "^17.0.0 || ^18.0.0 || ^19.0.0", + "react-dom": "^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@emotion/react": { + "optional": true + }, + "@emotion/styled": { + "optional": true + }, + "@mui/material-pigment-css": { + "optional": true + }, + "@types/react": { + "optional": true + } + } + }, + "node_modules/@mui/private-theming": { + "version": "6.4.6", + "resolved": "https://registry.npmjs.org/@mui/private-theming/-/private-theming-6.4.6.tgz", + "integrity": "sha512-T5FxdPzCELuOrhpA2g4Pi6241HAxRwZudzAuL9vBvniuB5YU82HCmrARw32AuCiyTfWzbrYGGpZ4zyeqqp9RvQ==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.26.0", + "@mui/utils": "^6.4.6", + "prop-types": "^15.8.1" + }, + "engines": { + "node": ">=14.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mui-org" + }, + "peerDependencies": { + "@types/react": "^17.0.0 || ^18.0.0 || ^19.0.0", + "react": "^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@mui/styled-engine": { + "version": "6.4.6", + "resolved": "https://registry.npmjs.org/@mui/styled-engine/-/styled-engine-6.4.6.tgz", + "integrity": "sha512-vSWYc9ZLX46be5gP+FCzWVn5rvDr4cXC5JBZwSIkYk9xbC7GeV+0kCvB8Q6XLFQJy+a62bbqtmdwS4Ghi9NBlQ==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.26.0", + "@emotion/cache": "^11.13.5", + "@emotion/serialize": "^1.3.3", + "@emotion/sheet": "^1.4.0", + "csstype": "^3.1.3", + "prop-types": "^15.8.1" + }, + "engines": { + "node": ">=14.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mui-org" + }, + "peerDependencies": { + "@emotion/react": "^11.4.1", + "@emotion/styled": "^11.3.0", + "react": "^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@emotion/react": { + "optional": true + }, + "@emotion/styled": { + "optional": true + } + } + }, + "node_modules/@mui/system": { + "version": "6.4.6", + "resolved": "https://registry.npmjs.org/@mui/system/-/system-6.4.6.tgz", + "integrity": "sha512-FQjWwPec7pMTtB/jw5f9eyLynKFZ6/Ej9vhm5kGdtmts1z5b7Vyn3Rz6kasfYm1j2TfrfGnSXRvvtwVWxjpz6g==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.26.0", + "@mui/private-theming": "^6.4.6", + "@mui/styled-engine": "^6.4.6", + "@mui/types": "^7.2.21", + "@mui/utils": "^6.4.6", + "clsx": "^2.1.1", + "csstype": "^3.1.3", + "prop-types": "^15.8.1" + }, + "engines": { + "node": ">=14.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mui-org" + }, + "peerDependencies": { + "@emotion/react": "^11.5.0", + "@emotion/styled": "^11.3.0", + "@types/react": "^17.0.0 || ^18.0.0 || ^19.0.0", + "react": "^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@emotion/react": { + "optional": true + }, + "@emotion/styled": { + "optional": true + }, + "@types/react": { + "optional": true + } + } + }, + "node_modules/@mui/types": { + "version": "7.2.21", + "resolved": "https://registry.npmjs.org/@mui/types/-/types-7.2.21.tgz", + "integrity": "sha512-6HstngiUxNqLU+/DPqlUJDIPbzUBxIVHb1MmXP0eTWDIROiCR2viugXpEif0PPe2mLqqakPzzRClWAnK+8UJww==", + "license": "MIT", + "peerDependencies": { + "@types/react": "^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@mui/utils": { + "version": "6.4.6", + "resolved": "https://registry.npmjs.org/@mui/utils/-/utils-6.4.6.tgz", + "integrity": "sha512-43nZeE1pJF2anGafNydUcYFPtHwAqiBiauRtaMvurdrZI3YrUjHkAu43RBsxef7OFtJMXGiHFvq43kb7lig0sA==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.26.0", + "@mui/types": "^7.2.21", + "@types/prop-types": "^15.7.14", + "clsx": "^2.1.1", + "prop-types": "^15.8.1", + "react-is": "^19.0.0" + }, + "engines": { + "node": ">=14.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mui-org" + }, + "peerDependencies": { + "@types/react": "^17.0.0 || ^18.0.0 || ^19.0.0", + "react": "^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@popperjs/core": { + "version": "2.11.8", + "resolved": "https://registry.npmjs.org/@popperjs/core/-/core-2.11.8.tgz", + "integrity": "sha512-P1st0aksCrn9sGZhp8GMYwBnQsbvAWsZAX44oXNNvLHGqAOcoVxmjZiohstwQ7SqKnbR47akdNi+uleWD8+g6A==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/popperjs" + } + }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.34.8", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.34.8.tgz", + "integrity": "sha512-q217OSE8DTp8AFHuNHXo0Y86e1wtlfVrXiAlwkIvGRQv9zbc6mE3sjIVfwI8sYUyNxwOg0j/Vm1RKM04JcWLJw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.34.8", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.34.8.tgz", + "integrity": "sha512-Gigjz7mNWaOL9wCggvoK3jEIUUbGul656opstjaUSGC3eT0BM7PofdAJaBfPFWWkXNVAXbaQtC99OCg4sJv70Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.34.8", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.34.8.tgz", + "integrity": "sha512-02rVdZ5tgdUNRxIUrFdcMBZQoaPMrxtwSb+/hOfBdqkatYHR3lZ2A2EGyHq2sGOd0Owk80oV3snlDASC24He3Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.34.8", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.34.8.tgz", + "integrity": "sha512-qIP/elwR/tq/dYRx3lgwK31jkZvMiD6qUtOycLhTzCvrjbZ3LjQnEM9rNhSGpbLXVJYQ3rq39A6Re0h9tU2ynw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.34.8", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.34.8.tgz", + "integrity": "sha512-IQNVXL9iY6NniYbTaOKdrlVP3XIqazBgJOVkddzJlqnCpRi/yAeSOa8PLcECFSQochzqApIOE1GHNu3pCz+BDA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.34.8", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.34.8.tgz", + "integrity": "sha512-TYXcHghgnCqYFiE3FT5QwXtOZqDj5GmaFNTNt3jNC+vh22dc/ukG2cG+pi75QO4kACohZzidsq7yKTKwq/Jq7Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.34.8", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.34.8.tgz", + "integrity": "sha512-A4iphFGNkWRd+5m3VIGuqHnG3MVnqKe7Al57u9mwgbyZ2/xF9Jio72MaY7xxh+Y87VAHmGQr73qoKL9HPbXj1g==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.34.8", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.34.8.tgz", + "integrity": "sha512-S0lqKLfTm5u+QTxlFiAnb2J/2dgQqRy/XvziPtDd1rKZFXHTyYLoVL58M/XFwDI01AQCDIevGLbQrMAtdyanpA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.34.8", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.34.8.tgz", + "integrity": "sha512-jpz9YOuPiSkL4G4pqKrus0pn9aYwpImGkosRKwNi+sJSkz+WU3anZe6hi73StLOQdfXYXC7hUfsQlTnjMd3s1A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.34.8", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.34.8.tgz", + "integrity": "sha512-KdSfaROOUJXgTVxJNAZ3KwkRc5nggDk+06P6lgi1HLv1hskgvxHUKZ4xtwHkVYJ1Rep4GNo+uEfycCRRxht7+Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loongarch64-gnu": { + "version": "4.34.8", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loongarch64-gnu/-/rollup-linux-loongarch64-gnu-4.34.8.tgz", + "integrity": "sha512-NyF4gcxwkMFRjgXBM6g2lkT58OWztZvw5KkV2K0qqSnUEqCVcqdh2jN4gQrTn/YUpAcNKyFHfoOZEer9nwo6uQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-powerpc64le-gnu": { + "version": "4.34.8", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.34.8.tgz", + "integrity": "sha512-LMJc999GkhGvktHU85zNTDImZVUCJ1z/MbAJTnviiWmmjyckP5aQsHtcujMjpNdMZPT2rQEDBlJfubhs3jsMfw==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.34.8", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.34.8.tgz", + "integrity": "sha512-xAQCAHPj8nJq1PI3z8CIZzXuXCstquz7cIOL73HHdXiRcKk8Ywwqtx2wrIy23EcTn4aZ2fLJNBB8d0tQENPCmw==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.34.8", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.34.8.tgz", + "integrity": "sha512-DdePVk1NDEuc3fOe3dPPTb+rjMtuFw89gw6gVWxQFAuEqqSdDKnrwzZHrUYdac7A7dXl9Q2Vflxpme15gUWQFA==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.34.8", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.34.8.tgz", + "integrity": "sha512-8y7ED8gjxITUltTUEJLQdgpbPh1sUQ0kMTmufRF/Ns5tI9TNMNlhWtmPKKHCU0SilX+3MJkZ0zERYYGIVBYHIA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.34.8", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.34.8.tgz", + "integrity": "sha512-SCXcP0ZpGFIe7Ge+McxY5zKxiEI5ra+GT3QRxL0pMMtxPfpyLAKleZODi1zdRHkz5/BhueUrYtYVgubqe9JBNQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.34.8", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.34.8.tgz", + "integrity": "sha512-YHYsgzZgFJzTRbth4h7Or0m5O74Yda+hLin0irAIobkLQFRQd1qWmnoVfwmKm9TXIZVAD0nZ+GEb2ICicLyCnQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.34.8", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.34.8.tgz", + "integrity": "sha512-r3NRQrXkHr4uWy5TOjTpTYojR9XmF0j/RYgKCef+Ag46FWUTltm5ziticv8LdNsDMehjJ543x/+TJAek/xBA2w==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.34.8", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.34.8.tgz", + "integrity": "sha512-U0FaE5O1BCpZSeE6gBl3c5ObhePQSfk9vDRToMmTkbhCOgW4jqvtS5LGyQ76L1fH8sM0keRp4uDTsbjiUyjk0g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@types/babel__core": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", + "integrity": "sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.20.7", + "@babel/types": "^7.20.7", + "@types/babel__generator": "*", + "@types/babel__template": "*", + "@types/babel__traverse": "*" + } + }, + "node_modules/@types/babel__generator": { + "version": "7.6.8", + "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.6.8.tgz", + "integrity": "sha512-ASsj+tpEDsEiFr1arWrlN6V3mdfjRMZt6LtK/Vp/kreFLnr5QH5+DhvD5nINYZXzwJvXeGq+05iUXcAzVrqWtw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__template": { + "version": "7.4.4", + "resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.4.4.tgz", + "integrity": "sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.1.0", + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__traverse": { + "version": "7.20.6", + "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.20.6.tgz", + "integrity": "sha512-r1bzfrm0tomOI8g1SzvCaQHo6Lcv6zu0EA+W2kHrt8dyrHQxGzBBL4kdkzIS+jBMV+EYcMAEAqXqYaLJq5rOZg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.20.7" + } + }, + "node_modules/@types/debug": { + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.12.tgz", + "integrity": "sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==", + "license": "MIT", + "dependencies": { + "@types/ms": "*" + } + }, + "node_modules/@types/estree": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.6.tgz", + "integrity": "sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw==", + "license": "MIT" + }, + "node_modules/@types/estree-jsx": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@types/estree-jsx/-/estree-jsx-1.0.5.tgz", + "integrity": "sha512-52CcUVNFyfb1A2ALocQw/Dd1BQFNmSdkuC3BkZ6iqhdMfQz7JWOFRuJFloOzjk+6WijU56m9oKXFAXc7o3Towg==", + "license": "MIT", + "dependencies": { + "@types/estree": "*" + } + }, + "node_modules/@types/hast": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/hast/-/hast-3.0.4.tgz", + "integrity": "sha512-WPs+bbQw5aCj+x6laNGWLH3wviHtoCv/P3+otBhbOhJgG8qtpdAMlTCxLtsTWA7LH1Oh/bFCHsBn0TPS5m30EQ==", + "license": "MIT", + "dependencies": { + "@types/unist": "*" + } + }, + "node_modules/@types/mdast": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-4.0.4.tgz", + "integrity": "sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==", + "license": "MIT", + "dependencies": { + "@types/unist": "*" + } + }, + "node_modules/@types/ms": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@types/ms/-/ms-2.1.0.tgz", + "integrity": "sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==", + "license": "MIT" + }, + "node_modules/@types/parse-json": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.2.tgz", + "integrity": "sha512-dISoDXWWQwUquiKsyZ4Ng+HX2KsPL7LyHKHQwgGFEA3IaKac4Obd+h2a/a6waisAoepJlBcx9paWqjA8/HVjCw==", + "license": "MIT" + }, + "node_modules/@types/prop-types": { + "version": "15.7.14", + "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.14.tgz", + "integrity": "sha512-gNMvNH49DJ7OJYv+KAKn0Xp45p8PLl6zo2YnvDIbTd4J6MER2BmWN49TG7n9LvkyihINxeKW8+3bfS2yDC9dzQ==", + "license": "MIT" + }, + "node_modules/@types/react": { + "version": "19.0.10", + "resolved": "https://registry.npmjs.org/@types/react/-/react-19.0.10.tgz", + "integrity": "sha512-JuRQ9KXLEjaUNjTWpzuR231Z2WpIwczOkBEIvbHNCzQefFIT0L8IqE6NV6ULLyC1SI/i234JnDoMkfg+RjQj2g==", + "license": "MIT", + "dependencies": { + "csstype": "^3.0.2" + } + }, + "node_modules/@types/react-dom": { + "version": "19.0.4", + "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.0.4.tgz", + "integrity": "sha512-4fSQ8vWFkg+TGhePfUzVmat3eC14TXYSsiiDSLI0dVLsrm9gZFABjPy/Qu6TKgl1tq1Bu1yDsuQgY3A3DOjCcg==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "@types/react": "^19.0.0" + } + }, + "node_modules/@types/react-transition-group": { + "version": "4.4.12", + "resolved": "https://registry.npmjs.org/@types/react-transition-group/-/react-transition-group-4.4.12.tgz", + "integrity": "sha512-8TV6R3h2j7a91c+1DXdJi3Syo69zzIZbz7Lg5tORM5LEJG7X/E6a1V3drRyBRZq7/utz7A+c4OgYLiLcYGHG6w==", + "license": "MIT", + "peerDependencies": { + "@types/react": "*" + } + }, + "node_modules/@types/unist": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", + "integrity": "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==", + "license": "MIT" + }, + "node_modules/@ungap/structured-clone": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.3.0.tgz", + "integrity": "sha512-WmoN8qaIAo7WTYWbAZuG8PYEhn5fkz7dZrqTBZ7dtt//lL2Gwms1IcnQ5yHqjDfX8Ft5j4YzDM23f87zBfDe9g==", + "license": "ISC" + }, + "node_modules/@vitejs/plugin-react": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-4.3.4.tgz", + "integrity": "sha512-SCCPBJtYLdE8PX/7ZQAs1QAZ8Jqwih+0VBLum1EGqmCCQal+MIUqLCzj3ZUy8ufbC0cAM4LRlSTm7IQJwWT4ug==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/core": "^7.26.0", + "@babel/plugin-transform-react-jsx-self": "^7.25.9", + "@babel/plugin-transform-react-jsx-source": "^7.25.9", + "@types/babel__core": "^7.20.5", + "react-refresh": "^0.14.2" + }, + "engines": { + "node": "^14.18.0 || >=16.0.0" + }, + "peerDependencies": { + "vite": "^4.2.0 || ^5.0.0 || ^6.0.0" + } + }, + "node_modules/babel-plugin-macros": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/babel-plugin-macros/-/babel-plugin-macros-3.1.0.tgz", + "integrity": "sha512-Cg7TFGpIr01vOQNODXOOaGz2NpCU5gl8x1qJFbb6hbZxR7XrcE2vtbAsTAbJ7/xwJtUuJEw8K8Zr/AE0LHlesg==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.12.5", + "cosmiconfig": "^7.0.0", + "resolve": "^1.19.0" + }, + "engines": { + "node": ">=10", + "npm": ">=6" + } + }, + "node_modules/bail": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/bail/-/bail-2.0.2.tgz", + "integrity": "sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/braces": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fill-range": "^7.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/browserslist": { + "version": "4.24.4", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.24.4.tgz", + "integrity": "sha512-KDi1Ny1gSePi1vm0q4oxSF8b4DR44GF4BbmS2YdhPLOEqd8pDviZOGH/GsmRwoWJ2+5Lr085X7naowMwKHDG1A==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "caniuse-lite": "^1.0.30001688", + "electron-to-chromium": "^1.5.73", + "node-releases": "^2.0.19", + "update-browserslist-db": "^1.1.1" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, + "node_modules/callsites": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", + "integrity": "sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001701", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001701.tgz", + "integrity": "sha512-faRs/AW3jA9nTwmJBSO1PQ6L/EOgsB5HMQQq4iCu5zhPgVVgO/pZRHlmatwijZKetFw8/Pr4q6dEN8sJuq8qTw==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "CC-BY-4.0" + }, + "node_modules/ccount": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/ccount/-/ccount-2.0.1.tgz", + "integrity": "sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-entities": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/character-entities/-/character-entities-2.0.2.tgz", + "integrity": "sha512-shx7oQ0Awen/BRIdkjkvz54PnEEI/EjwXDSIZp86/KKdbafHh1Df/RYGBhn4hbe2+uKC9FnT5UCEdyPz3ai9hQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-entities-html4": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/character-entities-html4/-/character-entities-html4-2.1.0.tgz", + "integrity": "sha512-1v7fgQRj6hnSwFpq1Eu0ynr/CDEw0rXo2B61qXrLNdHZmPKgb7fqS1a2JwF0rISo9q77jDI8VMEHoApn8qDoZA==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-entities-legacy": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-3.0.0.tgz", + "integrity": "sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/character-reference-invalid": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/character-reference-invalid/-/character-reference-invalid-2.0.1.tgz", + "integrity": "sha512-iBZ4F4wRbyORVsu0jPV7gXkOsGYjGHPmAyv+HiHG8gi5PtC9KI2j1+v8/tlibRvjoWX027ypmG/n0HtO5t7unw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/clsx": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz", + "integrity": "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/comma-separated-tokens": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-2.0.3.tgz", + "integrity": "sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/convert-source-map": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.9.0.tgz", + "integrity": "sha512-ASFBup0Mz1uyiIjANan1jzLQami9z1PoYSZCiiYW2FczPbenXc45FZdBZLzOT+r6+iciuEModtmCti+hjaAk0A==", + "license": "MIT" + }, + "node_modules/cosmiconfig": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-7.1.0.tgz", + "integrity": "sha512-AdmX6xUzdNASswsFtmwSt7Vj8po9IuqXm0UXz7QKPuEUmPB4XyjGfaAr2PSuELMwkRMVH1EpIkX5bTZGRB3eCA==", + "license": "MIT", + "dependencies": { + "@types/parse-json": "^4.0.0", + "import-fresh": "^3.2.1", + "parse-json": "^5.0.0", + "path-type": "^4.0.0", + "yaml": "^1.10.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/cosmiconfig/node_modules/yaml": { + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-1.10.2.tgz", + "integrity": "sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==", + "license": "ISC", + "engines": { + "node": ">= 6" + } + }, + "node_modules/csstype": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.3.tgz", + "integrity": "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==", + "license": "MIT" + }, + "node_modules/debug": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", + "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/decode-named-character-reference": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/decode-named-character-reference/-/decode-named-character-reference-1.0.2.tgz", + "integrity": "sha512-O8x12RzrUF8xyVcY0KJowWsmaJxQbmy0/EtnNtHRpsOcT7dFk5W598coHqBVpmWo1oQQfsCqfCmkZN5DJrZVdg==", + "license": "MIT", + "dependencies": { + "character-entities": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/devlop": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/devlop/-/devlop-1.1.0.tgz", + "integrity": "sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==", + "license": "MIT", + "dependencies": { + "dequal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/dom-helpers": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/dom-helpers/-/dom-helpers-5.2.1.tgz", + "integrity": "sha512-nRCa7CK3VTrM2NmGkIy4cbK7IZlgBE/PYMn55rrXefr5xXDP0LdtfPnblFDoVdcAfslJ7or6iqAUnx0CCGIWQA==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.8.7", + "csstype": "^3.0.2" + } + }, + "node_modules/electron-to-chromium": { + "version": "1.5.107", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.107.tgz", + "integrity": "sha512-dJr1o6yCntRkXElnhsHh1bAV19bo/hKyFf7tCcWgpXbuFIF0Lakjgqv5LRfSDaNzAII8Fnxg2tqgHkgCvxdbxw==", + "dev": true, + "license": "ISC" + }, + "node_modules/error-ex": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", + "integrity": "sha512-7dFHNmqeFSEt2ZBsCriorKnn3Z2pj+fd9kmI6QoWw4//DL+icEBfc0U7qJCisqrTsKTjw4fNFy2pW9OqStD84g==", + "license": "MIT", + "dependencies": { + "is-arrayish": "^0.2.1" + } + }, + "node_modules/esbuild": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.0.tgz", + "integrity": "sha512-BXq5mqc8ltbaN34cDqWuYKyNhX8D/Z0J1xdtdQ8UcIIIyJyz+ZMKUt58tF3SrZ85jcfN/PZYhjR5uDQAYNVbuw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.25.0", + "@esbuild/android-arm": "0.25.0", + "@esbuild/android-arm64": "0.25.0", + "@esbuild/android-x64": "0.25.0", + "@esbuild/darwin-arm64": "0.25.0", + "@esbuild/darwin-x64": "0.25.0", + "@esbuild/freebsd-arm64": "0.25.0", + "@esbuild/freebsd-x64": "0.25.0", + "@esbuild/linux-arm": "0.25.0", + "@esbuild/linux-arm64": "0.25.0", + "@esbuild/linux-ia32": "0.25.0", + "@esbuild/linux-loong64": "0.25.0", + "@esbuild/linux-mips64el": "0.25.0", + "@esbuild/linux-ppc64": "0.25.0", + "@esbuild/linux-riscv64": "0.25.0", + "@esbuild/linux-s390x": "0.25.0", + "@esbuild/linux-x64": "0.25.0", + "@esbuild/netbsd-arm64": "0.25.0", + "@esbuild/netbsd-x64": "0.25.0", + "@esbuild/openbsd-arm64": "0.25.0", + "@esbuild/openbsd-x64": "0.25.0", + "@esbuild/sunos-x64": "0.25.0", + "@esbuild/win32-arm64": "0.25.0", + "@esbuild/win32-ia32": "0.25.0", + "@esbuild/win32-x64": "0.25.0" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/estree-util-is-identifier-name": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/estree-util-is-identifier-name/-/estree-util-is-identifier-name-3.0.0.tgz", + "integrity": "sha512-hFtqIDZTIUZ9BXLb8y4pYGyk6+wekIivNVTcmvk8NoOh+VeRn5y6cEHzbURrWbfp1fIqdVipilzj+lfaadNZmg==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", + "license": "MIT" + }, + "node_modules/fill-range": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "dev": true, + "license": "MIT", + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/find-root": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/find-root/-/find-root-1.1.0.tgz", + "integrity": "sha512-NKfW6bec6GfKc0SGx1e07QZY9PE99u0Bft/0rzSD5k3sO/vwkVUpDUKVm5Gpp5Ue3YfShPFTX2070tDs5kB9Ng==", + "license": "MIT" + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/gensync": { + "version": "1.0.0-beta.2", + "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", + "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/globals": { + "version": "11.12.0", + "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", + "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/hast-util-to-jsx-runtime": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/hast-util-to-jsx-runtime/-/hast-util-to-jsx-runtime-2.3.5.tgz", + "integrity": "sha512-gHD+HoFxOMmmXLuq9f2dZDMQHVcplCVpMfBNRpJsF03yyLZvJGzsFORe8orVuYDX9k2w0VH0uF8oryFd1whqKQ==", + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0", + "@types/hast": "^3.0.0", + "@types/unist": "^3.0.0", + "comma-separated-tokens": "^2.0.0", + "devlop": "^1.0.0", + "estree-util-is-identifier-name": "^3.0.0", + "hast-util-whitespace": "^3.0.0", + "mdast-util-mdx-expression": "^2.0.0", + "mdast-util-mdx-jsx": "^3.0.0", + "mdast-util-mdxjs-esm": "^2.0.0", + "property-information": "^7.0.0", + "space-separated-tokens": "^2.0.0", + "style-to-object": "^1.0.0", + "unist-util-position": "^5.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-whitespace": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/hast-util-whitespace/-/hast-util-whitespace-3.0.0.tgz", + "integrity": "sha512-88JUN06ipLwsnv+dVn+OIYOvAuvBMy/Qoi6O7mQHxdPXpjy+Cd6xRkWwux7DKO+4sYILtLBRIKgsdpS2gQc7qw==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hoist-non-react-statics": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/hoist-non-react-statics/-/hoist-non-react-statics-3.3.2.tgz", + "integrity": "sha512-/gGivxi8JPKWNm/W0jSmzcMPpfpPLc3dY/6GxhX2hQ9iGj3aDfklV4ET7NjKpSinLpJ5vafa9iiGIEZg10SfBw==", + "license": "BSD-3-Clause", + "dependencies": { + "react-is": "^16.7.0" + } + }, + "node_modules/hoist-non-react-statics/node_modules/react-is": { + "version": "16.13.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz", + "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==", + "license": "MIT" + }, + "node_modules/html-url-attributes": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/html-url-attributes/-/html-url-attributes-3.0.1.tgz", + "integrity": "sha512-ol6UPyBWqsrO6EJySPz2O7ZSr856WDrEzM5zMqp+FJJLGMW35cLYmmZnl0vztAZxRUoNZJFTCohfjuIJ8I4QBQ==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/import-fresh": { + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/import-fresh/-/import-fresh-3.3.1.tgz", + "integrity": "sha512-TR3KfrTZTYLPB6jUjfx6MF9WcWrHL9su5TObK4ZkYgBdWKPOFoSoQIdEuTuR82pmtxH2spWG9h6etwfr1pLBqQ==", + "license": "MIT", + "dependencies": { + "parent-module": "^1.0.0", + "resolve-from": "^4.0.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/inline-style-parser": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/inline-style-parser/-/inline-style-parser-0.2.4.tgz", + "integrity": "sha512-0aO8FkhNZlj/ZIbNi7Lxxr12obT7cL1moPfE4tg1LkX7LlLfC6DeX4l2ZEud1ukP9jNQyNnfzQVqwbwmAATY4Q==", + "license": "MIT" + }, + "node_modules/is-alphabetical": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-alphabetical/-/is-alphabetical-2.0.1.tgz", + "integrity": "sha512-FWyyY60MeTNyeSRpkM2Iry0G9hpr7/9kD40mD/cGQEuilcZYS4okz8SN2Q6rLCJ8gbCt6fN+rC+6tMGS99LaxQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-alphanumerical": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-alphanumerical/-/is-alphanumerical-2.0.1.tgz", + "integrity": "sha512-hmbYhX/9MUMF5uh7tOXyK/n0ZvWpad5caBA17GsC6vyuCqaWliRG5K1qS9inmUhEMaOBIW7/whAnSwveW/LtZw==", + "license": "MIT", + "dependencies": { + "is-alphabetical": "^2.0.0", + "is-decimal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-arrayish": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", + "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", + "license": "MIT" + }, + "node_modules/is-core-module": { + "version": "2.16.1", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.1.tgz", + "integrity": "sha512-UfoeMA6fIJ8wTYFEUjelnaGI67v6+N7qXJEvQuIGa99l4xsCruSYOVSQ0uPANn4dAzm8lkYPaKLrrijLq7x23w==", + "license": "MIT", + "dependencies": { + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-decimal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-decimal/-/is-decimal-2.0.1.tgz", + "integrity": "sha512-AAB9hiomQs5DXWcRB1rqsxGUstbRroFOPPVAomNk/3XHR5JyEZChOyTWe2oayKnsSsr/kcGqF+z6yuH6HHpN0A==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-hexadecimal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-hexadecimal/-/is-hexadecimal-2.0.1.tgz", + "integrity": "sha512-DgZQp241c8oO6cA1SbTEWiXeoxV42vlcJxgH+B3hi1AiqqKruZR3ZGF8In3fj4+/y/7rHvlOZLZtgJ/4ttYGZg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/is-plain-obj": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-4.1.0.tgz", + "integrity": "sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "license": "MIT" + }, + "node_modules/jsesc": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", + "integrity": "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==", + "license": "MIT", + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/json-parse-even-better-errors": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", + "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", + "license": "MIT" + }, + "node_modules/json5": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", + "dev": true, + "license": "MIT", + "bin": { + "json5": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/lines-and-columns": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", + "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", + "license": "MIT" + }, + "node_modules/longest-streak": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/longest-streak/-/longest-streak-3.1.0.tgz", + "integrity": "sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/loose-envify": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", + "integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==", + "license": "MIT", + "dependencies": { + "js-tokens": "^3.0.0 || ^4.0.0" + }, + "bin": { + "loose-envify": "cli.js" + } + }, + "node_modules/lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "dev": true, + "license": "ISC", + "dependencies": { + "yallist": "^3.0.2" + } + }, + "node_modules/mdast-util-from-markdown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/mdast-util-from-markdown/-/mdast-util-from-markdown-2.0.2.tgz", + "integrity": "sha512-uZhTV/8NBuw0WHkPTrCqDOl0zVe1BIng5ZtHoDk49ME1qqcjYmmLmOf0gELgcRMxN4w2iuIeVso5/6QymSrgmA==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "@types/unist": "^3.0.0", + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "mdast-util-to-string": "^4.0.0", + "micromark": "^4.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-decode-string": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0", + "unist-util-stringify-position": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-mdx-expression": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-mdx-expression/-/mdast-util-mdx-expression-2.0.1.tgz", + "integrity": "sha512-J6f+9hUp+ldTZqKRSg7Vw5V6MqjATc+3E4gf3CFNcuZNWD8XdyI6zQ8GqH7f8169MM6P7hMBRDVGnn7oHB9kXQ==", + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-mdx-jsx": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/mdast-util-mdx-jsx/-/mdast-util-mdx-jsx-3.2.0.tgz", + "integrity": "sha512-lj/z8v0r6ZtsN/cGNNtemmmfoLAFZnjMbNyLzBafjzikOM+glrjNHPlf6lQDOTccj9n5b0PPihEBbhneMyGs1Q==", + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "@types/unist": "^3.0.0", + "ccount": "^2.0.0", + "devlop": "^1.1.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0", + "parse-entities": "^4.0.0", + "stringify-entities": "^4.0.0", + "unist-util-stringify-position": "^4.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-mdxjs-esm": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/mdast-util-mdxjs-esm/-/mdast-util-mdxjs-esm-2.0.1.tgz", + "integrity": "sha512-EcmOpxsZ96CvlP03NghtH1EsLtr0n9Tm4lPUJUBccV9RwUOneqSycg19n5HGzCf+10LozMRSObtVr3ee1WoHtg==", + "license": "MIT", + "dependencies": { + "@types/estree-jsx": "^1.0.0", + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "mdast-util-from-markdown": "^2.0.0", + "mdast-util-to-markdown": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-phrasing": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/mdast-util-phrasing/-/mdast-util-phrasing-4.1.0.tgz", + "integrity": "sha512-TqICwyvJJpBwvGAMZjj4J2n0X8QWp21b9l0o7eXyVJ25YNWYbJDVIyD1bZXE6WtV6RmKJVYmQAKWa0zWOABz2w==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "unist-util-is": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-to-hast": { + "version": "13.2.0", + "resolved": "https://registry.npmjs.org/mdast-util-to-hast/-/mdast-util-to-hast-13.2.0.tgz", + "integrity": "sha512-QGYKEuUsYT9ykKBCMOEDLsU5JRObWQusAolFMeko/tYPufNkRffBAQjIE+99jbA87xv6FgmjLtwjh9wBWajwAA==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "@ungap/structured-clone": "^1.0.0", + "devlop": "^1.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "trim-lines": "^3.0.0", + "unist-util-position": "^5.0.0", + "unist-util-visit": "^5.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-to-markdown": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/mdast-util-to-markdown/-/mdast-util-to-markdown-2.1.2.tgz", + "integrity": "sha512-xj68wMTvGXVOKonmog6LwyJKrYXZPvlwabaryTjLh9LuvovB/KAH+kvi8Gjj+7rJjsFi23nkUxRQv1KqSroMqA==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "@types/unist": "^3.0.0", + "longest-streak": "^3.0.0", + "mdast-util-phrasing": "^4.0.0", + "mdast-util-to-string": "^4.0.0", + "micromark-util-classify-character": "^2.0.0", + "micromark-util-decode-string": "^2.0.0", + "unist-util-visit": "^5.0.0", + "zwitch": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/mdast-util-to-string": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/mdast-util-to-string/-/mdast-util-to-string-4.0.0.tgz", + "integrity": "sha512-0H44vDimn51F0YwvxSJSm0eCDOJTRlmN0R1yBh4HLj9wiV1Dn0QoXGbvFAWj2hSItVTlCmBF1hqKlIyUBVFLPg==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/micromark": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/micromark/-/micromark-4.0.1.tgz", + "integrity": "sha512-eBPdkcoCNvYcxQOAKAlceo5SNdzZWfF+FcSupREAzdAh9rRmE239CEQAiTwIgblwnoM8zzj35sZ5ZwvSEOF6Kw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "@types/debug": "^4.0.0", + "debug": "^4.0.0", + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "micromark-core-commonmark": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-combine-extensions": "^2.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-encode": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-sanitize-uri": "^2.0.0", + "micromark-util-subtokenize": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-core-commonmark": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/micromark-core-commonmark/-/micromark-core-commonmark-2.0.2.tgz", + "integrity": "sha512-FKjQKbxd1cibWMM1P9N+H8TwlgGgSkWZMmfuVucLCHaYqeSvJ0hFeHsIa65pA2nYbes0f8LDHPMrd9X7Ujxg9w==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "decode-named-character-reference": "^1.0.0", + "devlop": "^1.0.0", + "micromark-factory-destination": "^2.0.0", + "micromark-factory-label": "^2.0.0", + "micromark-factory-space": "^2.0.0", + "micromark-factory-title": "^2.0.0", + "micromark-factory-whitespace": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-classify-character": "^2.0.0", + "micromark-util-html-tag-name": "^2.0.0", + "micromark-util-normalize-identifier": "^2.0.0", + "micromark-util-resolve-all": "^2.0.0", + "micromark-util-subtokenize": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-destination": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-destination/-/micromark-factory-destination-2.0.1.tgz", + "integrity": "sha512-Xe6rDdJlkmbFRExpTOmRj9N3MaWmbAgdpSrBQvCFqhezUn4AHqJHbaEnfbVYYiexVSs//tqOdY/DxhjdCiJnIA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-label": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-label/-/micromark-factory-label-2.0.1.tgz", + "integrity": "sha512-VFMekyQExqIW7xIChcXn4ok29YE3rnuyveW3wZQWWqF4Nv9Wk5rgJ99KzPvHjkmPXF93FXIbBp6YdW3t71/7Vg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-space": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-space/-/micromark-factory-space-2.0.1.tgz", + "integrity": "sha512-zRkxjtBxxLd2Sc0d+fbnEunsTj46SWXgXciZmHq0kDYGnck/ZSGj9/wULTV95uoeYiK5hRXP2mJ98Uo4cq/LQg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-title": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-title/-/micromark-factory-title-2.0.1.tgz", + "integrity": "sha512-5bZ+3CjhAd9eChYTHsjy6TGxpOFSKgKKJPJxr293jTbfry2KDoWkhBb6TcPVB4NmzaPhMs1Frm9AZH7OD4Cjzw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-factory-whitespace": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-factory-whitespace/-/micromark-factory-whitespace-2.0.1.tgz", + "integrity": "sha512-Ob0nuZ3PKt/n0hORHyvoD9uZhr+Za8sFoP+OnMcnWK5lngSzALgQYKMr9RJVOWLqQYuyn6ulqGWSXdwf6F80lQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-factory-space": "^2.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-character": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-2.1.1.tgz", + "integrity": "sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-chunked": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-chunked/-/micromark-util-chunked-2.0.1.tgz", + "integrity": "sha512-QUNFEOPELfmvv+4xiNg2sRYeS/P84pTW0TCgP5zc9FpXetHY0ab7SxKyAQCNCc1eK0459uoLI1y5oO5Vc1dbhA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-classify-character": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-classify-character/-/micromark-util-classify-character-2.0.1.tgz", + "integrity": "sha512-K0kHzM6afW/MbeWYWLjoHQv1sgg2Q9EccHEDzSkxiP/EaagNzCm7T/WMKZ3rjMbvIpvBiZgwR3dKMygtA4mG1Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-combine-extensions": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-combine-extensions/-/micromark-util-combine-extensions-2.0.1.tgz", + "integrity": "sha512-OnAnH8Ujmy59JcyZw8JSbK9cGpdVY44NKgSM7E9Eh7DiLS2E9RNQf0dONaGDzEG9yjEl5hcqeIsj4hfRkLH/Bg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-chunked": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-decode-numeric-character-reference": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/micromark-util-decode-numeric-character-reference/-/micromark-util-decode-numeric-character-reference-2.0.2.tgz", + "integrity": "sha512-ccUbYk6CwVdkmCQMyr64dXz42EfHGkPQlBj5p7YVGzq8I7CtjXZJrubAYezf7Rp+bjPseiROqe7G6foFd+lEuw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-decode-string": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-decode-string/-/micromark-util-decode-string-2.0.1.tgz", + "integrity": "sha512-nDV/77Fj6eH1ynwscYTOsbK7rR//Uj0bZXBwJZRfaLEJ1iGBR6kIfNmlNqaqJf649EP0F3NWNdeJi03elllNUQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "decode-named-character-reference": "^1.0.0", + "micromark-util-character": "^2.0.0", + "micromark-util-decode-numeric-character-reference": "^2.0.0", + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-encode": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-encode/-/micromark-util-encode-2.0.1.tgz", + "integrity": "sha512-c3cVx2y4KqUnwopcO9b/SCdo2O67LwJJ/UyqGfbigahfegL9myoEFoDYZgkT7f36T0bLrM9hZTAaAyH+PCAXjw==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-html-tag-name": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-html-tag-name/-/micromark-util-html-tag-name-2.0.1.tgz", + "integrity": "sha512-2cNEiYDhCWKI+Gs9T0Tiysk136SnR13hhO8yW6BGNyhOC4qYFnwF1nKfD3HFAIXA5c45RrIG1ub11GiXeYd1xA==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-normalize-identifier": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-normalize-identifier/-/micromark-util-normalize-identifier-2.0.1.tgz", + "integrity": "sha512-sxPqmo70LyARJs0w2UclACPUUEqltCkJ6PhKdMIDuJ3gSf/Q+/GIe3WKl0Ijb/GyH9lOpUkRAO2wp0GVkLvS9Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-resolve-all": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-resolve-all/-/micromark-util-resolve-all-2.0.1.tgz", + "integrity": "sha512-VdQyxFWFT2/FGJgwQnJYbe1jjQoNTS4RjglmSjTUlpUMa95Htx9NHeYW4rGDJzbjvCsl9eLjMQwGeElsqmzcHg==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-sanitize-uri": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-sanitize-uri/-/micromark-util-sanitize-uri-2.0.1.tgz", + "integrity": "sha512-9N9IomZ/YuGGZZmQec1MbgxtlgougxTodVwDzzEouPKo3qFWvymFHWcnDi2vzV1ff6kas9ucW+o3yzJK9YB1AQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "micromark-util-character": "^2.0.0", + "micromark-util-encode": "^2.0.0", + "micromark-util-symbol": "^2.0.0" + } + }, + "node_modules/micromark-util-subtokenize": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/micromark-util-subtokenize/-/micromark-util-subtokenize-2.0.4.tgz", + "integrity": "sha512-N6hXjrin2GTJDe3MVjf5FuXpm12PGm80BrUAeub9XFXca8JZbP+oIwY4LJSVwFUCL1IPm/WwSVUN7goFHmSGGQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT", + "dependencies": { + "devlop": "^1.0.0", + "micromark-util-chunked": "^2.0.0", + "micromark-util-symbol": "^2.0.0", + "micromark-util-types": "^2.0.0" + } + }, + "node_modules/micromark-util-symbol": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz", + "integrity": "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromark-util-types": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/micromark-util-types/-/micromark-util-types-2.0.1.tgz", + "integrity": "sha512-534m2WhVTddrcKVepwmVEVnUAmtrx9bfIjNoQHRqfnvdaHQiFytEhJoTgpWJvDEXCO5gLTQh3wYC1PgOJA4NSQ==", + "funding": [ + { + "type": "GitHub Sponsors", + "url": "https://github.com/sponsors/unifiedjs" + }, + { + "type": "OpenCollective", + "url": "https://opencollective.com/unified" + } + ], + "license": "MIT" + }, + "node_modules/micromatch": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", + "dev": true, + "license": "MIT", + "dependencies": { + "braces": "^3.0.3", + "picomatch": "^2.3.1" + }, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/nanoid": { + "version": "3.3.8", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.8.tgz", + "integrity": "sha512-WNLf5Sd8oZxOm+TzppcYk8gVOgP+l58xNy58D0nbUnOxOWRWvlcCV4kUF7ltmI6PsrLl/BgKEyS4mqsGChFN0w==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/node-releases": { + "version": "2.0.19", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.19.tgz", + "integrity": "sha512-xxOWJsBKtzAq7DY0J+DTzuz58K8e7sJbdgwkbMWQe8UYB6ekmsQ45q0M/tJDsGaZmbC+l7n57UV8Hl5tHxO9uw==", + "dev": true, + "license": "MIT" + }, + "node_modules/object-assign": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", + "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/parent-module": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", + "integrity": "sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==", + "license": "MIT", + "dependencies": { + "callsites": "^3.0.0" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/parse-entities": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/parse-entities/-/parse-entities-4.0.2.tgz", + "integrity": "sha512-GG2AQYWoLgL877gQIKeRPGO1xF9+eG1ujIb5soS5gPvLQ1y2o8FL90w2QWNdf9I361Mpp7726c+lj3U0qK1uGw==", + "license": "MIT", + "dependencies": { + "@types/unist": "^2.0.0", + "character-entities-legacy": "^3.0.0", + "character-reference-invalid": "^2.0.0", + "decode-named-character-reference": "^1.0.0", + "is-alphanumerical": "^2.0.0", + "is-decimal": "^2.0.0", + "is-hexadecimal": "^2.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/parse-entities/node_modules/@types/unist": { + "version": "2.0.11", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-2.0.11.tgz", + "integrity": "sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==", + "license": "MIT" + }, + "node_modules/parse-json": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", + "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.0.0", + "error-ex": "^1.3.1", + "json-parse-even-better-errors": "^2.3.0", + "lines-and-columns": "^1.1.6" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/path-parse": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", + "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==", + "license": "MIT" + }, + "node_modules/path-type": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", + "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/postcss": { + "version": "8.5.3", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.3.tgz", + "integrity": "sha512-dle9A3yYxlBSrt8Fu+IpjGT8SY8hN0mlaA6GY8t0P5PjIOZemULz/E2Bnm/2dcUOena75OTNkHI76uZBNUUq3A==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.8", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/prop-types": { + "version": "15.8.1", + "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz", + "integrity": "sha512-oj87CgZICdulUohogVAR7AjlC0327U4el4L6eAvOqCeudMDVU0NThNaV+b9Df4dXgSP1gXMTnPdhfe/2qDH5cg==", + "license": "MIT", + "dependencies": { + "loose-envify": "^1.4.0", + "object-assign": "^4.1.1", + "react-is": "^16.13.1" + } + }, + "node_modules/prop-types/node_modules/react-is": { + "version": "16.13.1", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz", + "integrity": "sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==", + "license": "MIT" + }, + "node_modules/property-information": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/property-information/-/property-information-7.0.0.tgz", + "integrity": "sha512-7D/qOz/+Y4X/rzSB6jKxKUsQnphO046ei8qxG59mtM3RG3DHgTK81HrxrmoDVINJb8NKT5ZsRbwHvQ6B68Iyhg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/react": { + "version": "19.0.0", + "resolved": "https://registry.npmjs.org/react/-/react-19.0.0.tgz", + "integrity": "sha512-V8AVnmPIICiWpGfm6GLzCR/W5FXLchHop40W4nXBmdlEceh16rCN8O8LNWm5bh5XUX91fh7KpA+W0TgMKmgTpQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-dom": { + "version": "19.0.0", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-19.0.0.tgz", + "integrity": "sha512-4GV5sHFG0e/0AD4X+ySy6UJd3jVl1iNsNHdpad0qhABJ11twS3TTBnseqsKurKcsNqCEFeGL3uLpVChpIO3QfQ==", + "license": "MIT", + "dependencies": { + "scheduler": "^0.25.0" + }, + "peerDependencies": { + "react": "^19.0.0" + } + }, + "node_modules/react-is": { + "version": "19.0.0", + "resolved": "https://registry.npmjs.org/react-is/-/react-is-19.0.0.tgz", + "integrity": "sha512-H91OHcwjZsbq3ClIDHMzBShc1rotbfACdWENsmEf0IFvZ3FgGPtdHMcsv45bQ1hAbgdfiA8SnxTKfDS+x/8m2g==", + "license": "MIT" + }, + "node_modules/react-markdown": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/react-markdown/-/react-markdown-10.0.0.tgz", + "integrity": "sha512-4mTz7Sya/YQ1jYOrkwO73VcFdkFJ8L8I9ehCxdcV0XrClHyOJGKbBk5FR4OOOG+HnyKw5u+C/Aby9TwinCteYA==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "devlop": "^1.0.0", + "hast-util-to-jsx-runtime": "^2.0.0", + "html-url-attributes": "^3.0.0", + "mdast-util-to-hast": "^13.0.0", + "remark-parse": "^11.0.0", + "remark-rehype": "^11.0.0", + "unified": "^11.0.0", + "unist-util-visit": "^5.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + }, + "peerDependencies": { + "@types/react": ">=18", + "react": ">=18" + } + }, + "node_modules/react-refresh": { + "version": "0.14.2", + "resolved": "https://registry.npmjs.org/react-refresh/-/react-refresh-0.14.2.tgz", + "integrity": "sha512-jCvmsr+1IUSMUyzOkRcvnVbX3ZYC6g9TDrDbFuFmRDq7PD4yaGbLKNQL6k2jnArV8hjYxh7hVhAZB6s9HDGpZA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-transition-group": { + "version": "4.4.5", + "resolved": "https://registry.npmjs.org/react-transition-group/-/react-transition-group-4.4.5.tgz", + "integrity": "sha512-pZcd1MCJoiKiBR2NRxeCRg13uCXbydPnmB4EOeRrY7480qNWO8IIgQG6zlDkm6uRMsURXPuKq0GWtiM59a5Q6g==", + "license": "BSD-3-Clause", + "dependencies": { + "@babel/runtime": "^7.5.5", + "dom-helpers": "^5.0.1", + "loose-envify": "^1.4.0", + "prop-types": "^15.6.2" + }, + "peerDependencies": { + "react": ">=16.6.0", + "react-dom": ">=16.6.0" + } + }, + "node_modules/regenerator-runtime": { + "version": "0.14.1", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz", + "integrity": "sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==", + "license": "MIT" + }, + "node_modules/remark-parse": { + "version": "11.0.0", + "resolved": "https://registry.npmjs.org/remark-parse/-/remark-parse-11.0.0.tgz", + "integrity": "sha512-FCxlKLNGknS5ba/1lmpYijMUzX2esxW5xQqjWxw2eHFfS2MSdaHVINFmhjo+qN1WhZhNimq0dZATN9pH0IDrpA==", + "license": "MIT", + "dependencies": { + "@types/mdast": "^4.0.0", + "mdast-util-from-markdown": "^2.0.0", + "micromark-util-types": "^2.0.0", + "unified": "^11.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/remark-rehype": { + "version": "11.1.1", + "resolved": "https://registry.npmjs.org/remark-rehype/-/remark-rehype-11.1.1.tgz", + "integrity": "sha512-g/osARvjkBXb6Wo0XvAeXQohVta8i84ACbenPpoSsxTOQH/Ae0/RGP4WZgnMH5pMLpsj4FG7OHmcIcXxpza8eQ==", + "license": "MIT", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/mdast": "^4.0.0", + "mdast-util-to-hast": "^13.0.0", + "unified": "^11.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/resolve": { + "version": "1.22.10", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.10.tgz", + "integrity": "sha512-NPRy+/ncIMeDlTAsuqwKIiferiawhefFJtkNSW0qZJEqMEb+qBt/77B/jGeeek+F0uOeN05CDa6HXbbIgtVX4w==", + "license": "MIT", + "dependencies": { + "is-core-module": "^2.16.0", + "path-parse": "^1.0.7", + "supports-preserve-symlinks-flag": "^1.0.0" + }, + "bin": { + "resolve": "bin/resolve" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/resolve-from": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-4.0.0.tgz", + "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/rollup": { + "version": "4.34.8", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.34.8.tgz", + "integrity": "sha512-489gTVMzAYdiZHFVA/ig/iYFllCcWFHMvUHI1rpFmkoUtRlQxqh6/yiNqnYibjMZ2b/+FUQwldG+aLsEt6bglQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "1.0.6" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@rollup/rollup-android-arm-eabi": "4.34.8", + "@rollup/rollup-android-arm64": "4.34.8", + "@rollup/rollup-darwin-arm64": "4.34.8", + "@rollup/rollup-darwin-x64": "4.34.8", + "@rollup/rollup-freebsd-arm64": "4.34.8", + "@rollup/rollup-freebsd-x64": "4.34.8", + "@rollup/rollup-linux-arm-gnueabihf": "4.34.8", + "@rollup/rollup-linux-arm-musleabihf": "4.34.8", + "@rollup/rollup-linux-arm64-gnu": "4.34.8", + "@rollup/rollup-linux-arm64-musl": "4.34.8", + "@rollup/rollup-linux-loongarch64-gnu": "4.34.8", + "@rollup/rollup-linux-powerpc64le-gnu": "4.34.8", + "@rollup/rollup-linux-riscv64-gnu": "4.34.8", + "@rollup/rollup-linux-s390x-gnu": "4.34.8", + "@rollup/rollup-linux-x64-gnu": "4.34.8", + "@rollup/rollup-linux-x64-musl": "4.34.8", + "@rollup/rollup-win32-arm64-msvc": "4.34.8", + "@rollup/rollup-win32-ia32-msvc": "4.34.8", + "@rollup/rollup-win32-x64-msvc": "4.34.8", + "fsevents": "~2.3.2" + } + }, + "node_modules/scheduler": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.25.0.tgz", + "integrity": "sha512-xFVuu11jh+xcO7JOAGJNOXld8/TcEHK/4CituBUeUb5hqxJLj9YuemAEuvm9gQ/+pgXYfbQuqAkiYu+u7YEsNA==", + "license": "MIT" + }, + "node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/source-map": { + "version": "0.5.7", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", + "integrity": "sha512-LbrmJOMUSdEVxIKvdcJzQC+nQhe8FUZQTXQy6+I75skNgn3OoQ0DZA8YnFa7gp8tqtL3KPf1kmo0R5DoApeSGQ==", + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/space-separated-tokens": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/space-separated-tokens/-/space-separated-tokens-2.0.2.tgz", + "integrity": "sha512-PEGlAwrG8yXGXRjW32fGbg66JAlOAwbObuqVoJpv/mRgoWDQfgH1wDPvtzWyUSNAXBGSk8h755YDbbcEy3SH2Q==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/stringify-entities": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/stringify-entities/-/stringify-entities-4.0.4.tgz", + "integrity": "sha512-IwfBptatlO+QCJUo19AqvrPNqlVMpW9YEL2LIVY+Rpv2qsjCGxaDLNRgeGsQWJhfItebuJhsGSLjaBbNSQ+ieg==", + "license": "MIT", + "dependencies": { + "character-entities-html4": "^2.0.0", + "character-entities-legacy": "^3.0.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/style-to-object": { + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/style-to-object/-/style-to-object-1.0.8.tgz", + "integrity": "sha512-xT47I/Eo0rwJmaXC4oilDGDWLohVhR6o/xAQcPQN8q6QBuZVL8qMYL85kLmST5cPjAorwvqIA4qXTRQoYHaL6g==", + "license": "MIT", + "dependencies": { + "inline-style-parser": "0.2.4" + } + }, + "node_modules/stylis": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/stylis/-/stylis-4.2.0.tgz", + "integrity": "sha512-Orov6g6BB1sDfYgzWfTHDOxamtX1bE/zo104Dh9e6fqJ3PooipYyfJ0pUmrZO2wAvO8YbEyeFrkV91XTsGMSrw==", + "license": "MIT" + }, + "node_modules/supports-preserve-symlinks-flag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", + "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/trim-lines": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/trim-lines/-/trim-lines-3.0.1.tgz", + "integrity": "sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/trough": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/trough/-/trough-2.2.0.tgz", + "integrity": "sha512-tmMpK00BjZiUyVyvrBK7knerNgmgvcV/KLVyuma/SC+TQN167GrMRciANTz09+k3zW8L8t60jWO1GpfkZdjTaw==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, + "node_modules/typescript": { + "version": "5.7.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.7.3.tgz", + "integrity": "sha512-84MVSjMEHP+FQRPy3pX9sTVV/INIex71s9TL2Gm5FG/WG1SqXeKyZ0k7/blY/4FdOzI12CBy1vGc4og/eus0fw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/unified": { + "version": "11.0.5", + "resolved": "https://registry.npmjs.org/unified/-/unified-11.0.5.tgz", + "integrity": "sha512-xKvGhPWw3k84Qjh8bI3ZeJjqnyadK+GEFtazSfZv/rKeTkTjOJho6mFqh2SM96iIcZokxiOpg78GazTSg8+KHA==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "bail": "^2.0.0", + "devlop": "^1.0.0", + "extend": "^3.0.0", + "is-plain-obj": "^4.0.0", + "trough": "^2.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-is": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-6.0.0.tgz", + "integrity": "sha512-2qCTHimwdxLfz+YzdGfkqNlH0tLi9xjTnHddPmJwtIG9MGsdbutfTc4P+haPD7l7Cjxf/WZj+we5qfVPvvxfYw==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-position": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/unist-util-position/-/unist-util-position-5.0.0.tgz", + "integrity": "sha512-fucsC7HjXvkB5R3kTCO7kUjRdrS0BJt3M/FPxmHMBOm8JQi2BsHAHFsy27E0EolP8rp0NzXsJ+jNPyDWvOJZPA==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-stringify-position": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/unist-util-stringify-position/-/unist-util-stringify-position-4.0.0.tgz", + "integrity": "sha512-0ASV06AAoKCDkS2+xw5RXJywruurpbC4JZSm7nr7MOt1ojAzvyyaO+UxZf18j8FCF6kmzCZKcAgN/yu2gm2XgQ==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-visit": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-5.0.0.tgz", + "integrity": "sha512-MR04uvD+07cwl/yhVuVWAtw+3GOR/knlL55Nd/wAdblk27GCVt3lqpTivy/tkJcZoNPzTwS1Y+KMojlLDhoTzg==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-is": "^6.0.0", + "unist-util-visit-parents": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/unist-util-visit-parents": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/unist-util-visit-parents/-/unist-util-visit-parents-6.0.1.tgz", + "integrity": "sha512-L/PqWzfTP9lzzEa6CKs0k2nARxTdZduw3zyh8d2NVBnsyvHjSX4TWse388YrrQKbvI8w20fGjGlhgT96WwKykw==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-is": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/update-browserslist-db": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.3.tgz", + "integrity": "sha512-UxhIZQ+QInVdunkDAaiazvvT/+fXL5Osr0JZlJulepYu6Jd7qJtDZjlur0emRlT71EN3ScPoE7gvsuIKKNavKw==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "escalade": "^3.2.0", + "picocolors": "^1.1.1" + }, + "bin": { + "update-browserslist-db": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, + "node_modules/vfile": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/vfile/-/vfile-6.0.3.tgz", + "integrity": "sha512-KzIbH/9tXat2u30jf+smMwFCsno4wHVdNmzFyL+T/L3UGqqk6JKfVqOFOZEpZSHADH1k40ab6NUIXZq422ov3Q==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "vfile-message": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/vfile-message": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/vfile-message/-/vfile-message-4.0.2.tgz", + "integrity": "sha512-jRDZ1IMLttGj41KcZvlrYAaI3CfqpLpfpf+Mfig13viT6NKvRzWZ+lXz0Y5D60w6uJIBAOGq9mSHf0gktF0duw==", + "license": "MIT", + "dependencies": { + "@types/unist": "^3.0.0", + "unist-util-stringify-position": "^4.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/vite": { + "version": "6.2.0", + "resolved": "https://registry.npmjs.org/vite/-/vite-6.2.0.tgz", + "integrity": "sha512-7dPxoo+WsT/64rDcwoOjk76XHj+TqNTIvHKcuMQ1k4/SeHDaQt5GFAeLYzrimZrMpn/O6DtdI03WUjdxuPM0oQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "^0.25.0", + "postcss": "^8.5.3", + "rollup": "^4.30.1" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^18.0.0 || ^20.0.0 || >=22.0.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", + "jiti": ">=1.21.0", + "less": "*", + "lightningcss": "^1.21.0", + "sass": "*", + "sass-embedded": "*", + "stylus": "*", + "sugarss": "*", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/vite-plugin-singlefile": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/vite-plugin-singlefile/-/vite-plugin-singlefile-2.1.0.tgz", + "integrity": "sha512-7tJo+UgZABlKpY/nubth/wxJ4+pUGREPnEwNOknxwl2MM0zTvF14KTU4Ln1lc140gjLLV5mjDrvuoquU7OZqCg==", + "dev": true, + "license": "MIT", + "dependencies": { + "micromatch": "^4.0.8" + }, + "engines": { + "node": ">18.0.0" + }, + "peerDependencies": { + "rollup": "^4.28.1", + "vite": "^5.4.11 || ^6.0.0" + } + }, + "node_modules/yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", + "dev": true, + "license": "ISC" + }, + "node_modules/yaml": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.7.0.tgz", + "integrity": "sha512-+hSoy/QHluxmC9kCIJyL/uyFmLmc+e5CFR5Wa+bpIhIj85LVb9ZH2nVnqrHoSvKogwODv0ClqZkmiSSaIH5LTA==", + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/zwitch": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/zwitch/-/zwitch-2.0.4.tgz", + "integrity": "sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + } + } +} diff --git a/scripts/settings/package.json b/scripts/settings/package.json new file mode 100644 index 0000000..fc7bb7b --- /dev/null +++ b/scripts/settings/package.json @@ -0,0 +1,31 @@ +{ + "name": "material-ui-vite-ts", + "private": true, + "version": "5.0.0", + "type": "module", + "scripts": { + "dev": "vite", + "build": "vite build", + "preview": "vite preview" + }, + "dependencies": { + "@dnd-kit/core": "^6.3.1", + "@dnd-kit/sortable": "^10.0.0", + "@emotion/react": "latest", + "@emotion/styled": "latest", + "@mui/icons-material": "latest", + "@mui/material": "latest", + "react": "19.0.0", + "react-dom": "19.0.0", + "react-markdown": "^10.0.0", + "yaml": "^2.7.0" + }, + "devDependencies": { + "@types/react": "latest", + "@types/react-dom": "latest", + "@vitejs/plugin-react": "latest", + "typescript": "latest", + "vite": "latest", + "vite-plugin-singlefile": "^2.1.0" + } +} diff --git a/scripts/settings/src/App.tsx b/scripts/settings/src/App.tsx new file mode 100644 index 0000000..4d98528 --- /dev/null +++ b/scripts/settings/src/App.tsx @@ -0,0 +1,450 @@ +import * as React from 'react'; +import { useEffect, useState, useRef } from 'react'; +import Container from '@mui/material/Container'; +import Typography from '@mui/material/Typography'; +import Box from '@mui/material/Box'; +import FileUploadIcon from '@mui/icons-material/FileUpload'; +// +import { + DndContext, + closestCenter, + KeyboardSensor, + PointerSensor, + useSensor, + useSensors, + DragOverlay +} from "@dnd-kit/core"; +import { + arrayMove, + SortableContext, + sortableKeyboardCoordinates, + rectSortingStrategy +} from "@dnd-kit/sortable"; + +import type { DragStartEvent, DragEndEvent, UniqueIdentifier } from "@dnd-kit/core"; + + +import { Module } from './types'; + +import { modules, steps, module_types, empty_config } from './schema.json'; +import { + Stack, + Button, +} from '@mui/material'; +import Grid from '@mui/material/Grid2'; + +import { parseDocument, Document, YAMLSeq, YAMLMap, Scalar } from 'yaml' +import StepCard from './StepCard'; + + +function FileDrop({ setYamlFile }: { setYamlFile: React.Dispatch> }) { + + const [showError, setShowError] = useState(false); + const [label, setLabel] = useState(<>Drag and drop your orchestration.yaml file here, or click to select a file.); + const wrapperRef = useRef(null); + + function openYAMLFile(event: any) { + let file = event.target.files[0]; + if (file.type.indexOf('yaml') === -1) { + setShowError(true); + setLabel(<>Invalid type, only YAML files are accepted.) + return; + } + let reader = new FileReader(); + reader.onload = function (e) { + let contents = e.target ? e.target.result : ''; + try { + let document = parseDocument(contents as string); + if (document.errors.length > 0) { + // not a valid yaml file + setShowError(true); + setLabel(<>Invalid file. Make sure your Orchestration is a valid YAML file with a 'steps' section in it.) + return; + } else { + setShowError(false); + setLabel(<>File loaded successfully.) + } + // do some basic validation of 'steps' + let steps = document.get('steps'); + if (!steps) { + setShowError(true); + setLabel(<>Invalid file. Your orchestration file must have a 'steps' section in it.) + return; + } + const replacements = { + feeder: 'feeders', + formatter: 'formatters', + archivers: 'extractors', + }; + + let error = false; + for (let stepType of Object.keys(replacements)) { + if (steps.get(stepType) !== undefined) { + setShowError(true); + setLabel(<>Invalid file. Your orchestration file appears to be in the old (v0.12) format with a '{stepType}' section.
You should manually update your orchestration file first (hint: {stepType} → {replacements[stepType]})); + error = true; + return; + } + }; + setYamlFile(document); + } catch (e) { + console.error(e); + } + } + reader.readAsText(file); + } + return ( + <> +
{ + e.currentTarget.style.backgroundColor = 'var(--mui-palette-LinearProgress-infoBg)'; + }} + onDragLeave={(e) => { + e.currentTarget.style.backgroundColor = ''; + }} + onDrop={(e) => { + e.currentTarget.style.backgroundColor = ''; + }} + > + + + + {label} + +
+ + ); +} + +function ModuleTypes({ stepType, setEnabledModules, enabledModules, configValues }: { stepType: string, setEnabledModules: any, enabledModules: any, configValues: any }) { + const [showError, setShowError] = useState(false); + const [activeId, setActiveId] = useState(); + const [items, setItems] = useState([]); + + useEffect(() => { + setItems(enabledModules[stepType].map(([name, enabled]: [string, boolean]) => name)); + } + , [enabledModules]); + + const toggleModule = (event: any) => { + // make sure that 'feeder' and 'formatter' types only have one value + let name = event.target.id; + let checked = event.target.checked; + if (stepType === 'feeders' || stepType === 'formatters') { + // check how many modules of this type are enabled + const checkedModules = enabledModules[stepType].filter(([m, enabled]: [string, boolean]) => { + return (m !== name && enabled) || (checked && m === name) + }); + if (checkedModules.length > 1) { + setShowError(true); + } else { + setShowError(false); + } + } else { + setShowError(false); + } + let newEnabledModules = { ...enabledModules }; + newEnabledModules[stepType] = enabledModules[stepType].map(([m, enabled]: [string, boolean]) => { + return (m === name) ? [m, checked] : [m, enabled]; + }); + setEnabledModules(newEnabledModules); + } + + const sensors = useSensors( + useSensor(PointerSensor), + useSensor(KeyboardSensor, { + coordinateGetter: sortableKeyboardCoordinates + }) + ); + + const handleDragStart = (event: DragStartEvent) => { + setActiveId(event.active.id); + }; + + const handleDragEnd = (event: DragEndEvent) => { + setActiveId(undefined); + const { active, over } = event; + + if (active.id !== over?.id) { + const oldIndex = items.indexOf(active.id as string); + const newIndex = items.indexOf(over?.id as string); + + let newArray = arrayMove(items, oldIndex, newIndex); + // set it also on steps + let newEnabledModules = { ...enabledModules }; + newEnabledModules[stepType] = enabledModules[stepType].sort((a, b) => { + return newArray.indexOf(a[0]) - newArray.indexOf(b[0]); + }) + setEnabledModules(newEnabledModules); + } + }; + return ( + <> + + + {stepType} + + + Select the {stepType} you wish to enable. Drag to reorder. + + + {showError ? Only one {stepType.slice(0,-1)} can be enabled at a time. : null} + + + + + {items.map((name: string) => { + let m: Module = modules[name]; + return ( + + ); + })} + + {activeId ? ( +
+ + ) : null} + + + + + + ); +} + + +export default function App() { + const [yamlFile, setYamlFile] = useState(new Document()); + const [enabledModules, setEnabledModules] = useState<{}>(Object.fromEntries(Object.keys(steps).map(type => [type, steps[type].map((name: string) => [name, false])]))); + const [configValues, setConfigValues] = useState<{ + [key: string]: { + [key: string + ]: any + } + }>( + Object.keys(modules).reduce((acc, module) => { + acc[module] = {}; + return acc; + }, {}) + ); + + const saveSettings = function (copy: boolean = false) { + // edit the yamlFile + + // generate the steps config + let stepsConfig = enabledModules; + + let finalYamlFile: Document = null; + if (!yamlFile || yamlFile.contents == null) { + // create the yaml file from + finalYamlFile = parseDocument(empty_config as string); + } else { + finalYamlFile = yamlFile; + } + + // set the steps + module_types.forEach((type: string) => { + let stepType = type + 's'; + let existingSteps = finalYamlFile.getIn(['steps', stepType]) as YAMLSeq; + stepsConfig[stepType].forEach(([name, enabled]: [string, boolean]) => { + let index = existingSteps.items.findIndex((item) => { + return (item.value || item) === name + }); + let stepItem = finalYamlFile.getIn(['steps', stepType], true) as YAMLSeq; + + if (enabled && index === -1) { + finalYamlFile.addIn(['steps', stepType], name); + stepItem.commentBefore = stepItem.commentBefore?.replace("\n - " + name, ''); + stepItem.comment = stepItem.comment?.replace("\n - " + name, ''); + } else if (!enabled && index !== -1) { + // set the value to empty and add a comment before with the commented value + finalYamlFile.deleteIn(['steps', stepType, index]); + stepItem.commentBefore += "\n - " + name; + finalYamlFile.setIn(['steps', stepType], stepItem); + } + }); + // sort the items + existingSteps.items.sort((a: Scalar | string, b: Scalar | string) => { + return (stepsConfig[stepType].findIndex((val: [string, boolean]) => {return val[0] === (a.value || a)}) - + stepsConfig[stepType].findIndex((val: [string, boolean]) => {return val[0] === (b.value || b)})) + }); + existingSteps.flow = existingSteps.items.length ? false : true; + }); + + // set all other settings + // loop through each item that isn't 'steps' in the finalYamlFile and check if it exists in configValues + + Object.keys(configValues).forEach((module_name: string) => { + // get an existing key + let existingConfig = finalYamlFile.get(module_name, true) as YAMLMap; + if (existingConfig) { + Object.keys(configValues[module_name]).forEach((config_name: string) => { + let existingConfigYAML = existingConfig.get(config_name, true) as Scalar; + if (existingConfigYAML) { + existingConfigYAML.value = configValues[module_name][config_name]; + existingConfig.set(config_name, existingConfigYAML); + } else { + existingConfig.set(config_name, configValues[module_name][config_name]); + } + }); + finalYamlFile.set(module_name, existingConfig); + } else { + if (configValues[module_name] && Object.keys(configValues[module_name]).length > 0) { + finalYamlFile.set(module_name, configValues[module_name]); + } + } + }); + + if (copy) { + navigator.clipboard.writeText(String(finalYamlFile)).then(() => { + alert("Settings copied to clipboard."); + }); + } else { + // offer the file for download + const blob = new Blob([String(finalYamlFile)], { type: 'application/x-yaml' }); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = 'orchestration.yaml'; + a.click(); + } + } + + useEffect(() => { + // load the configs, and set the default values if they exist + let newConfigValues = {}; + Object.keys(modules).map((module: string) => { + let m = modules[module]; + let configs = m.configs; + if (!configs) { + return; + } + newConfigValues[module] = {}; + Object.keys(configs).map((config: string) => { + let config_args = configs[config]; + if (config_args.default !== undefined) { + newConfigValues[module][config] = config_args.default; + } + }); + }) + setConfigValues(newConfigValues); + }, []); + + useEffect(() => { + if (!yamlFile || yamlFile.contents == null) { + return; + } + + let settings = yamlFile.toJS(); + // make a deep copy of settings + let stepSettings = settings['steps']; + + let newEnabledModules = Object.fromEntries(Object.keys(steps).map((type: string) => { + return [type, steps[type].map((name: string) => { + return [name, stepSettings[type].indexOf(name) !== -1]; + }).sort((a, b) => { + let aIndex = stepSettings[type].indexOf(a[0]); + let bIndex = stepSettings[type].indexOf(b[0]); + if (aIndex === -1 && bIndex === -1) { + return a - b; + } + if (bIndex === -1) { + return -1; + } + if (aIndex === -1) { + return 1; + } + return aIndex - bIndex; + })]; + }).sort((a, b) => { + return module_types.indexOf(a[0]) - module_types.indexOf(b[0]); + })); + setEnabledModules(newEnabledModules); + + // set the config values + let newConfigValues = settings; + delete newConfigValues['steps']; + + + setConfigValues(Object.keys(modules).reduce((acc, module) => { + acc[module] = newConfigValues[module] || {}; + return acc; + }, {})); + }, [yamlFile]); + + + + return ( + + + + + 1. Select your orchestration.yaml settings file. + + Or skip this step to start from scratch + + + + + 2. Choose the Modules you wish to enable/disable + + {Object.keys(steps).map((stepType: string) => { + return ( + + + + ); + })} + + + + 3. Configure your Enabled Modules + + + Next to each module you've enabled, you can click 'Configure' to set the module's settings. + + + + + 4. Save your settings + + + + + + + + + ); +} diff --git a/scripts/settings/src/StepCard.tsx b/scripts/settings/src/StepCard.tsx new file mode 100644 index 0000000..52ee76b --- /dev/null +++ b/scripts/settings/src/StepCard.tsx @@ -0,0 +1,258 @@ +import { useState } from "react"; +import { useSortable } from "@dnd-kit/sortable"; +import ReactMarkdown from 'react-markdown'; + +import { CSS } from "@dnd-kit/utilities"; + +import { + Card, + CardActions, + CardHeader, + Button, + Dialog, + DialogTitle, + DialogContent, + Box, + IconButton, + Checkbox, + Select, + MenuItem, + FormControl, + FormControlLabel, + FormHelperText, + TextField, + Stack, + Typography, + InputAdornment, +} from '@mui/material'; +import Grid from '@mui/material/Grid2'; +import DragIndicatorIcon from '@mui/icons-material/DragIndicator'; +import Visibility from '@mui/icons-material/Visibility'; +import VisibilityOff from '@mui/icons-material/VisibilityOff'; +import HelpIconOutlined from '@mui/icons-material/HelpOutline'; +import { Module, Config } from "./types"; + + +// adds 'capitalize' method to String prototype +declare global { + interface String { + capitalize(): string; + } +} +String.prototype.capitalize = function (this: string) { + return this.charAt(0).toUpperCase() + this.slice(1); +}; + +const StepCard = ({ + type, + module, + toggleModule, + enabledModules, + configValues +}: { + type: string, + module: Module, + toggleModule: any, + enabledModules: any, + configValues: any +}) => { + const { + attributes, + listeners, + setNodeRef, + transform, + transition, + isDragging + } = useSortable({ id: module.name }); + + + const style = { + ...Card.style, + transform: CSS.Transform.toString(transform), + transition, + zIndex: isDragging ? "100" : "auto", + opacity: isDragging ? 0.3 : 1 + }; + + let name = module.name; + const [helpOpen, setHelpOpen] = useState(false); + const [configOpen, setConfigOpen] = useState(false); + const enabled = enabledModules[type].find((m: any) => m[0] === name)[1]; + + return ( + + + } + label={module.display_name} /> + } + /> + + + + setHelpOpen(true)}> + + + {enabled && module.configs && name != 'cli_feeder' ? ( + + ) : null} + + + + + + + + setHelpOpen(false)} + maxWidth="lg" + > + + {module.display_name} + + + + {module.manifest.description.split("\n").map((line: string) => line.trim()).join("\n")} + + + + {module.configs && name != 'cli_feeder' && } + + ) +} + +function ConfigField({ config_value, module, configValues }: { config_value: any, module: Module, configValues: any }) { + const [showPassword, setShowPassword] = useState(false); + const handleClickShowPassword = () => setShowPassword((show) => !show); + + const handleMouseDownPassword = (event: React.MouseEvent) => { + event.preventDefault(); + }; + + const handleMouseUpPassword = (event: React.MouseEvent) => { + event.preventDefault(); + }; + + function setConfigValue(config: any, value: any) { + configValues[module.name][config] = value; + } + const config_args: Config = module.configs[config_value]; + const config_name: string = config_value.replace(/_/g, " "); + const config_display_name = config_name.capitalize(); + const value = configValues[module.name][config_value] || config_args.default; + + + const config_value_lower = config_value.toLowerCase(); + const is_password = config_value_lower.includes('password') || + config_value_lower.includes('secret') || + config_value_lower.includes('token') || + config_value_lower.includes('key') || + config_value_lower.includes('api_hash') || + config_args.type === 'password'; + + const text_input_type = is_password ? 'password' : (config_args.type === 'int' ? 'number' : 'text'); + + return ( + + {config_display_name} {config_args.required && (`(required)`)} + + {config_args.type === 'bool' ? + { + setConfigValue(config_value, e.target.checked); + }} + />} label={config_args.help.capitalize()} + /> + : + ( + config_args.choices !== undefined ? + + : + (config_args.type === 'json_loader' ? + { + try { + let val = JSON.parse(e.target.value); + setConfigValue(config_value, val); + } catch (e) { + console.log(e); + } + } + } /> + : + { + setConfigValue(config_value, e.target.value); + }} + required={config_args.required} + slotProps={ is_password ? { + input: { endAdornment: ( + + + {showPassword ? : } + + + )} + } : {}} + /> + ) + ) + } + {config_args.type !== 'bool' && ( + {config_args.help.capitalize()} + )} + + + ) +} + +function ConfigPanel({ module, open, setOpen, configValues }: { module: Module, open: boolean, setOpen: any, configValues: any }) { + + return ( + <> + setOpen(false)} + maxWidth="lg" + > + + {module.display_name} + + + + {Object.keys(module.configs).map((config_value: any) => { + return ( + + ); + })} + + + + + ); +} + +export default StepCard; \ No newline at end of file diff --git a/scripts/settings/src/main.tsx b/scripts/settings/src/main.tsx new file mode 100644 index 0000000..44c7951 --- /dev/null +++ b/scripts/settings/src/main.tsx @@ -0,0 +1,44 @@ +import * as React from 'react'; +import * as ReactDOM from 'react-dom/client'; +import { ThemeProvider } from '@mui/material/styles'; +import { CssBaseline } from '@mui/material'; +import App from './App'; +import { createTheme } from '@mui/material/styles'; +import { red } from '@mui/material/colors'; +import { useState, useEffect } from 'react'; + +function RootApp() { + const [mode, setMode] = useState('light'); + +useEffect(() => { + setMode(window.localStorage.getItem('theme') || 'light'); +}, []); + +var observer = new MutationObserver(function(mutations) { + setMode(window.localStorage.getItem('theme') || 'light'); + +}) +observer.observe(document.documentElement, {attributes: true, attributeFilter: ['data-theme']}); + +// A custom theme for this app +const theme = createTheme({ + palette: { + mode: mode == 'light' ? 'light' : 'dark', + }, + cssVariables: true +}); + + return ( + + + + + ); +} + +ReactDOM.createRoot(document.getElementById('root')!).render( + + + , +); + diff --git a/scripts/settings/src/schema.json b/scripts/settings/src/schema.json new file mode 100644 index 0000000..64a903a --- /dev/null +++ b/scripts/settings/src/schema.json @@ -0,0 +1,2118 @@ +{ + "modules": { + "gsheet_feeder": { + "name": "gsheet_feeder", + "display_name": "Google Sheets Feeder", + "manifest": { + "name": "Google Sheets Feeder", + "author": "Bellingcat", + "type": [ + "feeder" + ], + "requires_setup": true, + "description": "\n GsheetsFeeder \n A Google Sheets-based feeder for the Auto Archiver.\n\n This reads data from Google Sheets and filters rows based on user-defined rules.\n The filtered rows are processed into `Metadata` objects.\n\n ### Features\n - Validates the sheet structure and filters rows based on input configurations.\n - Processes only worksheets allowed by the `allow_worksheets` and `block_worksheets` configurations.\n - Ensures only rows with valid URLs and unprocessed statuses are included for archival.\n - Supports organizing stored files into folder paths based on sheet and worksheet names.\n\n ### Setup\n - Requires a Google Service Account JSON file for authentication, which should be stored in `secrets/gsheets_service_account.json`.\n To set up a service account, follow the instructions [here](https://gspread.readthedocs.io/en/latest/oauth2.html).\n - Define the `sheet` or `sheet_id` configuration to specify the sheet to archive.\n - Customize the column names in your Google sheet using the `columns` configuration.\n ", + "dependencies": { + "python": [ + "loguru", + "gspread", + "slugify" + ] + }, + "entry_point": "gsheet_feeder::GsheetsFeeder", + "version": "1.0", + "configs": { + "sheet": { + "default": null, + "help": "name of the sheet to archive" + }, + "sheet_id": { + "default": null, + "help": "the id of the sheet to archive (alternative to 'sheet' config)" + }, + "header": { + "default": 1, + "type": "int", + "help": "index of the header row (starts at 1)" + }, + "service_account": { + "default": "secrets/service_account.json", + "help": "service account JSON file path. Learn how to create one: https://gspread.readthedocs.io/en/latest/oauth2.html", + "required": true + }, + "columns": { + "default": { + "url": "link", + "status": "archive status", + "folder": "destination folder", + "archive": "archive location", + "date": "archive date", + "thumbnail": "thumbnail", + "timestamp": "upload timestamp", + "title": "upload title", + "text": "text content", + "screenshot": "screenshot", + "hash": "hash", + "pdq_hash": "perceptual hashes", + "wacz": "wacz", + "replaywebpage": "replaywebpage" + }, + "help": "Custom names for the columns in your Google sheet. If you don't want to use the default column names, change them with this setting", + "type": "json_loader" + }, + "allow_worksheets": { + "default": [], + "help": "A list of worksheet names that should be processed (overrides worksheet_block), leave empty so all are allowed" + }, + "block_worksheets": { + "default": [], + "help": "A list of worksheet names for worksheets that should be explicitly blocked from being processed" + }, + "use_sheet_names_in_stored_paths": { + "default": true, + "help": "if True the stored files path will include 'workbook_name/worksheet_name/...'", + "type": "bool" + } + } + }, + "configs": { + "sheet": { + "default": null, + "help": "name of the sheet to archive" + }, + "sheet_id": { + "default": null, + "help": "the id of the sheet to archive (alternative to 'sheet' config)" + }, + "header": { + "default": 1, + "type": "int", + "help": "index of the header row (starts at 1)" + }, + "service_account": { + "default": "secrets/service_account.json", + "help": "service account JSON file path. Learn how to create one: https://gspread.readthedocs.io/en/latest/oauth2.html", + "required": true + }, + "columns": { + "default": { + "url": "link", + "status": "archive status", + "folder": "destination folder", + "archive": "archive location", + "date": "archive date", + "thumbnail": "thumbnail", + "timestamp": "upload timestamp", + "title": "upload title", + "text": "text content", + "screenshot": "screenshot", + "hash": "hash", + "pdq_hash": "perceptual hashes", + "wacz": "wacz", + "replaywebpage": "replaywebpage" + }, + "help": "Custom names for the columns in your Google sheet. If you don't want to use the default column names, change them with this setting", + "type": "json_loader" + }, + "allow_worksheets": { + "default": [], + "help": "A list of worksheet names that should be processed (overrides worksheet_block), leave empty so all are allowed" + }, + "block_worksheets": { + "default": [], + "help": "A list of worksheet names for worksheets that should be explicitly blocked from being processed" + }, + "use_sheet_names_in_stored_paths": { + "default": true, + "help": "if True the stored files path will include 'workbook_name/worksheet_name/...'", + "type": "bool" + } + } + }, + "atlos_feeder": { + "name": "atlos_feeder", + "display_name": "Atlos Feeder", + "manifest": { + "name": "Atlos Feeder", + "author": "Bellingcat", + "type": [ + "feeder" + ], + "requires_setup": true, + "description": "\n AtlosFeeder: A feeder module that integrates with the Atlos API to fetch source material URLs for archival.\n\n ### Features\n - Connects to the Atlos API to retrieve a list of source material URLs.\n - Filters source materials based on visibility, processing status, and metadata.\n - Converts filtered source materials into `Metadata` objects with the relevant `atlos_id` and URL.\n - Iterates through paginated results using a cursor for efficient API interaction.\n\n ### Notes\n - Requires an Atlos API endpoint and a valid API token for authentication.\n - Ensures only unprocessed, visible, and ready-to-archive URLs are returned.\n - Handles pagination transparently when retrieving data from the Atlos API.\n ", + "dependencies": { + "python": [ + "loguru", + "requests" + ] + }, + "entry_point": "", + "version": "1.0", + "configs": { + "api_token": { + "type": "str", + "required": true, + "help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/" + }, + "atlos_url": { + "default": "https://platform.atlos.org", + "help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.", + "type": "str" + } + } + }, + "configs": { + "api_token": { + "type": "str", + "required": true, + "help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/" + }, + "atlos_url": { + "default": "https://platform.atlos.org", + "help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.", + "type": "str" + } + } + }, + "csv_feeder": { + "name": "csv_feeder", + "display_name": "CSV Feeder", + "manifest": { + "name": "CSV Feeder", + "author": "Bellingcat", + "type": [ + "feeder" + ], + "requires_setup": true, + "description": "\n Reads URLs from CSV files and feeds them into the archiving process.\n\n ### Features\n - Supports reading URLs from multiple input files, specified as a comma-separated list.\n - Allows specifying the column number or name to extract URLs from.\n - Skips header rows if the first value is not a valid URL.\n\n ### Setup\n - Input files should be formatted with one URL per line, with or without a header row.\n - If you have a header row, you can specify the column number or name to read URLs from using the 'column' config option.\n ", + "dependencies": { + "python": [ + "loguru" + ], + "bin": [ + "" + ] + }, + "entry_point": "csv_feeder::CSVFeeder", + "version": "1.0", + "configs": { + "files": { + "default": null, + "help": "Path to the input file(s) to read the URLs from, comma separated. Input files should be formatted with one URL per line", + "required": true, + "type": "valid_file", + "nargs": "+" + }, + "column": { + "default": null, + "help": "Column number or name to read the URLs from, 0-indexed" + } + } + }, + "configs": { + "files": { + "default": null, + "help": "Path to the input file(s) to read the URLs from, comma separated. Input files should be formatted with one URL per line", + "required": true, + "type": "valid_file", + "nargs": "+" + }, + "column": { + "default": null, + "help": "Column number or name to read the URLs from, 0-indexed" + } + } + }, + "cli_feeder": { + "name": "cli_feeder", + "display_name": "Command Line Feeder", + "manifest": { + "name": "Command Line Feeder", + "author": "Bellingcat", + "type": [ + "feeder" + ], + "requires_setup": false, + "description": "\nThe Command Line Feeder is the default enabled feeder for the Auto Archiver. It allows you to pass URLs directly to the orchestrator from the command line \nwithout the need to specify any additional configuration or command line arguments:\n\n`auto-archiver --feeder cli_feeder -- \"https://example.com/1/,https://example.com/2/\"`\n\nYou can pass multiple URLs by separating them with a space. The URLs will be processed in the order they are provided.\n\n`auto-archiver --feeder cli_feeder -- https://example.com/1/ https://example.com/2/`\n", + "dependencies": {}, + "entry_point": "cli_feeder::CLIFeeder", + "version": "1.0", + "configs": { + "urls": { + "default": null, + "help": "URL(s) to archive, either a single URL or a list of urls, should not come from config.yaml" + } + } + }, + "configs": { + "urls": { + "default": null, + "help": "URL(s) to archive, either a single URL or a list of urls, should not come from config.yaml" + } + } + }, + "instagram_api_extractor": { + "name": "instagram_api_extractor", + "display_name": "Instagram API Extractor", + "manifest": { + "name": "Instagram API Extractor", + "author": "Bellingcat", + "type": [ + "extractor" + ], + "requires_setup": true, + "description": "\nArchives various types of Instagram content using the Instagrapi API.\n\nRequires setting up an Instagrapi API deployment and providing an access token and API endpoint.\n\n### Features\n- Connects to an Instagrapi API deployment to fetch Instagram profiles, posts, stories, highlights, reels, and tagged content.\n- Supports advanced configuration options, including:\n - Full profile download (all posts, stories, highlights, and tagged content).\n - Limiting the number of posts to fetch for large profiles.\n - Minimising JSON output to remove empty fields and redundant data.\n- Provides robust error handling and retries for API calls.\n- Ensures efficient media scraping, including handling nested or carousel media items.\n- Adds downloaded media and metadata to the result for further processing.\n\n### Notes\n- Requires a valid Instagrapi API token (`access_token`) and API endpoint (`api_endpoint`).\n- Full-profile downloads can be limited by setting `full_profile_max_posts`.\n- Designed to fetch content in batches for large profiles, minimising API load.\n", + "dependencies": { + "python": [ + "requests", + "loguru", + "retrying", + "tqdm" + ] + }, + "entry_point": "instagram_api_extractor::InstagramAPIExtractor", + "version": "1.0", + "configs": { + "access_token": { + "default": null, + "help": "a valid instagrapi-api token" + }, + "api_endpoint": { + "required": true, + "help": "API endpoint to use" + }, + "full_profile": { + "default": false, + "type": "bool", + "help": "if true, will download all posts, tagged posts, stories, and highlights for a profile, if false, will only download the profile pic and information." + }, + "full_profile_max_posts": { + "default": 0, + "type": "int", + "help": "Use to limit the number of posts to download when full_profile is true. 0 means no limit. limit is applied softly since posts are fetched in batch, once to: posts, tagged posts, and highlights" + }, + "minimize_json_output": { + "default": true, + "type": "bool", + "help": "if true, will remove empty values from the json output" + } + } + }, + "configs": { + "access_token": { + "default": null, + "help": "a valid instagrapi-api token" + }, + "api_endpoint": { + "required": true, + "help": "API endpoint to use" + }, + "full_profile": { + "default": false, + "type": "bool", + "help": "if true, will download all posts, tagged posts, stories, and highlights for a profile, if false, will only download the profile pic and information." + }, + "full_profile_max_posts": { + "default": 0, + "type": "int", + "help": "Use to limit the number of posts to download when full_profile is true. 0 means no limit. limit is applied softly since posts are fetched in batch, once to: posts, tagged posts, and highlights" + }, + "minimize_json_output": { + "default": true, + "type": "bool", + "help": "if true, will remove empty values from the json output" + } + } + }, + "instagram_tbot_extractor": { + "name": "instagram_tbot_extractor", + "display_name": "Instagram Telegram Bot Extractor", + "manifest": { + "name": "Instagram Telegram Bot Extractor", + "author": "Bellingcat", + "type": [ + "extractor" + ], + "requires_setup": true, + "description": "\nThe `InstagramTbotExtractor` module uses a Telegram bot (`instagram_load_bot`) to fetch and archive Instagram content,\nsuch as posts and stories. It leverages the Telethon library to interact with the Telegram API, sending Instagram URLs\nto the bot and downloading the resulting media and metadata. The downloaded content is stored as `Media` objects and\nreturned as part of a `Metadata` object.\n\n### Features\n- Supports archiving Instagram posts and stories through the Telegram bot.\n- Downloads and saves media files (e.g., images, videos) in a temporary directory.\n- Captures and returns metadata, including titles and descriptions, as a `Metadata` object.\n- Automatically manages Telegram session files for secure access.\n\n### Setup\n\nTo use the `InstagramTbotExtractor`, you need to provide the following configuration settings:\n- **API ID and Hash**: Telegram API credentials obtained from [my.telegram.org/apps](https://my.telegram.org/apps).\n- **Session File**: Optional path to store the Telegram session file for future use.\n- The session file is created automatically and should be unique for each instance.\n- You may need to enter your Telegram credentials (phone) and use the a 2FA code sent to you the first time you run the extractor.:\n```2025-01-30 00:43:49.348 | INFO | auto_archiver.modules.instagram_tbot_extractor.instagram_tbot_extractor:setup:36 - SETUP instagram_tbot_extractor checking login...\nPlease enter your phone (or bot token): +447123456789\nPlease enter the code you received: 00000\nSigned in successfully as E C; remember to not break the ToS or you will risk an account ban!\n```\n ", + "dependencies": { + "python": [ + "loguru", + "telethon" + ] + }, + "entry_point": "", + "version": "1.0", + "configs": { + "api_id": { + "default": null, + "help": "telegram API_ID value, go to https://my.telegram.org/apps" + }, + "api_hash": { + "default": null, + "help": "telegram API_HASH value, go to https://my.telegram.org/apps" + }, + "session_file": { + "default": "secrets/anon-insta", + "help": "optional, records the telegram login session for future usage, '.session' will be appended to the provided value." + }, + "timeout": { + "default": 45, + "type": "int", + "help": "timeout to fetch the instagram content in seconds." + } + } + }, + "configs": { + "api_id": { + "default": null, + "help": "telegram API_ID value, go to https://my.telegram.org/apps" + }, + "api_hash": { + "default": null, + "help": "telegram API_HASH value, go to https://my.telegram.org/apps" + }, + "session_file": { + "default": "secrets/anon-insta", + "help": "optional, records the telegram login session for future usage, '.session' will be appended to the provided value." + }, + "timeout": { + "default": 45, + "type": "int", + "help": "timeout to fetch the instagram content in seconds." + } + } + }, + "twitter_api_extractor": { + "name": "twitter_api_extractor", + "display_name": "Twitter API Extractor", + "manifest": { + "name": "Twitter API Extractor", + "author": "Bellingcat", + "type": [ + "extractor" + ], + "requires_setup": true, + "description": "\n The `TwitterApiExtractor` fetches tweets and associated media using the Twitter API. \n It supports multiple API configurations for extended rate limits and reliable access. \n Features include URL expansion, media downloads (e.g., images, videos), and structured output \n via `Metadata` and `Media` objects. Requires Twitter API credentials such as bearer tokens \n or consumer key/secret and access token/secret.\n \n ### Features\n - Fetches tweets and their metadata, including text, creation timestamp, and author information.\n - Downloads media attachments (e.g., images, videos) in high quality.\n - Supports multiple API configurations for improved rate limiting.\n - Expands shortened URLs (e.g., `t.co` links).\n - Outputs structured metadata and media using `Metadata` and `Media` objects.\n \n ### Setup\n To use the `TwitterApiExtractor`, you must provide valid Twitter API credentials via configuration:\n - **Bearer Token(s)**: A single token or a list for rate-limited API access.\n - **Consumer Key and Secret**: Required for user-authenticated API access.\n - **Access Token and Secret**: Complements the consumer key for enhanced API capabilities.\n \n Credentials can be obtained by creating a Twitter developer account at [Twitter Developer Platform](https://developer.twitter.com/en).\n ", + "dependencies": { + "python": [ + "requests", + "loguru", + "pytwitter", + "slugify" + ], + "bin": [ + "" + ] + }, + "entry_point": "", + "version": "1.0", + "configs": { + "bearer_token": { + "default": null, + "help": "[deprecated: see bearer_tokens] twitter API bearer_token which is enough for archiving, if not provided you will need consumer_key, consumer_secret, access_token, access_secret" + }, + "bearer_tokens": { + "default": [], + "help": " a list of twitter API bearer_token which is enough for archiving, if not provided you will need consumer_key, consumer_secret, access_token, access_secret, if provided you can still add those for better rate limits. CSV of bearer tokens if provided via the command line" + }, + "consumer_key": { + "default": null, + "help": "twitter API consumer_key" + }, + "consumer_secret": { + "default": null, + "help": "twitter API consumer_secret" + }, + "access_token": { + "default": null, + "help": "twitter API access_token" + }, + "access_secret": { + "default": null, + "help": "twitter API access_secret" + } + } + }, + "configs": { + "bearer_token": { + "default": null, + "help": "[deprecated: see bearer_tokens] twitter API bearer_token which is enough for archiving, if not provided you will need consumer_key, consumer_secret, access_token, access_secret" + }, + "bearer_tokens": { + "default": [], + "help": " a list of twitter API bearer_token which is enough for archiving, if not provided you will need consumer_key, consumer_secret, access_token, access_secret, if provided you can still add those for better rate limits. CSV of bearer tokens if provided via the command line" + }, + "consumer_key": { + "default": null, + "help": "twitter API consumer_key" + }, + "consumer_secret": { + "default": null, + "help": "twitter API consumer_secret" + }, + "access_token": { + "default": null, + "help": "twitter API access_token" + }, + "access_secret": { + "default": null, + "help": "twitter API access_secret" + } + } + }, + "instagram_extractor": { + "name": "instagram_extractor", + "display_name": "Instagram Extractor", + "manifest": { + "name": "Instagram Extractor", + "author": "Bellingcat", + "type": [ + "extractor" + ], + "requires_setup": true, + "description": "\n Uses the [Instaloader library](https://instaloader.github.io/as-module.html) to download content from Instagram. This class handles both individual posts\n and user profiles, downloading as much information as possible, including images, videos, text, stories,\n highlights, and tagged posts. \n Authentication is required via username/password or a session file.\n \n ", + "dependencies": { + "python": [ + "instaloader", + "loguru" + ] + }, + "entry_point": "", + "version": "1.0", + "configs": { + "username": { + "required": true, + "help": "a valid Instagram username" + }, + "password": { + "required": true, + "help": "the corresponding Instagram account password" + }, + "download_folder": { + "default": "instaloader", + "help": "name of a folder to temporarily download content to" + }, + "session_file": { + "default": "secrets/instaloader.session", + "help": "path to the instagram session which saves session credentials" + } + } + }, + "configs": { + "username": { + "required": true, + "help": "a valid Instagram username" + }, + "password": { + "required": true, + "help": "the corresponding Instagram account password" + }, + "download_folder": { + "default": "instaloader", + "help": "name of a folder to temporarily download content to" + }, + "session_file": { + "default": "secrets/instaloader.session", + "help": "path to the instagram session which saves session credentials" + } + } + }, + "telethon_extractor": { + "name": "telethon_extractor", + "display_name": "Telethon Extractor", + "manifest": { + "name": "Telethon Extractor", + "author": "Bellingcat", + "type": [ + "extractor" + ], + "requires_setup": true, + "description": "\nThe `TelethonExtractor` uses the Telethon library to archive posts and media from Telegram channels and groups. \nIt supports private and public channels, downloading grouped posts with media, and can join channels using invite links \nif provided in the configuration. \n\n### Features\n- Fetches posts and metadata from Telegram channels and groups, including private channels.\n- Downloads media attachments (e.g., images, videos, audio) from individual posts or grouped posts.\n- Handles channel invites to join channels dynamically during setup.\n- Utilizes Telethon's capabilities for reliable Telegram interactions.\n- Outputs structured metadata and media using `Metadata` and `Media` objects.\n\n### Setup\nTo use the `TelethonExtractor`, you must configure the following:\n- **API ID and API Hash**: Obtain these from [my.telegram.org](https://my.telegram.org/apps).\n- **Session File**: Optional, but records login sessions for future use (default: `secrets/anon.session`).\n- **Bot Token**: Optional, allows access to additional content (e.g., large videos) but limits private channel archiving.\n- **Channel Invites**: Optional, specify a JSON string of invite links to join channels during setup.\n\n### First Time Login\nThe first time you run, you will be prompted to do a authentication with the phone number associated, alternatively you can put your `anon.session` in the root.\n\n\n", + "dependencies": { + "python": [ + "telethon", + "loguru", + "tqdm" + ], + "bin": [ + "" + ] + }, + "entry_point": "", + "version": "1.0", + "configs": { + "api_id": { + "default": null, + "help": "telegram API_ID value, go to https://my.telegram.org/apps" + }, + "api_hash": { + "default": null, + "help": "telegram API_HASH value, go to https://my.telegram.org/apps" + }, + "bot_token": { + "default": null, + "help": "optional, but allows access to more content such as large videos, talk to @botfather" + }, + "session_file": { + "default": "secrets/anon", + "help": "optional, records the telegram login session for future usage, '.session' will be appended to the provided value." + }, + "join_channels": { + "default": true, + "type": "bool", + "help": "disables the initial setup with channel_invites config, useful if you have a lot and get stuck" + }, + "channel_invites": { + "default": {}, + "help": "(JSON string) private channel invite links (format: t.me/joinchat/HASH OR t.me/+HASH) and (optional but important to avoid hanging for minutes on startup) channel id (format: CHANNEL_ID taken from a post url like https://t.me/c/CHANNEL_ID/1), the telegram account will join any new channels on setup", + "type": "json_loader" + } + } + }, + "configs": { + "api_id": { + "default": null, + "help": "telegram API_ID value, go to https://my.telegram.org/apps" + }, + "api_hash": { + "default": null, + "help": "telegram API_HASH value, go to https://my.telegram.org/apps" + }, + "bot_token": { + "default": null, + "help": "optional, but allows access to more content such as large videos, talk to @botfather" + }, + "session_file": { + "default": "secrets/anon", + "help": "optional, records the telegram login session for future usage, '.session' will be appended to the provided value." + }, + "join_channels": { + "default": true, + "type": "bool", + "help": "disables the initial setup with channel_invites config, useful if you have a lot and get stuck" + }, + "channel_invites": { + "default": {}, + "help": "(JSON string) private channel invite links (format: t.me/joinchat/HASH OR t.me/+HASH) and (optional but important to avoid hanging for minutes on startup) channel id (format: CHANNEL_ID taken from a post url like https://t.me/c/CHANNEL_ID/1), the telegram account will join any new channels on setup", + "type": "json_loader" + } + } + }, + "vk_extractor": { + "name": "vk_extractor", + "display_name": "VKontakte Extractor", + "manifest": { + "name": "VKontakte Extractor", + "author": "Bellingcat", + "type": [ + "extractor" + ], + "requires_setup": true, + "description": "\nThe `VkExtractor` fetches posts, text, and images from VK (VKontakte) social media pages. \nThis archiver is specialized for `/wall` posts and uses the `VkScraper` library to extract \nand download content. Note that VK videos are handled separately by the `YTDownloader`.\n\n### Features\n- Extracts text, timestamps, and metadata from VK `/wall` posts.\n- Downloads associated images and attaches them to the resulting `Metadata` object.\n- Processes multiple segments of VK URLs that contain mixed content (e.g., wall, photo).\n- Outputs structured metadata and media using `Metadata` and `Media` objects.\n\n### Setup\nTo use the `VkArchiver`, you must provide valid VKontakte login credentials and session information:\n- **Username**: A valid VKontakte account username.\n- **Password**: The corresponding password for the VKontakte account.\n- **Session File**: Optional. Path to a session configuration file (`.json`) for persistent VK login.\n\nCredentials can be set in the configuration file or directly via environment variables. Ensure you \nhave access to the VKontakte API by creating an account at [VKontakte](https://vk.com/).\n", + "dependencies": { + "python": [ + "loguru", + "vk_url_scraper" + ] + }, + "entry_point": "", + "version": "1.0", + "configs": { + "username": { + "required": true, + "help": "valid VKontakte username" + }, + "password": { + "required": true, + "help": "valid VKontakte password" + }, + "session_file": { + "default": "secrets/vk_config.v2.json", + "help": "valid VKontakte password" + } + }, + "depends": [ + "core", + "utils" + ] + }, + "configs": { + "username": { + "required": true, + "help": "valid VKontakte username" + }, + "password": { + "required": true, + "help": "valid VKontakte password" + }, + "session_file": { + "default": "secrets/vk_config.v2.json", + "help": "valid VKontakte password" + } + } + }, + "generic_extractor": { + "name": "generic_extractor", + "display_name": "Generic Extractor", + "manifest": { + "name": "Generic Extractor", + "author": "Bellingcat", + "type": [ + "extractor" + ], + "requires_setup": false, + "description": "\nThis is the generic extractor used by auto-archiver, which uses `yt-dlp` under the hood.\n\nThis module is responsible for downloading and processing media content from platforms\nsupported by `yt-dlp`, such as YouTube, Facebook, and others. It provides functionality\nfor retrieving videos, subtitles, comments, and other metadata, and it integrates with\nthe broader archiving framework.\n\n### Features\n- Supports downloading videos and playlists.\n- Retrieves metadata like titles, descriptions, upload dates, and durations.\n- Downloads subtitles and comments when enabled.\n- Configurable options for handling live streams, proxies, and more.\n- Supports authentication of websites using the 'authentication' settings from your orchestration.\n\n### Dropins\n- For websites supported by `yt-dlp` that also contain posts in addition to videos\n (e.g. Facebook, Twitter, Bluesky), dropins can be created to extract post data and create \n metadata objects. Some dropins are included in this generic_archiver by default, but\ncustom dropins can be created to handle additional websites and passed to the archiver\nvia the command line using the `--dropins` option (TODO!).\n", + "dependencies": { + "python": [ + "yt_dlp", + "requests", + "loguru", + "slugify" + ] + }, + "entry_point": "", + "version": "0.1.0", + "configs": { + "subtitles": { + "default": true, + "help": "download subtitles if available", + "type": "bool" + }, + "comments": { + "default": false, + "help": "download all comments if available, may lead to large metadata", + "type": "bool" + }, + "livestreams": { + "default": false, + "help": "if set, will download live streams, otherwise will skip them; see --max-filesize for more control", + "type": "bool" + }, + "live_from_start": { + "default": false, + "help": "if set, will download live streams from their earliest available moment, otherwise starts now.", + "type": "bool" + }, + "proxy": { + "default": "", + "help": "http/socks (https seems to not work atm) proxy to use for the webdriver, eg https://proxy-user:password@proxy-ip:port" + }, + "end_means_success": { + "default": true, + "help": "if True, any archived content will mean a 'success', if False this archiver will not return a 'success' stage; this is useful for cases when the yt-dlp will archive a video but ignore other types of content like images or text only pages that the subsequent archivers can retrieve.", + "type": "bool" + }, + "allow_playlist": { + "default": false, + "help": "If True will also download playlists, set to False if the expectation is to download a single video.", + "type": "bool" + }, + "max_downloads": { + "default": "inf", + "help": "Use to limit the number of videos to download when a channel or long page is being extracted. 'inf' means no limit." + } + } + }, + "configs": { + "subtitles": { + "default": true, + "help": "download subtitles if available", + "type": "bool" + }, + "comments": { + "default": false, + "help": "download all comments if available, may lead to large metadata", + "type": "bool" + }, + "livestreams": { + "default": false, + "help": "if set, will download live streams, otherwise will skip them; see --max-filesize for more control", + "type": "bool" + }, + "live_from_start": { + "default": false, + "help": "if set, will download live streams from their earliest available moment, otherwise starts now.", + "type": "bool" + }, + "proxy": { + "default": "", + "help": "http/socks (https seems to not work atm) proxy to use for the webdriver, eg https://proxy-user:password@proxy-ip:port" + }, + "end_means_success": { + "default": true, + "help": "if True, any archived content will mean a 'success', if False this archiver will not return a 'success' stage; this is useful for cases when the yt-dlp will archive a video but ignore other types of content like images or text only pages that the subsequent archivers can retrieve.", + "type": "bool" + }, + "allow_playlist": { + "default": false, + "help": "If True will also download playlists, set to False if the expectation is to download a single video.", + "type": "bool" + }, + "max_downloads": { + "default": "inf", + "help": "Use to limit the number of videos to download when a channel or long page is being extracted. 'inf' means no limit." + } + } + }, + "telegram_extractor": { + "name": "telegram_extractor", + "display_name": "Telegram Extractor", + "manifest": { + "name": "Telegram Extractor", + "author": "Bellingcat", + "type": [ + "extractor" + ], + "requires_setup": false, + "description": " \n The `TelegramExtractor` retrieves publicly available media content from Telegram message links without requiring login credentials. \n It processes URLs to fetch images and videos embedded in Telegram messages, ensuring a structured output using `Metadata` \n and `Media` objects. Recommended for scenarios where login-based archiving is not viable, although `telethon_archiver` \n is advised for more comprehensive functionality, and higher quality media extraction.\n \n ### Features\n- Extracts images and videos from public Telegram message links (`t.me`).\n- Processes HTML content of messages to retrieve embedded media.\n- Sets structured metadata, including timestamps, content, and media details.\n- Does not require user authentication for Telegram.\n\n ", + "dependencies": { + "python": [ + "requests", + "bs4", + "loguru" + ] + }, + "entry_point": "", + "version": "1.0", + "configs": {} + }, + "configs": null + }, + "wayback_extractor_enricher": { + "name": "wayback_extractor_enricher", + "display_name": "Wayback Machine Enricher (and Extractor)", + "manifest": { + "name": "Wayback Machine Enricher (and Extractor)", + "author": "Bellingcat", + "type": [ + "enricher", + "extractor" + ], + "requires_setup": true, + "description": "\n Submits the current URL to the Wayback Machine for archiving and returns either a job ID or the completed archive URL.\n\n ### Features\n - Archives URLs using the Internet Archive's Wayback Machine API.\n - Supports conditional archiving based on the existence of prior archives within a specified time range.\n - Provides proxies for HTTP and HTTPS requests.\n - Fetches and confirms the archive URL or provides a job ID for later status checks.\n\n ### Notes\n - Requires a valid Wayback Machine API key and secret.\n - Handles rate-limiting by Wayback Machine and retries status checks with exponential backoff.\n \n ### Steps to Get an Wayback API Key:\n - Sign up for an account at [Internet Archive](https://archive.org/account/signup).\n - Log in to your account.\n - Navigte to your [account settings](https://archive.org/account).\n - or: https://archive.org/developers/tutorial-get-ia-credentials.html\n - Under Wayback Machine API Keys, generate a new key.\n - Note down your API key and secret, as they will be required for authentication.\n ", + "dependencies": { + "python": [ + "loguru", + "requests" + ] + }, + "entry_point": "wayback_extractor_enricher::WaybackExtractorEnricher", + "version": "1.0", + "configs": { + "timeout": { + "default": 15, + "type": "int", + "help": "seconds to wait for successful archive confirmation from wayback, if more than this passes the result contains the job_id so the status can later be checked manually." + }, + "if_not_archived_within": { + "default": null, + "help": "only tell wayback to archive if no archive is available before the number of seconds specified, use None to ignore this option. For more information: https://docs.google.com/document/d/1Nsv52MvSjbLb2PCpHlat0gkzw0EvtSgpKHu4mk0MnrA" + }, + "key": { + "required": true, + "help": "wayback API key. to get credentials visit https://archive.org/account/s3.php" + }, + "secret": { + "required": true, + "help": "wayback API secret. to get credentials visit https://archive.org/account/s3.php" + }, + "proxy_http": { + "default": null, + "help": "http proxy to use for wayback requests, eg http://proxy-user:password@proxy-ip:port" + }, + "proxy_https": { + "default": null, + "help": "https proxy to use for wayback requests, eg https://proxy-user:password@proxy-ip:port" + } + } + }, + "configs": { + "timeout": { + "default": 15, + "type": "int", + "help": "seconds to wait for successful archive confirmation from wayback, if more than this passes the result contains the job_id so the status can later be checked manually." + }, + "if_not_archived_within": { + "default": null, + "help": "only tell wayback to archive if no archive is available before the number of seconds specified, use None to ignore this option. For more information: https://docs.google.com/document/d/1Nsv52MvSjbLb2PCpHlat0gkzw0EvtSgpKHu4mk0MnrA" + }, + "key": { + "required": true, + "help": "wayback API key. to get credentials visit https://archive.org/account/s3.php" + }, + "secret": { + "required": true, + "help": "wayback API secret. to get credentials visit https://archive.org/account/s3.php" + }, + "proxy_http": { + "default": null, + "help": "http proxy to use for wayback requests, eg http://proxy-user:password@proxy-ip:port" + }, + "proxy_https": { + "default": null, + "help": "https proxy to use for wayback requests, eg https://proxy-user:password@proxy-ip:port" + } + } + }, + "wacz_extractor_enricher": { + "name": "wacz_extractor_enricher", + "display_name": "WACZ Enricher (and Extractor)", + "manifest": { + "name": "WACZ Enricher (and Extractor)", + "author": "Bellingcat", + "type": [ + "enricher", + "extractor" + ], + "requires_setup": true, + "description": "\n Creates .WACZ archives of web pages using the `browsertrix-crawler` tool, with options for media extraction and screenshot saving.\n [Browsertrix-crawler](https://crawler.docs.browsertrix.com/user-guide/) is a headless browser-based crawler that archives web pages in WACZ format.\n\n ### Features\n - Archives web pages into .WACZ format using Docker or direct invocation of `browsertrix-crawler`.\n - Supports custom profiles for archiving private or dynamic content.\n - Extracts media (images, videos, audio) and screenshots from the archive, optionally adding them to the enrichment pipeline.\n - Generates metadata from the archived page's content and structure (e.g., titles, text).\n\n ### Notes\n - Requires Docker for running `browsertrix-crawler` .\n - Configurable via parameters for timeout, media extraction, screenshots, and proxy settings.\n ", + "dependencies": { + "python": [ + "loguru", + "jsonlines", + "warcio" + ], + "bin": [ + "docker" + ] + }, + "entry_point": "wacz_extractor_enricher::WaczExtractorEnricher", + "version": "1.0", + "configs": { + "profile": { + "default": null, + "help": "browsertrix-profile (for profile generation see https://github.com/webrecorder/browsertrix-crawler#creating-and-using-browser-profiles)." + }, + "docker_commands": { + "default": null, + "help": "if a custom docker invocation is needed" + }, + "timeout": { + "default": 120, + "type": "int", + "help": "timeout for WACZ generation in seconds" + }, + "extract_media": { + "default": false, + "type": "bool", + "help": "If enabled all the images/videos/audio present in the WACZ archive will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched." + }, + "extract_screenshot": { + "default": true, + "type": "bool", + "help": "If enabled the screenshot captured by browsertrix will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched." + }, + "socks_proxy_host": { + "default": null, + "help": "SOCKS proxy host for browsertrix-crawler, use in combination with socks_proxy_port. eg: user:password@host" + }, + "socks_proxy_port": { + "default": null, + "type": "int", + "help": "SOCKS proxy port for browsertrix-crawler, use in combination with socks_proxy_host. eg 1234" + }, + "proxy_server": { + "default": null, + "help": "SOCKS server proxy URL, in development" + } + } + }, + "configs": { + "profile": { + "default": null, + "help": "browsertrix-profile (for profile generation see https://github.com/webrecorder/browsertrix-crawler#creating-and-using-browser-profiles)." + }, + "docker_commands": { + "default": null, + "help": "if a custom docker invocation is needed" + }, + "timeout": { + "default": 120, + "type": "int", + "help": "timeout for WACZ generation in seconds" + }, + "extract_media": { + "default": false, + "type": "bool", + "help": "If enabled all the images/videos/audio present in the WACZ archive will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched." + }, + "extract_screenshot": { + "default": true, + "type": "bool", + "help": "If enabled the screenshot captured by browsertrix will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched." + }, + "socks_proxy_host": { + "default": null, + "help": "SOCKS proxy host for browsertrix-crawler, use in combination with socks_proxy_port. eg: user:password@host" + }, + "socks_proxy_port": { + "default": null, + "type": "int", + "help": "SOCKS proxy port for browsertrix-crawler, use in combination with socks_proxy_host. eg 1234" + }, + "proxy_server": { + "default": null, + "help": "SOCKS server proxy URL, in development" + } + } + }, + "metadata_enricher": { + "name": "metadata_enricher", + "display_name": "Media Metadata Enricher", + "manifest": { + "name": "Media Metadata Enricher", + "author": "Bellingcat", + "type": [ + "enricher" + ], + "requires_setup": true, + "description": "\n Extracts metadata information from files using ExifTool.\n\n ### Features\n - Uses ExifTool to extract detailed metadata from media files.\n - Processes file-specific data like camera settings, geolocation, timestamps, and other embedded metadata.\n - Adds extracted metadata to the corresponding `Media` object within the `Metadata`.\n\n ### Notes\n - Requires ExifTool to be installed and accessible via the system's PATH.\n - Skips enrichment for files where metadata extraction fails.\n ", + "dependencies": { + "python": [ + "loguru" + ], + "bin": [ + "exiftool" + ] + }, + "entry_point": "", + "version": "1.0", + "configs": {} + }, + "configs": null + }, + "timestamping_enricher": { + "name": "timestamping_enricher", + "display_name": "Timestamping Enricher", + "manifest": { + "name": "Timestamping Enricher", + "author": "Bellingcat", + "type": [ + "enricher" + ], + "requires_setup": true, + "description": "\n Generates RFC3161-compliant timestamp tokens using Time Stamp Authorities (TSA) for archived files.\n\n ### Features\n - Creates timestamp tokens to prove the existence of files at a specific time, useful for legal and authenticity purposes.\n - Aggregates file hashes into a text file and timestamps the concatenated data.\n - Uses multiple Time Stamp Authorities (TSAs) to ensure reliability and redundancy.\n - Validates timestamping certificates against trusted Certificate Authorities (CAs) using the `certifi` trust store.\n\n ### Notes\n - Should be run after the `hash_enricher` to ensure file hashes are available.\n - Requires internet access to interact with the configured TSAs.\n ", + "dependencies": { + "python": [ + "loguru", + "slugify", + "tsp_client", + "asn1crypto", + "certvalidator", + "certifi" + ] + }, + "entry_point": "", + "version": "1.0", + "configs": { + "tsa_urls": { + "default": [ + "http://timestamp.digicert.com", + "http://timestamp.identrust.com", + "http://timestamp.globalsign.com/tsa/r6advanced1", + "http://tss.accv.es:8318/tsa" + ], + "help": "List of RFC3161 Time Stamp Authorities to use, separate with commas if passed via the command line." + } + } + }, + "configs": { + "tsa_urls": { + "default": [ + "http://timestamp.digicert.com", + "http://timestamp.identrust.com", + "http://timestamp.globalsign.com/tsa/r6advanced1", + "http://tss.accv.es:8318/tsa" + ], + "help": "List of RFC3161 Time Stamp Authorities to use, separate with commas if passed via the command line." + } + } + }, + "screenshot_enricher": { + "name": "screenshot_enricher", + "display_name": "Screenshot Enricher", + "manifest": { + "name": "Screenshot Enricher", + "author": "Bellingcat", + "type": [ + "enricher" + ], + "requires_setup": true, + "description": "\n Captures screenshots and optionally saves web pages as PDFs using a WebDriver.\n\n ### Features\n - Takes screenshots of web pages, with configurable width, height, and timeout settings.\n - Optionally saves pages as PDFs, with additional configuration for PDF printing options.\n - Bypasses URLs detected as authentication walls.\n - Integrates seamlessly with the metadata enrichment pipeline, adding screenshots and PDFs as media.\n\n ### Notes\n - Requires a WebDriver (e.g., ChromeDriver) installed and accessible via the system's PATH.\n ", + "dependencies": { + "python": [ + "loguru", + "selenium" + ] + }, + "entry_point": "", + "version": "1.0", + "configs": { + "width": { + "default": 1280, + "type": "int", + "help": "width of the screenshots" + }, + "height": { + "default": 720, + "type": "int", + "help": "height of the screenshots" + }, + "timeout": { + "default": 60, + "type": "int", + "help": "timeout for taking the screenshot" + }, + "sleep_before_screenshot": { + "default": 4, + "type": "int", + "help": "seconds to wait for the pages to load before taking screenshot" + }, + "http_proxy": { + "default": "", + "help": "http proxy to use for the webdriver, eg http://proxy-user:password@proxy-ip:port" + }, + "save_to_pdf": { + "default": false, + "type": "bool", + "help": "save the page as pdf along with the screenshot. PDF saving options can be adjusted with the 'print_options' parameter" + }, + "print_options": { + "default": {}, + "help": "options to pass to the pdf printer, in JSON format. See https://www.selenium.dev/documentation/webdriver/interactions/print_page/ for more information", + "type": "json_loader" + } + } + }, + "configs": { + "width": { + "default": 1280, + "type": "int", + "help": "width of the screenshots" + }, + "height": { + "default": 720, + "type": "int", + "help": "height of the screenshots" + }, + "timeout": { + "default": 60, + "type": "int", + "help": "timeout for taking the screenshot" + }, + "sleep_before_screenshot": { + "default": 4, + "type": "int", + "help": "seconds to wait for the pages to load before taking screenshot" + }, + "http_proxy": { + "default": "", + "help": "http proxy to use for the webdriver, eg http://proxy-user:password@proxy-ip:port" + }, + "save_to_pdf": { + "default": false, + "type": "bool", + "help": "save the page as pdf along with the screenshot. PDF saving options can be adjusted with the 'print_options' parameter" + }, + "print_options": { + "default": {}, + "help": "options to pass to the pdf printer, in JSON format. See https://www.selenium.dev/documentation/webdriver/interactions/print_page/ for more information", + "type": "json_loader" + } + } + }, + "whisper_enricher": { + "name": "whisper_enricher", + "display_name": "Whisper Enricher", + "manifest": { + "name": "Whisper Enricher", + "author": "Bellingcat", + "type": [ + "enricher" + ], + "requires_setup": true, + "description": "\n Integrates with a Whisper API service to transcribe, translate, or detect the language of audio and video files.\n\n ### Features\n - Submits audio or video files to a Whisper API deployment for processing.\n - Supports operations such as transcription, translation, and language detection.\n - Optionally generates SRT subtitle files for video content.\n - Integrates with S3-compatible storage systems to make files publicly accessible for processing.\n - Handles job submission, status checking, artifact retrieval, and cleanup.\n\n ### Notes\n - Requires a Whisper API endpoint and API key for authentication.\n - Only compatible with S3-compatible storage systems for media file accessibility.\n - ** This stores the media files in S3 prior to enriching them as Whisper requires public URLs to access the media files.\n - Handles multiple jobs and retries for failed or incomplete processing.\n ", + "dependencies": { + "python": [ + "s3_storage", + "loguru", + "requests" + ] + }, + "entry_point": "", + "version": "1.0", + "configs": { + "api_endpoint": { + "required": true, + "help": "WhisperApi api endpoint, eg: https://whisperbox-api.com/api/v1, a deployment of https://github.com/bellingcat/whisperbox-transcribe." + }, + "api_key": { + "required": true, + "help": "WhisperApi api key for authentication" + }, + "include_srt": { + "default": false, + "type": "bool", + "help": "Whether to include a subtitle SRT (SubRip Subtitle file) for the video (can be used in video players)." + }, + "timeout": { + "default": 90, + "type": "int", + "help": "How many seconds to wait at most for a successful job completion." + }, + "action": { + "default": "translate", + "help": "which Whisper operation to execute", + "choices": [ + "transcribe", + "translate", + "language_detection" + ] + } + } + }, + "configs": { + "api_endpoint": { + "required": true, + "help": "WhisperApi api endpoint, eg: https://whisperbox-api.com/api/v1, a deployment of https://github.com/bellingcat/whisperbox-transcribe." + }, + "api_key": { + "required": true, + "help": "WhisperApi api key for authentication" + }, + "include_srt": { + "default": false, + "type": "bool", + "help": "Whether to include a subtitle SRT (SubRip Subtitle file) for the video (can be used in video players)." + }, + "timeout": { + "default": 90, + "type": "int", + "help": "How many seconds to wait at most for a successful job completion." + }, + "action": { + "default": "translate", + "help": "which Whisper operation to execute", + "choices": [ + "transcribe", + "translate", + "language_detection" + ] + } + } + }, + "thumbnail_enricher": { + "name": "thumbnail_enricher", + "display_name": "Thumbnail Enricher", + "manifest": { + "name": "Thumbnail Enricher", + "author": "Bellingcat", + "type": [ + "enricher" + ], + "requires_setup": false, + "description": "\n Generates thumbnails for video files to provide visual previews.\n\n ### Features\n - Processes video files and generates evenly distributed thumbnails.\n - Calculates the number of thumbnails based on video duration, `thumbnails_per_minute`, and `max_thumbnails`.\n - Distributes thumbnails equally across the video's duration and stores them as media objects.\n - Adds metadata for each thumbnail, including timestamps and IDs.\n\n ### Notes\n - Requires `ffmpeg` to be installed and accessible via the system's PATH.\n - Handles videos without pre-existing duration metadata by probing with `ffmpeg`.\n - Skips enrichment for non-video media files.\n ", + "dependencies": { + "python": [ + "loguru", + "ffmpeg" + ], + "bin": [ + "ffmpeg" + ] + }, + "entry_point": "", + "version": "1.0", + "configs": { + "thumbnails_per_minute": { + "default": 60, + "type": "int", + "help": "how many thumbnails to generate per minute of video, can be limited by max_thumbnails" + }, + "max_thumbnails": { + "default": 16, + "type": "int", + "help": "limit the number of thumbnails to generate per video, 0 means no limit" + } + } + }, + "configs": { + "thumbnails_per_minute": { + "default": 60, + "type": "int", + "help": "how many thumbnails to generate per minute of video, can be limited by max_thumbnails" + }, + "max_thumbnails": { + "default": 16, + "type": "int", + "help": "limit the number of thumbnails to generate per video, 0 means no limit" + } + } + }, + "meta_enricher": { + "name": "meta_enricher", + "display_name": "Archive Metadata Enricher", + "manifest": { + "name": "Archive Metadata Enricher", + "author": "Bellingcat", + "type": [ + "enricher" + ], + "requires_setup": false, + "description": " \n Adds metadata information about the archive operations, Adds metadata about archive operations, including file sizes and archive duration./\n To be included at the end of all enrichments.\n \n ### Features\n- Calculates the total size of all archived media files, storing the result in human-readable and byte formats.\n- Computes the duration of the archival process, storing the elapsed time in seconds.\n- Ensures all enrichments are performed only if the `Metadata` object contains valid data.\n- Adds detailed metadata to provide insights into file sizes and archival performance.\n\n### Notes\n- Skips enrichment if no media or metadata is available in the `Metadata` object.\n- File sizes are calculated using the `os.stat` module, ensuring accurate byte-level reporting.\n", + "dependencies": { + "python": [ + "loguru" + ] + }, + "entry_point": "", + "version": "1.0", + "configs": {} + }, + "configs": null + }, + "pdq_hash_enricher": { + "name": "pdq_hash_enricher", + "display_name": "PDQ Hash Enricher", + "manifest": { + "name": "PDQ Hash Enricher", + "author": "Bellingcat", + "type": [ + "enricher" + ], + "requires_setup": false, + "description": "\n PDQ Hash Enricher for generating perceptual hashes of media files.\n\n ### Features\n - Calculates perceptual hashes for image files using the PDQ hashing algorithm.\n - Enables detection of duplicate or near-duplicate visual content.\n - Processes images stored in `Metadata` objects, adding computed hashes to the corresponding `Media` entries.\n - Skips non-image media or files unsuitable for hashing (e.g., corrupted or unsupported formats).\n\n ### Notes\n - Best used after enrichers like `thumbnail_enricher` or `screenshot_enricher` to ensure images are available.\n - Uses the `pdqhash` library to compute 256-bit perceptual hashes, which are stored as hexadecimal strings.\n ", + "dependencies": { + "python": [ + "loguru", + "pdqhash", + "numpy", + "PIL" + ] + }, + "entry_point": "", + "version": "1.0", + "configs": {} + }, + "configs": null + }, + "ssl_enricher": { + "name": "ssl_enricher", + "display_name": "SSL Certificate Enricher", + "manifest": { + "name": "SSL Certificate Enricher", + "author": "Bellingcat", + "type": [ + "enricher" + ], + "requires_setup": false, + "description": "\n Retrieves SSL certificate information for a domain and stores it as a file.\n\n ### Features\n - Fetches SSL certificates for domains using the HTTPS protocol.\n - Stores certificates in PEM format and adds them as media to the metadata.\n - Skips enrichment if no media has been archived, based on the `skip_when_nothing_archived` configuration.\n\n ### Notes\n - Requires the target URL to use the HTTPS scheme; other schemes are not supported.\n ", + "dependencies": { + "python": [ + "loguru", + "slugify" + ] + }, + "entry_point": "ssl_enricher::SSLEnricher", + "version": "1.0", + "configs": { + "skip_when_nothing_archived": { + "default": true, + "type": "bool", + "help": "if true, will skip enriching when no media is archived" + } + } + }, + "configs": { + "skip_when_nothing_archived": { + "default": true, + "type": "bool", + "help": "if true, will skip enriching when no media is archived" + } + } + }, + "hash_enricher": { + "name": "hash_enricher", + "display_name": "Hash Enricher", + "manifest": { + "name": "Hash Enricher", + "author": "Bellingcat", + "type": [ + "enricher" + ], + "requires_setup": false, + "description": "\nGenerates cryptographic hashes for media files to ensure data integrity and authenticity.\n\n### Features\n- Calculates cryptographic hashes (SHA-256 or SHA3-512) for media files stored in `Metadata` objects.\n- Ensures content authenticity, integrity validation, and duplicate identification.\n- Efficiently processes large files by reading file bytes in configurable chunk sizes.\n- Supports dynamic configuration of hash algorithms and chunk sizes.\n- Updates media metadata with the computed hash value in the format `:`.\n\n### Notes\n- Default hash algorithm is SHA-256, but SHA3-512 is also supported.\n- Chunk size defaults to 16 MB but can be adjusted based on memory requirements.\n- Useful for workflows requiring hash-based content validation or deduplication.\n", + "dependencies": { + "python": [ + "loguru" + ] + }, + "entry_point": "", + "version": "1.0", + "configs": { + "algorithm": { + "default": "SHA-256", + "help": "hash algorithm to use", + "choices": [ + "SHA-256", + "SHA3-512" + ] + }, + "chunksize": { + "default": 16000000, + "help": "number of bytes to use when reading files in chunks (if this value is too large you will run out of RAM), default is 16MB", + "type": "int" + } + } + }, + "configs": { + "algorithm": { + "default": "SHA-256", + "help": "hash algorithm to use", + "choices": [ + "SHA-256", + "SHA3-512" + ] + }, + "chunksize": { + "default": 16000000, + "help": "number of bytes to use when reading files in chunks (if this value is too large you will run out of RAM), default is 16MB", + "type": "int" + } + } + }, + "atlos_db": { + "name": "atlos_db", + "display_name": "Atlos Database", + "manifest": { + "name": "Atlos Database", + "author": "Bellingcat", + "type": [ + "database" + ], + "requires_setup": true, + "description": "\nHandles integration with the Atlos platform for managing archival results.\n\n### Features\n- Outputs archival results to the Atlos API for storage and tracking.\n- Updates failure status with error details when archiving fails.\n- Processes and formats metadata, including ISO formatting for datetime fields.\n- Skips processing for items without an Atlos ID.\n\n### Setup\nRequired configs:\n- atlos_url: Base URL for the Atlos API.\n- api_token: Authentication token for API access.\n", + "dependencies": { + "python": [ + "loguru", + "" + ], + "bin": [ + "" + ] + }, + "entry_point": "atlos_db::AtlosDb", + "version": "1.0", + "configs": { + "api_token": { + "default": null, + "help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/", + "required": true, + "type": "str" + }, + "atlos_url": { + "default": "https://platform.atlos.org", + "help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.", + "type": "str" + } + } + }, + "configs": { + "api_token": { + "default": null, + "help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/", + "required": true, + "type": "str" + }, + "atlos_url": { + "default": "https://platform.atlos.org", + "help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.", + "type": "str" + } + } + }, + "api_db": { + "name": "api_db", + "display_name": "Auto Archiver API Database", + "manifest": { + "name": "Auto Archiver API Database", + "author": "Bellingcat", + "type": [ + "database" + ], + "requires_setup": true, + "description": "\n Provides integration with the Auto Archiver API for querying and storing archival data.\n\n### Features\n- **API Integration**: Supports querying for existing archives and submitting results.\n- **Duplicate Prevention**: Avoids redundant archiving when `use_api_cache` is disabled.\n- **Configurable**: Supports settings like API endpoint, authentication token, tags, and permissions.\n- **Tagging and Metadata**: Adds tags and manages metadata for archives.\n- **Optional Storage**: Archives results conditionally based on configuration.\n\n### Setup\nRequires access to an Auto Archiver API instance and a valid API token.\n ", + "dependencies": { + "python": [ + "requests", + "loguru" + ] + }, + "entry_point": "api_db::AAApiDb", + "version": "1.0", + "configs": { + "api_endpoint": { + "required": true, + "help": "API endpoint where calls are made to" + }, + "api_token": { + "default": null, + "help": "API Bearer token." + }, + "public": { + "default": false, + "type": "bool", + "help": "whether the URL should be publicly available via the API" + }, + "author_id": { + "default": null, + "help": "which email to assign as author" + }, + "group_id": { + "default": null, + "help": "which group of users have access to the archive in case public=false as author" + }, + "use_api_cache": { + "default": true, + "type": "bool", + "help": "if False then the API database will be queried prior to any archiving operations and stop if the link has already been archived" + }, + "store_results": { + "default": true, + "type": "bool", + "help": "when set, will send the results to the API database." + }, + "tags": { + "default": [], + "help": "what tags to add to the archived URL" + } + } + }, + "configs": { + "api_endpoint": { + "required": true, + "help": "API endpoint where calls are made to" + }, + "api_token": { + "default": null, + "help": "API Bearer token." + }, + "public": { + "default": false, + "type": "bool", + "help": "whether the URL should be publicly available via the API" + }, + "author_id": { + "default": null, + "help": "which email to assign as author" + }, + "group_id": { + "default": null, + "help": "which group of users have access to the archive in case public=false as author" + }, + "use_api_cache": { + "default": true, + "type": "bool", + "help": "if False then the API database will be queried prior to any archiving operations and stop if the link has already been archived" + }, + "store_results": { + "default": true, + "type": "bool", + "help": "when set, will send the results to the API database." + }, + "tags": { + "default": [], + "help": "what tags to add to the archived URL" + } + } + }, + "gsheet_db": { + "name": "gsheet_db", + "display_name": "Google Sheets Database", + "manifest": { + "name": "Google Sheets Database", + "author": "Bellingcat", + "type": [ + "database" + ], + "requires_setup": true, + "description": "\n GsheetsDatabase:\n Handles integration with Google Sheets for tracking archival tasks.\n\n### Features\n- Updates a Google Sheet with the status of the archived URLs, including in progress, success or failure, and method used.\n- Saves metadata such as title, text, timestamp, hashes, screenshots, and media URLs to designated columns.\n- Formats media-specific metadata, such as thumbnails and PDQ hashes for the sheet.\n- Skips redundant updates for empty or invalid data fields.\n\n### Notes\n- Currently works only with metadata provided by GsheetFeeder. \n- Requires configuration of a linked Google Sheet and appropriate API credentials.\n ", + "dependencies": { + "python": [ + "loguru", + "gspread", + "slugify" + ] + }, + "entry_point": "gsheet_db::GsheetsDb", + "version": "1.0", + "configs": { + "allow_worksheets": { + "default": [], + "help": "(CSV) only worksheets whose name is included in allow are included (overrides worksheet_block), leave empty so all are allowed" + }, + "block_worksheets": { + "default": [], + "help": "(CSV) explicitly block some worksheets from being processed" + }, + "use_sheet_names_in_stored_paths": { + "default": true, + "type": "bool", + "help": "if True the stored files path will include 'workbook_name/worksheet_name/...'" + } + } + }, + "configs": { + "allow_worksheets": { + "default": [], + "help": "(CSV) only worksheets whose name is included in allow are included (overrides worksheet_block), leave empty so all are allowed" + }, + "block_worksheets": { + "default": [], + "help": "(CSV) explicitly block some worksheets from being processed" + }, + "use_sheet_names_in_stored_paths": { + "default": true, + "type": "bool", + "help": "if True the stored files path will include 'workbook_name/worksheet_name/...'" + } + } + }, + "console_db": { + "name": "console_db", + "display_name": "Console Database", + "manifest": { + "name": "Console Database", + "author": "Bellingcat", + "type": [ + "database" + ], + "requires_setup": false, + "description": "\nProvides a simple database implementation that outputs archival results and status updates to the console.\n\n### Features\n- Logs the status of archival tasks directly to the console, including:\n - started\n - failed (with error details)\n - aborted\n - done (with optional caching status)\n- Useful for debugging or lightweight setups where no external database is required.\n\n### Setup\nNo additional configuration is required.\n", + "dependencies": { + "python": [ + "loguru" + ] + }, + "entry_point": "", + "version": "1.0", + "configs": {} + }, + "configs": null + }, + "csv_db": { + "name": "csv_db", + "display_name": "CSV Database", + "manifest": { + "name": "CSV Database", + "author": "Bellingcat", + "type": [ + "database" + ], + "requires_setup": false, + "description": "\nHandles exporting archival results to a CSV file.\n\n### Features\n- Saves archival metadata as rows in a CSV file.\n- Automatically creates the CSV file with a header if it does not exist.\n- Appends new metadata entries to the existing file.\n\n### Setup\nRequired config:\n- csv_file: Path to the CSV file where results will be stored (default: \"db.csv\").\n", + "dependencies": { + "python": [ + "loguru" + ] + }, + "entry_point": "csv_db::CSVDb", + "version": "1.0", + "configs": { + "csv_file": { + "default": "db.csv", + "help": "CSV file name to save metadata to" + } + } + }, + "configs": { + "csv_file": { + "default": "db.csv", + "help": "CSV file name to save metadata to" + } + } + }, + "gdrive_storage": { + "name": "gdrive_storage", + "display_name": "Google Drive Storage", + "manifest": { + "name": "Google Drive Storage", + "author": "Dave Mateer", + "type": [ + "storage" + ], + "requires_setup": true, + "description": "\n \n GDriveStorage: A storage module for saving archived content to Google Drive.\n\n Source Documentation: https://davemateer.com/2022/04/28/google-drive-with-python\n\n ### Features\n - Saves media files to Google Drive, organizing them into folders based on the provided path structure.\n - Supports OAuth token-based authentication or service account credentials for API access.\n - Automatically creates folders in Google Drive if they don't exist.\n - Retrieves CDN URLs for stored files, enabling easy sharing and access.\n\n ### Notes\n - Requires setup with either a Google OAuth token or a service account JSON file.\n - Files are uploaded to the specified `root_folder_id` and organized by the `media.key` structure.\n - Automatically handles Google Drive API token refreshes for long-running jobs.\n \n ## Overview\nThis module integrates Google Drive as a storage backend, enabling automatic folder creation and file uploads. It supports authentication via **service accounts** (recommended for automation) or **OAuth tokens** (for user-based authentication).\n\n## Features\n- Saves files to Google Drive, organizing them into structured folders.\n- Supports both **service account** and **OAuth token** authentication.\n- Automatically creates folders if they don't exist.\n- Generates public URLs for easy file sharing.\n\n## Setup Guide\n1. **Enable Google Drive API**\n - Create a Google Cloud project at [Google Cloud Console](https://console.cloud.google.com/)\n - Enable the **Google Drive API**.\n\n2. **Set Up a Google Drive Folder**\n - Create a folder in **Google Drive** and copy its **folder ID** from the URL.\n - Add the **folder ID** to your configuration (`orchestration.yaml`):\n ```yaml\n root_folder_id: \"FOLDER_ID\"\n ```\n\n3. **Authentication Options**\n - **Option 1: Service Account (Recommended)**\n - Create a **service account** in Google Cloud IAM.\n - Download the JSON key file and save it as:\n ```\n secrets/service_account.json\n ```\n - **Share your Drive folder** with the service account\u2019s `client_email` (found in the JSON file).\n \n - **Option 2: OAuth Token (User Authentication)**\n - Create OAuth **Desktop App credentials** in Google Cloud.\n - Save the credentials as:\n ```\n secrets/oauth_credentials.json\n ```\n - Generate an OAuth token by running:\n ```sh\n python scripts/create_update_gdrive_oauth_token.py -c secrets/oauth_credentials.json\n ```\n\n \n Notes on the OAuth token:\n Tokens are refreshed after 1 hour however keep working for 7 days (tbc)\n so as long as the job doesn't last for 7 days then this method of refreshing only once per run will work\n see this link for details on the token:\n https://davemateer.com/2022/04/28/google-drive-with-python#tokens\n \n \n", + "dependencies": { + "python": [ + "loguru", + "googleapiclient", + "google" + ] + }, + "entry_point": "gdrive_storage::GDriveStorage", + "version": "1.0", + "configs": { + "path_generator": { + "default": "url", + "help": "how to store the file in terms of directory structure: 'flat' sets to root; 'url' creates a directory based on the provided URL; 'random' creates a random directory.", + "choices": [ + "flat", + "url", + "random" + ] + }, + "filename_generator": { + "default": "static", + "help": "how to name stored files: 'random' creates a random string; 'static' uses a replicable strategy such as a hash.", + "choices": [ + "random", + "static" + ] + }, + "root_folder_id": { + "required": true, + "help": "root google drive folder ID to use as storage, found in URL: 'https://drive.google.com/drive/folders/FOLDER_ID'" + }, + "oauth_token": { + "default": null, + "help": "JSON filename with Google Drive OAuth token: check auto-archiver repository scripts folder for create_update_gdrive_oauth_token.py. NOTE: storage used will count towards owner of GDrive folder, therefore it is best to use oauth_token_filename over service_account." + }, + "service_account": { + "default": "secrets/service_account.json", + "help": "service account JSON file path, same as used for Google Sheets. NOTE: storage used will count towards the developer account." + } + } + }, + "configs": { + "path_generator": { + "default": "url", + "help": "how to store the file in terms of directory structure: 'flat' sets to root; 'url' creates a directory based on the provided URL; 'random' creates a random directory.", + "choices": [ + "flat", + "url", + "random" + ] + }, + "filename_generator": { + "default": "static", + "help": "how to name stored files: 'random' creates a random string; 'static' uses a replicable strategy such as a hash.", + "choices": [ + "random", + "static" + ] + }, + "root_folder_id": { + "required": true, + "help": "root google drive folder ID to use as storage, found in URL: 'https://drive.google.com/drive/folders/FOLDER_ID'" + }, + "oauth_token": { + "default": null, + "help": "JSON filename with Google Drive OAuth token: check auto-archiver repository scripts folder for create_update_gdrive_oauth_token.py. NOTE: storage used will count towards owner of GDrive folder, therefore it is best to use oauth_token_filename over service_account." + }, + "service_account": { + "default": "secrets/service_account.json", + "help": "service account JSON file path, same as used for Google Sheets. NOTE: storage used will count towards the developer account." + } + } + }, + "atlos_storage": { + "name": "atlos_storage", + "display_name": "Atlos Storage", + "manifest": { + "name": "Atlos Storage", + "author": "Bellingcat", + "type": [ + "storage" + ], + "requires_setup": true, + "description": "\n Stores media files in a [Atlos](https://www.atlos.org/).\n\n ### Features\n - Saves media files to Atlos, organizing them into folders based on the provided path structure.\n\n ### Notes\n - Requires setup with Atlos credentials.\n - Files are uploaded to the specified `root_folder_id` and organized by the `media.key` structure.\n ", + "dependencies": { + "python": [ + "loguru", + "boto3" + ], + "bin": [] + }, + "entry_point": "", + "version": "1.0", + "configs": { + "api_token": { + "default": null, + "help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/", + "required": true, + "type": "str" + }, + "atlos_url": { + "default": "https://platform.atlos.org", + "help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.", + "type": "str" + } + } + }, + "configs": { + "api_token": { + "default": null, + "help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/", + "required": true, + "type": "str" + }, + "atlos_url": { + "default": "https://platform.atlos.org", + "help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.", + "type": "str" + } + } + }, + "s3_storage": { + "name": "s3_storage", + "display_name": "S3 Storage", + "manifest": { + "name": "S3 Storage", + "author": "Bellingcat", + "type": [ + "storage" + ], + "requires_setup": true, + "description": "\n S3Storage: A storage module for saving media files to an S3-compatible object storage.\n\n ### Features\n - Uploads media files to an S3 bucket with customizable configurations.\n - Supports `random_no_duplicate` mode to avoid duplicate uploads by checking existing files based on SHA-256 hashes.\n - Automatically generates unique paths for files when duplicates are found.\n - Configurable endpoint and CDN URL for different S3-compatible providers.\n - Supports both private and public file storage, with public files being readable online.\n\n ### Notes\n - Requires S3 credentials (API key and secret) and a bucket name to function.\n - The `random_no_duplicate` option ensures no duplicate uploads by leveraging hash-based folder structures.\n - Uses `boto3` for interaction with the S3 API.\n - Depends on the `HashEnricher` module for hash calculation.\n ", + "dependencies": { + "python": [ + "hash_enricher", + "boto3", + "loguru" + ] + }, + "entry_point": "", + "version": "1.0", + "configs": { + "path_generator": { + "default": "flat", + "help": "how to store the file in terms of directory structure: 'flat' sets to root; 'url' creates a directory based on the provided URL; 'random' creates a random directory.", + "choices": [ + "flat", + "url", + "random" + ] + }, + "filename_generator": { + "default": "static", + "help": "how to name stored files: 'random' creates a random string; 'static' uses a replicable strategy such as a hash.", + "choices": [ + "random", + "static" + ] + }, + "bucket": { + "default": null, + "help": "S3 bucket name" + }, + "region": { + "default": null, + "help": "S3 region name" + }, + "key": { + "default": null, + "help": "S3 API key" + }, + "secret": { + "default": null, + "help": "S3 API secret" + }, + "random_no_duplicate": { + "default": false, + "type": "bool", + "help": "if set, it will override `path_generator`, `filename_generator` and `folder`. It will check if the file already exists and if so it will not upload it again. Creates a new root folder path `no-dups/`" + }, + "endpoint_url": { + "default": "https://{region}.digitaloceanspaces.com", + "help": "S3 bucket endpoint, {region} are inserted at runtime" + }, + "cdn_url": { + "default": "https://{bucket}.{region}.cdn.digitaloceanspaces.com/{key}", + "help": "S3 CDN url, {bucket}, {region} and {key} are inserted at runtime" + }, + "private": { + "default": false, + "type": "bool", + "help": "if true S3 files will not be readable online" + } + } + }, + "configs": { + "path_generator": { + "default": "flat", + "help": "how to store the file in terms of directory structure: 'flat' sets to root; 'url' creates a directory based on the provided URL; 'random' creates a random directory.", + "choices": [ + "flat", + "url", + "random" + ] + }, + "filename_generator": { + "default": "static", + "help": "how to name stored files: 'random' creates a random string; 'static' uses a replicable strategy such as a hash.", + "choices": [ + "random", + "static" + ] + }, + "bucket": { + "default": null, + "help": "S3 bucket name" + }, + "region": { + "default": null, + "help": "S3 region name" + }, + "key": { + "default": null, + "help": "S3 API key" + }, + "secret": { + "default": null, + "help": "S3 API secret" + }, + "random_no_duplicate": { + "default": false, + "type": "bool", + "help": "if set, it will override `path_generator`, `filename_generator` and `folder`. It will check if the file already exists and if so it will not upload it again. Creates a new root folder path `no-dups/`" + }, + "endpoint_url": { + "default": "https://{region}.digitaloceanspaces.com", + "help": "S3 bucket endpoint, {region} are inserted at runtime" + }, + "cdn_url": { + "default": "https://{bucket}.{region}.cdn.digitaloceanspaces.com/{key}", + "help": "S3 CDN url, {bucket}, {region} and {key} are inserted at runtime" + }, + "private": { + "default": false, + "type": "bool", + "help": "if true S3 files will not be readable online" + } + } + }, + "local_storage": { + "name": "local_storage", + "display_name": "Local Storage", + "manifest": { + "name": "Local Storage", + "author": "Bellingcat", + "type": [ + "storage" + ], + "requires_setup": false, + "description": "\n LocalStorage: A storage module for saving archived content locally on the filesystem.\n\n ### Features\n - Saves archived media files to a specified folder on the local filesystem.\n - Maintains file metadata during storage using `shutil.copy2`.\n - Supports both absolute and relative paths for stored files, configurable via `save_absolute`.\n - Automatically creates directories as needed for storing files.\n\n ### Notes\n - Default storage folder is `./archived`, but this can be changed via the `save_to` configuration.\n - The `save_absolute` option can reveal the file structure in output formats; use with caution.\n ", + "dependencies": { + "python": [ + "loguru" + ] + }, + "entry_point": "", + "version": "1.0", + "configs": { + "path_generator": { + "default": "flat", + "help": "how to store the file in terms of directory structure: 'flat' sets to root; 'url' creates a directory based on the provided URL; 'random' creates a random directory.", + "choices": [ + "flat", + "url", + "random" + ] + }, + "filename_generator": { + "default": "static", + "help": "how to name stored files: 'random' creates a random string; 'static' uses a replicable strategy such as a hash.", + "choices": [ + "random", + "static" + ] + }, + "save_to": { + "default": "./local_archive", + "help": "folder where to save archived content" + }, + "save_absolute": { + "default": false, + "type": "bool", + "help": "whether the path to the stored file is absolute or relative in the output result inc. formatters (WARN: leaks the file structure)" + } + } + }, + "configs": { + "path_generator": { + "default": "flat", + "help": "how to store the file in terms of directory structure: 'flat' sets to root; 'url' creates a directory based on the provided URL; 'random' creates a random directory.", + "choices": [ + "flat", + "url", + "random" + ] + }, + "filename_generator": { + "default": "static", + "help": "how to name stored files: 'random' creates a random string; 'static' uses a replicable strategy such as a hash.", + "choices": [ + "random", + "static" + ] + }, + "save_to": { + "default": "./local_archive", + "help": "folder where to save archived content" + }, + "save_absolute": { + "default": false, + "type": "bool", + "help": "whether the path to the stored file is absolute or relative in the output result inc. formatters (WARN: leaks the file structure)" + } + } + }, + "mute_formatter": { + "name": "mute_formatter", + "display_name": "Mute Formatter", + "manifest": { + "name": "Mute Formatter", + "author": "Bellingcat", + "type": [ + "formatter" + ], + "requires_setup": true, + "description": " Default formatter.\n ", + "dependencies": {}, + "entry_point": "", + "version": "1.0", + "configs": {} + }, + "configs": null + }, + "html_formatter": { + "name": "html_formatter", + "display_name": "HTML Formatter", + "manifest": { + "name": "HTML Formatter", + "author": "Bellingcat", + "type": [ + "formatter" + ], + "requires_setup": false, + "description": " ", + "dependencies": { + "python": [ + "hash_enricher", + "loguru", + "jinja2" + ], + "bin": [ + "" + ] + }, + "entry_point": "", + "version": "1.0", + "configs": { + "detect_thumbnails": { + "default": true, + "help": "if true will group by thumbnails generated by thumbnail enricher by id 'thumbnail_00'", + "type": "bool" + } + } + }, + "configs": { + "detect_thumbnails": { + "default": true, + "help": "if true will group by thumbnails generated by thumbnail enricher by id 'thumbnail_00'", + "type": "bool" + } + } + } + }, + "steps": { + "feeders": [ + "cli_feeder", + "gsheet_feeder", + "atlos_feeder", + "csv_feeder" + ], + "extractors": [ + "wayback_extractor_enricher", + "wacz_extractor_enricher", + "instagram_api_extractor", + "instagram_tbot_extractor", + "generic_extractor", + "twitter_api_extractor", + "instagram_extractor", + "telethon_extractor", + "vk_extractor", + "telegram_extractor" + ], + "enrichers": [ + "wayback_extractor_enricher", + "wacz_extractor_enricher", + "metadata_enricher", + "timestamping_enricher", + "thumbnail_enricher", + "screenshot_enricher", + "meta_enricher", + "pdq_hash_enricher", + "whisper_enricher", + "ssl_enricher", + "hash_enricher" + ], + "databases": [ + "console_db", + "atlos_db", + "api_db", + "csv_db", + "gsheet_db" + ], + "storages": [ + "local_storage", + "gdrive_storage", + "atlos_storage", + "s3_storage" + ], + "formatters": [ + "html_formatter", + "mute_formatter" + ] + }, + "configs": [ + "gsheet_feeder", + "atlos_feeder", + "csv_feeder", + "cli_feeder", + "instagram_api_extractor", + "instagram_tbot_extractor", + "twitter_api_extractor", + "instagram_extractor", + "telethon_extractor", + "vk_extractor", + "generic_extractor", + "wayback_extractor_enricher", + "wacz_extractor_enricher", + "timestamping_enricher", + "screenshot_enricher", + "whisper_enricher", + "thumbnail_enricher", + "ssl_enricher", + "hash_enricher", + "atlos_db", + "api_db", + "gsheet_db", + "csv_db", + "gdrive_storage", + "atlos_storage", + "s3_storage", + "local_storage", + "html_formatter" + ], + "module_types": [ + "feeder", + "extractor", + "enricher", + "database", + "storage", + "formatter" + ], + "empty_config": "# Auto Archiver Configuration\n\n# Steps are the modules that will be run in the order they are defined\nsteps:\n feeders: []\n extractors: []\n enrichers: []\n databases: []\n storages: []\n formatters: []\n\n# Global configuration\n\n# Authentication\n# a dictionary of authentication information that can be used by extractors to login to website. \n# you can use a comma separated list for multiple domains on the same line (common usecase: x.com,twitter.com)\n# Common login 'types' are username/password, cookie, api key/token.\n# There are two special keys for using cookies, they are: cookies_file and cookies_from_browser. \n# Some Examples:\n# facebook.com:\n# username: \"my_username\"\n# password: \"my_password\"\n# or for a site that uses an API key:\n# twitter.com,x.com:\n# api_key\n# api_secret\n# youtube.com:\n# cookie: \"login_cookie=value ; other_cookie=123\" # multiple 'key=value' pairs should be separated by ;\n\nauthentication: {}\n\n# These are the global configurations that are used by the modules\n\nlogging:\n level: INFO\n\n" +} \ No newline at end of file diff --git a/scripts/settings/src/types.d.ts b/scripts/settings/src/types.d.ts new file mode 100644 index 0000000..fdf80fc --- /dev/null +++ b/scripts/settings/src/types.d.ts @@ -0,0 +1,21 @@ +export interface Config { + name: string; + description: string; + type: string?; + default: any; + help: string; + choices: string[]; + required: boolean; +} + +interface Manifest { + description: string; +} + +export interface Module { + name: string; + description: string; + configs: { [key: string]: Config }; + manifest: Manifest; + display_name: string; +} diff --git a/scripts/settings/tsconfig.json b/scripts/settings/tsconfig.json new file mode 100644 index 0000000..3d0a51a --- /dev/null +++ b/scripts/settings/tsconfig.json @@ -0,0 +1,21 @@ +{ + "compilerOptions": { + "target": "ESNext", + "useDefineForClassFields": true, + "lib": ["DOM", "DOM.Iterable", "ESNext"], + "allowJs": false, + "skipLibCheck": true, + "esModuleInterop": false, + "allowSyntheticDefaultImports": true, + "strict": true, + "forceConsistentCasingInFileNames": true, + "module": "ESNext", + "moduleResolution": "Node", + "resolveJsonModule": true, + "isolatedModules": true, + "noEmit": true, + "jsx": "react-jsx" + }, + "include": ["src"], + "references": [{ "path": "./tsconfig.node.json" }] +} diff --git a/scripts/settings/tsconfig.node.json b/scripts/settings/tsconfig.node.json new file mode 100644 index 0000000..9d31e2a --- /dev/null +++ b/scripts/settings/tsconfig.node.json @@ -0,0 +1,9 @@ +{ + "compilerOptions": { + "composite": true, + "module": "ESNext", + "moduleResolution": "Node", + "allowSyntheticDefaultImports": true + }, + "include": ["vite.config.ts"] +} diff --git a/scripts/settings/vite.config.ts b/scripts/settings/vite.config.ts new file mode 100644 index 0000000..a04d8c7 --- /dev/null +++ b/scripts/settings/vite.config.ts @@ -0,0 +1,12 @@ +import { defineConfig } from 'vite'; +import react from '@vitejs/plugin-react'; +import { viteSingleFile } from "vite-plugin-singlefile" + +// https://vite.dev/config/ +export default defineConfig({ + plugins: [react(), viteSingleFile()], + build: { + minify: false, + sourcemap: true, + } +}); diff --git a/src/auto_archiver/core/base_module.py b/src/auto_archiver/core/base_module.py index 8d520d1..d6e4455 100644 --- a/src/auto_archiver/core/base_module.py +++ b/src/auto_archiver/core/base_module.py @@ -105,8 +105,8 @@ class BaseModule(ABC): for key in self.authentication.keys(): if key in site or site in key: logger.debug(f"Could not find exact authentication information for site '{site}'. \ - did find information for '{key}' which is close, is this what you meant? \ - If so, edit your authentication settings to make sure it exactly matches.") +did find information for '{key}' which is close, is this what you meant? \ +If so, edit your authentication settings to make sure it exactly matches.") def get_ytdlp_cookiejar(args): import yt_dlp diff --git a/src/auto_archiver/core/module.py b/src/auto_archiver/core/module.py index 5442e71..2c6617d 100644 --- a/src/auto_archiver/core/module.py +++ b/src/auto_archiver/core/module.py @@ -80,7 +80,10 @@ class ModuleFactory: available = self.available_modules(limit_to_modules=[module_name], suppress_warnings=suppress_warnings) if not available: - raise IndexError(f"Module '{module_name}' not found. Are you sure it's installed/exists?") + message = f"Module '{module_name}' not found. Are you sure it's installed/exists?" + if 'archiver' in module_name: + message += f" Did you mean {module_name.replace('archiver', 'extractor')}?" + raise IndexError(message) return available[0] def available_modules(self, limit_to_modules: List[str]= [], suppress_warnings: bool = False) -> List[LazyBaseModule]: diff --git a/src/auto_archiver/core/orchestrator.py b/src/auto_archiver/core/orchestrator.py index 274fa9e..b78c7e7 100644 --- a/src/auto_archiver/core/orchestrator.py +++ b/src/auto_archiver/core/orchestrator.py @@ -15,6 +15,7 @@ from copy import copy from rich_argparse import RichHelpFormatter from loguru import logger +import requests from .metadata import Metadata, Media from auto_archiver.version import __version__ @@ -72,10 +73,20 @@ class ArchivingOrchestrator: self.basic_parser = parser return parser + + def check_steps(self, config): + for module_type in MODULE_TYPES: + if not config['steps'].get(f"{module_type}s", []): + if module_type == 'feeder' or module_type == 'formatter' and config['steps'].get(f"{module_type}"): + raise SetupError(f"It appears you have '{module_type}' set under 'steps' in your configuration file, but as of version 0.13.0 of Auto Archiver, you must use '{module_type}s'. Change this in your configuration file and try again. \ +Here's how that would look: \n\nsteps:\n {module_type}s:\n - [your_{module_type}_name_here]\n {'extractors:...' if module_type == 'feeder' else '...'}\n") + if module_type == 'extractor' and config['steps'].get('archivers'): + raise SetupError(f"As of version 0.13.0 of Auto Archiver, the 'archivers' step name has been changed to 'extractors'. Change this in your configuration file and try again. \ +Here's how that would look: \n\nsteps:\n extractors:\n - [your_extractor_name_here]\n enrichers:...\n") + raise SetupError(f"No {module_type}s were configured. Make sure to set at least one {module_type} in your configuration file or on the command line (using --{module_type}s)") def setup_complete_parser(self, basic_config: dict, yaml_config: dict, unused_args: list[str]) -> None: - # modules parser to get the overridden 'steps' values modules_parser = argparse.ArgumentParser( add_help=False, @@ -100,6 +111,7 @@ class ArchivingOrchestrator: # but should we add them? Or should we just add them to the 'complete' parser? if is_valid_config(yaml_config): + self.check_steps(yaml_config) # only load the modules enabled in config # TODO: if some steps are empty (e.g. 'feeders' is empty), should we default to the 'simple' ones? Or only if they are ALL empty? enabled_modules = [] @@ -115,10 +127,6 @@ class ArchivingOrchestrator: simple_modules = [module for module in self.module_factory.available_modules() if not module.requires_setup] self.add_individual_module_args(simple_modules, parser) - # for simple mode, we use the cli_feeder and any modules that don't require setup - if not yaml_config['steps']['feeders']: - yaml_config['steps']['feeders'] = ['cli_feeder'] - # add them to the config for module in simple_modules: for module_type in module.type: @@ -171,9 +179,6 @@ class ArchivingOrchestrator: if not parser: parser = self.parser - # allow passing URLs directly on the command line - parser.add_argument('urls', nargs='*', default=[], help='URL(s) to archive, either a single URL or a list of urls, should not come from config.yaml') - parser.add_argument('--authentication', dest='authentication', help='A dictionary of sites and their authentication methods \ (token, username etc.) that extractors can use to log into \ a website. If passing this on the command line, use a JSON string. \ @@ -193,7 +198,11 @@ class ArchivingOrchestrator: modules = self.module_factory.available_modules() for module in modules: - + if module.name == 'cli_feeder': + # special case. For the CLI feeder, allow passing URLs directly on the command line without setting --cli_feeder.urls= + parser.add_argument('urls', nargs='*', default=[], help='URL(s) to archive, either a single URL or a list of urls, should not come from config.yaml') + continue + if not module.configs: # this module has no configs, don't show anything in the help # (TODO: do we want to show something about this module though, like a description?) @@ -277,36 +286,16 @@ class ArchivingOrchestrator: raise SetupError(f"Only one {module_type} is allowed, found {len(step_items)} {module_type}s. Please remove one of the following from your configuration file: {modules_to_load}") for module in modules_to_load: - if module == 'cli_feeder': - # cli_feeder is a pseudo module, it just takes the command line args for [URLS] - urls = self.config['urls'] - if not urls: - raise SetupError("No URLs provided. Please provide at least one URL via the command line, or set up an alternative feeder. Use --help for more information.") - - def feed(self) -> Generator[Metadata]: - for url in urls: - logger.debug(f"Processing URL: '{url}'") - yield Metadata().set_url(url) - - pseudo_module = type('CLIFeeder', (Feeder,), { - 'name': 'cli_feeder', - 'display_name': 'CLI Feeder', - '__iter__': feed - - })() - - pseudo_module.__iter__ = feed - step_items.append(pseudo_module) - continue if module in invalid_modules: continue + loaded_module = None try: loaded_module: BaseModule = self.module_factory.get_module(module, self.config) except (KeyboardInterrupt, Exception) as e: logger.error(f"Error during setup of modules: {e}\n{traceback.format_exc()}") - if module_type == 'extractor' and loaded_module.name == module: + if loaded_module and module_type == 'extractor': loaded_module.cleanup() raise e @@ -348,7 +337,23 @@ class ArchivingOrchestrator: yaml_config = self.load_config(basic_config.config_file) return self.setup_complete_parser(basic_config, yaml_config, unused_args) + + def check_for_updates(self): + response = requests.get("https://pypi.org/pypi/auto-archiver/json").json() + latest_version = response['info']['version'] + # check version compared to current version + if latest_version != __version__: + if os.environ.get('RUNNING_IN_DOCKER'): + update_cmd = "`docker pull bellingcat/auto-archiver:latest`" + else: + update_cmd = "`pip install --upgrade auto-archiver`" + logger.warning("") + logger.warning("********* IMPORTANT: UPDATE AVAILABLE ********") + logger.warning(f"A new version of auto-archiver is available (v{latest_version}, you have {__version__})") + logger.warning(f"Make sure to update to the latest version using: {update_cmd}") + logger.warning("") + def setup(self, args: list): """ Function to configure all setup of the orchestrator: setup configs and load modules. @@ -356,6 +361,8 @@ class ArchivingOrchestrator: This method should only ever be called once """ + self.check_for_updates() + if self.setup_finished: logger.warning("The `setup_config()` function should only ever be run once. \ If you need to re-run the setup, please re-instantiate a new instance of the orchestrator. \ diff --git a/src/auto_archiver/modules/atlos_db/__init__.py b/src/auto_archiver/modules/atlos_db/__init__.py deleted file mode 100644 index e14d202..0000000 --- a/src/auto_archiver/modules/atlos_db/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .atlos_db import AtlosDb \ No newline at end of file diff --git a/src/auto_archiver/modules/atlos_db/__manifest__.py b/src/auto_archiver/modules/atlos_db/__manifest__.py deleted file mode 100644 index d23ff23..0000000 --- a/src/auto_archiver/modules/atlos_db/__manifest__.py +++ /dev/null @@ -1,38 +0,0 @@ -{ - "name": "Atlos Database", - "type": ["database"], - "entry_point": "atlos_db::AtlosDb", - "requires_setup": True, - "dependencies": - {"python": ["loguru", - ""], - "bin": [""]}, - "configs": { - "api_token": { - "default": None, - "help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/", - "required": True, - "type": "str", - }, - "atlos_url": { - "default": "https://platform.atlos.org", - "help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.", - "type": "str" - }, - }, - "description": """ -Handles integration with the Atlos platform for managing archival results. - -### Features -- Outputs archival results to the Atlos API for storage and tracking. -- Updates failure status with error details when archiving fails. -- Processes and formats metadata, including ISO formatting for datetime fields. -- Skips processing for items without an Atlos ID. - -### Setup -Required configs: -- atlos_url: Base URL for the Atlos API. -- api_token: Authentication token for API access. -""" -, -} diff --git a/src/auto_archiver/modules/atlos_db/atlos_db.py b/src/auto_archiver/modules/atlos_db/atlos_db.py deleted file mode 100644 index baa9fef..0000000 --- a/src/auto_archiver/modules/atlos_db/atlos_db.py +++ /dev/null @@ -1,66 +0,0 @@ -from typing import Union - -import requests -from loguru import logger - -from auto_archiver.core import Database -from auto_archiver.core import Metadata - - -class AtlosDb(Database): - """ - Outputs results to Atlos - """ - - def failed(self, item: Metadata, reason: str) -> None: - """Update DB accordingly for failure""" - # If the item has no Atlos ID, there's nothing for us to do - if not item.metadata.get("atlos_id"): - logger.info(f"Item {item.get_url()} has no Atlos ID, skipping") - return - - requests.post( - f"{self.atlos_url}/api/v2/source_material/metadata/{item.metadata['atlos_id']}/auto_archiver", - headers={"Authorization": f"Bearer {self.api_token}"}, - json={"metadata": {"processed": True, "status": "error", "error": reason}}, - ).raise_for_status() - logger.info( - f"Stored failure for {item.get_url()} (ID {item.metadata['atlos_id']}) on Atlos: {reason}" - ) - - def fetch(self, item: Metadata) -> Union[Metadata, bool]: - """check and fetch if the given item has been archived already, each - database should handle its own caching, and configuration mechanisms""" - return False - - def _process_metadata(self, item: Metadata) -> dict: - """Process metadata for storage on Atlos. Will convert any datetime - objects to ISO format.""" - - return { - k: v.isoformat() if hasattr(v, "isoformat") else v - for k, v in item.metadata.items() - } - - def done(self, item: Metadata, cached: bool = False) -> None: - """archival result ready - should be saved to DB""" - - if not item.metadata.get("atlos_id"): - logger.info(f"Item {item.get_url()} has no Atlos ID, skipping") - return - - requests.post( - f"{self.atlos_url}/api/v2/source_material/metadata/{item.metadata['atlos_id']}/auto_archiver", - headers={"Authorization": f"Bearer {self.api_token}"}, - json={ - "metadata": dict( - processed=True, - status="success", - results=self._process_metadata(item), - ) - }, - ).raise_for_status() - - logger.info( - f"Stored success for {item.get_url()} (ID {item.metadata['atlos_id']}) on Atlos" - ) diff --git a/src/auto_archiver/modules/atlos_feeder/__init__.py b/src/auto_archiver/modules/atlos_feeder/__init__.py deleted file mode 100644 index 67b243a..0000000 --- a/src/auto_archiver/modules/atlos_feeder/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .atlos_feeder import AtlosFeeder \ No newline at end of file diff --git a/src/auto_archiver/modules/atlos_feeder/__manifest__.py b/src/auto_archiver/modules/atlos_feeder/__manifest__.py deleted file mode 100644 index d59f420..0000000 --- a/src/auto_archiver/modules/atlos_feeder/__manifest__.py +++ /dev/null @@ -1,34 +0,0 @@ -{ - "name": "Atlos Feeder", - "type": ["feeder"], - "requires_setup": True, - "dependencies": { - "python": ["loguru", "requests"], - }, - "configs": { - "api_token": { - "type": "str", - "required": True, - "help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/", - }, - "atlos_url": { - "default": "https://platform.atlos.org", - "help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.", - "type": "str" - }, - }, - "description": """ - AtlosFeeder: A feeder module that integrates with the Atlos API to fetch source material URLs for archival. - - ### Features - - Connects to the Atlos API to retrieve a list of source material URLs. - - Filters source materials based on visibility, processing status, and metadata. - - Converts filtered source materials into `Metadata` objects with the relevant `atlos_id` and URL. - - Iterates through paginated results using a cursor for efficient API interaction. - - ### Notes - - Requires an Atlos API endpoint and a valid API token for authentication. - - Ensures only unprocessed, visible, and ready-to-archive URLs are returned. - - Handles pagination transparently when retrieving data from the Atlos API. - """ -} diff --git a/src/auto_archiver/modules/atlos_feeder/atlos_feeder.py b/src/auto_archiver/modules/atlos_feeder/atlos_feeder.py deleted file mode 100644 index 8c8f9cb..0000000 --- a/src/auto_archiver/modules/atlos_feeder/atlos_feeder.py +++ /dev/null @@ -1,42 +0,0 @@ -import requests -from loguru import logger - -from auto_archiver.core import Feeder -from auto_archiver.core import Metadata - - -class AtlosFeeder(Feeder): - - def __iter__(self) -> Metadata: - # Get all the urls from the Atlos API - count = 0 - cursor = None - while True: - response = requests.get( - f"{self.atlos_url}/api/v2/source_material", - headers={"Authorization": f"Bearer {self.api_token}"}, - params={"cursor": cursor}, - ) - data = response.json() - response.raise_for_status() - cursor = data["next"] - - for item in data["results"]: - if ( - item["source_url"] not in [None, ""] - and ( - item["metadata"] - .get("auto_archiver", {}) - .get("processed", False) - != True - ) - and item["visibility"] == "visible" - and item["status"] not in ["processing", "pending"] - ): - yield Metadata().set_url(item["source_url"]).set( - "atlos_id", item["id"] - ) - count += 1 - - if len(data["results"]) == 0 or cursor is None: - break diff --git a/src/auto_archiver/modules/atlos_feeder_db_storage/__init__.py b/src/auto_archiver/modules/atlos_feeder_db_storage/__init__.py new file mode 100644 index 0000000..8d62823 --- /dev/null +++ b/src/auto_archiver/modules/atlos_feeder_db_storage/__init__.py @@ -0,0 +1 @@ +from .atlos_feeder_db_storage import AtlosFeederDbStorage \ No newline at end of file diff --git a/src/auto_archiver/modules/atlos_feeder_db_storage/__manifest__.py b/src/auto_archiver/modules/atlos_feeder_db_storage/__manifest__.py new file mode 100644 index 0000000..2ea8f8f --- /dev/null +++ b/src/auto_archiver/modules/atlos_feeder_db_storage/__manifest__.py @@ -0,0 +1,46 @@ +{ + "name": "Atlos Feeder Database Storage", + "type": ["feeder", "database", "storage"], +"entry_point": "atlos_feeder_db_storage::AtlosFeederDbStorage", + "requires_setup": True, + "dependencies": { + "python": ["loguru", "requests"], + }, + "configs": { + "api_token": { + "type": "str", + "required": True, + "help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/", + }, + "atlos_url": { + "default": "https://platform.atlos.org", + "help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.", + "type": "str" + }, + }, + "description": """ + A module that integrates with the Atlos API to fetch source material URLs for archival, uplaod extracted media, + + [Atlos](https://www.atlos.org/) is a visual investigation and archiving platform designed for investigative research, journalism, and open-source intelligence (OSINT). + It helps users organize, analyze, and store media from various sources, making it easier to track and investigate digital evidence. + + To get started create a new project and obtain an API token from the settings page. You can group event's into Atlos's 'incidents'. + Here you can add 'source material' by URLn and the Atlos feeder will fetch these URLs for archival. + + You can use Atlos only as a 'feeder', however you can also implement the 'database' and 'storage' features to store the media files in Atlos which is recommended. + The Auto Archiver will retain the Atlos ID for each item, ensuring that the media and database outputs are uplaoded back into the relevant media item. + + + ### Features + - Connects to the Atlos API to retrieve a list of source material URLs. + - Iterates through the URLs from all source material items which are unprocessed, visible, and ready to archive. + - If the storage option is selected, it will store the media files alongside the original source material item in Atlos. + - Is the database option is selected it will output the results to the media item, as well as updating failure status with error details when archiving fails. + - Skips Storege/ database upload for items without an Atlos ID - restricting that you must use the Atlos feeder so that it has the Atlos ID to store the results with. + + ### Notes + - Requires an Atlos account with a project and a valid API token for authentication. + - Ensures only unprocessed, visible, and ready-to-archive URLs are returned. + - Feches any media items within an Atlos project, regardless of separation into incidents. + """ +} diff --git a/src/auto_archiver/modules/atlos_feeder_db_storage/atlos_feeder_db_storage.py b/src/auto_archiver/modules/atlos_feeder_db_storage/atlos_feeder_db_storage.py new file mode 100644 index 0000000..87b4f82 --- /dev/null +++ b/src/auto_archiver/modules/atlos_feeder_db_storage/atlos_feeder_db_storage.py @@ -0,0 +1,153 @@ +import hashlib +import os +from typing import IO, Iterator, Optional, Union + +import requests +from loguru import logger + +from auto_archiver.core import Database, Feeder, Media, Metadata, Storage +from auto_archiver.utils import calculate_file_hash + + +class AtlosFeederDbStorage(Feeder, Database, Storage): + + def setup(self) -> requests.Session: + """create and return a persistent session.""" + self.session = requests.Session() + + def _get(self, endpoint: str, params: Optional[dict] = None) -> dict: + """Wrapper for GET requests to the Atlos API.""" + url = f"{self.atlos_url}{endpoint}" + response = self.session.get( + url, headers={"Authorization": f"Bearer {self.api_token}"}, params=params + ) + response.raise_for_status() + return response.json() + + def _post( + self, + endpoint: str, + json: Optional[dict] = None, + params: Optional[dict] = None, + files: Optional[dict] = None, + ) -> dict: + """Wrapper for POST requests to the Atlos API.""" + url = f"{self.atlos_url}{endpoint}" + response = self.session.post( + url, + headers={"Authorization": f"Bearer {self.api_token}"}, + json=json, + params=params, + files=files, + ) + response.raise_for_status() + return response.json() + + # ! Atlos Module - Feeder Methods + + def __iter__(self) -> Iterator[Metadata]: + """Iterate over unprocessed, visible source materials from Atlos.""" + cursor = None + while True: + data = self._get("/api/v2/source_material", params={"cursor": cursor}) + cursor = data.get("next") + results = data.get("results", []) + for item in results: + if ( + item.get("source_url") not in [None, ""] + and not item.get("metadata", {}).get("auto_archiver", {}).get("processed", False) + and item.get("visibility") == "visible" + and item.get("status") not in ["processing", "pending"] + ): + yield Metadata().set_url(item["source_url"]).set("atlos_id", item["id"]) + if not results or cursor is None: + break + + # ! Atlos Module - Database Methods + + def failed(self, item: Metadata, reason: str) -> None: + """Mark an item as failed in Atlos, if the ID exists.""" + atlos_id = item.metadata.get("atlos_id") + if not atlos_id: + logger.info(f"Item {item.get_url()} has no Atlos ID, skipping") + return + self._post( + f"/api/v2/source_material/metadata/{atlos_id}/auto_archiver", + json={"metadata": {"processed": True, "status": "error", "error": reason}}, + ) + logger.info(f"Stored failure for {item.get_url()} (ID {atlos_id}) on Atlos: {reason}") + + def fetch(self, item: Metadata) -> Union[Metadata, bool]: + """check and fetch if the given item has been archived already, each + database should handle its own caching, and configuration mechanisms""" + return False + + def _process_metadata(self, item: Metadata) -> dict: + """Process metadata for storage on Atlos. Will convert any datetime + objects to ISO format.""" + return { + k: v.isoformat() if hasattr(v, "isoformat") else v + for k, v in item.metadata.items() + } + + def done(self, item: Metadata, cached: bool = False) -> None: + """Mark an item as successfully archived in Atlos.""" + atlos_id = item.metadata.get("atlos_id") + if not atlos_id: + logger.info(f"Item {item.get_url()} has no Atlos ID, skipping") + return + self._post( + f"/api/v2/source_material/metadata/{atlos_id}/auto_archiver", + json={ + "metadata": { + "processed": True, + "status": "success", + "results": self._process_metadata(item), + } + }, + ) + logger.info(f"Stored success for {item.get_url()} (ID {atlos_id}) on Atlos") + + # ! Atlos Module - Storage Methods + + def get_cdn_url(self, _media: Media) -> str: + """Return the base Atlos URL as the CDN URL.""" + return self.atlos_url + + def upload(self, media: Media, metadata: Optional[Metadata] = None, **_kwargs) -> bool: + """Upload a media file to Atlos if it has not been uploaded already.""" + if metadata is None: + logger.error(f"No metadata provided for {media.filename}") + return False + + atlos_id = metadata.get("atlos_id") + if not atlos_id: + logger.error(f"No Atlos ID found in metadata; can't store {media.filename} in Atlos.") + return False + + media_hash = calculate_file_hash(media.filename, hash_algo=hashlib.sha256, chunksize=4096) + + # Check whether the media has already been uploaded + source_material = self._get(f"/api/v2/source_material/{atlos_id}")["result"] + existing_media = [ + artifact.get("file_hash_sha256") + for artifact in source_material.get("artifacts", []) + ] + if media_hash in existing_media: + logger.info(f"{media.filename} with SHA256 {media_hash} already uploaded to Atlos") + return True + + # Upload the media to the Atlos API + with open(media.filename, "rb") as file_obj: + self._post( + f"/api/v2/source_material/upload/{atlos_id}", + params={"title": media.properties}, + files={"file": (os.path.basename(media.filename), file_obj)}, + ) + logger.info(f"Uploaded {media.filename} to Atlos with ID {atlos_id} and title {media.key}") + return True + + def uploadf(self, file: IO[bytes], key: str, **kwargs: dict) -> bool: + """Upload a file-like object; not implemented.""" + pass + diff --git a/src/auto_archiver/modules/atlos_storage/__init__.py b/src/auto_archiver/modules/atlos_storage/__init__.py deleted file mode 100644 index 9e815c7..0000000 --- a/src/auto_archiver/modules/atlos_storage/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .atlos_storage import AtlosStorage \ No newline at end of file diff --git a/src/auto_archiver/modules/atlos_storage/__manifest__.py b/src/auto_archiver/modules/atlos_storage/__manifest__.py deleted file mode 100644 index 55b5120..0000000 --- a/src/auto_archiver/modules/atlos_storage/__manifest__.py +++ /dev/null @@ -1,32 +0,0 @@ -{ - "name": "Atlos Storage", - "type": ["storage"], - "requires_setup": True, - "dependencies": { - "python": ["loguru", "boto3"], - "bin": [] - }, - "description": """ - Stores media files in a [Atlos](https://www.atlos.org/). - - ### Features - - Saves media files to Atlos, organizing them into folders based on the provided path structure. - - ### Notes - - Requires setup with Atlos credentials. - - Files are uploaded to the specified `root_folder_id` and organized by the `media.key` structure. - """, - "configs": { - "api_token": { - "default": None, - "help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/", - "required": True, - "type": "str" - }, - "atlos_url": { - "default": "https://platform.atlos.org", - "help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.", - "type": "str" - }, - } -} \ No newline at end of file diff --git a/src/auto_archiver/modules/atlos_storage/atlos_storage.py b/src/auto_archiver/modules/atlos_storage/atlos_storage.py deleted file mode 100644 index f8eef68..0000000 --- a/src/auto_archiver/modules/atlos_storage/atlos_storage.py +++ /dev/null @@ -1,66 +0,0 @@ -import hashlib -import os -from typing import IO, Optional - -import requests -from loguru import logger - -from auto_archiver.core import Media, Metadata -from auto_archiver.core import Storage - - -class AtlosStorage(Storage): - - def get_cdn_url(self, _media: Media) -> str: - # It's not always possible to provide an exact URL, because it's - # possible that the media once uploaded could have been copied to - # another project. - return self.atlos_url - - def _hash(self, media: Media) -> str: - # Hash the media file using sha-256. We don't use the existing auto archiver - # hash because there's no guarantee that the configuerer is using sha-256, which - # is how Atlos hashes files. - - sha256 = hashlib.sha256() - with open(media.filename, "rb") as f: - while True: - buf = f.read(4096) - if not buf: break - sha256.update(buf) - return sha256.hexdigest() - - def upload(self, media: Media, metadata: Optional[Metadata]=None, **_kwargs) -> bool: - atlos_id = metadata.get("atlos_id") - if atlos_id is None: - logger.error(f"No Atlos ID found in metadata; can't store {media.filename} on Atlos") - return False - - media_hash = self._hash(media) - - # Check whether the media has already been uploaded - source_material = requests.get( - f"{self.atlos_url}/api/v2/source_material/{atlos_id}", - headers={"Authorization": f"Bearer {self.api_token}"}, - ).json()["result"] - existing_media = [x["file_hash_sha256"] for x in source_material.get("artifacts", [])] - if media_hash in existing_media: - logger.info(f"{media.filename} with SHA256 {media_hash} already uploaded to Atlos") - return True - - # Upload the media to the Atlos API - requests.post( - f"{self.atlos_url}/api/v2/source_material/upload/{atlos_id}", - headers={"Authorization": f"Bearer {self.api_token}"}, - params={ - "title": media.properties - }, - files={"file": (os.path.basename(media.filename), open(media.filename, "rb"))}, - ).raise_for_status() - - logger.info(f"Uploaded {media.filename} to Atlos with ID {atlos_id} and title {media.key}") - - return True - - # must be implemented even if unused - def uploadf(self, file: IO[bytes], key: str, **kwargs: dict) -> bool: pass diff --git a/src/auto_archiver/modules/cli_feeder/__manifest__.py b/src/auto_archiver/modules/cli_feeder/__manifest__.py new file mode 100644 index 0000000..609aa3e --- /dev/null +++ b/src/auto_archiver/modules/cli_feeder/__manifest__.py @@ -0,0 +1,23 @@ +{ + 'name': 'Command Line Feeder', + 'type': ['feeder'], + 'entry_point': 'cli_feeder::CLIFeeder', + 'requires_setup': False, + 'description': 'Feeds URLs to orchestrator from the command line', + 'configs': { + 'urls': { + 'default': None, + 'help': 'URL(s) to archive, either a single URL or a list of urls, should not come from config.yaml', + }, + }, + 'description': """ +The Command Line Feeder is the default enabled feeder for the Auto Archiver. It allows you to pass URLs directly to the orchestrator from the command line +without the need to specify any additional configuration or command line arguments: + +`auto-archiver --feeder cli_feeder -- "https://example.com/1/,https://example.com/2/"` + +You can pass multiple URLs by separating them with a space. The URLs will be processed in the order they are provided. + +`auto-archiver --feeder cli_feeder -- https://example.com/1/ https://example.com/2/` +""", +} \ No newline at end of file diff --git a/src/auto_archiver/modules/cli_feeder/cli_feeder.py b/src/auto_archiver/modules/cli_feeder/cli_feeder.py new file mode 100644 index 0000000..20ca6ae --- /dev/null +++ b/src/auto_archiver/modules/cli_feeder/cli_feeder.py @@ -0,0 +1,21 @@ +from loguru import logger + +from auto_archiver.core.feeder import Feeder +from auto_archiver.core.metadata import Metadata + +class CLIFeeder(Feeder): + + def setup(self) -> None: + self.urls = self.config['urls'] + if not self.urls: + raise ValueError("No URLs provided. Please provide at least one URL via the command line, or set up an alternative feeder. Use --help for more information.") + + def __iter__(self) -> Metadata: + urls = self.config['urls'] + for url in urls: + logger.debug(f"Processing {url}") + m = Metadata().set_url(url) + m.set_context("folder", "cli") + yield m + + logger.success(f"Processed {len(urls)} URL(s)") \ No newline at end of file diff --git a/src/auto_archiver/modules/console_db/console_db.py b/src/auto_archiver/modules/console_db/console_db.py index 48609b0..b26a605 100644 --- a/src/auto_archiver/modules/console_db/console_db.py +++ b/src/auto_archiver/modules/console_db/console_db.py @@ -10,7 +10,7 @@ class ConsoleDb(Database): """ def started(self, item: Metadata) -> None: - logger.warning(f"STARTED {item}") + logger.info(f"STARTED {item}") def failed(self, item: Metadata, reason:str) -> None: logger.error(f"FAILED {item}: {reason}") diff --git a/src/auto_archiver/modules/csv_db/__manifest__.py b/src/auto_archiver/modules/csv_db/__manifest__.py index 507ce14..d9733b2 100644 --- a/src/auto_archiver/modules/csv_db/__manifest__.py +++ b/src/auto_archiver/modules/csv_db/__manifest__.py @@ -6,7 +6,7 @@ }, 'entry_point': 'csv_db::CSVDb', "configs": { - "csv_file": {"default": "db.csv", "help": "CSV file name"} + "csv_file": {"default": "db.csv", "help": "CSV file name to save metadata to"}, }, "description": """ Handles exporting archival results to a CSV file. diff --git a/src/auto_archiver/modules/generic_extractor/__manifest__.py b/src/auto_archiver/modules/generic_extractor/__manifest__.py index 2936983..5a5f94f 100644 --- a/src/auto_archiver/modules/generic_extractor/__manifest__.py +++ b/src/auto_archiver/modules/generic_extractor/__manifest__.py @@ -28,6 +28,13 @@ the broader archiving framework. metadata objects. Some dropins are included in this generic_archiver by default, but custom dropins can be created to handle additional websites and passed to the archiver via the command line using the `--dropins` option (TODO!). + +### Auto-Updates + +The Generic Extractor will also automatically check for updates to `yt-dlp` (every 5 days by default). +This can be configured using the `ytdlp_update_interval` setting (or disabled by setting it to -1). +If you are having issues with the extractor, you can review the version of `yt-dlp` being used with `yt-dlp --version`. + """, "configs": { "subtitles": {"default": True, "help": "download subtitles if available", "type": "bool"}, @@ -69,5 +76,10 @@ via the command line using the `--dropins` option (TODO!). "help": "Additional arguments to pass to the yt-dlp extractor. See https://github.com/yt-dlp/yt-dlp/blob/master/README.md#extractor-arguments.", "type": "json_loader", }, + "ytdlp_update_interval": { + "default": 5, + "help": "How often to check for yt-dlp updates (days). If positive, will check and update yt-dlp every [num] days. Set it to -1 to disable, or 0 to always update on every run.", + "type": "int", + }, }, } diff --git a/src/auto_archiver/modules/generic_extractor/generic_extractor.py b/src/auto_archiver/modules/generic_extractor/generic_extractor.py index c3fcc30..917b382 100644 --- a/src/auto_archiver/modules/generic_extractor/generic_extractor.py +++ b/src/auto_archiver/modules/generic_extractor/generic_extractor.py @@ -1,7 +1,11 @@ -import datetime, os, yt_dlp, pysubs2 +import datetime, os import importlib +import subprocess from typing import Generator, Type + +import yt_dlp from yt_dlp.extractor.common import InfoExtractor +import pysubs2 from loguru import logger @@ -11,6 +15,44 @@ from auto_archiver.core import Metadata, Media class GenericExtractor(Extractor): _dropins = {} + def setup(self): + # check for file .ytdlp-update in the secrets folder + if self.ytdlp_update_interval < 0: + return + + use_secrets = os.path.exists('secrets') + path = os.path.join('secrets' if use_secrets else '', '.ytdlp-update') + next_update_check = None + if os.path.exists(path): + with open(path, "r") as f: + next_update_check = datetime.datetime.fromisoformat(f.read()) + + if not next_update_check or next_update_check < datetime.datetime.now(): + self.update_ytdlp() + + next_update_check = datetime.datetime.now() + datetime.timedelta(days=self.ytdlp_update_interval) + with open(path, "w") as f: + f.write(next_update_check.isoformat()) + + def update_ytdlp(self): + logger.info("Checking and updating yt-dlp...") + logger.info(f"Tip: change the 'ytdlp_update_interval' setting to control how often yt-dlp is updated. Set to -1 to disable or 0 to enable on every run. Current setting: {self.ytdlp_update_interval}") + from importlib.metadata import version as get_version + old_version = get_version("yt-dlp") + try: + # try and update with pip (this works inside poetry environment and in a normal virtualenv) + result = subprocess.run(["pip", "install", "--upgrade", "yt-dlp"], check=True, capture_output=True) + + if "Successfully installed yt-dlp" in result.stdout.decode(): + new_version = importlib.metadata.version("yt-dlp") + logger.info(f"yt-dlp successfully (from {old_version} to {new_version})") + importlib.reload(yt_dlp) + else: + logger.info("yt-dlp already up to date") + + except Exception as e: + logger.error(f"Error updating yt-dlp: {e}") + def suitable_extractors(self, url: str) -> Generator[str, None, None]: """ Returns a list of valid extractors for the given URL""" diff --git a/src/auto_archiver/modules/gsheet_db/__init__.py b/src/auto_archiver/modules/gsheet_db/__init__.py deleted file mode 100644 index 01fdee6..0000000 --- a/src/auto_archiver/modules/gsheet_db/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .gsheet_db import GsheetsDb \ No newline at end of file diff --git a/src/auto_archiver/modules/gsheet_db/__manifest__.py b/src/auto_archiver/modules/gsheet_db/__manifest__.py deleted file mode 100644 index cf95245..0000000 --- a/src/auto_archiver/modules/gsheet_db/__manifest__.py +++ /dev/null @@ -1,38 +0,0 @@ -{ - "name": "Google Sheets Database", - "type": ["database"], - "entry_point": "gsheet_db::GsheetsDb", - "requires_setup": True, - "dependencies": { - "python": ["loguru", "gspread", "slugify"], - }, - "configs": { - "allow_worksheets": { - "default": set(), - "help": "(CSV) only worksheets whose name is included in allow are included (overrides worksheet_block), leave empty so all are allowed", - }, - "block_worksheets": { - "default": set(), - "help": "(CSV) explicitly block some worksheets from being processed", - }, - "use_sheet_names_in_stored_paths": { - "default": True, - "type": "bool", - "help": "if True the stored files path will include 'workbook_name/worksheet_name/...'", - } - }, - "description": """ - GsheetsDatabase: - Handles integration with Google Sheets for tracking archival tasks. - -### Features -- Updates a Google Sheet with the status of the archived URLs, including in progress, success or failure, and method used. -- Saves metadata such as title, text, timestamp, hashes, screenshots, and media URLs to designated columns. -- Formats media-specific metadata, such as thumbnails and PDQ hashes for the sheet. -- Skips redundant updates for empty or invalid data fields. - -### Notes -- Currently works only with metadata provided by GsheetFeeder. -- Requires configuration of a linked Google Sheet and appropriate API credentials. - """ -} diff --git a/src/auto_archiver/modules/gsheet_db/gsheet_db.py b/src/auto_archiver/modules/gsheet_db/gsheet_db.py deleted file mode 100644 index c19f2ae..0000000 --- a/src/auto_archiver/modules/gsheet_db/gsheet_db.py +++ /dev/null @@ -1,114 +0,0 @@ -from typing import Union, Tuple -from urllib.parse import quote - -from loguru import logger - -from auto_archiver.core import Database -from auto_archiver.core import Metadata, Media -from auto_archiver.modules.gsheet_feeder import GWorksheet -from auto_archiver.utils.misc import get_current_timestamp - - -class GsheetsDb(Database): - """ - NB: only works if GsheetFeeder is used. - could be updated in the future to support non-GsheetFeeder metadata - """ - - def started(self, item: Metadata) -> None: - logger.warning(f"STARTED {item}") - gw, row = self._retrieve_gsheet(item) - gw.set_cell(row, "status", "Archive in progress") - - def failed(self, item: Metadata, reason: str) -> None: - logger.error(f"FAILED {item}") - self._safe_status_update(item, f"Archive failed {reason}") - - def aborted(self, item: Metadata) -> None: - logger.warning(f"ABORTED {item}") - self._safe_status_update(item, "") - - def fetch(self, item: Metadata) -> Union[Metadata, bool]: - """check if the given item has been archived already""" - return False - - def done(self, item: Metadata, cached: bool = False) -> None: - """archival result ready - should be saved to DB""" - logger.success(f"DONE {item.get_url()}") - gw, row = self._retrieve_gsheet(item) - # self._safe_status_update(item, 'done') - - cell_updates = [] - row_values = gw.get_row(row) - - def batch_if_valid(col, val, final_value=None): - final_value = final_value or val - try: - if val and gw.col_exists(col) and gw.get_cell(row_values, col) == "": - cell_updates.append((row, col, final_value)) - except Exception as e: - logger.error(f"Unable to batch {col}={final_value} due to {e}") - - status_message = item.status - if cached: - status_message = f"[cached] {status_message}" - cell_updates.append((row, "status", status_message)) - - media: Media = item.get_final_media() - if hasattr(media, "urls"): - batch_if_valid("archive", "\n".join(media.urls)) - batch_if_valid("date", True, get_current_timestamp()) - batch_if_valid("title", item.get_title()) - batch_if_valid("text", item.get("content", "")) - batch_if_valid("timestamp", item.get_timestamp()) - if media: - batch_if_valid("hash", media.get("hash", "not-calculated")) - - # merge all pdq hashes into a single string, if present - pdq_hashes = [] - all_media = item.get_all_media() - for m in all_media: - if pdq := m.get("pdq_hash"): - pdq_hashes.append(pdq) - if len(pdq_hashes): - batch_if_valid("pdq_hash", ",".join(pdq_hashes)) - - if (screenshot := item.get_media_by_id("screenshot")) and hasattr( - screenshot, "urls" - ): - batch_if_valid("screenshot", "\n".join(screenshot.urls)) - - if thumbnail := item.get_first_image("thumbnail"): - if hasattr(thumbnail, "urls"): - batch_if_valid("thumbnail", f'=IMAGE("{thumbnail.urls[0]}")') - - if browsertrix := item.get_media_by_id("browsertrix"): - batch_if_valid("wacz", "\n".join(browsertrix.urls)) - batch_if_valid( - "replaywebpage", - "\n".join( - [ - f"https://replayweb.page/?source={quote(wacz)}#view=pages&url={quote(item.get_url())}" - for wacz in browsertrix.urls - ] - ), - ) - - gw.batch_set_cell(cell_updates) - - def _safe_status_update(self, item: Metadata, new_status: str) -> None: - try: - gw, row = self._retrieve_gsheet(item) - gw.set_cell(row, "status", new_status) - except Exception as e: - logger.debug(f"Unable to update sheet: {e}") - - def _retrieve_gsheet(self, item: Metadata) -> Tuple[GWorksheet, int]: - - if gsheet := item.get_context("gsheet"): - gw: GWorksheet = gsheet.get("worksheet") - row: int = gsheet.get("row") - elif self.sheet_id: - logger.error(f"Unable to retrieve Gsheet for {item.get_url()}, GsheetDB must be used alongside GsheetFeeder.") - - return gw, row diff --git a/src/auto_archiver/modules/gsheet_feeder/__init__.py b/src/auto_archiver/modules/gsheet_feeder/__init__.py deleted file mode 100644 index bb4230a..0000000 --- a/src/auto_archiver/modules/gsheet_feeder/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .gworksheet import GWorksheet -from .gsheet_feeder import GsheetsFeeder \ No newline at end of file diff --git a/src/auto_archiver/modules/gsheet_feeder/gsheet_feeder.py b/src/auto_archiver/modules/gsheet_feeder/gsheet_feeder.py deleted file mode 100644 index 2026804..0000000 --- a/src/auto_archiver/modules/gsheet_feeder/gsheet_feeder.py +++ /dev/null @@ -1,95 +0,0 @@ -""" -GsheetsFeeder: A Google Sheets-based feeder for the Auto Archiver. - -This reads data from Google Sheets and filters rows based on user-defined rules. -The filtered rows are processed into `Metadata` objects. - -### Key properties -- validates the sheet's structure and filters rows based on input configurations. -- Ensures only rows with valid URLs and unprocessed statuses are included. -""" -import os -import gspread - -from loguru import logger -from slugify import slugify - -from auto_archiver.core import Feeder -from auto_archiver.core import Metadata -from . import GWorksheet - - -class GsheetsFeeder(Feeder): - - def setup(self) -> None: - self.gsheets_client = gspread.service_account(filename=self.service_account) - # TODO mv to validators - if not self.sheet and not self.sheet_id: - raise ValueError("You need to define either a 'sheet' name or a 'sheet_id' in your manifest.") - - def open_sheet(self): - if self.sheet: - return self.gsheets_client.open(self.sheet) - else: # self.sheet_id - return self.gsheets_client.open_by_key(self.sheet_id) - - def __iter__(self) -> Metadata: - sh = self.open_sheet() - for ii, worksheet in enumerate(sh.worksheets()): - if not self.should_process_sheet(worksheet.title): - logger.debug(f"SKIPPED worksheet '{worksheet.title}' due to allow/block rules") - continue - logger.info(f'Opening worksheet {ii=}: {worksheet.title=} header={self.header}') - gw = GWorksheet(worksheet, header_row=self.header, columns=self.columns) - if len(missing_cols := self.missing_required_columns(gw)): - logger.warning(f"SKIPPED worksheet '{worksheet.title}' due to missing required column(s) for {missing_cols}") - continue - - # process and yield metadata here: - yield from self._process_rows(gw) - logger.success(f'Finished worksheet {worksheet.title}') - - def _process_rows(self, gw: GWorksheet): - for row in range(1 + self.header, gw.count_rows() + 1): - url = gw.get_cell(row, 'url').strip() - if not len(url): continue - original_status = gw.get_cell(row, 'status') - status = gw.get_cell(row, 'status', fresh=original_status in ['', None]) - # TODO: custom status parser(?) aka should_retry_from_status - if status not in ['', None]: continue - - # All checks done - archival process starts here - m = Metadata().set_url(url) - self._set_context(m, gw, row) - yield m - - def _set_context(self, m: Metadata, gw: GWorksheet, row: int) -> Metadata: - - m.set_context("gsheet", {"row": row, "worksheet": gw}) - - if gw.get_cell_or_default(row, 'folder', "") is None: - folder = '' - else: - folder = slugify(gw.get_cell_or_default(row, 'folder', "").strip()) - if len(folder): - if self.use_sheet_names_in_stored_paths: - m.set_context("folder", os.path.join(folder, slugify(self.sheet), slugify(gw.wks.title))) - else: - m.set_context("folder", folder) - - - def should_process_sheet(self, sheet_name: str) -> bool: - if len(self.allow_worksheets) and sheet_name not in self.allow_worksheets: - # ALLOW rules exist AND sheet name not explicitly allowed - return False - if len(self.block_worksheets) and sheet_name in self.block_worksheets: - # BLOCK rules exist AND sheet name is blocked - return False - return True - - def missing_required_columns(self, gw: GWorksheet) -> list: - missing = [] - for required_col in ['url', 'status']: - if not gw.col_exists(required_col): - missing.append(required_col) - return missing diff --git a/src/auto_archiver/modules/gsheet_feeder_db/__init__.py b/src/auto_archiver/modules/gsheet_feeder_db/__init__.py new file mode 100644 index 0000000..2e9ac02 --- /dev/null +++ b/src/auto_archiver/modules/gsheet_feeder_db/__init__.py @@ -0,0 +1,2 @@ +from .gworksheet import GWorksheet +from .gsheet_feeder_db import GsheetsFeederDB \ No newline at end of file diff --git a/src/auto_archiver/modules/gsheet_feeder/__manifest__.py b/src/auto_archiver/modules/gsheet_feeder_db/__manifest__.py similarity index 66% rename from src/auto_archiver/modules/gsheet_feeder/__manifest__.py rename to src/auto_archiver/modules/gsheet_feeder_db/__manifest__.py index 130b9f6..9dd6b87 100644 --- a/src/auto_archiver/modules/gsheet_feeder/__manifest__.py +++ b/src/auto_archiver/modules/gsheet_feeder_db/__manifest__.py @@ -1,7 +1,7 @@ { - "name": "Google Sheets Feeder", - "type": ["feeder"], - "entry_point": "gsheet_feeder::GsheetsFeeder", + "name": "Google Sheets Feeder Database", + "type": ["feeder", "database"], + "entry_point": "gsheet_feeder_db::GsheetsFeederDB", "requires_setup": True, "dependencies": { "python": ["loguru", "gspread", "slugify"], @@ -12,7 +12,9 @@ "default": None, "help": "the id of the sheet to archive (alternative to 'sheet' config)", }, - "header": {"default": 1, "help": "index of the header row (starts at 1)", "type": "int"}, + "header": {"default": 1, + "type": "int", + "help": "index of the header row (starts at 1)", "type": "int"}, "service_account": { "default": "secrets/service_account.json", "help": "service account JSON file path. Learn how to create one: https://gspread.readthedocs.io/en/latest/oauth2.html", @@ -51,10 +53,23 @@ "help": "if True the stored files path will include 'workbook_name/worksheet_name/...'", "type": "bool", }, + "allow_worksheets": { + "default": set(), + "help": "(CSV) only worksheets whose name is included in allow are included (overrides worksheet_block), leave empty so all are allowed", + }, + "block_worksheets": { + "default": set(), + "help": "(CSV) explicitly block some worksheets from being processed", + }, + "use_sheet_names_in_stored_paths": { + "default": True, + "type": "bool", + "help": "if True the stored files path will include 'workbook_name/worksheet_name/...'", + } }, "description": """ - GsheetsFeeder - A Google Sheets-based feeder for the Auto Archiver. + GsheetsFeederDatabase + A Google Sheets-based feeder and optional database for the Auto Archiver. This reads data from Google Sheets and filters rows based on user-defined rules. The filtered rows are processed into `Metadata` objects. @@ -64,11 +79,16 @@ - Processes only worksheets allowed by the `allow_worksheets` and `block_worksheets` configurations. - Ensures only rows with valid URLs and unprocessed statuses are included for archival. - Supports organizing stored files into folder paths based on sheet and worksheet names. + - If the database is enabled, this updates the Google Sheet with the status of the archived URLs, including in progress, success or failure, and method used. + - Saves metadata such as title, text, timestamp, hashes, screenshots, and media URLs to designated columns. + - Formats media-specific metadata, such as thumbnails and PDQ hashes for the sheet. + - Skips redundant updates for empty or invalid data fields. ### Setup - Requires a Google Service Account JSON file for authentication, which should be stored in `secrets/gsheets_service_account.json`. To set up a service account, follow the instructions [here](https://gspread.readthedocs.io/en/latest/oauth2.html). - Define the `sheet` or `sheet_id` configuration to specify the sheet to archive. - Customize the column names in your Google sheet using the `columns` configuration. + - The Google Sheet can be used soley as a feeder or as a feeder and database, but note you can't currently feed into the database from an alternate feeder. """, } diff --git a/src/auto_archiver/modules/gsheet_feeder_db/gsheet_feeder_db.py b/src/auto_archiver/modules/gsheet_feeder_db/gsheet_feeder_db.py new file mode 100644 index 0000000..406eeb4 --- /dev/null +++ b/src/auto_archiver/modules/gsheet_feeder_db/gsheet_feeder_db.py @@ -0,0 +1,196 @@ +""" +GsheetsFeeder: A Google Sheets-based feeder for the Auto Archiver. + +This reads data from Google Sheets and filters rows based on user-defined rules. +The filtered rows are processed into `Metadata` objects. + +### Key properties +- validates the sheet's structure and filters rows based on input configurations. +- Ensures only rows with valid URLs and unprocessed statuses are included. +""" +import os +from typing import Tuple, Union +from urllib.parse import quote + +import gspread +from loguru import logger +from slugify import slugify + +from auto_archiver.core import Feeder, Database, Media +from auto_archiver.core import Metadata +from auto_archiver.modules.gsheet_feeder_db import GWorksheet +from auto_archiver.utils.misc import calculate_file_hash, get_current_timestamp + + +class GsheetsFeederDB(Feeder, Database): + + def setup(self) -> None: + self.gsheets_client = gspread.service_account(filename=self.service_account) + # TODO mv to validators + if not self.sheet and not self.sheet_id: + raise ValueError("You need to define either a 'sheet' name or a 'sheet_id' in your manifest.") + + def open_sheet(self): + if self.sheet: + return self.gsheets_client.open(self.sheet) + else: # self.sheet_id + return self.gsheets_client.open_by_key(self.sheet_id) + + def __iter__(self) -> Metadata: + sh = self.open_sheet() + for ii, worksheet in enumerate(sh.worksheets()): + if not self.should_process_sheet(worksheet.title): + logger.debug(f"SKIPPED worksheet '{worksheet.title}' due to allow/block rules") + continue + logger.info(f'Opening worksheet {ii=}: {worksheet.title=} header={self.header}') + gw = GWorksheet(worksheet, header_row=self.header, columns=self.columns) + if len(missing_cols := self.missing_required_columns(gw)): + logger.warning(f"SKIPPED worksheet '{worksheet.title}' due to missing required column(s) for {missing_cols}") + continue + + # process and yield metadata here: + yield from self._process_rows(gw) + logger.success(f'Finished worksheet {worksheet.title}') + + def _process_rows(self, gw: GWorksheet): + for row in range(1 + self.header, gw.count_rows() + 1): + url = gw.get_cell(row, 'url').strip() + if not len(url): continue + original_status = gw.get_cell(row, 'status') + status = gw.get_cell(row, 'status', fresh=original_status in ['', None]) + # TODO: custom status parser(?) aka should_retry_from_status + if status not in ['', None]: continue + + # All checks done - archival process starts here + m = Metadata().set_url(url) + self._set_context(m, gw, row) + yield m + + def _set_context(self, m: Metadata, gw: GWorksheet, row: int) -> Metadata: + # TODO: Check folder value not being recognised + m.set_context("gsheet", {"row": row, "worksheet": gw}) + + if gw.get_cell_or_default(row, 'folder', "") is None: + folder = '' + else: + folder = slugify(gw.get_cell_or_default(row, 'folder', "").strip()) + if len(folder): + if self.use_sheet_names_in_stored_paths: + m.set_context("folder", os.path.join(folder, slugify(self.sheet), slugify(gw.wks.title))) + else: + m.set_context("folder", folder) + + def should_process_sheet(self, sheet_name: str) -> bool: + if len(self.allow_worksheets) and sheet_name not in self.allow_worksheets: + # ALLOW rules exist AND sheet name not explicitly allowed + return False + if len(self.block_worksheets) and sheet_name in self.block_worksheets: + # BLOCK rules exist AND sheet name is blocked + return False + return True + + def missing_required_columns(self, gw: GWorksheet) -> list: + missing = [] + for required_col in ['url', 'status']: + if not gw.col_exists(required_col): + missing.append(required_col) + return missing + + + def started(self, item: Metadata) -> None: + logger.warning(f"STARTED {item}") + gw, row = self._retrieve_gsheet(item) + gw.set_cell(row, "status", "Archive in progress") + + def failed(self, item: Metadata, reason: str) -> None: + logger.error(f"FAILED {item}") + self._safe_status_update(item, f"Archive failed {reason}") + + def aborted(self, item: Metadata) -> None: + logger.warning(f"ABORTED {item}") + self._safe_status_update(item, "") + + def fetch(self, item: Metadata) -> Union[Metadata, bool]: + """check if the given item has been archived already""" + return False + + def done(self, item: Metadata, cached: bool = False) -> None: + """archival result ready - should be saved to DB""" + logger.success(f"DONE {item.get_url()}") + gw, row = self._retrieve_gsheet(item) + # self._safe_status_update(item, 'done') + + cell_updates = [] + row_values = gw.get_row(row) + + def batch_if_valid(col, val, final_value=None): + final_value = final_value or val + try: + if val and gw.col_exists(col) and gw.get_cell(row_values, col) == "": + cell_updates.append((row, col, final_value)) + except Exception as e: + logger.error(f"Unable to batch {col}={final_value} due to {e}") + + status_message = item.status + if cached: + status_message = f"[cached] {status_message}" + cell_updates.append((row, "status", status_message)) + + media: Media = item.get_final_media() + if hasattr(media, "urls"): + batch_if_valid("archive", "\n".join(media.urls)) + batch_if_valid("date", True, get_current_timestamp()) + batch_if_valid("title", item.get_title()) + batch_if_valid("text", item.get("content", "")) + batch_if_valid("timestamp", item.get_timestamp()) + if media: + batch_if_valid("hash", media.get("hash", "not-calculated")) + + # merge all pdq hashes into a single string, if present + pdq_hashes = [] + all_media = item.get_all_media() + for m in all_media: + if pdq := m.get("pdq_hash"): + pdq_hashes.append(pdq) + if len(pdq_hashes): + batch_if_valid("pdq_hash", ",".join(pdq_hashes)) + + if (screenshot := item.get_media_by_id("screenshot")) and hasattr( + screenshot, "urls" + ): + batch_if_valid("screenshot", "\n".join(screenshot.urls)) + + if thumbnail := item.get_first_image("thumbnail"): + if hasattr(thumbnail, "urls"): + batch_if_valid("thumbnail", f'=IMAGE("{thumbnail.urls[0]}")') + + if browsertrix := item.get_media_by_id("browsertrix"): + batch_if_valid("wacz", "\n".join(browsertrix.urls)) + batch_if_valid( + "replaywebpage", + "\n".join( + [ + f"https://replayweb.page/?source={quote(wacz)}#view=pages&url={quote(item.get_url())}" + for wacz in browsertrix.urls + ] + ), + ) + + gw.batch_set_cell(cell_updates) + + def _safe_status_update(self, item: Metadata, new_status: str) -> None: + try: + gw, row = self._retrieve_gsheet(item) + gw.set_cell(row, "status", new_status) + except Exception as e: + logger.debug(f"Unable to update sheet: {e}") + + def _retrieve_gsheet(self, item: Metadata) -> Tuple[GWorksheet, int]: + + if gsheet := item.get_context("gsheet"): + gw: GWorksheet = gsheet.get("worksheet") + row: int = gsheet.get("row") + elif self.sheet_id: + logger.error(f"Unable to retrieve Gsheet for {item.get_url()}, GsheetDB must be used alongside GsheetFeeder.") + + return gw, row diff --git a/src/auto_archiver/modules/gsheet_feeder/gworksheet.py b/src/auto_archiver/modules/gsheet_feeder_db/gworksheet.py similarity index 100% rename from src/auto_archiver/modules/gsheet_feeder/gworksheet.py rename to src/auto_archiver/modules/gsheet_feeder_db/gworksheet.py diff --git a/src/auto_archiver/modules/html_formatter/__manifest__.py b/src/auto_archiver/modules/html_formatter/__manifest__.py index ec19cf8..6e51c7a 100644 --- a/src/auto_archiver/modules/html_formatter/__manifest__.py +++ b/src/auto_archiver/modules/html_formatter/__manifest__.py @@ -7,7 +7,9 @@ "bin": [""] }, "configs": { - "detect_thumbnails": {"default": True, "help": "if true will group by thumbnails generated by thumbnail enricher by id 'thumbnail_00'"} + "detect_thumbnails": {"default": True, + "help": "if true will group by thumbnails generated by thumbnail enricher by id 'thumbnail_00'", + "type": "bool"}, }, "description": """ """, } diff --git a/src/auto_archiver/modules/instagram_extractor/__manifest__.py b/src/auto_archiver/modules/instagram_extractor/__manifest__.py index 05cae19..a66389f 100644 --- a/src/auto_archiver/modules/instagram_extractor/__manifest__.py +++ b/src/auto_archiver/modules/instagram_extractor/__manifest__.py @@ -10,25 +10,30 @@ "requires_setup": True, "configs": { "username": {"required": True, - "help": "a valid Instagram username"}, + "help": "A valid Instagram username."}, "password": { "required": True, - "help": "the corresponding Instagram account password", + "help": "The corresponding Instagram account password.", }, "download_folder": { "default": "instaloader", - "help": "name of a folder to temporarily download content to", + "help": "Name of a folder to temporarily download content to.", }, "session_file": { "default": "secrets/instaloader.session", - "help": "path to the instagram session which saves session credentials", + "help": "Path to the instagram session file which saves session credentials. If one doesn't exist this gives the path to store a new one.", }, # TODO: fine-grain # "download_stories": {"default": True, "help": "if the link is to a user profile: whether to get stories information"}, }, "description": """ - Uses the [Instaloader library](https://instaloader.github.io/as-module.html) to download content from Instagram. This class handles both individual posts - and user profiles, downloading as much information as possible, including images, videos, text, stories, + Uses the [Instaloader library](https://instaloader.github.io/as-module.html) to download content from Instagram. + + > ⚠️ **Warning** + > This module is not actively maintained due to known issues with blocking. + > Prioritise usage of the [Instagram Tbot Extractor](./instagram_tbot_extractor.md) and [Instagram API Extractor](./instagram_api_extractor.md) + + This class handles both individual posts and user profiles, downloading as much information as possible, including images, videos, text, stories, highlights, and tagged posts. Authentication is required via username/password or a session file. diff --git a/src/auto_archiver/modules/instagram_extractor/instagram_extractor.py b/src/auto_archiver/modules/instagram_extractor/instagram_extractor.py index 0af2c32..7e195ad 100644 --- a/src/auto_archiver/modules/instagram_extractor/instagram_extractor.py +++ b/src/auto_archiver/modules/instagram_extractor/instagram_extractor.py @@ -3,7 +3,7 @@ highlights, and tagged posts. Authentication is required via username/password or a session file. """ -import re, os, shutil, traceback +import re, os, shutil import instaloader from loguru import logger @@ -15,10 +15,9 @@ class InstagramExtractor(Extractor): """ Uses Instaloader to download either a post (inc images, videos, text) or as much as possible from a profile (posts, stories, highlights, ...) """ + # NB: post regex should be tested before profile - valid_url = re.compile(r"(?:(?:http|https):\/\/)?(?:www.)?(?:instagram.com|instagr.am|instagr.com)\/") - # https://regex101.com/r/MGPquX/1 post_pattern = re.compile(r"{valid_url}(?:p|reel)\/(\w+)".format(valid_url=valid_url)) # https://regex101.com/r/6Wbsxa/1 @@ -28,19 +27,22 @@ class InstagramExtractor(Extractor): def setup(self) -> None: self.insta = instaloader.Instaloader( - download_geotags=True, download_comments=True, compress_json=False, dirname_pattern=self.download_folder, filename_pattern="{date_utc}_UTC_{target}__{typename}" + download_geotags=True, + download_comments=True, + compress_json=False, + dirname_pattern=self.download_folder, + filename_pattern="{date_utc}_UTC_{target}__{typename}" ) try: self.insta.load_session_from_file(self.username, self.session_file) except Exception as e: - logger.error(f"Unable to login from session file: {e}\n{traceback.format_exc()}") try: - self.insta.login(self.username, config.instagram_self.password) - # TODO: wait for this issue to be fixed https://github.com/instaloader/instaloader/issues/1758 + logger.debug(f"Session file failed", exc_info=True) + logger.info("No valid session file found - Attempting login with use and password.") + self.insta.login(self.username, self.password) self.insta.save_session_to_file(self.session_file) - except Exception as e2: - logger.error(f"Unable to finish login (retrying from file): {e2}\n{traceback.format_exc()}") - + except Exception as e: + logger.error(f"Failed to setup Instagram Extractor with Instagrapi. {e}") def download(self, item: Metadata) -> Metadata: diff --git a/src/auto_archiver/modules/instagram_tbot_extractor/instagram_tbot_extractor.py b/src/auto_archiver/modules/instagram_tbot_extractor/instagram_tbot_extractor.py index 4404d07..1416da9 100644 --- a/src/auto_archiver/modules/instagram_tbot_extractor/instagram_tbot_extractor.py +++ b/src/auto_archiver/modules/instagram_tbot_extractor/instagram_tbot_extractor.py @@ -104,7 +104,7 @@ class InstagramTbotExtractor(Extractor): message = "" time.sleep(3) # media is added before text by the bot so it can be used as a stop-logic mechanism - while attempts < (self.timeout - 3) and (not message or not len(seen_media)): + while attempts < max(self.timeout - 3, 3) and (not message or not len(seen_media)): attempts += 1 time.sleep(1) for post in self.client.iter_messages(chat, min_id=since_id): diff --git a/src/auto_archiver/modules/local_storage/__manifest__.py b/src/auto_archiver/modules/local_storage/__manifest__.py index 6d9cf53..8ad6381 100644 --- a/src/auto_archiver/modules/local_storage/__manifest__.py +++ b/src/auto_archiver/modules/local_storage/__manifest__.py @@ -17,7 +17,9 @@ "choices": ["random", "static"], }, "save_to": {"default": "./local_archive", "help": "folder where to save archived content"}, - "save_absolute": {"default": False, "help": "whether the path to the stored file is absolute or relative in the output result inc. formatters (WARN: leaks the file structure)"}, + "save_absolute": {"default": False, + "type": "bool", + "help": "whether the path to the stored file is absolute or relative in the output result inc. formatters (WARN: leaks the file structure)"}, }, "description": """ LocalStorage: A storage module for saving archived content locally on the filesystem. diff --git a/src/auto_archiver/modules/screenshot_enricher/__manifest__.py b/src/auto_archiver/modules/screenshot_enricher/__manifest__.py index 9829844..92c0883 100644 --- a/src/auto_archiver/modules/screenshot_enricher/__manifest__.py +++ b/src/auto_archiver/modules/screenshot_enricher/__manifest__.py @@ -6,13 +6,25 @@ "python": ["loguru", "selenium"], }, "configs": { - "width": {"default": 1280, "help": "width of the screenshots"}, - "height": {"default": 720, "help": "height of the screenshots"}, - "timeout": {"default": 60, "help": "timeout for taking the screenshot"}, - "sleep_before_screenshot": {"default": 4, "help": "seconds to wait for the pages to load before taking screenshot"}, + "width": {"default": 1280, + "type": "int", + "help": "width of the screenshots"}, + "height": {"default": 1024, + "type": "int", + "help": "height of the screenshots"}, + "timeout": {"default": 60, + "type": "int", + "help": "timeout for taking the screenshot"}, + "sleep_before_screenshot": {"default": 4, + "type": "int", + "help": "seconds to wait for the pages to load before taking screenshot"}, "http_proxy": {"default": "", "help": "http proxy to use for the webdriver, eg http://proxy-user:password@proxy-ip:port"}, - "save_to_pdf": {"default": False, "help": "save the page as pdf along with the screenshot. PDF saving options can be adjusted with the 'print_options' parameter"}, - "print_options": {"default": {}, "help": "options to pass to the pdf printer"} + "save_to_pdf": {"default": False, + "type": "bool", + "help": "save the page as pdf along with the screenshot. PDF saving options can be adjusted with the 'print_options' parameter"}, + "print_options": {"default": {}, + "help": "options to pass to the pdf printer, in JSON format. See https://www.selenium.dev/documentation/webdriver/interactions/print_page/ for more information", + "type": "json_loader"}, }, "description": """ Captures screenshots and optionally saves web pages as PDFs using a WebDriver. diff --git a/src/auto_archiver/modules/ssl_enricher/__manifest__.py b/src/auto_archiver/modules/ssl_enricher/__manifest__.py index 9028f14..097cd21 100644 --- a/src/auto_archiver/modules/ssl_enricher/__manifest__.py +++ b/src/auto_archiver/modules/ssl_enricher/__manifest__.py @@ -7,7 +7,9 @@ }, 'entry_point': 'ssl_enricher::SSLEnricher', "configs": { - "skip_when_nothing_archived": {"default": True, "help": "if true, will skip enriching when no media is archived"}, + "skip_when_nothing_archived": {"default": True, + "type": 'bool', + "help": "if true, will skip enriching when no media is archived"}, }, "description": """ Retrieves SSL certificate information for a domain and stores it as a file. diff --git a/src/auto_archiver/modules/telethon_extractor/__manifest__.py b/src/auto_archiver/modules/telethon_extractor/__manifest__.py index 458428b..5e58203 100644 --- a/src/auto_archiver/modules/telethon_extractor/__manifest__.py +++ b/src/auto_archiver/modules/telethon_extractor/__manifest__.py @@ -14,7 +14,9 @@ "api_hash": {"default": None, "help": "telegram API_HASH value, go to https://my.telegram.org/apps"}, "bot_token": {"default": None, "help": "optional, but allows access to more content such as large videos, talk to @botfather"}, "session_file": {"default": "secrets/anon", "help": "optional, records the telegram login session for future usage, '.session' will be appended to the provided value."}, - "join_channels": {"default": True, "help": "disables the initial setup with channel_invites config, useful if you have a lot and get stuck"}, + "join_channels": {"default": True, + "type": "bool", + "help": "disables the initial setup with channel_invites config, useful if you have a lot and get stuck"}, "channel_invites": { "default": {}, "help": "(JSON string) private channel invite links (format: t.me/joinchat/HASH OR t.me/+HASH) and (optional but important to avoid hanging for minutes on startup) channel id (format: CHANNEL_ID taken from a post url like https://t.me/c/CHANNEL_ID/1), the telegram account will join any new channels on setup", diff --git a/src/auto_archiver/modules/wacz_extractor_enricher/__manifest__.py b/src/auto_archiver/modules/wacz_extractor_enricher/__manifest__.py index b8d6201..9b373b9 100644 --- a/src/auto_archiver/modules/wacz_extractor_enricher/__manifest__.py +++ b/src/auto_archiver/modules/wacz_extractor_enricher/__manifest__.py @@ -17,11 +17,19 @@ "configs": { "profile": {"default": None, "help": "browsertrix-profile (for profile generation see https://github.com/webrecorder/browsertrix-crawler#creating-and-using-browser-profiles)."}, "docker_commands": {"default": None, "help":"if a custom docker invocation is needed"}, - "timeout": {"default": 120, "help": "timeout for WACZ generation in seconds"}, - "extract_media": {"default": False, "help": "If enabled all the images/videos/audio present in the WACZ archive will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched."}, - "extract_screenshot": {"default": True, "help": "If enabled the screenshot captured by browsertrix will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched."}, + "timeout": {"default": 120, + "type": "int", + "help": "timeout for WACZ generation in seconds", "type": "int"}, + "extract_media": {"default": False, + "type": 'bool', + "help": "If enabled all the images/videos/audio present in the WACZ archive will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched." + }, + "extract_screenshot": {"default": True, + "type": 'bool', + "help": "If enabled the screenshot captured by browsertrix will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched." + }, "socks_proxy_host": {"default": None, "help": "SOCKS proxy host for browsertrix-crawler, use in combination with socks_proxy_port. eg: user:password@host"}, - "socks_proxy_port": {"default": None, "help": "SOCKS proxy port for browsertrix-crawler, use in combination with socks_proxy_host. eg 1234"}, + "socks_proxy_port": {"default": None, "type":"int", "help": "SOCKS proxy port for browsertrix-crawler, use in combination with socks_proxy_host. eg 1234"}, "proxy_server": {"default": None, "help": "SOCKS server proxy URL, in development"}, }, "description": """ diff --git a/src/auto_archiver/modules/wayback_extractor_enricher/__manifest__.py b/src/auto_archiver/modules/wayback_extractor_enricher/__manifest__.py index 38a5610..62a7e8a 100644 --- a/src/auto_archiver/modules/wayback_extractor_enricher/__manifest__.py +++ b/src/auto_archiver/modules/wayback_extractor_enricher/__manifest__.py @@ -9,6 +9,7 @@ "configs": { "timeout": { "default": 15, + "type": "int", "help": "seconds to wait for successful archive confirmation from wayback, if more than this passes the result contains the job_id so the status can later be checked manually.", }, "if_not_archived_within": { diff --git a/src/auto_archiver/modules/whisper_enricher/__manifest__.py b/src/auto_archiver/modules/whisper_enricher/__manifest__.py index 98e743e..0e09d03 100644 --- a/src/auto_archiver/modules/whisper_enricher/__manifest__.py +++ b/src/auto_archiver/modules/whisper_enricher/__manifest__.py @@ -10,8 +10,12 @@ "help": "WhisperApi api endpoint, eg: https://whisperbox-api.com/api/v1, a deployment of https://github.com/bellingcat/whisperbox-transcribe."}, "api_key": {"required": True, "help": "WhisperApi api key for authentication"}, - "include_srt": {"default": False, "help": "Whether to include a subtitle SRT (SubRip Subtitle file) for the video (can be used in video players)."}, - "timeout": {"default": 90, "help": "How many seconds to wait at most for a successful job completion."}, + "include_srt": {"default": False, + "type": "bool", + "help": "Whether to include a subtitle SRT (SubRip Subtitle file) for the video (can be used in video players)."}, + "timeout": {"default": 90, + "type": "int", + "help": "How many seconds to wait at most for a successful job completion."}, "action": {"default": "translate", "help": "which Whisper operation to execute", "choices": ["transcribe", "translate", "language_detection"]}, diff --git a/src/auto_archiver/utils/webdriver.py b/src/auto_archiver/utils/webdriver.py index db26d04..cb4e2a9 100644 --- a/src/auto_archiver/utils/webdriver.py +++ b/src/auto_archiver/utils/webdriver.py @@ -1,18 +1,23 @@ """ This Webdriver class acts as a context manager for the selenium webdriver. """ from __future__ import annotations -from selenium import webdriver -from selenium.common.exceptions import TimeoutException -from selenium.webdriver.common.proxy import Proxy, ProxyType -from selenium.webdriver.common.print_page_options import PrintOptions -from loguru import logger -from selenium.webdriver.common.by import By +import os import time #import domain_for_url from urllib.parse import urlparse, urlunparse from http.cookiejar import MozillaCookieJar +from selenium import webdriver +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support import expected_conditions as EC +from selenium.common import exceptions as selenium_exceptions +from selenium.webdriver.common.print_page_options import PrintOptions +from selenium.webdriver.common.by import By + +from loguru import logger + + class CookieSettingDriver(webdriver.Firefox): facebook_accept_cookies: bool @@ -20,6 +25,10 @@ class CookieSettingDriver(webdriver.Firefox): cookiejar: MozillaCookieJar def __init__(self, cookies, cookiejar, facebook_accept_cookies, *args, **kwargs): + if os.environ.get('RUNNING_IN_DOCKER'): + # Selenium doesn't support linux-aarch64 driver, we need to set this manually + kwargs['service'] = webdriver.FirefoxService(executable_path='/usr/local/bin/geckodriver') + super(CookieSettingDriver, self).__init__(*args, **kwargs) self.cookies = cookies self.cookiejar = cookiejar @@ -64,14 +73,29 @@ class CookieSettingDriver(webdriver.Firefox): time.sleep(2) except Exception as e: logger.warning(f'Failed on fb accept cookies.', e) + + # now get the actual URL super(CookieSettingDriver, self).get(url) if self.facebook_accept_cookies: # try and click the 'close' button on the 'login' window to close it - close_button = self.find_element(By.XPATH, "//div[@role='dialog']//div[@aria-label='Close']") - if close_button: - close_button.click() + try: + xpath = "//div[@role='dialog']//div[@aria-label='Close']" + WebDriverWait(self, 5).until(EC.element_to_be_clickable((By.XPATH, xpath))).click() + except selenium_exceptions.NoSuchElementException: + logger.warning("Unable to find the 'close' button on the facebook login window") + pass + else: + + # for all other sites, try and use some common button text to reject/accept cookies + for text in ["Refuse non-essential cookies", "Decline optional cookies", "Reject additional cookies", "Reject all", "Accept all cookies"]: + try: + xpath = f"//*[contains(translate(text(), 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz'), '{text.lower()}')]" + WebDriverWait(self, 5).until(EC.element_to_be_clickable((By.XPATH, xpath))).click() + break + except selenium_exceptions.WebDriverException: + pass class Webdriver: @@ -90,7 +114,6 @@ class Webdriver: setattr(self.print_options, k, v) def __enter__(self) -> webdriver: - options = webdriver.FirefoxOptions() options.add_argument("--headless") options.add_argument(f'--proxy-server={self.http_proxy}') @@ -105,7 +128,7 @@ class Webdriver: self.driver.set_window_size(self.width, self.height) self.driver.set_page_load_timeout(self.timeout_seconds) self.driver.print_options = self.print_options - except TimeoutException as e: + except selenium_exceptions.TimeoutException as e: logger.error(f"failed to get new webdriver, possibly due to insufficient system resources or timeout settings: {e}") return self.driver diff --git a/tests/databases/test_atlos_db.py b/tests/databases/test_atlos_db.py index 82c07ef..a73f1df 100644 --- a/tests/databases/test_atlos_db.py +++ b/tests/databases/test_atlos_db.py @@ -2,7 +2,7 @@ import pytest from datetime import datetime from auto_archiver.core import Metadata -from auto_archiver.modules.atlos_db import AtlosDb +from auto_archiver.modules.atlos_feeder_db_storage import AtlosFeederDbStorage as AtlosDb class FakeAPIResponse: @@ -12,19 +12,28 @@ class FakeAPIResponse: self._data = data self.raise_error = raise_error + def json(self) -> dict: + return self._data + def raise_for_status(self) -> None: if self.raise_error: raise Exception("HTTP error") @pytest.fixture -def atlos_db(setup_module) -> AtlosDb: +def atlos_db(setup_module, mocker) -> AtlosDb: """Fixture for AtlosDb.""" configs: dict = { "api_token": "abc123", "atlos_url": "https://platform.atlos.org", } - return setup_module("atlos_db", configs) + mocker.patch("requests.Session") + atlos_feeder = setup_module("atlos_feeder_db_storage", configs) + fake_session = mocker.MagicMock() + # Configure the default response to have no results so that __iter__ terminates + fake_session.get.return_value = FakeAPIResponse({"next": None, "results": []}) + atlos_feeder.session = fake_session + return atlos_feeder def test_failed_no_atlos_id(atlos_db, metadata, mocker): @@ -38,25 +47,20 @@ def test_failed_with_atlos_id(atlos_db, metadata, mocker): """Test failed() posts failure when atlos_id is present.""" metadata.set("atlos_id", 42) fake_resp = FakeAPIResponse({}, raise_error=False) - post_mock = mocker.patch("requests.post", return_value=fake_resp) + post_mock = mocker.patch.object(atlos_db, "_post", return_value=fake_resp) atlos_db.failed(metadata, "failure reason") - expected_url = ( - f"{atlos_db.atlos_url}/api/v2/source_material/metadata/42/auto_archiver" - ) - expected_headers = {"Authorization": f"Bearer {atlos_db.api_token}"} + expected_endpoint = f"/api/v2/source_material/metadata/42/auto_archiver" expected_json = { "metadata": {"processed": True, "status": "error", "error": "failure reason"} } - post_mock.assert_called_once_with( - expected_url, headers=expected_headers, json=expected_json - ) + post_mock.assert_called_once_with(expected_endpoint, json=expected_json) def test_failed_http_error(atlos_db, metadata, mocker): """Test failed() raises exception on HTTP error.""" metadata.set("atlos_id", 42) - fake_resp = FakeAPIResponse({}, raise_error=True) - mocker.patch("requests.post", return_value=fake_resp) + # Patch _post to raise an exception instead of returning a fake response. + mocker.patch.object(atlos_db, "_post", side_effect=Exception("HTTP error")) with pytest.raises(Exception, match="HTTP error"): atlos_db.failed(metadata, "failure reason") @@ -81,12 +85,9 @@ def test_done_with_atlos_id(atlos_db, metadata, mocker): now = datetime.now() metadata.set("timestamp", now) fake_resp = FakeAPIResponse({}, raise_error=False) - post_mock = mocker.patch("requests.post", return_value=fake_resp) + post_mock = mocker.patch.object(atlos_db, "_post", return_value=fake_resp) atlos_db.done(metadata) - expected_url = ( - f"{atlos_db.atlos_url}/api/v2/source_material/metadata/99/auto_archiver" - ) - expected_headers = {"Authorization": f"Bearer {atlos_db.api_token}"} + expected_endpoint = f"/api/v2/source_material/metadata/99/auto_archiver" expected_results = metadata.metadata.copy() expected_results["timestamp"] = now.isoformat() expected_json = { @@ -96,15 +97,13 @@ def test_done_with_atlos_id(atlos_db, metadata, mocker): "results": expected_results, } } - post_mock.assert_called_once_with( - expected_url, headers=expected_headers, json=expected_json - ) + post_mock.assert_called_once_with(expected_endpoint, json=expected_json) def test_done_http_error(atlos_db, metadata, mocker): - """Test done() raises exception on HTTP error.""" + """Test done() raises an exception on HTTP error.""" metadata.set("atlos_id", 123) - fake_resp = FakeAPIResponse({}, raise_error=True) - mocker.patch("requests.post", return_value=fake_resp) + # Patch _post to raise an exception. + mocker.patch.object(atlos_db, "_post", side_effect=Exception("HTTP error")) with pytest.raises(Exception, match="HTTP error"): atlos_db.done(metadata) diff --git a/tests/databases/test_gsheet_db.py b/tests/databases/test_gsheet_db.py index 8b49e5a..2f1202d 100644 --- a/tests/databases/test_gsheet_db.py +++ b/tests/databases/test_gsheet_db.py @@ -2,8 +2,7 @@ from datetime import datetime, timezone import pytest from auto_archiver.core import Metadata, Media -from auto_archiver.modules.gsheet_db import GsheetsDb -from auto_archiver.modules.gsheet_feeder import GWorksheet +from auto_archiver.modules.gsheet_feeder_db import GsheetsFeederDB, GWorksheet @pytest.fixture @@ -32,8 +31,9 @@ def mock_metadata(mocker): @pytest.fixture def metadata(): metadata = Metadata() - metadata.add_media(Media(filename="screenshot.png", urls=["http://example.com/screenshot.png"]).set("id", "screenshot")) - metadata.add_media(Media(filename="browsertrix", urls=["http://example.com/browsertrix.wacz"]).set("id", "browsertrix")) + metadata.add_media(Media(filename="screenshot", urls=["http://example.com/screenshot.png"])) + metadata.add_media(Media(filename="browsertrix", urls=["http://example.com/browsertrix.wacz"])) + metadata.add_media(Media(filename="thumbnail", urls=["http://example.com/thumbnail.png"])) metadata.set_url("http://example.com") metadata.set_title("Example Title") metadata.set_content("Example Content") @@ -52,12 +52,19 @@ def mock_media(mocker): return mock_media @pytest.fixture -def gsheets_db(mock_gworksheet, setup_module, mocker) -> GsheetsDb: - db = setup_module("gsheet_db", { - "allow_worksheets": "set()", - "block_worksheets": "set()", - "use_sheet_names_in_stored_paths": "True", - }) +def gsheets_db(mock_gworksheet, setup_module, mocker): + mocker.patch("gspread.service_account") + config: dict = { + "sheet": "testsheet", + "sheet_id": None, + "header": 1, + "service_account": "test/service_account.json", + "columns": {'url': 'link', 'status': 'archive status', 'folder': 'destination folder', 'archive': 'archive location', 'date': 'archive date', 'thumbnail': 'thumbnail', 'timestamp': 'upload timestamp', 'title': 'upload title', 'text': 'text content', 'screenshot': 'screenshot', 'hash': 'hash', 'pdq_hash': 'perceptual hashes', 'wacz': 'wacz', 'replaywebpage': 'replaywebpage'}, + "allow_worksheets": set(), + "block_worksheets": set(), + "use_sheet_names_in_stored_paths": True, + } + db = setup_module("gsheet_feeder_db", config) db._retrieve_gsheet = mocker.MagicMock(return_value=(mock_gworksheet, 1)) return db @@ -79,10 +86,10 @@ def expected_calls(mock_media, fixed_timestamp): (1, 'text', 'Example Content'), (1, 'timestamp', '2025-01-01T00:00:00+00:00'), (1, 'hash', 'not-calculated'), - (1, 'screenshot', 'http://example.com/screenshot.png'), - (1, 'thumbnail', '=IMAGE("http://example.com/screenshot.png")'), - (1, 'wacz', 'http://example.com/browsertrix.wacz'), - (1, 'replaywebpage', 'https://replayweb.page/?source=http%3A//example.com/browsertrix.wacz#view=pages&url=http%3A//example.com') + # (1, 'screenshot', 'http://example.com/screenshot.png'), + # (1, 'thumbnail', '=IMAGE("http://example.com/thumbnail.png")'), + # (1, 'wacz', 'http://example.com/browsertrix.wacz'), + # (1, 'replaywebpage', 'https://replayweb.page/?source=http%3A%2F%2Fexample.com%2Fbrowsertrix.wacz#view=pages&url=') ] def test_retrieve_gsheet(gsheets_db, metadata, mock_gworksheet): @@ -107,13 +114,13 @@ def test_aborted(gsheets_db, mock_metadata, mock_gworksheet): def test_done(gsheets_db, metadata, mock_gworksheet, expected_calls, mocker): - mocker.patch("auto_archiver.modules.gsheet_db.gsheet_db.get_current_timestamp", return_value='2025-02-01T00:00:00+00:00') + mocker.patch("auto_archiver.modules.gsheet_feeder_db.gsheet_feeder_db.get_current_timestamp", return_value='2025-02-01T00:00:00+00:00') gsheets_db.done(metadata) mock_gworksheet.batch_set_cell.assert_called_once_with(expected_calls) def test_done_cached(gsheets_db, metadata, mock_gworksheet, mocker): - mocker.patch("auto_archiver.modules.gsheet_db.gsheet_db.get_current_timestamp", return_value='2025-02-01T00:00:00+00:00') + mocker.patch("auto_archiver.modules.gsheet_feeder_db.gsheet_feeder_db.get_current_timestamp", return_value='2025-02-01T00:00:00+00:00') gsheets_db.done(metadata, cached=True) # Verify the status message includes "[cached]" @@ -124,7 +131,7 @@ def test_done_cached(gsheets_db, metadata, mock_gworksheet, mocker): def test_done_missing_media(gsheets_db, metadata, mock_gworksheet, mocker): # clear media from metadata metadata.media = [] - mocker.patch("auto_archiver.modules.gsheet_db.gsheet_db.get_current_timestamp", return_value='2025-02-01T00:00:00+00:00') + mocker.patch("auto_archiver.modules.gsheet_feeder_db.gsheet_feeder_db.get_current_timestamp", return_value='2025-02-01T00:00:00+00:00') gsheets_db.done(metadata) # Verify nothing media-related gets updated call_args = mock_gworksheet.batch_set_cell.call_args[0][0] diff --git a/tests/extractors/test_instagram_extractor.py b/tests/extractors/test_instagram_extractor.py index 7efe1b1..647cab4 100644 --- a/tests/extractors/test_instagram_extractor.py +++ b/tests/extractors/test_instagram_extractor.py @@ -1,21 +1,36 @@ import pytest from auto_archiver.modules.instagram_extractor import InstagramExtractor -from .test_extractor_base import TestExtractorBase -class TestInstagramExtractor(TestExtractorBase): + +@pytest.fixture +def instagram_extractor(setup_module, mocker): extractor_module: str = 'instagram_extractor' - config: dict = {} + config: dict = { + "username": "user_name", + "password": "password123", + "download_folder": "instaloader", + "session_file": "secrets/instaloader.session", + } + fake_loader = mocker.MagicMock() + fake_loader.load_session_from_file.return_value = None + fake_loader.login.return_value = None + fake_loader.save_session_to_file.return_value = None + mocker.patch("instaloader.Instaloader", return_value=fake_loader,) + return setup_module(extractor_module, config) - @pytest.mark.parametrize("url", [ - "https://www.instagram.com/p/", - "https://www.instagram.com/p/1234567890/", - "https://www.instagram.com/reel/1234567890/", - "https://www.instagram.com/username/", - "https://www.instagram.com/username/stories/", - "https://www.instagram.com/username/highlights/", - ]) - def test_regex_matches(self, url): - # post - assert InstagramExtractor.valid_url.match(url) + +@pytest.mark.parametrize("url", [ + "https://www.instagram.com/p/", + "https://www.instagram.com/p/1234567890/", + "https://www.instagram.com/reel/1234567890/", + "https://www.instagram.com/username/", + "https://www.instagram.com/username/stories/", + "https://www.instagram.com/username/highlights/", +]) +def test_regex_matches(url: str, instagram_extractor: InstagramExtractor) -> None: + """ + Ensure that the valid_url regex matches all provided Instagram URLs. + """ + assert instagram_extractor.valid_url.match(url) \ No newline at end of file diff --git a/tests/feeders/test_atlos_feeder.py b/tests/feeders/test_atlos_feeder.py index f26bdc9..1ef9fab 100644 --- a/tests/feeders/test_atlos_feeder.py +++ b/tests/feeders/test_atlos_feeder.py @@ -1,5 +1,5 @@ import pytest -from auto_archiver.modules.atlos_feeder import AtlosFeeder +from auto_archiver.modules.atlos_feeder_db_storage import AtlosFeederDbStorage as AtlosFeeder class FakeAPIResponse: @@ -18,23 +18,26 @@ class FakeAPIResponse: @pytest.fixture -def atlos_feeder(setup_module) -> AtlosFeeder: +def atlos_feeder(setup_module, mocker) -> AtlosFeeder: """Fixture for AtlosFeeder.""" configs: dict = { "api_token": "abc123", "atlos_url": "https://platform.atlos.org", } - return setup_module("atlos_feeder", configs) + mocker.patch("requests.Session") + atlos_feeder = setup_module("atlos_feeder_db_storage", configs) + fake_session = mocker.MagicMock() + # Configure the default response to have no results so that __iter__ terminates + fake_session.get.return_value = FakeAPIResponse({"next": None, "results": []}) + atlos_feeder.session = fake_session + return atlos_feeder @pytest.fixture -def mock_atlos_api(mocker): - """Fixture to mock requests to Atlos API.""" +def mock_atlos_api(atlos_feeder): + """Fixture to update the atlos_feeder.session.get side_effect.""" def _mock_responses(responses): - mocker.patch( - "requests.get", - side_effect=[FakeAPIResponse(data) for data in responses], - ) + atlos_feeder.session.get.side_effect = [FakeAPIResponse(data) for data in responses] return _mock_responses @@ -100,9 +103,7 @@ def test_atlos_feeder_no_results(atlos_feeder, mock_atlos_api): def test_atlos_feeder_http_error(atlos_feeder, mocker): """Test raises an exception on HTTP error.""" - mocker.patch( - "requests.get", - return_value=FakeAPIResponse({"next": None, "results": []}, raise_error=True), - ) + fake_response = FakeAPIResponse({"next": None, "results": []}, raise_error=True) + atlos_feeder.session.get.side_effect = [fake_response] with pytest.raises(Exception, match="HTTP error"): list(atlos_feeder) diff --git a/tests/feeders/test_gsheet_feeder.py b/tests/feeders/test_gsheet_feeder.py index ef150d1..9ca81b0 100644 --- a/tests/feeders/test_gsheet_feeder.py +++ b/tests/feeders/test_gsheet_feeder.py @@ -2,7 +2,7 @@ from typing import Type import gspread import pytest -from auto_archiver.modules.gsheet_feeder import GsheetsFeeder +from auto_archiver.modules.gsheet_feeder_db import GsheetsFeederDB from auto_archiver.core import Metadata, Feeder @@ -11,13 +11,13 @@ def test_setup_without_sheet_and_sheet_id(setup_module, mocker): mocker.patch("gspread.service_account") with pytest.raises(ValueError): setup_module( - "gsheet_feeder", + "gsheet_feeder_db", {"service_account": "dummy.json", "sheet": None, "sheet_id": None}, ) @pytest.fixture -def gsheet_feeder(setup_module, mocker) -> GsheetsFeeder: +def gsheet_feeder(setup_module, mocker) -> GsheetsFeederDB: config: dict = { "service_account": "dummy.json", "sheet": "test-auto-archiver", @@ -45,7 +45,7 @@ def gsheet_feeder(setup_module, mocker) -> GsheetsFeeder: } mocker.patch("gspread.service_account") feeder = setup_module( - "gsheet_feeder", + "gsheet_feeder_db", config ) feeder.gsheets_client = mocker.MagicMock() @@ -90,7 +90,7 @@ class MockWorksheet: return matching.get(col_name, default) -def test__process_rows(gsheet_feeder: GsheetsFeeder): +def test__process_rows(gsheet_feeder: GsheetsFeederDB): testworksheet = MockWorksheet() metadata_items = list(gsheet_feeder._process_rows(testworksheet)) assert len(metadata_items) == 3 @@ -98,7 +98,7 @@ def test__process_rows(gsheet_feeder: GsheetsFeeder): assert metadata_items[0].get("url") == "http://example.com" -def test__set_metadata(gsheet_feeder: GsheetsFeeder): +def test__set_metadata(gsheet_feeder: GsheetsFeederDB): worksheet = MockWorksheet() metadata = Metadata() gsheet_feeder._set_context(metadata, worksheet, 1) @@ -106,12 +106,12 @@ def test__set_metadata(gsheet_feeder: GsheetsFeeder): @pytest.mark.skip(reason="Not recognising folder column") -def test__set_metadata_with_folder_pickled(gsheet_feeder: GsheetsFeeder, worksheet): +def test__set_metadata_with_folder_pickled(gsheet_feeder: GsheetsFeederDB, worksheet): gsheet_feeder._set_context(worksheet, 7) assert Metadata.get_context("gsheet") == {"row": 1, "worksheet": worksheet} -def test__set_metadata_with_folder(gsheet_feeder: GsheetsFeeder): +def test__set_metadata_with_folder(gsheet_feeder: GsheetsFeederDB): testworksheet = MockWorksheet() metadata = Metadata() testworksheet.wks.title = "TestSheet" @@ -140,7 +140,7 @@ def test_open_sheet_with_name_or_id( # Setup module with parameterized values feeder = setup_module( - "gsheet_feeder", + "gsheet_feeder_db", {"service_account": "dummy.json", "sheet": sheet, "sheet_id": sheet_id}, ) sheet_result = feeder.open_sheet() @@ -159,7 +159,7 @@ def test_open_sheet_with_sheet_id(setup_module, mocker): mock_service_account.return_value = mock_client mock_client.open_by_key.return_value = "MockSheet" feeder = setup_module( - "gsheet_feeder", + "gsheet_feeder_db", {"service_account": "dummy.json", "sheet": None, "sheet_id": "ABC123"}, ) sheet = feeder.open_sheet() @@ -170,7 +170,7 @@ def test_open_sheet_with_sheet_id(setup_module, mocker): def test_should_process_sheet(setup_module, mocker): mocker.patch("gspread.service_account") gdb = setup_module( - "gsheet_feeder", + "gsheet_feeder_db", { "service_account": "dummy.json", "sheet": "TestSheet", @@ -187,10 +187,10 @@ def test_should_process_sheet(setup_module, mocker): @pytest.mark.skip(reason="Requires a real connection") class TestGSheetsFeederReal: - """Testing GSheetsFeeder class""" + """Testing GsheetsFeeder class""" - module_name: str = "gsheet_feeder" - feeder: GsheetsFeeder + module_name: str = "gsheet_feeder_db" + feeder: GsheetsFeederDB # You must follow the setup process explain in the docs for this to work config: dict = { "service_account": "secrets/service_account.json", diff --git a/tests/feeders/test_gworksheet.py b/tests/feeders/test_gworksheet.py index 2b05504..b6a0b5c 100644 --- a/tests/feeders/test_gworksheet.py +++ b/tests/feeders/test_gworksheet.py @@ -1,7 +1,7 @@ # Note this isn't a feeder, but contained as utility of the gsheet feeder module import pytest -from auto_archiver.modules.gsheet_feeder import GWorksheet +from auto_archiver.modules.gsheet_feeder_db import GWorksheet class TestGWorksheet: diff --git a/tests/storages/test_atlos_storage.py b/tests/storages/test_atlos_storage.py index 7528456..bcd8f18 100644 --- a/tests/storages/test_atlos_storage.py +++ b/tests/storages/test_atlos_storage.py @@ -2,7 +2,7 @@ import os import hashlib import pytest from auto_archiver.core import Media, Metadata -from auto_archiver.modules.atlos_storage import AtlosStorage +from auto_archiver.modules.atlos_feeder_db_storage import AtlosFeederDbStorage as AtlosStorage class FakeAPIResponse: @@ -21,13 +21,19 @@ class FakeAPIResponse: @pytest.fixture -def atlos_storage(setup_module) -> AtlosStorage: +def atlos_storage(setup_module, mocker) -> AtlosStorage: """Fixture for AtlosStorage.""" configs: dict = { "api_token": "abc123", "atlos_url": "https://platform.atlos.org", } - return setup_module("atlos_storage", configs) + mocker.patch("requests.Session") + atlos_feeder = setup_module("atlos_feeder_db_storage", configs) + mock_session = mocker.MagicMock() + # Configure the default response to have no results so that __iter__ terminates + mock_session.get.return_value = FakeAPIResponse({"next": None, "results": []}) + atlos_feeder.session = mock_session + return atlos_feeder @pytest.fixture @@ -49,17 +55,6 @@ def test_get_cdn_url(atlos_storage: AtlosStorage) -> None: assert url == atlos_storage.atlos_url -def test_hash(tmp_path, atlos_storage: AtlosStorage) -> None: - """Test _hash() computes the correct SHA-256 hash of a file.""" - content = b"hello world" - file_path = tmp_path / "test.txt" - file_path.write_bytes(content) - media = Media(filename="dummy.mp4") - media.filename = str(file_path) - expected_hash = hashlib.sha256(content).hexdigest() - assert atlos_storage._hash(media) == expected_hash - - def test_upload_no_atlos_id(tmp_path, atlos_storage: AtlosStorage, media: Media, mocker) -> None: """Test upload() returns False when metadata lacks atlos_id.""" metadata = Metadata() # atlos_id not set @@ -69,74 +64,49 @@ def test_upload_no_atlos_id(tmp_path, atlos_storage: AtlosStorage, media: Media, post_mock.assert_not_called() -def test_upload_already_uploaded(atlos_storage: AtlosStorage, - metadata: Metadata, - media: Media, - tmp_path, - mocker) -> None: +def test_upload_already_uploaded(atlos_storage: AtlosStorage, metadata: Metadata, media: Media, mocker) -> None: """Test upload() returns True if media hash already exists.""" content = b"media content" metadata.set("atlos_id", 101) media_hash = hashlib.sha256(content).hexdigest() - fake_get = FakeAPIResponse({ - "result": {"artifacts": [{"file_hash_sha256": media_hash}]} - }) - get_mock = mocker.patch("requests.get", return_value=fake_get) - post_mock = mocker.patch("requests.post") + fake_get_response = {"result": {"artifacts": [{"file_hash_sha256": media_hash}]}} + get_mock = mocker.patch.object(atlos_storage, "_get", return_value=fake_get_response) + post_mock = mocker.patch.object(atlos_storage, "_post") result = atlos_storage.upload(media, metadata) assert result is True get_mock.assert_called_once() post_mock.assert_not_called() -def test_upload_not_uploaded(tmp_path, atlos_storage: AtlosStorage, - metadata: Metadata, - media: Media, - mocker) -> None: +def test_upload_not_uploaded(tmp_path, atlos_storage: AtlosStorage, metadata: Metadata, media: Media, mocker) -> None: """Test upload() uploads media when not already present.""" metadata.set("atlos_id", 202) - fake_get = FakeAPIResponse({ - "result": {"artifacts": [{"file_hash_sha256": "different_hash"}]} - }) - get_mock = mocker.patch("requests.get", return_value=fake_get) - fake_post = FakeAPIResponse({}, raise_error=False) - post_mock = mocker.patch("requests.post", return_value=fake_post) + fake_get_response = {"result": {"artifacts": [{"file_hash_sha256": "different_hash"}]}} + get_mock = mocker.patch.object(atlos_storage, "_get", return_value=fake_get_response) + fake_post_response = {"result": "uploaded"} + post_mock = mocker.patch.object(atlos_storage, "_post", return_value=fake_post_response) result = atlos_storage.upload(media, metadata) assert result is True + get_mock.assert_called_once() post_mock.assert_called_once() - expected_url = f"{atlos_storage.atlos_url}/api/v2/source_material/upload/202" + expected_endpoint = f"/api/v2/source_material/upload/202" + call_args = post_mock.call_args[0] + assert call_args[0] == expected_endpoint + call_kwargs = post_mock.call_args[1] expected_headers = {"Authorization": f"Bearer {atlos_storage.api_token}"} expected_params = {"title": media.properties} - call_kwargs = post_mock.call_args.kwargs - assert call_kwargs["headers"] == expected_headers assert call_kwargs["params"] == expected_params - # Verify the URL passed to requests.post. - posted_url = call_kwargs.get("url") or post_mock.call_args.args[0] - assert posted_url == expected_url - # Verify files parameter contains the correct filename. file_tuple = call_kwargs["files"]["file"] assert file_tuple[0] == os.path.basename(media.filename) -def test_upload_post_http_error(tmp_path, - atlos_storage: AtlosStorage, - metadata: Metadata, - media: Media, - mocker) -> None: +def test_upload_post_http_error(tmp_path, atlos_storage: AtlosStorage, metadata: Metadata, media: Media, mocker) -> None: """Test upload() propagates HTTP error during POST.""" metadata.set("atlos_id", 303) - fake_get = FakeAPIResponse({ - "result": {"artifacts": []} - }) - mocker.patch("requests.get", return_value=fake_get) - fake_post = FakeAPIResponse({}, raise_error=True) - mocker.patch("requests.post", return_value=fake_post) + fake_get_response = {"result": {"artifacts": []}} + mocker.patch.object(atlos_storage, "_get", return_value=fake_get_response) + mocker.patch.object(atlos_storage, "_post", side_effect=Exception("HTTP error")) with pytest.raises(Exception, match="HTTP error"): atlos_storage.upload(media, metadata) - -def test_uploadf_not_implemented(atlos_storage: AtlosStorage) -> None: - """Test uploadf() returns None (not implemented).""" - result = atlos_storage.uploadf(None, "dummy") - assert result is None diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py index 752adb8..72f4949 100644 --- a/tests/test_orchestrator.py +++ b/tests/test_orchestrator.py @@ -78,7 +78,7 @@ def test_help(orchestrator, basic_parser, capsys): assert "--logging.level" in logs # individual module configs - assert "--gsheet_feeder.sheet_id" in logs + assert "--gsheet_feeder_db.sheet_id" in logs def test_add_custom_modules_path(orchestrator, test_args): @@ -154,22 +154,22 @@ def test_load_modules_from_commandline(orchestrator, test_args): assert orchestrator.formatters[0].name == "example_module" def test_load_settings_for_module_from_commandline(orchestrator, test_args): - args = test_args + ["--feeders", "gsheet_feeder", "--gsheet_feeder.sheet_id", "123", "--gsheet_feeder.service_account", "tests/data/test_service_account.json"] + args = test_args + ["--feeders", "gsheet_feeder_db", "--gsheet_feeder_db.sheet_id", "123", "--gsheet_feeder_db.service_account", "tests/data/test_service_account.json"] orchestrator.setup(args) assert len(orchestrator.feeders) == 1 - assert orchestrator.feeders[0].name == "gsheet_feeder" - assert orchestrator.config['gsheet_feeder']['sheet_id'] == "123" + assert orchestrator.feeders[0].name == "gsheet_feeder_db" + assert orchestrator.config['gsheet_feeder_db']['sheet_id'] == "123" def test_multiple_orchestrator(test_args): - o1_args = test_args + ["--feeders", "gsheet_feeder", "--gsheet_feeder.service_account", "tests/data/test_service_account.json"] + o1_args = test_args + ["--feeders", "gsheet_feeder_db", "--gsheet_feeder_db.service_account", "tests/data/test_service_account.json"] o1 = ArchivingOrchestrator() with pytest.raises(ValueError) as exit_error: - # this should fail because the gsheet_feeder requires a sheet_id / sheet + # this should fail because the gsheet_feeder_db requires a sheet_id / sheet o1.setup(o1_args)