Merge pull request #240 from bellingcat/update_release

Update project version to: 0.13.5.

Update the release process docs and the latest version in pyproject.toml

.github/workflows/docker-publish.yaml

@@ -11,7 +11,7 @@ on:
 
 env:
   # Use docker.io for Docker Hub if empty
-  REGISTRY: ghcr.io
+  REGISTRY: docker.io
   # github.repository as <account>/<repo>
   IMAGE_NAME: ${{ github.repository }}
 
@@ -45,10 +45,12 @@ jobs:
           images: bellingcat/auto-archiver
       
       - name: Build and push Docker image
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v6
         with:
           context: .
           platforms: linux/amd64,linux/arm64
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:cache
+          cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:cache,mode=max

.github/workflows/tests-core.yaml

@@ -5,9 +5,13 @@ on:
     branches: [ main ]
     paths:
       - src/**
+      - poetry.lock
+      - pyproject.toml
   pull_request:
     paths:
       - src/**
+      - poetry.lock
+      - pyproject.toml
 
 jobs:
   tests:

.gitignore

@@ -33,3 +33,5 @@ dist*
 docs/_build/
 docs/source/autoapi/
 docs/source/modules/autogen/
+scripts/settings_page.html
+.vite

.readthedocs.yaml

@@ -9,6 +9,7 @@ build:
   os: ubuntu-22.04
   tools:
     python: "3.10"
+    nodejs: "22"
   jobs:
     post_install:
       - pip install poetry
@@ -17,6 +18,11 @@ build:
       # See https://github.com/readthedocs/readthedocs.org/pull/11152/
       - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH poetry install --with docs
 
+      # generate the config editor page. Schema then HTML
+      - VIRTUAL_ENV=$READTHEDOCS_VIRTUALENV_PATH poetry run python scripts/generate_settings_schema.py
+      # install node dependencies and build the settings
+      - cd scripts/settings && npm install && npm run build && yes | cp dist/index.html ../../docs/source/installation/settings_base.html && cd ../..
+
 
 sphinx:
   configuration: docs/source/conf.py

Dockerfile

@@ -7,13 +7,24 @@ ENV RUNNING_IN_DOCKER=1 \
     PYTHONFAULTHANDLER=1 \
     PATH="/root/.local/bin:$PATH"
 
+
+ARG TARGETARCH
+
 # Installing system dependencies
 RUN add-apt-repository ppa:mozillateam/ppa && \
 	apt-get update && \
     apt-get install -y --no-install-recommends gcc ffmpeg fonts-noto exiftool && \
 	apt-get install -y --no-install-recommends firefox-esr && \
-    ln -s /usr/bin/firefox-esr /usr/bin/firefox && \
-    wget https://github.com/mozilla/geckodriver/releases/download/v0.35.0/geckodriver-v0.35.0-linux64.tar.gz && \
+    ln -s /usr/bin/firefox-esr /usr/bin/firefox
+
+ARG GECKODRIVER_VERSION=0.36.0
+
+RUN if [ $(uname -m) = "aarch64" ]; then \
+        GECKODRIVER_ARCH=linux-aarch64; \
+    else \
+        GECKODRIVER_ARCH=linux64; \
+    fi && \
+    wget https://github.com/mozilla/geckodriver/releases/download/v${GECKODRIVER_VERSION}/geckodriver-v${GECKODRIVER_VERSION}-${GECKODRIVER_ARCH}.tar.gz && \
     tar -xvzf geckodriver* -C /usr/local/bin && \
     chmod +x /usr/local/bin/geckodriver && \
     rm geckodriver-v* && \

README.md

@@ -23,11 +23,13 @@ Read the [article about Auto Archiver on bellingcat.com](https://www.bellingcat.
 
 ## Installation
 
-View the [Installation Guide](installation/installation.md) for full instructions
+View the [Installation Guide](https://auto-archiver.readthedocs.io/en/latest/installation/installation.html) for full instructions
+
+**Advanced:**
 
 To get started quickly using Docker:
 
-`docker pull bellingcat/auto-archiver && docker run`
+`docker pull bellingcat/auto-archiver && docker run --rm -v secrets:/app/secrets bellingcat/auto-archiver --config secrets/orchestration.yaml`
 
 Or pip:
 

docs/scripts/scripts.py

@@ -1,8 +1,9 @@
 # iterate through all the modules in auto_archiver.modules and turn the __manifest__.py file into a markdown table
 from pathlib import Path
-from auto_archiver.core.module import available_modules
+from auto_archiver.core.module import ModuleFactory
 from auto_archiver.core.base_module import BaseModule
 from ruamel.yaml import YAML
+from ruamel.yaml.comments import CommentedMap
 import io
 
 MODULES_FOLDER = Path(__file__).parent.parent.parent.parent / "src" / "auto_archiver" / "modules"
@@ -30,6 +31,7 @@ steps:
 ...
 
 {config_string}
+
 """
 
 def generate_module_docs():
@@ -38,10 +40,11 @@ def generate_module_docs():
     modules_by_type = {}
 
     header_row = "| " + " | ".join(TABLE_HEADER) + "|\n" + "| --- " * len(TABLE_HEADER) + "|\n"
-    configs_cheatsheet = "\n## Configuration Options\n"
-    configs_cheatsheet += header_row
+    global_table = "\n## Configuration Options\n" + header_row
 
-    for module in sorted(available_modules(with_manifest=True), key=lambda x: (x.requires_setup, x.name)):
+    global_yaml = yaml.load("""\n# Module configuration\nplaceholder: {}""")
+
+    for module in sorted(ModuleFactory().available_modules(), key=lambda x: (x.requires_setup, x.name)):
         # generate the markdown file from the __manifest__.py file.
 
         manifest = module.manifest
@@ -66,19 +69,30 @@ def generate_module_docs():
 
             config_table = header_row
             config_yaml = {}
+
+            global_yaml[module.name] = CommentedMap()
+            global_yaml.yaml_set_comment_before_after_key(module.name, f"\n\n{module.display_name} configuration options")
+
+
             for key, value in manifest['configs'].items():
                 type = value.get('type', 'string')
-                if type == 'auto_archiver.utils.json_loader':
+                if type == 'json_loader':
                     value['type'] = 'json'
                 elif type == 'str':
                     type = "string"
                 
                 default = value.get('default', '')
                 config_yaml[key] = default
+
+                global_yaml[module.name][key] = default
+
+                if value.get('help', ''):
+                    global_yaml[module.name].yaml_add_eol_comment(value.get('help', ''), key)
+
                 help = "**Required**. " if value.get('required', False) else "Optional. "
                 help += value.get('help', '')
                 config_table += f"| `{module.name}.{key}` | {help} | {value.get('default', '')} | {type} |\n"
-                configs_cheatsheet += f"| `{module.name}.{key}` | {help} | {default} | {type} |\n"
+                global_table += f"| `{module.name}.{key}` | {help} | {default} | {type} |\n"
             readme_str += "\n## Configuration Options\n"
             readme_str += "\n### YAML\n"
 
@@ -103,8 +117,13 @@ def generate_module_docs():
                 f.write(readme_str)
         generate_index(modules_by_type)
 
+    del global_yaml['placeholder']
+    global_string = io.BytesIO()
+    global_yaml = yaml.dump(global_yaml, global_string)
+    global_string = global_string.getvalue().decode('utf-8')
+    global_yaml = f"```yaml\n{global_string}\n```"
     with open(SAVE_FOLDER / "configs_cheatsheet.md", "w") as f:
-        f.write(configs_cheatsheet)
+        f.write("### Configuration File\n" + global_yaml + "\n### Command Line\n" + global_table)
 
 
 def generate_index(modules_by_type):

docs/source/conf.py

@@ -3,9 +3,11 @@
 import sys
 import os
 from importlib.metadata import metadata
+from datetime import datetime
 
 sys.path.append(os.path.abspath('../scripts'))
 from scripts import generate_module_docs
+from auto_archiver.version import __version__
 
 # -- Project Hooks -----------------------------------------------------------
 # convert the module __manifest__.py files into markdown files
@@ -15,7 +17,8 @@ generate_module_docs()
 # -- Project information -----------------------------------------------------
 package_metadata = metadata("auto-archiver")
 project = package_metadata["name"]
-authors = "Bellingcat"
+copyright = str(datetime.now().year)
+author = "Bellingcat"
 release = package_metadata["version"]
 language = 'en'
 
@@ -32,7 +35,7 @@ extensions = [
 ]
 
 templates_path = ['_templates']
-exclude_patterns = []
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", ""]
 
 
 # -- AutoAPI Configuration ---------------------------------------------------
@@ -76,6 +79,14 @@ source_suffix = {
 html_theme = 'sphinx_book_theme'
 html_static_path = ["../_static"]
 html_css_files = ["custom.css"]
+html_title = f"Auto Archiver v{__version__}"
+html_logo = "bc.png"
+html_theme_options = {
+    "repository_url": "https://github.com/bellingcat/auto-archiver",
+    "use_repository_button": True,
+}
+
+
 
 copybutton_prompt_text = r">>> |\.\.\."
 copybutton_prompt_is_regexp = True

docs/source/core_modules.md

@@ -1,8 +1,8 @@
 # Module Documentation
 
-These pages describe the core modules that come with `auto-archiver` and provide the main functionality for archiving websites on the internet. There are five core module types:
+These pages describe the core modules that come with Auto Archiver and provide the main functionality for archiving websites on the internet. There are five core module types:
 
-1. Feeders - these 'feed' information (the URLs) from various sources to the `auto-archiver` for processing
+1. Feeders - these 'feed' information (the URLs) from various sources to the Auto Archiver for processing
 2. Extractors - these 'extract' the page data for a given URL that is fed in by a feeder
 3. Enrichers - these 'enrich' the data extracted in the previous step with additional information
 4. Storage - these 'store' the data in a persistent location (on disk, Google Drive etc.)

docs/source/development/creating_modules.md

@@ -1,6 +1,6 @@
 # Creating Your Own Modules
 
-Modules are what's used to extend `auto-archiver` to process different websites or media, and/or transform the data in a way that suits your needs. In most cases, the [Core Modules](../core_modules.md) should be sufficient for every day use, but the most common use-cases for making your own Modules include:
+Modules are what's used to extend Auto Archiver to process different websites or media, and/or transform the data in a way that suits your needs. In most cases, the [Core Modules](../core_modules.md) should be sufficient for every day use, but the most common use-cases for making your own Modules include:
 
 1. Extracting data from a website which doesn't work with the current core extractors.
 2. Enriching or altering the data before saving with additional information that the core enrichers do not offer.
@@ -21,7 +21,7 @@ When done, you should have a module structure as follows:
 │   └── awesome_extractor.py
 ``` 
 
-Check out the [core modules](https://github.com/bellingcat/auto-archiver/tree/main/src/auto_archiver/modules) in the `auto-archiver` repository for examples of the folder structure for real-world modules.
+Check out the [core modules](https://github.com/bellingcat/auto-archiver/tree/main/src/auto_archiver/modules) in the Auto Archiver repository for examples of the folder structure for real-world modules.
 
 ## Populating the Manifest File
 

docs/source/development/developer_guidelines.md

@@ -31,4 +31,5 @@ docker_development
 testing
 docs
 release
+settings_page
 ```

docs/source/development/release.md

@@ -2,14 +2,32 @@
 
 ```{note} This is a work in progress.
 ```
+### Update the project version
 
-1. Update the version number in [version.py](src/auto_archiver/version.py)
-2. Go to github releases > new release > use `vx.y.z` for matching version notation
-   1. package is automatically updated in pypi
-   2. docker image is automatically pushed to dockerhup
+Update the version number in the project file: [pyproject.toml](../../pyproject.toml) following SemVer:
+```toml
+[project]
+name = "auto-archiver"
+version = "0.1.1"
+```
+Then commit and push the changes.
 
+* The package version is automatically updated in PyPi using the workflow [python-publish.yml](../../.github/workflows/python-publish.yml)
+* A Docker image is automatically pushed with the git tag to dockerhub using the workflow [docker-publish.yml](../../.github/workflows/docker-publish.yml)
+
+### Create the release on Git
+
+The release needs a git tag which should match the project version number, prefixed with a 'v'. For example, if the project version is `0.1.1`, the git tag should be `v0.1.1`.
+This can be done the usual way, or created within the Github UI when you create the release.
+
+Go to GitHub releases > new release > create the release with the new tag and the release notes.
 
 
 manual release to docker hub
   * `docker image tag auto-archiver bellingcat/auto-archiver:latest`
   * `docker push bellingcat/auto-archiver`
+
+
+### Building the Settings Page
+
+The Settings page is built as part of the python-publish workflow and packaged within the app.

docs/source/development/settings_page.md

@@ -0,0 +1,31 @@
+# Configuration Editor
+
+The [configuration editor](../installation/config_editor.md), is an easy-to-use UI for users to edit their auto-archiver settings.
+
+The single-file app is built using React and vite. To get started developing the package, follow these steps:
+
+1. Make sure you have Node v22 installed.
+
+```{note} Tip: if you don't have node installed:
+
+Use `nvm` to manage your node installations. Use: 
+`curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.1/install.sh | bash` to install `nvm` and then `nvm i 22` to install Node v22
+```
+
+2. Generate the `schema.json` file for the currently installed modules using `python scripts/generate_settings_schema.py`
+3. Go to the settings folder `cd scripts/settings/` and build your environment with `npm i`
+4. Run a development version of the page with `npm run dev` and then open localhost:5173.
+5. Build a release version of the page with `npm run build`
+
+A release version creates a single-file app called `dist/index.html`. This file should be copied to `docs/source/installation/settings_base.html` so that it can be integrated into the sphinx docs.
+
+```{note}
+
+The single-file app dist/index.html does not include any `<html>` or `<head>` tags as it is designed to be built into a RTD docs page. Edit `index.html` in the settings folder if you wish to modify the built page.
+```
+
+## Readthedocs Integration
+
+The configuration editor is built as part of the RTD deployment (see `.readthedocs.yaml` file). This command is run every time RTD is built:
+
+`cd scripts/settings && npm install && npm run build && yes | cp dist/index.html ../../docs/source/installation/settings_base.html && cd ../..`

docs/source/how_to.md

@@ -1,49 +1,6 @@
 # How-To Guides
 
-## How to use Google Sheets to load and store archive information
-The `--gsheet_feeder.sheet` property is the name of the Google Sheet to check for URLs. 
-This sheet must have been shared with the Google Service account used by `gspread`. 
-This sheet must also have specific columns (case-insensitive) in the `header` - see the [Gsheet Feeder Docs](modules/autogen/feeder/gsheet_feeder.md) for more info. The default names of these columns and their purpose is:
-
-Inputs:
-
-* **Link** *(required)*: the URL of the post to archive
-* **Destination folder**: custom folder for archived file (regardless of storage)
-
-Outputs:
-* **Archive status** *(required)*: Status of archive operation
-* **Archive location**: URL of archived post
-* **Archive date**: Date archived
-* **Thumbnail**: Embeds a thumbnail for the post in the spreadsheet
-* **Timestamp**: Timestamp of original post
-* **Title**: Post title
-* **Text**: Post text
-* **Screenshot**: Link to screenshot of post
-* **Hash**: Hash of archived HTML file (which contains hashes of post media) - for checksums/verification
-* **Perceptual Hash**: Perceptual hashes of found images - these can be used for de-duplication of content
-* **WACZ**: Link to a WACZ web archive of post
-* **ReplayWebpage**: Link to a ReplayWebpage viewer of the WACZ archive
-
-For example, this is a spreadsheet configured with all of the columns for the auto archiver and a few URLs to archive. (Note that the column names are not case sensitive.)
-
-![A screenshot of a Google Spreadsheet with column headers defined as above, and several Youtube and Twitter URLs in the "Link" column](../demo-before.png)
-
-Now the auto archiver can be invoked, with this command in this example: `docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver:dockerize --config secrets/orchestration-global.yaml --gsheet_feeder.sheet "Auto archive test 2023-2"`. Note that the sheet name has been overridden/specified in the command line invocation.
-
-When the auto archiver starts running, it updates the "Archive status" column.
-
-![A screenshot of a Google Spreadsheet with column headers defined as above, and several Youtube and Twitter URLs in the "Link" column. The auto archiver has added "archive in progress" to one of the status columns.](../demo-progress.png)
-
-The links are downloaded and archived, and the spreadsheet is updated to the following:
-
-![A screenshot of a Google Spreadsheet with videos archived and metadata added per the description of the columns above.](../demo-after.png)
-
-Note that the first row is skipped, as it is assumed to be a header row (`--gsheet_feeder.header=1` and you can change it if you use more rows above). Rows with an empty URL column, or a non-empty archive column are also skipped. All sheets in the document will be checked.
-
-The "archive location" link contains the path of the archived file, in local storage, S3, or in Google Drive.
-
-![The archive result for a link in the demo sheet.](../demo-archive.png)
-
+The follow pages contain helpful how-to guides for common use cases of the Auto Archiver.
 ---
 
 ```{toctree}
@@ -51,4 +8,5 @@ The "archive location" link contains the path of the archived file, in local sto
 :glob:
 
 how_to/*
+
 ```

docs/source/how_to/authentication_how_to.md

@@ -0,0 +1,110 @@
+# Logging in to sites
+
+This how-to guide shows you how you can use various authentication methods to allow you to login to a site you are trying to archive. This is useful for websites that require a user to be logged in to browse them, or for sites that restrict bots.
+
+In this How-To, we will authenticate on use Twitter/X.com using cookies, and on XXXX using username/password.
+
+
+
+## Using cookies to authenticate on Twitter/X
+
+It can be useful to archive tweets after logging in, since some tweets are only visible to authenticated users. One case is Tweets marked as 'Sensitive'.
+
+Take this tweet as an example: [https://x.com/SozinhoRamalho/status/1876710769913450647](https://x.com/SozinhoRamalho/status/1876710769913450647)
+
+This tweet has been marked as sensitive, so a normal run of Auto Archiver without a logged in session will fail to extract the tweet:
+
+```{code-block} console
+:emphasize-lines: 3,4,5,6
+
+>>> auto-archiver https://x.com/SozinhoRamalho/status/1876710769913450647                                                                                     ✭ ✱
+ ...
+ERROR: [twitter] 1876710769913450647: NSFW tweet requires authentication. Use --cookies, 
+--cookies-from-browser, --username and --password, --netrc-cmd, or --netrc (twitter) to
+ provide account credentials. See https://github.com/yt-dlp/yt-dlp/wiki/FAQ#how-do-i-pass-cookies-to-yt-dlp 
+ for how to manually pass cookies
+[twitter] 1876710769913450647: Downloading guest token
+[twitter] 1876710769913450647: Downloading GraphQL JSON
+2025-02-20 15:06:13.362 | ERROR    | auto_archiver.modules.generic_extractor.generic_extractor:download_for_extractor:248 - Error downloading metadata for post: NSFW tweet requires authentication. Use --cookies, --cookies-from-browser, --username and --password, --netrc-cmd, or --netrc (twitter) to provide account credentials. See  https://github.com/yt-dlp/yt-dlp/wiki/FAQ#how-do-i-pass-cookies-to-yt-dlp  for how to manually pass cookies
+[generic] Extracting URL: https://x.com/SozinhoRamalho/status/1876710769913450647
+[generic] 1876710769913450647: Downloading webpage
+WARNING: [generic] Falling back on generic information extractor
+[generic] 1876710769913450647: Extracting information
+ERROR: Unsupported URL: https://x.com/SozinhoRamalho/status/1876710769913450647
+2025-02-20 15:06:13.744 | INFO     | auto_archiver.core.orchestrator:archive:483 - Trying extractor telegram_extractor for https://x.com/SozinhoRamalho/status/1876710769913450647
+2025-02-20 15:06:13.744 | SUCCESS  | auto_archiver.modules.console_db.console_db:done:23 - DONE Metadata(status='nothing archived', metadata={'_processed_at': datetime.datetime(2025, 2, 20, 15, 6, 12, 473979, tzinfo=datetime.timezone.utc), 'url': 'https://x.com/SozinhoRamalho/status/1876710769913450647'}, media=[])
+...
+```
+
+To get round this limitation, we can use **cookies** (information about a logged in user) to mimic being logged in to Twitter. There are two ways to pass cookies to Auto Archiver. One is from a file, and the other is from a browser profile on your computer.
+
+In this tutorial, we will export the Twitter cookies from our browser and add them to Auto Archiver
+
+**1. Installing a cookie exporter extension**
+
+First, we need to install an extension in our browser to export the cookies for a certain site. The [FAQ on yt-dlp](https://github.com/yt-dlp/yt-dlp/wiki/FAQ#how-do-i-pass-cookies-to-yt-dlp) provides some suggestions: Get [cookies.txt LOCALLY](https://chromewebstore.google.com/detail/get-cookiestxt-locally/cclelndahbckbenkjhflpdbgdldlbecc) for Chrome or [cookies.txt](https://addons.mozilla.org/en-US/firefox/addon/cookies-txt/) for Firefox.
+
+**2. Export the cookies**
+
+```{note} See the note [here](../installation/authentication.md#recommendations-for-authentication) on why you shouldn't use your own personal account for archiving.
+```
+
+Once the extension is installed in your preferred browser, login to Twitter in this browser, and then activate the extension and export the cookies. You can choose to export all your cookies for your browser, or just cookies for this specific site. In the image below, we're only exporting cookies for Twitter/x.com:
+
+![extract cookies](extract_cookies.png)
+
+
+**3. Adding the cookies file to Auto Archiver**
+
+You now will have a file called `cookies.txt` (tip: name it `twitter_cookies.txt` if you only exported cookies for Twitter), which needs to be added to Auto Archiver.
+
+Do this by going into your Auto Archiver configuration file, and editing the `authentication` section. We will add the `cookies_file` option for the site `x.com,twitter.com`.
+
+```{note} For websites that have multiple URLs (like x.com and twitter.com) you can 'reuse' the same login information without duplicating it using a comma separated list of domain names.
+```
+
+I've saved my `twitter_cookies.txt` file in a `secrets` folder, so here's how my authentication section looks now:
+
+```{code} yaml
+:caption: orchestration.yaml
+
+...
+
+authentication:
+   x.com,twitter.com:
+      cookies_file: secrets/twitter_cookies.txt
+...
+```
+
+**4. Re-run your archiving with the cookies enabled**
+
+Now, the next time we re-run Auto Archiver, the cookies from our logged-in session will be used by Auto Archiver, and restricted/sensitive tweets can be downloaded!
+
+```{code} console
+>>> auto-archiver https://x.com/SozinhoRamalho/status/1876710769913450647                                                                                   ✭ ✱ ◼
+...
+2025-02-20 15:27:46.785 | WARNING  | auto_archiver.modules.console_db.console_db:started:13 - STARTED Metadata(status='no archiver', metadata={'_processed_at': datetime.datetime(2025, 2, 20, 15, 27, 46, 785304, tzinfo=datetime.timezone.utc), 'url': 'https://x.com/SozinhoRamalho/status/1876710769913450647'}, media=[])
+2025-02-20 15:27:46.785 | INFO     | auto_archiver.core.orchestrator:archive:483 - Trying extractor generic_extractor for https://x.com/SozinhoRamalho/status/1876710769913450647
+[twitter] Extracting URL: https://x.com/SozinhoRamalho/status/1876710769913450647
+...
+2025-02-20 15:27:53.134 | INFO     | auto_archiver.modules.local_storage.local_storage:upload:26 - ./local_archive/https-x-com-sozinhoramalho-status-1876710769913450647/06e8bacf27ac4bb983bf6280.html
+2025-02-20 15:27:53.135 | SUCCESS  | auto_archiver.modules.console_db.console_db:done:23 - DONE Metadata(status='yt-dlp_Twitter: success', 
+metadata={'_processed_at': datetime.datetime(2025, 2, 20, 15, 27, 48, 564738, tzinfo=datetime.timezone.utc), 'url': 
+'https://x.com/SozinhoRamalho/status/1876710769913450647', 'title': 'ignore tweet, testing sensitivity warning nudity https://t.co/t3u0hQsSB1', 
+...
+```
+
+
+### Finishing Touches
+
+You've now successfully exported your cookies from a logged-in session in your browser, and used them to authenticate with Twitter and download a sensitive tweet. Congratulations!
+
+Finally,Some important things to remember:
+
+1. It's best not to use your own personal account for archiving. [Here's why](../installation/authentication.md#recommendations-for-authentication).
+2. Cookies can be short-lived, so may need updating. Sometimes, a website session may 'expire' or a website may force you to login again. In these instances, you'll need to repeat the export step (step 2) after logging in again to update your cookies.
+
+## Authenticating on XXXX site with username/password
+
+```{note} This section is still under construction 🚧
+```

docs/source/how_to/gsheets_setup.md

@@ -0,0 +1,159 @@
+# Using Google Sheets
+
+This guide explains how to set up Google Sheets to process URLs automatically and then store the archiving status back into the Google sheet. It is broadly split into 3 steps:
+
+1. Setting up your Google Sheet
+2. Setting up a service account so Auto Archiver can access the sheet
+3. Setting the Auto Archiver settings
+
+### 1. Setting up your Google Sheet
+
+Any Google sheet must have at least *one* column, with the name 'link' (you can change this name afterwards). This is the column with the URLs that you want the Auto Archiver to archive. 
+Your sheet can have many other columns that the Auto Archiver can use, and you can also include any additional columns for your own personal use. The order of the columns does not matter, the naming just needs to be correctly assigned to its corresponding value in the configuration file.
+
+We recommend copying [this template Google Sheet](https://docs.google.com/spreadsheets/d/1NJZo_XZUBKTI1Ghlgi4nTPVvCfb0HXAs6j5tNGas72k/edit?usp=sharing) as a starting point for your project, as this matches the default column names.
+
+Here's an overview of all the columns, and what a complete sheet would look like.
+
+**Inputs:**
+
+These are processed by the Gsheet Feeder and passed to the Auto Archiver.
+
+* **Link** *(required)*: the URL of the post that is to be archived
+* **Destination folder**: custom folder for archived file (regardless of storage)
+
+**Outputs:**
+
+These are updated by the Gsheet DB module during the archiving process.
+Note the required columns are only required if you are using the Gsheet DB module as well as the feeder.
+
+* **Archive status** *(required)*: Status of archive operation
+* **Archive location**: URL of archived post
+* **Archive date**: Date archived
+* **Thumbnail**: Embeds a thumbnail for the post in the spreadsheet
+* **Timestamp**: Timestamp of original post
+* **Title**: Post title
+* **Text**: Post text
+* **Screenshot**: Link to screenshot of post
+* **Hash**: Hash of archived HTML file (which contains hashes of post media) - for checksums/verification
+* **Perceptual Hash**: Perceptual hashes of found images - these can be used for de-duplication of content
+* **WACZ**: Link to a WACZ web archive of post
+* **ReplayWebpage**: Link to a ReplayWebpage viewer of the WACZ archive
+
+For example, this is a spreadsheet configured with all of the columns for the auto archiver and a few URLs to archive. 
+In this example the Ghseet Feeder and Gsheet DB are being used, and the archive is in progress.
+(Note that the column names are not case sensitive.)
+
+![A screenshot of a Google Spreadsheet with column headers defined as above, and several Youtube and Twitter URLs in the "Link" column](../../demo-before.png)
+
+We'll change the name of the 'Destination Folder' column in step 3.
+
+## 2. Setting up your Service Account
+
+Once your Google Sheet is set up, you need to create what's called a 'service account' that will allow the Auto Archiver to access it.
+
+To do this, follow the steps in [this guide](https://gspread.readthedocs.io/en/latest/oauth2.html) all the way up until step 8. You should have downloaded a file called `service_account.json` and shared the Google Sheet with the log 'client_email' email address in this file.
+
+Once you've downloaded the file, save it to `secrets/service_account.json`
+
+## 3. Setting up the configuration file
+
+Now that you've set up your Google sheet, and you've set up the service account so Auto Archiver can access the sheet, the final step is to set your configuration.
+
+First, make sure you have `gsheet_feeder_db` set in the `steps.feeders` section of your config. If you wish to store the results of the archiving process back in your Google sheet, make sure to also set the `ghseet_db` settig in the `steps.databases` section. Here's how this might look:
+
+```{code} yaml
+steps:
+    feeders:
+    - gsheet_feeder_db
+    ...
+    databases:
+    - gsheet_feeder_db # optional, if you also want to store the results in the Google sheet and tract the status of active archivals.
+    ...
+```
+
+Next, set up the `gsheet_feeder_db` configuration settings in the 'Configurations' part of the config `orchestration.yaml` file. Open up the file, and set the `gsheet_feeder_db.sheet` setting or the `gsheet_feeder_db.sheet_id` setting. The `sheet` should be the name of your sheet, as it shows in the top left of the sheet. 
+For example, the sheet [here](https://docs.google.com/spreadsheets/d/1NJZo_XZUBKTI1Ghlgi4nTPVvCfb0HXAs6j5tNGas72k/edit?gid=0#gid=0) is called 'Public Auto Archiver template'.
+
+Here's how this might look:
+
+```{code} yaml
+...
+gsheet_feeder_db:
+    sheet: 'My Awesome Sheet'
+    ...
+```
+
+You can also pass these settings directly on the command line without having to edit the file, here'a an example of how to do that (using docker):
+
+`docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver:dockerize --gsheet_feeder_db.sheet "My Awesome Sheet 2"`. 
+
+Here, the sheet name has been overridden/specified in the command line invocation.
+
+### 3a. (Optional) Changing the column names
+
+In step 1, we said we would change the name of the 'Destination Folder'. Perhaps you don't like this name, or already have a sheet with a different name. In our example here, we want to name this column 'Save Folder'. To do this, we need to edit the `ghseet_feeder_db.column` setting in the configuration file. 
+For more information on this setting, see the [Gsheet Feeder Database docs](../modules/autogen/feeder/gsheet_feeder_db.md#configuration-options). We will first copy the default settings from the Gsheet Feeder docs for the 'column' settings, and then edit the 'Destination Folder' section to rename it 'Save Folder'. Our final configuration section looks like:
+
+```{code} yaml
+...
+gsheet_feeder_db:
+    sheet: 'My Awesome Sheet'
+    header: 1
+    service_account: secrets/service_account.json
+    columns:
+      url: link
+      status: archive status
+      folder: save folder # <-- note how this value has been changed
+      archive: archive location
+      date: archive date
+      thumbnail: thumbnail
+      timestamp: upload timestamp
+      title: upload title
+      text: text content
+      screenshot: screenshot
+      hash: hash
+      pdq_hash: perceptual hashes
+      wacz: wacz
+      replaywebpage: replaywebpage
+    
+```
+## 4. Running the Auto Archiver
+### Feeding the URLs to the Auto Archiver
+
+The URLs to be archived should be added to the Google Sheet, and optionally a folder value. Leave all the other configured columns empty (but you may add additional columns for your own use, as long as they don't conflict with the column names mapped in the configuration file).
+The Auto Archiver will archive  any URLs which have an empty 'status' column
+
+### Viewing the Results after archiving
+
+With the `ghseet_feeder_db` installed, once you start running the Auto Archiver, it will update the "Archive status" column.
+The status will be set to "Archive in progress" once the archival starts. If the archival is stopped during a run, either manually or because an error is raised the status value should be cleared.
+
+![A screenshot of a Google Spreadsheet with column headers defined as above, and several Youtube and Twitter URLs in the "Link" column. The auto archiver has added "archive in progress" to one of the status columns.](../../demo-progress.png)
+
+The links are downloaded and archived, and the spreadsheet is updated to the following:
+
+![A screenshot of a Google Spreadsheet with videos archived and metadata added per the description of the columns above.](../../demo-after.png)
+
+Note that the first row is skipped, as it is assumed to be a header row (`--gsheet_feeder_db.header=1` and you can change it if you use more rows above). Rows with an empty URL column, or a non-empty archive column are also skipped. All sheets in the document will be checked.
+
+The "archive location" link contains the path of the archived file, in local storage, S3, or in Google Drive.
+
+![The archive result for a link in the demo sheet.](../../demo-archive.png)
+
+### Troubleshooting
+
+**Hanging Archival in progress status**
+
+Occasionally system crashes or other unexpected events can cause the Auto Archiver to exit without cleaning up the status value.
+If you are sure that all archival processes have stopped but you still see "Archive in progress" in the status column, you can manually clear the status column to allow the Auto Archiver to retry that archival on the next run.
+
+**Nothing archived status**
+
+Sometimes this means the tool is genuinely unable to extract the content at this point in time, but sometimes it can be resolved with different configurations. 
+Try:
+  - Turning on additional 'extractor' types in the configuration file (this can appear as 'no archiver' in the status column). 
+  - Changing credentials or refreshing session files for extractors which require them
+  - Check if the extractors can accept any additional configurations such as adding a cookie file.
+
+

docs/source/how_to/logging.md

@@ -0,0 +1,71 @@
+# Keeping Logs
+
+Auto Archiver's logs can be helpful for debugging problematic archiving processes. This guide shows you how to use the logs to 
+
+## Setting up logging
+
+Logging settings can be set on the command line or using the orchestration config file ([learn more](../installation/configuration)). A special `logging` section defines the logging options.
+
+#### Enabling or Disabling Logging
+
+Logging to the console is enabled by default. If you want to globally disable Auto Archiver's logging, then you can set `enabled: false` in your `logging` config:
+
+```{code} yaml
+
+...
+logging:
+   enabled: false
+...
+```
+
+```{note}
+This will disable all logs from Auto Archiver, but it does not disable logs for other tools that the Auto Archiver uses (for example: yt-dlp, firefox or ffmpeg). These logs will still appear in your console.
+```
+
+#### Logging Level
+
+There are 7 logging levels in total, with 4 commonly used levels. They are: `DEBUG`, `INFO`, `WARNING` and `ERROR`.
+
+Change the warning level by setting the value in your orchestration config file:
+
+```{code} yaml
+:caption: orchestration.yaml
+
+...
+logging:
+    level: DEBUG # or INFO / WARNING / ERROR
+...
+```
+
+For normal usage, it is recommended to use the `INFO` level, or if you prefer quieter logs with less information, you can use the `WARNING` level. If you encounter issues with the archiving, then it's recommended to enable the `DEBUG` level.
+
+```{note} To learn about all logging levels, see the [loguru documentation](https://loguru.readthedocs.io/en/stable/api/logger.html)
+```
+
+### Logging to a file
+
+As default, auto-archiver will log to the console. But if you wish to store your logs for future reference, or you are running the auto-archiver from within code a implementation, then you may with to enable file logging. This can be done by setting the `file:` config value in the logging settings.
+
+**Rotation:** For file logging, you can choose to 'rotate' your log files (creating new log files) so they do not get too large. Change this by setting the 'rotation' option in your logging settings. For a full list of rotation options, see the [loguru docs](https://loguru.readthedocs.io/en/stable/overview.html#easier-file-logging-with-rotation-retention-compression).
+
+```{code} yaml
+:caption: orchestration.yaml
+
+logging:
+    ...
+    file: /my/log/file.log
+    rotation: 1 day
+```
+
+### Full logging example
+
+The below example logs only `WARNING` logs to the console and to the file `/my/file.log`, rotating that file once per week:
+
+```{code} yaml
+:caption: orchestration.yaml
+
+logging:
+    level: WARNING
+    file: /my/file.log
+    rotation: 1 week
+```

docs/source/how_to/new_config_format.md

@@ -0,0 +1,146 @@
+# Upgrading from v0.12
+
+```{note} This how-to is only relevant for people who used Auto Archiver before February 2025 (versions prior to 0.13).
+
+If you are new to Auto Archiver, then you are already using the latest configuration format and this how-to is not relevant for you.
+```
+
+Versions 0.13+ of Auto Archiver has breaking changes in the configuration format, which means earlier configuration formats will not work without slight modifications.
+
+## How do I know if I need to update my configuration format?
+
+There are two simple ways to check if you need to update your format:
+
+1. When you try and run auto-archiver using your existing configuration file, you get an error about no feeders or formatters being configured, like:
+
+```{code} console
+AssertionError: No feeders were configured. Make sure to set at least one feeder in
+your configuration file or on the command line (using --feeders)
+```
+
+2. Within your configuration file, you have a `feeder:` option. This is the old format. An example old format:
+```{code} yaml
+
+steps:
+  feeder: cli_feeder
+...
+```
+
+The next two sections outline the two methods you have for updating your file.
+
+## 1. Manually edit the configuration file and change the values.
+
+This is recommended if you want to keep all your old settings. Follow the steps below to change the relevant settings:
+
+#### a) Feeder & Formatter Steps Settings
+
+The feeder and formatter settings have been changed from a single string to a list.
+
+- `steps.feeder (string)` → `steps.feeders (list)`
+- `steps.formatter (string)` → `steps.formatters (list)`
+
+Example:
+
+```{code} yaml
+
+steps:
+   feeder: cli_feeder
+   ...
+   formatter: html_formatter
+
+# the above should be changed to:
+steps:
+   feeders:
+   - cli_feeder
+   ...
+   formatters:
+   - html_formatter
+```
+
+```{note} Auto Archiver still only supports one feeder and formatter, but from v0.13 onwards they must be added to the configuration file as a list.
+```
+
+#### b) Extractor (formerly Archiver) Steps Settings
+
+With v0.13 of Auto Archiver, `archivers` have been renamed to `extractors` to better reflect what they actually do - extract information from a URL. Change the configuration by renaming:
+
+- `steps.archivers` → `steps.extractors`
+
+The names of the actual modules have also changed, so for any extractor modules you have enabled, you will need to rename the `archiver` part to `extractor`. Some examples:
+
+- `telethon_archiver` → `telethon_extractor`
+- `wacz_archiver_enricher` → `wacz_extractor_enricher`
+- `wayback_archiver_enricher` → `wayback_extractor_enricher`
+- `vk_archiver` → `vk_extractor`
+
+
+#### c) Module Renaming
+
+
+The `youtube_archiver` has been renamed to `generic_extractor` as it is considered the default/fallback extractor. Read more about the [generic extractor](../modules/autogen/extractor/generic_extractor.md).
+
+The `atlos` modules have been merged into one, as have the `gsheets` feeder and database.
+
+- `atlos_feeder` → `atlos_feeder_db_storage`
+- `atlos_storage` → `atlos_feeder_db_storage`
+- `atlos_db` → `atlos_feeder_db_storage`
+- `gsheet_feeder` → `gsheet_feeder_db`
+- `gsheet_db` → `gsheet_feeder_db`
+
+
+Example:
+```{code} yaml
+steps:
+   feeders:
+   - gsheet_feeder_db # formerly gsheet_feeder
+   ...
+   extractors: # formerly 'archivers'
+   - telethon_extractor # formerly telethon_archiver
+   - generic_extractor # formerly youtube_archiver
+   - vk_extractor # formerly vk_archiver
+   databases:
+   - gsheet_feeder_db # formerly gsheet_db
+   ...
+
+```
+
+```{note}
+
+Don't forget to also rename the configuration settings. For example:
+
+```{code} yaml
+gsheet_feeder_db: # formerly gsheet_feeder
+  service_account: secrets/service_account.json
+  sheet: My Google Sheet
+...
+```
+
+#### d) Redundant / Obsolete Modules
+
+With v0.13 of Auto Archiver, the following modules have been removed and their features have been built in to the generic_extractor. You should remove them from the 'steps' section of your configuration file:
+
+* `twitter_archiver` - use the `generic_extractor` for general extraction, or the `twitter_api_extractor` for API access.
+* `tiktok_archiver` - use the `generic_extractor` to extract TikTok videos.
+
+
+## 2. Auto-generate a new config, then copy over your settings.
+
+Using this method, you can have Auto Archiver auto-generate a configuration file for you, then you can copy over the desired settings from your old config file. This is probably the easiest method and quickest to setup, but it may require some trial and error as you copy over your settings.
+
+First, move your existing `orchestration.yaml` file to a different folder or rename it.
+
+Then, you can generate a `simple` or `full` config using:
+
+```{code} console
+>>> # generate a simple config
+>>> auto-archiver 
+>>> # config will be written to orchestration.yaml
+>>> 
+>>> # generate a full config
+>>> auto-archiver --mode=full
+>>> 
+```
+
+After this, copy over any settings from your old config to the new config.
+
+

docs/source/index.md

@@ -8,10 +8,10 @@
 :caption: Contents:
 
 Overview <self>
-contributing
-installation/installation.rst
+installation/setup
 core_modules.md
 how_to
+contributing
 development/developer_guidelines
 autoapi/index.rst
 ```

docs/source/installation/authentication.md

@@ -4,22 +4,42 @@ The Authentication framework for auto-archiver allows you to add login details f
 
 There are two main use cases for authentication:
 * Some websites require some kind of authentication in order to view the content. Examples include Facebook, Telegram etc.
-* Some websites use anti-bot systems to block bot-like tools from accessig the website. Adding real login information to auto-archiver can sometimes bypass this.
+* Some websites use anti-bot systems to block bot-like tools from accessing the website. Adding real login information to auto-archiver can sometimes bypass this.
 
 ## The Authentication Config
 
-You can save your authentication information directly inside your orchestration config file, or as a separate file (for security/multi-deploy purposes). Whether storing your settings inside the orchestration file, or as a separate file, the configuration format is the same.
+You can save your authentication information directly inside your orchestration config file, or as a separate file (for security/multi-deploy purposes). Whether storing your settings inside the orchestration file, or as a separate file, the configuration format is the same. Currently, auto-archiver supports the following authentication types:
+
+**Username & Password:**
+- `username`: str - the username to use for login
+- `password`: str - the password to use for login
+
+**API**
+- `api_key`: str - the API key to use for login
+- `api_secret`: str - the API secret to use for login
+  
+**Cookies**
+- `cookie`: str - a cookie string to use for login (specific to this site)
+- `cookies_from_browser`: str - load cookies from this browser, for this site only.
+- `cookies_file`: str - load cookies from this file, for this site only.
+
+```{note} 
+
+The Username & Password, and API settings only work with the Generic Extractor. Other modules (like the screenshot enricher) can only use the `cookies` options. Furthermore, many sites can still detect bots and block username/password logins. Twitter/X and YouTube are two prominent ones that block username/password logging.
+
+One of the 'Cookies' options is recommended for the most robust archiving.
+```
 
 ```{code} yaml
 authentication:
    # optional file to load authentication information from, for security or multi-system deploy purposes
    load_from_file: path/to/authentication/file.txt
-   # optional setting to load cookies from the named browser on the system.
+   # optional setting to load cookies from the named browser on the system, for **ALL** websites
    cookies_from_browser: firefox
-   # optional setting to load cookies from a cookies.txt/cookies.jar file. See note below on extracting these
+   # optional setting to load cookies from a cookies.txt/cookies.jar file, for **ALL** websites. See note below on extracting these
    cookies_file: path/to/cookies.jar
 
-   twitter.com,x.com:
+   mysite.com:
       username: myusername
       password: 123
     
@@ -29,15 +49,10 @@ authentication:
     othersite.com:
        api_key: 123
        api_secret: 1234
-
-# All available options:
-  # - username: str - the username to use for login
-  # - password: str - the password to use for login
-  # - api_key: str - the API key to use for login
-  # - api_secret: str - the API secret to use for login
-  # - cookie: str - a cookie string to use for login (specific to this site)
+  
 ```
 
+
 ### Recommendations for authentication
 
 1. **Store authentication information separately:**

docs/source/installation/config_editor.md

@@ -0,0 +1,5 @@
+# Configuration Editor
+
+```{raw} html
+:file: settings.html
+```

docs/source/installation/configurations.md

@@ -1,13 +1,18 @@
 
 # Configuration
 
-This section of the documentation provides guidelines for configuring the tool.
+The recommended way to configure auto-archiver for first-time users is to [run the Auto Archiver](setup.md#running) and have it auto-generate a default configuration for you. Then, if needed, you can edit the configuration file using one of the following methods.
 
-## Configuring using a file
 
-The recommended way to configure auto-archiver for long-term and deployed projects is a configuration file, typically called `orchestration.yaml`. This is a YAML file containing all the settings for your entire workflow.
+## 1. Configuration file
 
-The structure of orchestration file is split into 2 parts: `steps` (what [steps](../flow_overview.md) to use) and `configurations` (settings for different modules), here's a simplification:
+The configuration file is typically called `orchestration.yaml` and stored in the `secrets` folder on your desktop. The configuration file contains all the settings for your entire Auto Archiver workflow in one easy-to-find place.
+
+If you want to have Auto Archiver run with the recommended 'basic' setup, 
+
+### Advanced Configuration
+
+The structure of orchestration file is split into 2 parts: `steps` (what [steps](../flow_overview.md) to use) and `configurations` (settings for individual modules).
 
 A default `orchestration.yaml` will be created for you the first time you run auto-archiver (without any arguments). Here's what it looks like:
 
@@ -21,9 +26,9 @@ A default `orchestration.yaml` will be created for you the first time you run au
 
 </details>
 
-## Configuring from the Command Line
+## 2. Command Line configuration
 
-You can run auto-archiver directy from the command line, without the need for a configuration file, command line arguments are parsed using the format `module_name.config_value`. For example, a config value of `api_key` in the `instagram_extractor` module would be passed on the command line with the flag `--instagram_extractor.api_key=API_KEY`.
+You can run auto-archiver directly from the command line, without the need for a configuration file, command line arguments are parsed using the format `module_name.config_value`. For example, a config value of `api_key` in the `instagram_extractor` module would be passed on the command line with the flag `--instagram_extractor.api_key=API_KEY`.
 
 The command line arguments are useful for testing or editing config values and enabling/disabling modules on the fly. When you are happy with your settings, you can store them back in your configuration file by passing the `-s/--store` flag on the command line.
 

docs/source/installation/installation.md

@@ -1,80 +1,44 @@
-# Installing Auto Archiver
+# Installation
 
-```{toctree}
-:depth: 1
-:hidden:
+There are 3 main ways to use the auto-archiver. We recommend the 'docker' method for most uses. This installs all the requirements in one command.
 
-configurations.md
-config_cheatsheet.md
-```
-
-There are 3  main ways to use the auto-archiver:
-1. Easiest: [via docker](#installing-with-docker)
+1. Easiest (recommended): [via docker](#installing-with-docker)
 2. Local Install: [using pip](#installing-locally-with-pip)
 3. Developer Install: [see the developer guidelines](../development/developer_guidelines)
 
-
-But **you always need a configuration/orchestration file**, which is where you'll configure where/what/how to archive. Make sure you read [orchestration](#orchestration).
-
-
-## Installing with Docker
+## 1. Installing with Docker
 
 [![dockeri.co](https://dockerico.blankenship.io/image/bellingcat/auto-archiver)](https://hub.docker.com/r/bellingcat/auto-archiver)
 
-Docker works like a virtual machine running inside your computer, it isolates everything and makes installation simple. Since it is an isolated environment when you need to pass it your orchestration file or get downloaded media out of docker you will need to connect folders on your machine with folders inside docker with the `-v` volume flag.
+Docker works like a virtual machine running inside your computer, making installation simple. You'll need to first set up Docker, and then download the Auto Archiver 'image':
 
 
-1. Install [docker](https://docs.docker.com/get-docker/)
-2. Pull the auto-archiver docker [image](https://hub.docker.com/r/bellingcat/auto-archiver) with `docker pull bellingcat/auto-archiver`
-3. Run the docker image locally in a container: `docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver --config secrets/orchestration.yaml` breaking this command down:
-   1. `docker run` tells docker to start a new container (an instance of the image)
-   2. `--rm` makes sure this container is removed after execution (less garbage locally)
-   3. `-v $PWD/secrets:/app/secrets` - your secrets folder
-      1. `-v` is a volume flag which means a folder that you have on your computer will be connected to a folder inside the docker container
-      2. `$PWD/secrets` points to a `secrets/` folder in your current working directory (where your console points to), we use this folder as a best practice to hold all the secrets/tokens/passwords/... you use
-      3. `/app/secrets` points to the path the docker container where this image can be found
-   4.  `-v $PWD/local_archive:/app/local_archive` - (optional) if you use local_storage
-       1.  `-v` same as above, this is a volume instruction
-       2.  `$PWD/local_archive` is a folder `local_archive/` in case you want to archive locally and have the files accessible outside docker
-       3.  `/app/local_archive` is a folder inside docker that you can reference in your orchestration.yml file 
+**a) Download and install docker**
 
-### Example invocations
+Go to the [Docker website](https://docs.docker.com/get-docker/) and download right version for your operating system. 
 
-The invocations below will run the auto-archiver Docker image using a configuration file that you have specified
+**b) Pull the Auto Archiver docker image**
+
+Open your command line terminal, and copy-paste / type:
 
 ```bash
-# all the configurations come from ./secrets/orchestration.yaml
-docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver --config secrets/orchestration.yaml
-# uses the same configurations but for another google docs sheet 
-# with a header on row 2 and with some different column names
-# notice that columns is a dictionary so you need to pass it as JSON and it will override only the values provided
-docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver --config secrets/orchestration.yaml --gsheet_feeder.sheet="use it on another sheets doc" --gsheet_feeder.header=2 --gsheet_feeder.columns='{"url": "link"}'
-# all the configurations come from orchestration.yaml and specifies that s3 files should be private
-docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver --config secrets/orchestration.yaml --s3_storage.private=1
+docker pull bellingcat/auto-archiver
 ```
 
-## Installing Locally with Pip
+This will download the docker image, which may take a while.
+
+That's it, all done! You're now ready to set up [your configuration file](configurations.md). Or, if you want to use the recommended defaults, then you can [run Auto Archiver immediately](setup.md#running-a-docker-install).
+
+------------
+
+## 2. Installing Locally with Pip
 
 1. Make sure you have python 3.10 or higher installed
 2. Install the package with your preferred package manager: `pip/pipenv/conda install auto-archiver` or `poetry add auto-archiver`
 3. Test it's installed with `auto-archiver --help`
-4. Install other local dependency requirements (for )
-5. Run it with your orchestration file and pass any flags you want in the command line `auto-archiver --config secrets/orchestration.yaml` if your orchestration file is inside a `secrets/`, which we advise
+4. Install other local dependency requirements (for example `ffmpeg`, `firefox`)
 
-### Example invocations
-
-Once all your [local requirements](#installing-local-requirements) are correctly installed, the
-
-```bash
-# all the configurations come from ./secrets/orchestration.yaml
-auto-archiver --config secrets/orchestration.yaml
-# uses the same configurations but for another google docs sheet 
-# with a header on row 2 and with some different column names
-# notice that columns is a dictionary so you need to pass it as JSON and it will override only the values provided
-auto-archiver --config secrets/orchestration.yaml --gsheet_feeder.sheet="use it on another sheets doc" --gsheet_feeder.header=2 --gsheet_feeder.columns='{"url": "link"}'
-# all the configurations come from orchestration.yaml and specifies that s3 files should be private
-auto-archiver --config secrets/orchestration.yaml --s3_storage.private=1
-```
+After this, you're ready to set up your [your configuration file](configurations.md), or if you want to use the recommended defaults, then you can [run Auto Archiver immediately](setup.md#running-a-local-install).
 
 ### Installing Local Requirements
 

docs/source/installation/requirements.md

@@ -0,0 +1,14 @@
+# Requirements
+
+Using the Auto Archiver is very simple, but ideally you have some familiarity with using the command line to run programs. ([Command line crash course](https://developer.mozilla.org/en-US/docs/Learn_web_development/Getting_started/Environment_setup/Command_line)).
+
+### System Requirements
+
+* Auto Archiver works on any Windows, macOS and Linux computer
+* If you're using the **local install** method, then you should make sure to have python3.10+ installed
+
+### Storage Requirements
+
+By default, Auto Archiver uses your local computer storage for any downloaded media (videos, images etc.). If you're downloading large files, this may take up a lot of your local computer's space (more than 5GB of space).
+
+If your storage space is limited, then you may want to set up an [alternative storage method](../modules/storage.md) for your media.

docs/source/installation/setup.md

@@ -0,0 +1,77 @@
+# Getting Started
+
+```{toctree}
+:maxdepth: 1
+:hidden:
+
+installation.md
+configurations.md
+config_editor.md
+authentication.md
+requirements.md
+config_cheatsheet.md
+```
+
+## Getting Started
+
+To get started with Auto Archiver, there are 3 main steps you need to complete.
+
+1. [Install Auto Archiver](installation.md)
+2. [Setup up your configuration](configurations.md) (if you are ok with the default settings, you can skip this step)
+3. Run the archiving process<a id="running"></a>
+
+The way you run the Auto Archiver depends on how you installed it (docker install or local install)
+
+### Running a Docker Install
+
+If you installed Auto Archiver using docker, open up your terminal, and copy-paste / type the following command:
+
+```bash
+docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver
+ ```
+
+breaking this command down:
+   1. `docker run` tells docker to start a new container (an instance of the image)
+   2. `--rm` makes sure this container is removed after execution (less garbage locally)
+   3. `-v $PWD/secrets:/app/secrets` - your secrets folder with settings
+      1. `-v` is a volume flag which means a folder that you have on your computer will be connected to a folder inside the docker container
+      2. `$PWD/secrets` points to a `secrets/` folder in your current working directory (where your console points to), we use this folder as a best practice to hold all the secrets/tokens/passwords/... you use
+      3. `/app/secrets` points to the path the docker container where this image can be found
+   4.  `-v $PWD/local_archive:/app/local_archive` - (optional) if you use local_storage
+       1.  `-v` same as above, this is a volume instruction
+       2.  `$PWD/local_archive` is a folder `local_archive/` in case you want to archive locally and have the files accessible outside docker
+       3.  `/app/local_archive` is a folder inside docker that you can reference in your orchestration.yml file 
+
+### Example invocations
+
+The invocations below will run the auto-archiver Docker image using a configuration file that you have specified
+
+```bash
+# Have auto-archiver run with the default settings, generating a settings file in ./secrets/orchestration.yaml
+docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver
+
+# uses the same configuration, but with the `gsheet_feeder`, a header on row 2 and with some different column names
+# notice that columns is a dictionary so you need to pass it as JSON and it will override only the values provided
+docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver --feeders=gsheet_feeder --gsheet_feeder.sheet="use it on another sheets doc" --gsheet_feeder.header=2 --gsheet_feeder.columns='{"url": "link"}'
+# Runs auto-archiver for the first time, but in 'full' mode, enabling all modules to get a full settings file
+docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver --mode full
+```
+
+------------
+
+### Running a Local Install
+
+### Example invocations
+
+Once all your [local requirements](#installing-local-requirements) are correctly installed, the
+
+```bash
+# all the configurations come from ./secrets/orchestration.yaml
+auto-archiver --config secrets/orchestration.yaml
+# uses the same configurations but for another google docs sheet 
+# with a header on row 2 and with some different column names
+# notice that columns is a dictionary so you need to pass it as JSON and it will override only the values provided
+auto-archiver --config secrets/orchestration.yaml --gsheet_feeder.sheet="use it on another sheets doc" --gsheet_feeder.header=2 --gsheet_feeder.columns='{"url": "link"}'
+# all the configurations come from orchestration.yaml and specifies that s3 files should be private
+auto-archiver --config secrets/orchestration.yaml --s3_storage.private=1
+```

docs/source/modules/database.md

@@ -8,7 +8,7 @@ The default (enabled) databases are the CSV Database and the Console Database.
 ```
 
 ```{toctree}
-:depth: 1
+:maxdepth: 1
 :hidden:
 :glob:
 autogen/database/*

docs/source/modules/enricher.md

@@ -7,7 +7,7 @@ Enricher modules are used to add additional information to the items  that have
 ```
 
 ```{toctree}
-:depth: 1
+:maxdepth: 1
 :hidden:
 :glob:
 autogen/enricher/*

docs/source/modules/extractor.md

@@ -4,14 +4,14 @@ Extractor modules are used to extract the content of a given URL. Typically, one
 
 Extractors that are able to extract content from a wide range of websites include:
 1. Generic Extractor: parses videos and images on sites using the powerful yt-dlp library.
-2. Wayback Machine Extractor: sends pages to the Waygback machine for archiving, and stores the link.
+2. Wayback Machine Extractor: sends pages to the Wayback machine for archiving, and stores the link.
 3. WACZ Extractor: runs a web browser to 'browse' the URL and save a copy of the page in WACZ format. 
 
 ```{include} autogen/extractor.md
 ```
 
 ```{toctree}
-:depth: 1
+:maxdepth: 1
 :hidden:
 :glob:
 autogen/extractor/*

docs/source/modules/feeder.md

@@ -1,8 +1,8 @@
 # Feeder Modules
 
-Feeder modules are used to feed URLs into the `auto-archiver` for processing. Feeders can take these URLs from a variety of sources, such as a file, a database, or the command line.
+Feeder modules are used to feed URLs into the Auto Archiver for processing. Feeders can take these URLs from a variety of sources, such as a file, a database, or the command line.
 
-The default feeder is the command line feeder (`cli_feeder`), which allows you to input URLs directly into the `auto-archiver` from the command line.
+The default feeder is the command line feeder (`cli_feeder`), which allows you to input URLs directly into `auto-archiver` from the command line.
 
 Command line feeder usage:
 ```{code} bash
@@ -13,7 +13,7 @@ auto-archiver [options] -- URL1 URL2 ...
 ```
 
 ```{toctree}
-:depth: 1
+:maxdepth: 1
 :glob:
 :hidden:
 autogen/feeder/*

docs/source/modules/formatter.md

@@ -6,7 +6,7 @@ Formatter modules are used to format the data extracted from a URL into a specif
 ```
 
 ```{toctree}
-:depth: 1
+:maxdepth: 1
 :hidden:
 :glob:
 autogen/formatter/*

docs/source/modules/storage.md

@@ -8,7 +8,7 @@ The default is to store the files downloaded (e.g. images, videos) in a local di
 ```
 
 ```{toctree}
-:depth: 1
+:maxdepth: 1
 :hidden:
 :glob:
 autogen/storage/*

poetry.lock

@@ -103,14 +103,14 @@ tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"]
 
 [[package]]
 name = "authlib"
-version = "1.4.0"
+version = "1.5.1"
 description = "The ultimate Python library in building OAuth and OpenID Connect servers and clients."
 optional = false
 python-versions = ">=3.9"
 groups = ["main"]
 files = [
-    {file = "Authlib-1.4.0-py2.py3-none-any.whl", hash = "sha256:4bb20b978c8b636222b549317c1815e1fe62234fc1c5efe8855d84aebf3a74e3"},
-    {file = "authlib-1.4.0.tar.gz", hash = "sha256:1c1e6608b5ed3624aeeee136ca7f8c120d6f51f731aa152b153d54741840e1f2"},
+    {file = "authlib-1.5.1-py2.py3-none-any.whl", hash = "sha256:8408861cbd9b4ea2ff759b00b6f02fd7d81ac5a56d0b2b22c08606c6049aae11"},
+    {file = "authlib-1.5.1.tar.gz", hash = "sha256:5cbc85ecb0667312c1cdc2f9095680bb735883b123fb509fde1e65b1c5df972e"},
 ]
 
 [package.dependencies]
@@ -134,33 +134,34 @@ tomli = {version = "*", markers = "python_version < \"3.11\""}
 
 [[package]]
 name = "babel"
-version = "2.16.0"
+version = "2.17.0"
 description = "Internationalization utilities"
 optional = false
 python-versions = ">=3.8"
 groups = ["docs"]
 files = [
-    {file = "babel-2.16.0-py3-none-any.whl", hash = "sha256:368b5b98b37c06b7daf6696391c3240c938b37767d4584413e8438c5c435fa8b"},
-    {file = "babel-2.16.0.tar.gz", hash = "sha256:d1f3554ca26605fe173f3de0c65f750f5a42f924499bf134de6423582298e316"},
+    {file = "babel-2.17.0-py3-none-any.whl", hash = "sha256:4d0b53093fdfb4b21c92b5213dba5a1b23885afa8383709427046b21c366e5f2"},
+    {file = "babel-2.17.0.tar.gz", hash = "sha256:0c54cffb19f690cdcc52a3b50bcbf71e07a808d1c80d549f2459b9d2cf0afb9d"},
 ]
 
 [package.extras]
-dev = ["freezegun (>=1.0,<2.0)", "pytest (>=6.0)", "pytest-cov"]
+dev = ["backports.zoneinfo", "freezegun (>=1.0,<2.0)", "jinja2 (>=3.0)", "pytest (>=6.0)", "pytest-cov", "pytz", "setuptools", "tzdata"]
 
 [[package]]
 name = "beautifulsoup4"
-version = "4.12.3"
+version = "4.13.3"
 description = "Screen-scraping library"
 optional = false
-python-versions = ">=3.6.0"
+python-versions = ">=3.7.0"
 groups = ["main", "docs"]
 files = [
-    {file = "beautifulsoup4-4.12.3-py3-none-any.whl", hash = "sha256:b80878c9f40111313e55da8ba20bdba06d8fa3969fc68304167741bbf9e082ed"},
-    {file = "beautifulsoup4-4.12.3.tar.gz", hash = "sha256:74e3d1928edc070d21748185c46e3fb33490f22f52a3addee9aee0f4f7781051"},
+    {file = "beautifulsoup4-4.13.3-py3-none-any.whl", hash = "sha256:99045d7d3f08f91f0d656bc9b7efbae189426cd913d830294a15eefa0ea4df16"},
+    {file = "beautifulsoup4-4.13.3.tar.gz", hash = "sha256:1bd32405dacc920b42b83ba01644747ed77456a65760e285fbc47633ceddaf8b"},
 ]
 
 [package.dependencies]
 soupsieve = ">1.2"
+typing-extensions = ">=4.0.0"
 
 [package.extras]
 cchardet = ["cchardet"]
@@ -171,18 +172,18 @@ lxml = ["lxml"]
 
 [[package]]
 name = "boto3"
-version = "1.36.6"
+version = "1.37.8"
 description = "The AWS SDK for Python"
 optional = false
 python-versions = ">=3.8"
 groups = ["main"]
 files = [
-    {file = "boto3-1.36.6-py3-none-any.whl", hash = "sha256:6d473f0f340d02b4e9ad5b8e68786a09728101a8b950231b89ebdaf72b6dca21"},
-    {file = "boto3-1.36.6.tar.gz", hash = "sha256:b36feae061dc0793cf311468956a0a9e99215ce38bc99a1a4e55a5b105f16297"},
+    {file = "boto3-1.37.8-py3-none-any.whl", hash = "sha256:b9f506e08c9f54687d6c073ef1c550a24a62cc2d1e0bc7cda9f13112a38818bf"},
+    {file = "boto3-1.37.8.tar.gz", hash = "sha256:9448f4a079189e19c3253cfdc5b8ef6dc51a3b82431e8347a51f4c1b2d9dab42"},
 ]
 
 [package.dependencies]
-botocore = ">=1.36.6,<1.37.0"
+botocore = ">=1.37.8,<1.38.0"
 jmespath = ">=0.7.1,<2.0.0"
 s3transfer = ">=0.11.0,<0.12.0"
 
@@ -191,14 +192,14 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"]
 
 [[package]]
 name = "botocore"
-version = "1.36.6"
+version = "1.37.8"
 description = "Low-level, data-driven core of boto 3."
 optional = false
 python-versions = ">=3.8"
 groups = ["main"]
 files = [
-    {file = "botocore-1.36.6-py3-none-any.whl", hash = "sha256:f77bbbb03fb420e260174650fb5c0cc142ec20a96967734eed2b0ef24334ef34"},
-    {file = "botocore-1.36.6.tar.gz", hash = "sha256:4864c53d638da191a34daf3ede3ff1371a3719d952cc0c6bd24ce2836a38dd77"},
+    {file = "botocore-1.37.8-py3-none-any.whl", hash = "sha256:a6c94f33de12f4b10b10684019e554c980469b8394c6d82448a738cbd8452cef"},
+    {file = "botocore-1.37.8.tar.gz", hash = "sha256:b5825e08dd3e25642aa22a0d7d92bf81fef1ef857117e4155f923bbccf5aba63"},
 ]
 
 [package.dependencies]
@@ -207,7 +208,7 @@ python-dateutil = ">=2.1,<3.0.0"
 urllib3 = {version = ">=1.25.4,<2.2.0 || >2.2.0,<3", markers = "python_version >= \"3.10\""}
 
 [package.extras]
-crt = ["awscrt (==0.23.4)"]
+crt = ["awscrt (==0.23.8)"]
 
 [[package]]
 name = "brotli"
@@ -362,14 +363,14 @@ beautifulsoup4 = "*"
 
 [[package]]
 name = "cachetools"
-version = "5.5.1"
+version = "5.5.2"
 description = "Extensible memoizing collections and decorators"
 optional = false
 python-versions = ">=3.7"
 groups = ["main"]
 files = [
-    {file = "cachetools-5.5.1-py3-none-any.whl", hash = "sha256:b76651fdc3b24ead3c648bbdeeb940c1b04d365b38b4af66788f9ec4a81d42bb"},
-    {file = "cachetools-5.5.1.tar.gz", hash = "sha256:70f238fbba50383ef62e55c6aff6d9673175fe59f7c6782c7a0b9e38f4a9df95"},
+    {file = "cachetools-5.5.2-py3-none-any.whl", hash = "sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a"},
+    {file = "cachetools-5.5.2.tar.gz", hash = "sha256:1a661caa9175d26759571b2e19580f9d6393969e5dfca11fdb1f947a23e640d4"},
 ]
 
 [[package]]
@@ -674,26 +675,26 @@ typing-inspect = ">=0.4.0,<1"
 
 [[package]]
 name = "dateparser"
-version = "1.2.0"
+version = "1.2.1"
 description = "Date parsing library designed to parse dates from HTML pages"
 optional = false
-python-versions = ">=3.7"
+python-versions = ">=3.8"
 groups = ["main"]
 files = [
-    {file = "dateparser-1.2.0-py2.py3-none-any.whl", hash = "sha256:0b21ad96534e562920a0083e97fd45fa959882d4162acc358705144520a35830"},
-    {file = "dateparser-1.2.0.tar.gz", hash = "sha256:7975b43a4222283e0ae15be7b4999d08c9a70e2d378ac87385b1ccf2cffbbb30"},
+    {file = "dateparser-1.2.1-py3-none-any.whl", hash = "sha256:bdcac262a467e6260030040748ad7c10d6bacd4f3b9cdb4cfd2251939174508c"},
+    {file = "dateparser-1.2.1.tar.gz", hash = "sha256:7e4919aeb48481dbfc01ac9683c8e20bfe95bb715a38c1e9f6af889f4f30ccc3"},
 ]
 
 [package.dependencies]
-python-dateutil = "*"
-pytz = "*"
-regex = "<2019.02.19 || >2019.02.19,<2021.8.27 || >2021.8.27"
-tzlocal = "*"
+python-dateutil = ">=2.7.0"
+pytz = ">=2024.2"
+regex = ">=2015.06.24,<2019.02.19 || >2019.02.19,<2021.8.27 || >2021.8.27"
+tzlocal = ">=0.2"
 
 [package.extras]
-calendars = ["convertdate", "hijri-converter"]
-fasttext = ["fasttext"]
-langdetect = ["langdetect"]
+calendars = ["convertdate (>=2.2.1)", "hijridate"]
+fasttext = ["fasttext (>=0.9.1)", "numpy (>=1.19.3,<2)"]
+langdetect = ["langdetect (>=1.0.0)"]
 
 [[package]]
 name = "docutils"
@@ -755,14 +756,14 @@ files = [
 
 [[package]]
 name = "google-api-core"
-version = "2.24.0"
+version = "2.24.1"
 description = "Google API client core library"
 optional = false
 python-versions = ">=3.7"
 groups = ["main"]
 files = [
-    {file = "google_api_core-2.24.0-py3-none-any.whl", hash = "sha256:10d82ac0fca69c82a25b3efdeefccf6f28e02ebb97925a8cce8edbfe379929d9"},
-    {file = "google_api_core-2.24.0.tar.gz", hash = "sha256:e255640547a597a4da010876d333208ddac417d60add22b6851a0c66a831fcaf"},
+    {file = "google_api_core-2.24.1-py3-none-any.whl", hash = "sha256:bc78d608f5a5bf853b80bd70a795f703294de656c096c0968320830a4bc280f1"},
+    {file = "google_api_core-2.24.1.tar.gz", hash = "sha256:f8b36f5456ab0dd99a1b693a40a31d1e7757beea380ad1b38faaf8941eae9d8a"},
 ]
 
 [package.dependencies]
@@ -780,14 +781,14 @@ grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"]
 
 [[package]]
 name = "google-api-python-client"
-version = "2.159.0"
+version = "2.163.0"
 description = "Google API Client Library for Python"
 optional = false
 python-versions = ">=3.7"
 groups = ["main"]
 files = [
-    {file = "google_api_python_client-2.159.0-py2.py3-none-any.whl", hash = "sha256:baef0bb631a60a0bd7c0bf12a5499e3a40cd4388484de7ee55c1950bf820a0cf"},
-    {file = "google_api_python_client-2.159.0.tar.gz", hash = "sha256:55197f430f25c907394b44fa078545ffef89d33fd4dca501b7db9f0d8e224bd6"},
+    {file = "google_api_python_client-2.163.0-py2.py3-none-any.whl", hash = "sha256:080e8bc0669cb4c1fb8efb8da2f5b91a2625d8f0e7796cfad978f33f7016c6c4"},
+    {file = "google_api_python_client-2.163.0.tar.gz", hash = "sha256:88dee87553a2d82176e2224648bf89272d536c8f04dcdda37ef0a71473886dd7"},
 ]
 
 [package.dependencies]
@@ -859,14 +860,14 @@ tool = ["click (>=6.0.0)"]
 
 [[package]]
 name = "googleapis-common-protos"
-version = "1.66.0"
+version = "1.69.1"
 description = "Common protobufs used in Google APIs"
 optional = false
 python-versions = ">=3.7"
 groups = ["main"]
 files = [
-    {file = "googleapis_common_protos-1.66.0-py2.py3-none-any.whl", hash = "sha256:d7abcd75fabb2e0ec9f74466401f6c119a0b498e27370e9be4c94cb7e382b8ed"},
-    {file = "googleapis_common_protos-1.66.0.tar.gz", hash = "sha256:c3e7b33d15fdca5374cc0a7346dd92ffa847425cc4ea941d970f13680052ec8c"},
+    {file = "googleapis_common_protos-1.69.1-py2.py3-none-any.whl", hash = "sha256:4077f27a6900d5946ee5a369fab9c8ded4c0ef1c6e880458ea2f70c14f7b70d5"},
+    {file = "googleapis_common_protos-1.69.1.tar.gz", hash = "sha256:e20d2d8dda87da6fe7340afbbdf4f0bcb4c8fae7e6cadf55926c31f946b0b9b1"},
 ]
 
 [package.dependencies]
@@ -877,14 +878,14 @@ grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"]
 
 [[package]]
 name = "gspread"
-version = "6.1.4"
+version = "6.2.0"
 description = "Google Spreadsheets Python API"
 optional = false
 python-versions = ">=3.8"
 groups = ["main"]
 files = [
-    {file = "gspread-6.1.4-py3-none-any.whl", hash = "sha256:c34781c426031a243ad154952b16f21ac56a5af90687885fbee3d1fba5280dcd"},
-    {file = "gspread-6.1.4.tar.gz", hash = "sha256:b8eec27de7cadb338bb1b9f14a9be168372dee8965c0da32121816b5050ac1de"},
+    {file = "gspread-6.2.0-py3-none-any.whl", hash = "sha256:7fa1a11e1ecacc6c5946fa016be05941baca8540404314f59aec963dd8ae5db3"},
+    {file = "gspread-6.2.0.tar.gz", hash = "sha256:bc3d02d1c39e0b40bfc8035b4fec407aa71a17f343fc81cc7e3f75bfa6555de6"},
 ]
 
 [package.dependencies]
@@ -977,14 +978,14 @@ browser-cookie3 = ["browser_cookie3 (>=0.19.1)"]
 
 [[package]]
 name = "jinja2"
-version = "3.1.5"
+version = "3.1.6"
 description = "A very fast and expressive template engine."
 optional = false
 python-versions = ">=3.7"
 groups = ["main", "docs"]
 files = [
-    {file = "jinja2-3.1.5-py3-none-any.whl", hash = "sha256:aba0f4dc9ed8013c424088f68a5c226f7d6097ed89b246d7749c2ec4175c6adb"},
-    {file = "jinja2-3.1.5.tar.gz", hash = "sha256:8fefff8dc3034e27bb80d67c671eb8a9bc424c0ef4c0826edbff304cceff43bb"},
+    {file = "jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67"},
+    {file = "jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d"},
 ]
 
 [package.dependencies]
@@ -1158,14 +1159,14 @@ files = [
 
 [[package]]
 name = "marshmallow"
-version = "3.26.0"
+version = "3.26.1"
 description = "A lightweight library for converting complex datatypes to and from native Python datatypes."
 optional = false
 python-versions = ">=3.9"
 groups = ["main"]
 files = [
-    {file = "marshmallow-3.26.0-py3-none-any.whl", hash = "sha256:1287bca04e6a5f4094822ac153c03da5e214a0a60bcd557b140f3e66991b8ca1"},
-    {file = "marshmallow-3.26.0.tar.gz", hash = "sha256:eb36762a1cc76d7abf831e18a3a1b26d3d481bbc74581b8e532a3d3a8115e1cb"},
+    {file = "marshmallow-3.26.1-py3-none-any.whl", hash = "sha256:3350409f20a70a7e4e11a27661187b77cdcaeb20abca41c1454fe33636bea09c"},
+    {file = "marshmallow-3.26.1.tar.gz", hash = "sha256:e6d8affb6cb61d39d26402096dc0aee12d5a26d490a121f118d2e81dc0719dc6"},
 ]
 
 [package.dependencies]
@@ -1234,14 +1235,14 @@ files = [
 
 [[package]]
 name = "myst-parser"
-version = "4.0.0"
+version = "4.0.1"
 description = "An extended [CommonMark](https://spec.commonmark.org/) compliant parser,"
 optional = false
 python-versions = ">=3.10"
 groups = ["docs"]
 files = [
-    {file = "myst_parser-4.0.0-py3-none-any.whl", hash = "sha256:b9317997552424448c6096c2558872fdb6f81d3ecb3a40ce84a7518798f3f28d"},
-    {file = "myst_parser-4.0.0.tar.gz", hash = "sha256:851c9dfb44e36e56d15d05e72f02b80da21a9e0d07cba96baf5e2d476bb91531"},
+    {file = "myst_parser-4.0.1-py3-none-any.whl", hash = "sha256:9134e88959ec3b5780aedf8a99680ea242869d012e8821db3126d427edc9c95d"},
+    {file = "myst_parser-4.0.1.tar.gz", hash = "sha256:5cfea715e4f3574138aecbf7d54132296bfd72bb614d31168f48c477a830a7c4"},
 ]
 
 [package.dependencies]
@@ -1253,10 +1254,10 @@ pyyaml = "*"
 sphinx = ">=7,<9"
 
 [package.extras]
-code-style = ["pre-commit (>=3.0,<4.0)"]
+code-style = ["pre-commit (>=4.0,<5.0)"]
 linkify = ["linkify-it-py (>=2.0,<3.0)"]
 rtd = ["ipython", "sphinx (>=7)", "sphinx-autodoc2 (>=0.5.0,<0.6.0)", "sphinx-book-theme (>=1.1,<2.0)", "sphinx-copybutton", "sphinx-design", "sphinx-pyscript", "sphinx-tippy (>=0.4.3)", "sphinx-togglebutton", "sphinxext-opengraph (>=0.9.0,<0.10.0)", "sphinxext-rediraffe (>=0.2.7,<0.3.0)"]
-testing = ["beautifulsoup4", "coverage[toml]", "defusedxml", "pytest (>=8,<9)", "pytest-cov", "pytest-param-files (>=0.6.0,<0.7.0)", "pytest-regressions", "sphinx-pytest"]
+testing = ["beautifulsoup4", "coverage[toml]", "defusedxml", "pygments (<2.19)", "pytest (>=8,<9)", "pytest-cov", "pytest-param-files (>=0.6.0,<0.7.0)", "pytest-regressions", "sphinx-pytest"]
 testing-docutils = ["pygments", "pytest (>=8,<9)", "pytest-param-files (>=0.6.0,<0.7.0)"]
 
 [[package]]
@@ -1530,14 +1531,14 @@ testing = ["pytest", "pytest-benchmark"]
 
 [[package]]
 name = "proto-plus"
-version = "1.25.0"
-description = "Beautiful, Pythonic protocol buffers."
+version = "1.26.0"
+description = "Beautiful, Pythonic protocol buffers"
 optional = false
 python-versions = ">=3.7"
 groups = ["main"]
 files = [
-    {file = "proto_plus-1.25.0-py3-none-any.whl", hash = "sha256:c91fc4a65074ade8e458e95ef8bac34d4008daa7cce4a12d6707066fca648961"},
-    {file = "proto_plus-1.25.0.tar.gz", hash = "sha256:fbb17f57f7bd05a68b7707e745e26528b0b3c34e378db91eef93912c54982d91"},
+    {file = "proto_plus-1.26.0-py3-none-any.whl", hash = "sha256:bf2dfaa3da281fc3187d12d224c707cb57214fb2c22ba854eb0c105a3fb2d4d7"},
+    {file = "proto_plus-1.26.0.tar.gz", hash = "sha256:6e93d5f5ca267b54300880fff156b6a3386b3fa3f43b1da62e680fc0c586ef22"},
 ]
 
 [package.dependencies]
@@ -1673,14 +1674,14 @@ files = [
 
 [[package]]
 name = "pydata-sphinx-theme"
-version = "0.16.1"
+version = "0.15.4"
 description = "Bootstrap-based Sphinx theme from the PyData community"
 optional = false
 python-versions = ">=3.9"
 groups = ["docs"]
 files = [
-    {file = "pydata_sphinx_theme-0.16.1-py3-none-any.whl", hash = "sha256:225331e8ac4b32682c18fcac5a57a6f717c4e632cea5dd0e247b55155faeccde"},
-    {file = "pydata_sphinx_theme-0.16.1.tar.gz", hash = "sha256:a08b7f0b7f70387219dc659bff0893a7554d5eb39b59d3b8ef37b8401b7642d7"},
+    {file = "pydata_sphinx_theme-0.15.4-py3-none-any.whl", hash = "sha256:2136ad0e9500d0949f96167e63f3e298620040aea8f9c74621959eda5d4cf8e6"},
+    {file = "pydata_sphinx_theme-0.15.4.tar.gz", hash = "sha256:7762ec0ac59df3acecf49fd2f889e1b4565dbce8b88b2e29ee06fdd90645a06d"},
 ]
 
 [package.dependencies]
@@ -1688,8 +1689,9 @@ accessible-pygments = "*"
 Babel = "*"
 beautifulsoup4 = "*"
 docutils = "!=0.17.0"
+packaging = "*"
 pygments = ">=2.7"
-sphinx = ">=6.1"
+sphinx = ">=5"
 typing-extensions = "*"
 
 [package.extras]
@@ -1775,14 +1777,14 @@ files = [
 
 [[package]]
 name = "pytest"
-version = "8.3.4"
+version = "8.3.5"
 description = "pytest: simple powerful testing with Python"
 optional = false
 python-versions = ">=3.8"
 groups = ["dev"]
 files = [
-    {file = "pytest-8.3.4-py3-none-any.whl", hash = "sha256:50e16d954148559c9a74109af1eaf0c945ba2d8f30f0a3d3335edde19788b6f6"},
-    {file = "pytest-8.3.4.tar.gz", hash = "sha256:965370d062bce11e73868e0335abac31b4d3de0e82f4007408d242b4f8610761"},
+    {file = "pytest-8.3.5-py3-none-any.whl", hash = "sha256:c69214aa47deac29fad6c2a4f590b9c4a9fdb16a403176fe154b79c0b4d4d820"},
+    {file = "pytest-8.3.5.tar.gz", hash = "sha256:f4efe70cc14e511565ac476b57c279e12a855b11f48f212af1080ef2263d3845"},
 ]
 
 [package.dependencies]
@@ -1814,6 +1816,24 @@ loguru = "*"
 [package.extras]
 test = ["pytest", "pytest-cov"]
 
+[[package]]
+name = "pytest-mock"
+version = "3.14.0"
+description = "Thin-wrapper around the mock package for easier use with pytest"
+optional = false
+python-versions = ">=3.8"
+groups = ["dev"]
+files = [
+    {file = "pytest-mock-3.14.0.tar.gz", hash = "sha256:2719255a1efeceadbc056d6bf3df3d1c5015530fb40cf347c0f9afac88410bd0"},
+    {file = "pytest_mock-3.14.0-py3-none-any.whl", hash = "sha256:0b72c38033392a5f4621342fe11e9219ac11ec9d375f8e2a0c164539e0d70f6f"},
+]
+
+[package.dependencies]
+pytest = ">=6.2.5"
+
+[package.extras]
+dev = ["pre-commit", "pytest-asyncio", "tox"]
+
 [[package]]
 name = "python-dateutil"
 version = "2.9.0.post0"
@@ -1866,14 +1886,14 @@ requests = ">=2.28"
 
 [[package]]
 name = "pytz"
-version = "2024.2"
+version = "2025.1"
 description = "World timezone definitions, modern and historical"
 optional = false
 python-versions = "*"
 groups = ["main"]
 files = [
-    {file = "pytz-2024.2-py2.py3-none-any.whl", hash = "sha256:31c7c1817eb7fae7ca4b8c7ee50c72f93aa2dd863de768e1ef4245d426aa0725"},
-    {file = "pytz-2024.2.tar.gz", hash = "sha256:2aa355083c50a0f93fa581709deac0c9ad65cca8a9e9beac660adcbd493c798a"},
+    {file = "pytz-2025.1-py2.py3-none-any.whl", hash = "sha256:89dd22dca55b46eac6eda23b2d72721bf1bdfef212645d81513ef5d03038de57"},
+    {file = "pytz-2025.1.tar.gz", hash = "sha256:c2db42be2a2518b28e65f9207c4d05e6ff547d1efa4086469ef855e4ab70178e"},
 ]
 
 [[package]]
@@ -2122,14 +2142,14 @@ jupyter = ["ipywidgets (>=7.5.1,<9)"]
 
 [[package]]
 name = "rich-argparse"
-version = "1.6.0"
+version = "1.7.0"
 description = "Rich help formatters for argparse and optparse"
 optional = false
 python-versions = ">=3.8"
 groups = ["main"]
 files = [
-    {file = "rich_argparse-1.6.0-py3-none-any.whl", hash = "sha256:fbe70a1d821b3f2fa8958cddf0cae131870a6e9faa04ab52b409cb1eda809bd7"},
-    {file = "rich_argparse-1.6.0.tar.gz", hash = "sha256:092083c30da186f25bcdff8b1d47fdfb571288510fb051e0488a72cc3128de13"},
+    {file = "rich_argparse-1.7.0-py3-none-any.whl", hash = "sha256:b8ec8943588e9731967f4f97b735b03dc127c416f480a083060433a97baf2fd3"},
+    {file = "rich_argparse-1.7.0.tar.gz", hash = "sha256:f31d809c465ee43f367d599ccaf88b73bc2c4d75d74ed43f2d538838c53544ba"},
 ]
 
 [package.dependencies]
@@ -2228,32 +2248,32 @@ files = [
 
 [[package]]
 name = "s3transfer"
-version = "0.11.2"
+version = "0.11.4"
 description = "An Amazon S3 Transfer Manager"
 optional = false
 python-versions = ">=3.8"
 groups = ["main"]
 files = [
-    {file = "s3transfer-0.11.2-py3-none-any.whl", hash = "sha256:be6ecb39fadd986ef1701097771f87e4d2f821f27f6071c872143884d2950fbc"},
-    {file = "s3transfer-0.11.2.tar.gz", hash = "sha256:3b39185cb72f5acc77db1a58b6e25b977f28d20496b6e58d6813d75f464d632f"},
+    {file = "s3transfer-0.11.4-py3-none-any.whl", hash = "sha256:ac265fa68318763a03bf2dc4f39d5cbd6a9e178d81cc9483ad27da33637e320d"},
+    {file = "s3transfer-0.11.4.tar.gz", hash = "sha256:559f161658e1cf0a911f45940552c696735f5c74e64362e515f333ebed87d679"},
 ]
 
 [package.dependencies]
-botocore = ">=1.36.0,<2.0a.0"
+botocore = ">=1.37.4,<2.0a.0"
 
 [package.extras]
-crt = ["botocore[crt] (>=1.36.0,<2.0a.0)"]
+crt = ["botocore[crt] (>=1.37.4,<2.0a.0)"]
 
 [[package]]
 name = "selenium"
-version = "4.28.1"
+version = "4.29.0"
 description = "Official Python bindings for Selenium WebDriver"
 optional = false
 python-versions = ">=3.9"
 groups = ["main"]
 files = [
-    {file = "selenium-4.28.1-py3-none-any.whl", hash = "sha256:4238847e45e24e4472cfcf3554427512c7aab9443396435b1623ef406fff1cc1"},
-    {file = "selenium-4.28.1.tar.gz", hash = "sha256:0072d08670d7ec32db901bd0107695a330cecac9f196e3afb3fa8163026e022a"},
+    {file = "selenium-4.29.0-py3-none-any.whl", hash = "sha256:ce5d26f1ddc1111641113653af33694c13947dd36c2df09cdd33f554351d372e"},
+    {file = "selenium-4.29.0.tar.gz", hash = "sha256:3a62f7ec33e669364a6c0562a701deb69745b569c50d55f1a912bf8eb33358ba"},
 ]
 
 [package.dependencies]
@@ -2362,24 +2382,24 @@ test = ["cython (>=3.0)", "defusedxml (>=0.7.1)", "pytest (>=8.0)", "setuptools
 
 [[package]]
 name = "sphinx-autoapi"
-version = "3.4.0"
+version = "3.6.0"
 description = "Sphinx API documentation generator"
 optional = false
-python-versions = ">=3.8"
+python-versions = ">=3.9"
 groups = ["docs"]
 files = [
-    {file = "sphinx_autoapi-3.4.0-py3-none-any.whl", hash = "sha256:4027fef2875a22c5f2a57107c71641d82f6166bf55beb407a47aaf3ef14e7b92"},
-    {file = "sphinx_autoapi-3.4.0.tar.gz", hash = "sha256:e6d5371f9411bbb9fca358c00a9e57aef3ac94cbfc5df4bab285946462f69e0c"},
+    {file = "sphinx_autoapi-3.6.0-py3-none-any.whl", hash = "sha256:f3b66714493cab140b0e896d33ce7137654a16ac1edb6563edcbd47bf975f711"},
+    {file = "sphinx_autoapi-3.6.0.tar.gz", hash = "sha256:c685f274e41d0842ae7e199460c322c4bd7fec816ccc2da8d806094b4f64af06"},
 ]
 
 [package.dependencies]
 astroid = [
     {version = ">=2.7", markers = "python_version < \"3.12\""},
-    {version = ">=3.0.0a1", markers = "python_version >= \"3.12\""},
+    {version = ">=3", markers = "python_version >= \"3.12\""},
 ]
 Jinja2 = "*"
 PyYAML = "*"
-sphinx = ">=6.1.0"
+sphinx = ">=7.4.0"
 
 [[package]]
 name = "sphinx-autobuild"
@@ -2406,19 +2426,19 @@ test = ["httpx", "pytest (>=6)"]
 
 [[package]]
 name = "sphinx-book-theme"
-version = "1.1.3"
+version = "1.1.4"
 description = "A clean book theme for scientific explanations and documentation with Sphinx"
 optional = false
 python-versions = ">=3.9"
 groups = ["docs"]
 files = [
-    {file = "sphinx_book_theme-1.1.3-py3-none-any.whl", hash = "sha256:a554a9a7ac3881979a87a2b10f633aa2a5706e72218a10f71be38b3c9e831ae9"},
-    {file = "sphinx_book_theme-1.1.3.tar.gz", hash = "sha256:1f25483b1846cb3d353a6bc61b3b45b031f4acf845665d7da90e01ae0aef5b4d"},
+    {file = "sphinx_book_theme-1.1.4-py3-none-any.whl", hash = "sha256:843b3f5c8684640f4a2d01abd298beb66452d1b2394cd9ef5be5ebd5640ea0e1"},
+    {file = "sphinx_book_theme-1.1.4.tar.gz", hash = "sha256:73efe28af871d0a89bd05856d300e61edce0d5b2fbb7984e84454be0fedfe9ed"},
 ]
 
 [package.dependencies]
-pydata-sphinx-theme = ">=0.15.2"
-sphinx = ">=5"
+pydata-sphinx-theme = "0.15.4"
+sphinx = ">=6.1"
 
 [package.extras]
 code-style = ["pre-commit"]
@@ -2565,14 +2585,14 @@ test = ["pytest"]
 
 [[package]]
 name = "starlette"
-version = "0.45.3"
+version = "0.46.0"
 description = "The little ASGI library that shines."
 optional = false
 python-versions = ">=3.9"
 groups = ["docs"]
 files = [
-    {file = "starlette-0.45.3-py3-none-any.whl", hash = "sha256:dfb6d332576f136ec740296c7e8bb8c8a7125044e7c6da30744718880cdd059d"},
-    {file = "starlette-0.45.3.tar.gz", hash = "sha256:2cbcba2a75806f8a41c722141486f37c28e30a0921c5f6fe4346cb0dcee1302f"},
+    {file = "starlette-0.46.0-py3-none-any.whl", hash = "sha256:913f0798bd90ba90a9156383bcf1350a17d6259451d0d8ee27fc0cf2db609038"},
+    {file = "starlette-0.46.0.tar.gz", hash = "sha256:b359e4567456b28d473d0193f34c0de0ed49710d75ef183a74a5ce0499324f50"},
 ]
 
 [package.dependencies]
@@ -2583,14 +2603,14 @@ full = ["httpx (>=0.27.0,<0.29.0)", "itsdangerous", "jinja2", "python-multipart
 
 [[package]]
 name = "telethon"
-version = "1.38.1"
+version = "1.39.0"
 description = "Full-featured Telegram client library for Python 3"
 optional = false
 python-versions = ">=3.5"
 groups = ["main"]
 files = [
-    {file = "Telethon-1.38.1-py3-none-any.whl", hash = "sha256:30c187017501bfb982b8af5659f864dda4108f77ea49cfce61e8f6fdb8a18d6e"},
-    {file = "Telethon-1.38.1.tar.gz", hash = "sha256:f9866c1e37197a0894e0c02aa56a6359bffb14a585e88e18e3e819df4fda399a"},
+    {file = "Telethon-1.39.0-py3-none-any.whl", hash = "sha256:aa9f394b94be144799a6f6a93ab463867bc7c63503ede9631751940a98f6c703"},
+    {file = "telethon-1.39.0.tar.gz", hash = "sha256:35d4795d8c91deac515fb0bcb3723866b924de1c724e1d5c230460e96f284a63"},
 ]
 
 [package.dependencies]
@@ -2679,14 +2699,14 @@ telegram = ["requests"]
 
 [[package]]
 name = "trio"
-version = "0.28.0"
+version = "0.29.0"
 description = "A friendly Python library for async concurrency and I/O"
 optional = false
 python-versions = ">=3.9"
 groups = ["main"]
 files = [
-    {file = "trio-0.28.0-py3-none-any.whl", hash = "sha256:56d58977acc1635735a96581ec70513cc781b8b6decd299c487d3be2a721cd94"},
-    {file = "trio-0.28.0.tar.gz", hash = "sha256:4e547896fe9e8a5658e54e4c7c5fa1db748cbbbaa7c965e7d40505b928c73c05"},
+    {file = "trio-0.29.0-py3-none-any.whl", hash = "sha256:d8c463f1a9cc776ff63e331aba44c125f423a5a13c684307e828d930e625ba66"},
+    {file = "trio-0.29.0.tar.gz", hash = "sha256:ea0d3967159fc130acb6939a0be0e558e364fee26b5deeecc893a6b08c361bdf"},
 ]
 
 [package.dependencies]
@@ -2700,18 +2720,19 @@ sortedcontainers = "*"
 
 [[package]]
 name = "trio-websocket"
-version = "0.11.1"
+version = "0.12.2"
 description = "WebSocket library for Trio"
 optional = false
-python-versions = ">=3.7"
+python-versions = ">=3.8"
 groups = ["main"]
 files = [
-    {file = "trio-websocket-0.11.1.tar.gz", hash = "sha256:18c11793647703c158b1f6e62de638acada927344d534e3c7628eedcb746839f"},
-    {file = "trio_websocket-0.11.1-py3-none-any.whl", hash = "sha256:520d046b0d030cf970b8b2b2e00c4c2245b3807853ecd44214acd33d74581638"},
+    {file = "trio_websocket-0.12.2-py3-none-any.whl", hash = "sha256:df605665f1db533f4a386c94525870851096a223adcb97f72a07e8b4beba45b6"},
+    {file = "trio_websocket-0.12.2.tar.gz", hash = "sha256:22c72c436f3d1e264d0910a3951934798dcc5b00ae56fc4ee079d46c7cf20fae"},
 ]
 
 [package.dependencies]
 exceptiongroup = {version = "*", markers = "python_version < \"3.11\""}
+outcome = ">=1.2.0"
 trio = ">=0.11"
 wsproto = ">=0.14"
 
@@ -2778,14 +2799,14 @@ files = [
 
 [[package]]
 name = "tzlocal"
-version = "5.2"
+version = "5.3.1"
 description = "tzinfo object for the local timezone"
 optional = false
-python-versions = ">=3.8"
+python-versions = ">=3.9"
 groups = ["main"]
 files = [
-    {file = "tzlocal-5.2-py3-none-any.whl", hash = "sha256:49816ef2fe65ea8ac19d19aa7a1ae0551c834303d5014c6d5a62e4cbda8047b8"},
-    {file = "tzlocal-5.2.tar.gz", hash = "sha256:8d399205578f1a9342816409cc1e46a93ebd5755e39ea2d85334bea911bf0e6e"},
+    {file = "tzlocal-5.3.1-py3-none-any.whl", hash = "sha256:eb1a66c3ef5847adf7a834f1be0800581b683b5608e74f86ecbcef8ab91bb85d"},
+    {file = "tzlocal-5.3.1.tar.gz", hash = "sha256:cceffc7edecefea1f595541dbd6e990cb1ea3d19bf01b2809f362a03dd7921fd"},
 ]
 
 [package.dependencies]
@@ -3031,81 +3052,81 @@ test = ["websockets"]
 
 [[package]]
 name = "websockets"
-version = "14.2"
+version = "15.0.1"
 description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)"
 optional = false
 python-versions = ">=3.9"
 groups = ["main", "docs"]
 files = [
-    {file = "websockets-14.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e8179f95323b9ab1c11723e5d91a89403903f7b001828161b480a7810b334885"},
-    {file = "websockets-14.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0d8c3e2cdb38f31d8bd7d9d28908005f6fa9def3324edb9bf336d7e4266fd397"},
-    {file = "websockets-14.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:714a9b682deb4339d39ffa674f7b674230227d981a37d5d174a4a83e3978a610"},
-    {file = "websockets-14.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2e53c72052f2596fb792a7acd9704cbc549bf70fcde8a99e899311455974ca3"},
-    {file = "websockets-14.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e3fbd68850c837e57373d95c8fe352203a512b6e49eaae4c2f4088ef8cf21980"},
-    {file = "websockets-14.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b27ece32f63150c268593d5fdb82819584831a83a3f5809b7521df0685cd5d8"},
-    {file = "websockets-14.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4daa0faea5424d8713142b33825fff03c736f781690d90652d2c8b053345b0e7"},
-    {file = "websockets-14.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:bc63cee8596a6ec84d9753fd0fcfa0452ee12f317afe4beae6b157f0070c6c7f"},
-    {file = "websockets-14.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:7a570862c325af2111343cc9b0257b7119b904823c675b22d4ac547163088d0d"},
-    {file = "websockets-14.2-cp310-cp310-win32.whl", hash = "sha256:75862126b3d2d505e895893e3deac0a9339ce750bd27b4ba515f008b5acf832d"},
-    {file = "websockets-14.2-cp310-cp310-win_amd64.whl", hash = "sha256:cc45afb9c9b2dc0852d5c8b5321759cf825f82a31bfaf506b65bf4668c96f8b2"},
-    {file = "websockets-14.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3bdc8c692c866ce5fefcaf07d2b55c91d6922ac397e031ef9b774e5b9ea42166"},
-    {file = "websockets-14.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c93215fac5dadc63e51bcc6dceca72e72267c11def401d6668622b47675b097f"},
-    {file = "websockets-14.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1c9b6535c0e2cf8a6bf938064fb754aaceb1e6a4a51a80d884cd5db569886910"},
-    {file = "websockets-14.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a52a6d7cf6938e04e9dceb949d35fbdf58ac14deea26e685ab6368e73744e4c"},
-    {file = "websockets-14.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9f05702e93203a6ff5226e21d9b40c037761b2cfb637187c9802c10f58e40473"},
-    {file = "websockets-14.2-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22441c81a6748a53bfcb98951d58d1af0661ab47a536af08920d129b4d1c3473"},
-    {file = "websockets-14.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:efd9b868d78b194790e6236d9cbc46d68aba4b75b22497eb4ab64fa640c3af56"},
-    {file = "websockets-14.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1a5a20d5843886d34ff8c57424cc65a1deda4375729cbca4cb6b3353f3ce4142"},
-    {file = "websockets-14.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:34277a29f5303d54ec6468fb525d99c99938607bc96b8d72d675dee2b9f5bf1d"},
-    {file = "websockets-14.2-cp311-cp311-win32.whl", hash = "sha256:02687db35dbc7d25fd541a602b5f8e451a238ffa033030b172ff86a93cb5dc2a"},
-    {file = "websockets-14.2-cp311-cp311-win_amd64.whl", hash = "sha256:862e9967b46c07d4dcd2532e9e8e3c2825e004ffbf91a5ef9dde519ee2effb0b"},
-    {file = "websockets-14.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:1f20522e624d7ffbdbe259c6b6a65d73c895045f76a93719aa10cd93b3de100c"},
-    {file = "websockets-14.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:647b573f7d3ada919fd60e64d533409a79dcf1ea21daeb4542d1d996519ca967"},
-    {file = "websockets-14.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6af99a38e49f66be5a64b1e890208ad026cda49355661549c507152113049990"},
-    {file = "websockets-14.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:091ab63dfc8cea748cc22c1db2814eadb77ccbf82829bac6b2fbe3401d548eda"},
-    {file = "websockets-14.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b374e8953ad477d17e4851cdc66d83fdc2db88d9e73abf755c94510ebddceb95"},
-    {file = "websockets-14.2-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a39d7eceeea35db85b85e1169011bb4321c32e673920ae9c1b6e0978590012a3"},
-    {file = "websockets-14.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0a6f3efd47ffd0d12080594f434faf1cd2549b31e54870b8470b28cc1d3817d9"},
-    {file = "websockets-14.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:065ce275e7c4ffb42cb738dd6b20726ac26ac9ad0a2a48e33ca632351a737267"},
-    {file = "websockets-14.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e9d0e53530ba7b8b5e389c02282f9d2aa47581514bd6049d3a7cffe1385cf5fe"},
-    {file = "websockets-14.2-cp312-cp312-win32.whl", hash = "sha256:20e6dd0984d7ca3037afcb4494e48c74ffb51e8013cac71cf607fffe11df7205"},
-    {file = "websockets-14.2-cp312-cp312-win_amd64.whl", hash = "sha256:44bba1a956c2c9d268bdcdf234d5e5ff4c9b6dc3e300545cbe99af59dda9dcce"},
-    {file = "websockets-14.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:6f1372e511c7409a542291bce92d6c83320e02c9cf392223272287ce55bc224e"},
-    {file = "websockets-14.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4da98b72009836179bb596a92297b1a61bb5a830c0e483a7d0766d45070a08ad"},
-    {file = "websockets-14.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f8a86a269759026d2bde227652b87be79f8a734e582debf64c9d302faa1e9f03"},
-    {file = "websockets-14.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:86cf1aaeca909bf6815ea714d5c5736c8d6dd3a13770e885aafe062ecbd04f1f"},
-    {file = "websockets-14.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9b0f6c3ba3b1240f602ebb3971d45b02cc12bd1845466dd783496b3b05783a5"},
-    {file = "websockets-14.2-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:669c3e101c246aa85bc8534e495952e2ca208bd87994650b90a23d745902db9a"},
-    {file = "websockets-14.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:eabdb28b972f3729348e632ab08f2a7b616c7e53d5414c12108c29972e655b20"},
-    {file = "websockets-14.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:2066dc4cbcc19f32c12a5a0e8cc1b7ac734e5b64ac0a325ff8353451c4b15ef2"},
-    {file = "websockets-14.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ab95d357cd471df61873dadf66dd05dd4709cae001dd6342edafc8dc6382f307"},
-    {file = "websockets-14.2-cp313-cp313-win32.whl", hash = "sha256:a9e72fb63e5f3feacdcf5b4ff53199ec8c18d66e325c34ee4c551ca748623bbc"},
-    {file = "websockets-14.2-cp313-cp313-win_amd64.whl", hash = "sha256:b439ea828c4ba99bb3176dc8d9b933392a2413c0f6b149fdcba48393f573377f"},
-    {file = "websockets-14.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7cd5706caec1686c5d233bc76243ff64b1c0dc445339bd538f30547e787c11fe"},
-    {file = "websockets-14.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ec607328ce95a2f12b595f7ae4c5d71bf502212bddcea528290b35c286932b12"},
-    {file = "websockets-14.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:da85651270c6bfb630136423037dd4975199e5d4114cae6d3066641adcc9d1c7"},
-    {file = "websockets-14.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c3ecadc7ce90accf39903815697917643f5b7cfb73c96702318a096c00aa71f5"},
-    {file = "websockets-14.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1979bee04af6a78608024bad6dfcc0cc930ce819f9e10342a29a05b5320355d0"},
-    {file = "websockets-14.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2dddacad58e2614a24938a50b85969d56f88e620e3f897b7d80ac0d8a5800258"},
-    {file = "websockets-14.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:89a71173caaf75fa71a09a5f614f450ba3ec84ad9fca47cb2422a860676716f0"},
-    {file = "websockets-14.2-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:6af6a4b26eea4fc06c6818a6b962a952441e0e39548b44773502761ded8cc1d4"},
-    {file = "websockets-14.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:80c8efa38957f20bba0117b48737993643204645e9ec45512579132508477cfc"},
-    {file = "websockets-14.2-cp39-cp39-win32.whl", hash = "sha256:2e20c5f517e2163d76e2729104abc42639c41cf91f7b1839295be43302713661"},
-    {file = "websockets-14.2-cp39-cp39-win_amd64.whl", hash = "sha256:b4c8cef610e8d7c70dea92e62b6814a8cd24fbd01d7103cc89308d2bfe1659ef"},
-    {file = "websockets-14.2-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:d7d9cafbccba46e768be8a8ad4635fa3eae1ffac4c6e7cb4eb276ba41297ed29"},
-    {file = "websockets-14.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:c76193c1c044bd1e9b3316dcc34b174bbf9664598791e6fb606d8d29000e070c"},
-    {file = "websockets-14.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fd475a974d5352390baf865309fe37dec6831aafc3014ffac1eea99e84e83fc2"},
-    {file = "websockets-14.2-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2c6c0097a41968b2e2b54ed3424739aab0b762ca92af2379f152c1aef0187e1c"},
-    {file = "websockets-14.2-pp310-pypy310_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d7ff794c8b36bc402f2e07c0b2ceb4a2424147ed4785ff03e2a7af03711d60a"},
-    {file = "websockets-14.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:dec254fcabc7bd488dab64846f588fc5b6fe0d78f641180030f8ea27b76d72c3"},
-    {file = "websockets-14.2-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:bbe03eb853e17fd5b15448328b4ec7fb2407d45fb0245036d06a3af251f8e48f"},
-    {file = "websockets-14.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:a3c4aa3428b904d5404a0ed85f3644d37e2cb25996b7f096d77caeb0e96a3b42"},
-    {file = "websockets-14.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:577a4cebf1ceaf0b65ffc42c54856214165fb8ceeba3935852fc33f6b0c55e7f"},
-    {file = "websockets-14.2-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ad1c1d02357b7665e700eca43a31d52814ad9ad9b89b58118bdabc365454b574"},
-    {file = "websockets-14.2-pp39-pypy39_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f390024a47d904613577df83ba700bd189eedc09c57af0a904e5c39624621270"},
-    {file = "websockets-14.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:3c1426c021c38cf92b453cdf371228d3430acd775edee6bac5a4d577efc72365"},
-    {file = "websockets-14.2-py3-none-any.whl", hash = "sha256:7a6ceec4ea84469f15cf15807a747e9efe57e369c384fa86e022b3bea679b79b"},
-    {file = "websockets-14.2.tar.gz", hash = "sha256:5059ed9c54945efb321f097084b4c7e52c246f2c869815876a69d1efc4ad6eb5"},
+    {file = "websockets-15.0.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d63efaa0cd96cf0c5fe4d581521d9fa87744540d4bc999ae6e08595a1014b45b"},
+    {file = "websockets-15.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ac60e3b188ec7574cb761b08d50fcedf9d77f1530352db4eef1707fe9dee7205"},
+    {file = "websockets-15.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5756779642579d902eed757b21b0164cd6fe338506a8083eb58af5c372e39d9a"},
+    {file = "websockets-15.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0fdfe3e2a29e4db3659dbd5bbf04560cea53dd9610273917799f1cde46aa725e"},
+    {file = "websockets-15.0.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c2529b320eb9e35af0fa3016c187dffb84a3ecc572bcee7c3ce302bfeba52bf"},
+    {file = "websockets-15.0.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac1e5c9054fe23226fb11e05a6e630837f074174c4c2f0fe442996112a6de4fb"},
+    {file = "websockets-15.0.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5df592cd503496351d6dc14f7cdad49f268d8e618f80dce0cd5a36b93c3fc08d"},
+    {file = "websockets-15.0.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:0a34631031a8f05657e8e90903e656959234f3a04552259458aac0b0f9ae6fd9"},
+    {file = "websockets-15.0.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3d00075aa65772e7ce9e990cab3ff1de702aa09be3940d1dc88d5abf1ab8a09c"},
+    {file = "websockets-15.0.1-cp310-cp310-win32.whl", hash = "sha256:1234d4ef35db82f5446dca8e35a7da7964d02c127b095e172e54397fb6a6c256"},
+    {file = "websockets-15.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:39c1fec2c11dc8d89bba6b2bf1556af381611a173ac2b511cf7231622058af41"},
+    {file = "websockets-15.0.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:823c248b690b2fd9303ba00c4f66cd5e2d8c3ba4aa968b2779be9532a4dad431"},
+    {file = "websockets-15.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678999709e68425ae2593acf2e3ebcbcf2e69885a5ee78f9eb80e6e371f1bf57"},
+    {file = "websockets-15.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d50fd1ee42388dcfb2b3676132c78116490976f1300da28eb629272d5d93e905"},
+    {file = "websockets-15.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d99e5546bf73dbad5bf3547174cd6cb8ba7273062a23808ffea025ecb1cf8562"},
+    {file = "websockets-15.0.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:66dd88c918e3287efc22409d426c8f729688d89a0c587c88971a0faa2c2f3792"},
+    {file = "websockets-15.0.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8dd8327c795b3e3f219760fa603dcae1dcc148172290a8ab15158cf85a953413"},
+    {file = "websockets-15.0.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8fdc51055e6ff4adeb88d58a11042ec9a5eae317a0a53d12c062c8a8865909e8"},
+    {file = "websockets-15.0.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:693f0192126df6c2327cce3baa7c06f2a117575e32ab2308f7f8216c29d9e2e3"},
+    {file = "websockets-15.0.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:54479983bd5fb469c38f2f5c7e3a24f9a4e70594cd68cd1fa6b9340dadaff7cf"},
+    {file = "websockets-15.0.1-cp311-cp311-win32.whl", hash = "sha256:16b6c1b3e57799b9d38427dda63edcbe4926352c47cf88588c0be4ace18dac85"},
+    {file = "websockets-15.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:27ccee0071a0e75d22cb35849b1db43f2ecd3e161041ac1ee9d2352ddf72f065"},
+    {file = "websockets-15.0.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:3e90baa811a5d73f3ca0bcbf32064d663ed81318ab225ee4f427ad4e26e5aff3"},
+    {file = "websockets-15.0.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:592f1a9fe869c778694f0aa806ba0374e97648ab57936f092fd9d87f8bc03665"},
+    {file = "websockets-15.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0701bc3cfcb9164d04a14b149fd74be7347a530ad3bbf15ab2c678a2cd3dd9a2"},
+    {file = "websockets-15.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8b56bdcdb4505c8078cb6c7157d9811a85790f2f2b3632c7d1462ab5783d215"},
+    {file = "websockets-15.0.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0af68c55afbd5f07986df82831c7bff04846928ea8d1fd7f30052638788bc9b5"},
+    {file = "websockets-15.0.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64dee438fed052b52e4f98f76c5790513235efaa1ef7f3f2192c392cd7c91b65"},
+    {file = "websockets-15.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d5f6b181bb38171a8ad1d6aa58a67a6aa9d4b38d0f8c5f496b9e42561dfc62fe"},
+    {file = "websockets-15.0.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:5d54b09eba2bada6011aea5375542a157637b91029687eb4fdb2dab11059c1b4"},
+    {file = "websockets-15.0.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3be571a8b5afed347da347bfcf27ba12b069d9d7f42cb8c7028b5e98bbb12597"},
+    {file = "websockets-15.0.1-cp312-cp312-win32.whl", hash = "sha256:c338ffa0520bdb12fbc527265235639fb76e7bc7faafbb93f6ba80d9c06578a9"},
+    {file = "websockets-15.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:fcd5cf9e305d7b8338754470cf69cf81f420459dbae8a3b40cee57417f4614a7"},
+    {file = "websockets-15.0.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ee443ef070bb3b6ed74514f5efaa37a252af57c90eb33b956d35c8e9c10a1931"},
+    {file = "websockets-15.0.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5a939de6b7b4e18ca683218320fc67ea886038265fd1ed30173f5ce3f8e85675"},
+    {file = "websockets-15.0.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:746ee8dba912cd6fc889a8147168991d50ed70447bf18bcda7039f7d2e3d9151"},
+    {file = "websockets-15.0.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:595b6c3969023ecf9041b2936ac3827e4623bfa3ccf007575f04c5a6aa318c22"},
+    {file = "websockets-15.0.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c714d2fc58b5ca3e285461a4cc0c9a66bd0e24c5da9911e30158286c9b5be7f"},
+    {file = "websockets-15.0.1-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f3c1e2ab208db911594ae5b4f79addeb3501604a165019dd221c0bdcabe4db8"},
+    {file = "websockets-15.0.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:229cf1d3ca6c1804400b0a9790dc66528e08a6a1feec0d5040e8b9eb14422375"},
+    {file = "websockets-15.0.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:756c56e867a90fb00177d530dca4b097dd753cde348448a1012ed6c5131f8b7d"},
+    {file = "websockets-15.0.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:558d023b3df0bffe50a04e710bc87742de35060580a293c2a984299ed83bc4e4"},
+    {file = "websockets-15.0.1-cp313-cp313-win32.whl", hash = "sha256:ba9e56e8ceeeedb2e080147ba85ffcd5cd0711b89576b83784d8605a7df455fa"},
+    {file = "websockets-15.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:e09473f095a819042ecb2ab9465aee615bd9c2028e4ef7d933600a8401c79561"},
+    {file = "websockets-15.0.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5f4c04ead5aed67c8a1a20491d54cdfba5884507a48dd798ecaf13c74c4489f5"},
+    {file = "websockets-15.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:abdc0c6c8c648b4805c5eacd131910d2a7f6455dfd3becab248ef108e89ab16a"},
+    {file = "websockets-15.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a625e06551975f4b7ea7102bc43895b90742746797e2e14b70ed61c43a90f09b"},
+    {file = "websockets-15.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d591f8de75824cbb7acad4e05d2d710484f15f29d4a915092675ad3456f11770"},
+    {file = "websockets-15.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:47819cea040f31d670cc8d324bb6435c6f133b8c7a19ec3d61634e62f8d8f9eb"},
+    {file = "websockets-15.0.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac017dd64572e5c3bd01939121e4d16cf30e5d7e110a119399cf3133b63ad054"},
+    {file = "websockets-15.0.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:4a9fac8e469d04ce6c25bb2610dc535235bd4aa14996b4e6dbebf5e007eba5ee"},
+    {file = "websockets-15.0.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:363c6f671b761efcb30608d24925a382497c12c506b51661883c3e22337265ed"},
+    {file = "websockets-15.0.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:2034693ad3097d5355bfdacfffcbd3ef5694f9718ab7f29c29689a9eae841880"},
+    {file = "websockets-15.0.1-cp39-cp39-win32.whl", hash = "sha256:3b1ac0d3e594bf121308112697cf4b32be538fb1444468fb0a6ae4feebc83411"},
+    {file = "websockets-15.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:b7643a03db5c95c799b89b31c036d5f27eeb4d259c798e878d6937d71832b1e4"},
+    {file = "websockets-15.0.1-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0c9e74d766f2818bb95f84c25be4dea09841ac0f734d1966f415e4edfc4ef1c3"},
+    {file = "websockets-15.0.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:1009ee0c7739c08a0cd59de430d6de452a55e42d6b522de7aa15e6f67db0b8e1"},
+    {file = "websockets-15.0.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76d1f20b1c7a2fa82367e04982e708723ba0e7b8d43aa643d3dcd404d74f1475"},
+    {file = "websockets-15.0.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f29d80eb9a9263b8d109135351caf568cc3f80b9928bccde535c235de55c22d9"},
+    {file = "websockets-15.0.1-pp310-pypy310_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b359ed09954d7c18bbc1680f380c7301f92c60bf924171629c5db97febb12f04"},
+    {file = "websockets-15.0.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:cad21560da69f4ce7658ca2cb83138fb4cf695a2ba3e475e0559e05991aa8122"},
+    {file = "websockets-15.0.1-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7f493881579c90fc262d9cdbaa05a6b54b3811c2f300766748db79f098db9940"},
+    {file = "websockets-15.0.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:47b099e1f4fbc95b701b6e85768e1fcdaf1630f3cbe4765fa216596f12310e2e"},
+    {file = "websockets-15.0.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:67f2b6de947f8c757db2db9c71527933ad0019737ec374a8a6be9a956786aaf9"},
+    {file = "websockets-15.0.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d08eb4c2b7d6c41da6ca0600c077e93f5adcfd979cd777d747e9ee624556da4b"},
+    {file = "websockets-15.0.1-pp39-pypy39_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b826973a4a2ae47ba357e4e82fa44a463b8f168e1ca775ac64521442b19e87f"},
+    {file = "websockets-15.0.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:21c1fa28a6a7e3cbdc171c694398b6df4744613ce9b36b1a498e816787e28123"},
+    {file = "websockets-15.0.1-py3-none-any.whl", hash = "sha256:f7a866fbc1e97b5c617ee4116daaa09b722101d4a3c170c787450ba409f9736f"},
+    {file = "websockets-15.0.1.tar.gz", hash = "sha256:82544de02076bafba038ce055ee6412d68da13ab47f0c60cab827346de828dee"},
 ]
 
 [[package]]
@@ -3141,14 +3162,14 @@ h11 = ">=0.9.0,<1"
 
 [[package]]
 name = "yt-dlp"
-version = "2025.1.26"
+version = "2025.2.19"
 description = "A feature-rich command-line audio/video downloader"
 optional = false
 python-versions = ">=3.9"
 groups = ["main"]
 files = [
-    {file = "yt_dlp-2025.1.26-py3-none-any.whl", hash = "sha256:3e76bd896b9f96601021ca192ca0fbdd195e3c3dcc28302a3a34c9bc4979da7b"},
-    {file = "yt_dlp-2025.1.26.tar.gz", hash = "sha256:1c9738266921ad43c568ad01ac3362fb7c7af549276fbec92bd72f140da16240"},
+    {file = "yt_dlp-2025.2.19-py3-none-any.whl", hash = "sha256:3ed218eaeece55e9d715afd41abc450dc406ee63bf79355169dfde312d38fdb8"},
+    {file = "yt_dlp-2025.2.19.tar.gz", hash = "sha256:f33ca76df2e4db31880f2fe408d44f5058d9f135015b13e50610dfbe78245bea"},
 ]
 
 [package.extras]
@@ -3164,4 +3185,4 @@ test = ["pytest (>=8.1,<9.0)", "pytest-rerunfailures (>=14.0,<15.0)"]
 [metadata]
 lock-version = "2.1"
 python-versions = ">=3.10,<3.13"
-content-hash = "b3a6142d6495bc4c8741e9411d29352af219851e4b84b263f991e1bb6db1614e"
+content-hash = "2d0a953383901fe12e97f6f56a76a9d8008788695425792eedbf739a18585188"

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [project]
 name = "auto-archiver"
-version = "0.13.3"
+version = "0.13.5"
 description = "Automatically archive links to videos, images, and social media content from Google Sheets (and more)."
 
 requires-python = ">=3.10,<3.13"
@@ -63,6 +63,7 @@ dependencies = [
 pytest = "^8.3.4"
 autopep8 = "^2.3.1"
 pytest-loguru = "^0.4.0"
+pytest-mock = "^3.14.0"
 
 [tool.poetry.group.docs.dependencies]
 sphinx = "^8.1.3"

scripts/generate_settings_schema.py

@@ -0,0 +1,52 @@
+import json
+import os
+import io
+
+from ruamel.yaml import YAML
+
+from auto_archiver.core.module import ModuleFactory
+from auto_archiver.core.consts import MODULE_TYPES
+from auto_archiver.core.config import EMPTY_CONFIG
+
+class SchemaEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if isinstance(obj, set):
+            return list(obj)
+        return json.JSONEncoder.default(self, obj)
+
+# Get available modules
+module_factory = ModuleFactory()
+available_modules = module_factory.available_modules()
+
+modules_by_type = {}
+# Categorize modules by type
+for module in available_modules:
+    for type in module.manifest.get('type', []):
+        modules_by_type.setdefault(type, []).append(module)
+
+all_modules_ordered_by_type = sorted(available_modules, key=lambda x: (MODULE_TYPES.index(x.type[0]), not x.requires_setup))
+
+yaml: YAML = YAML()
+
+config_string = io.BytesIO()
+yaml.dump(EMPTY_CONFIG, config_string)
+config_string = config_string.getvalue().decode('utf-8')
+output_schema = {
+    'modules': dict((module.name, 
+                     {
+                         'name': module.name,
+                         'display_name': module.display_name,
+                         'manifest': module.manifest,
+                         'configs': module.configs or None
+                         }
+                         ) for module in all_modules_ordered_by_type),
+    'steps': dict((f"{module_type}s", [module.name for module in modules_by_type[module_type]]) for module_type in MODULE_TYPES),
+    'configs': [m.name for m in all_modules_ordered_by_type if m.configs],
+    'module_types': MODULE_TYPES,
+    'empty_config': config_string
+}
+
+current_file_dir = os.path.dirname(os.path.abspath(__file__))
+output_file = os.path.join(current_file_dir, 'settings/src/schema.json')
+with open(output_file, 'w') as file:
+    json.dump(output_schema, file, indent=4, cls=SchemaEncoder)

scripts/settings/.gitignore

@@ -0,0 +1,24 @@
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+pnpm-debug.log*
+lerna-debug.log*
+
+node_modules
+dist
+dist-ssr
+*.local
+
+# Editor directories and files
+.vscode/*
+!.vscode/extensions.json
+.idea
+.DS_Store
+*.suo
+*.ntvs*
+*.njsproj
+*.sln
+*.sw?

scripts/settings/index.html

@@ -0,0 +1,3 @@
+
+    <div id="root"></div>
+    <script type="module" src="/src/main.tsx"></script>

scripts/settings/package.json

@@ -0,0 +1,31 @@
+{
+  "name": "material-ui-vite-ts",
+  "private": true,
+  "version": "5.0.0",
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "preview": "vite preview"
+  },
+  "dependencies": {
+    "@dnd-kit/core": "^6.3.1",
+    "@dnd-kit/sortable": "^10.0.0",
+    "@emotion/react": "latest",
+    "@emotion/styled": "latest",
+    "@mui/icons-material": "latest",
+    "@mui/material": "latest",
+    "react": "19.0.0",
+    "react-dom": "19.0.0",
+    "react-markdown": "^10.0.0",
+    "yaml": "^2.7.0"
+  },
+  "devDependencies": {
+    "@types/react": "latest",
+    "@types/react-dom": "latest",
+    "@vitejs/plugin-react": "latest",
+    "typescript": "latest",
+    "vite": "latest",
+    "vite-plugin-singlefile": "^2.1.0"
+  }
+}

scripts/settings/src/App.tsx

@@ -0,0 +1,450 @@
+import * as React from 'react';
+import { useEffect, useState, useRef } from 'react';
+import Container from '@mui/material/Container';
+import Typography from '@mui/material/Typography';
+import Box from '@mui/material/Box';
+import FileUploadIcon from '@mui/icons-material/FileUpload';
+// 
+import {
+  DndContext,
+  closestCenter,
+  KeyboardSensor,
+  PointerSensor,
+  useSensor,
+  useSensors,
+  DragOverlay
+} from "@dnd-kit/core";
+import {
+  arrayMove,
+  SortableContext,
+  sortableKeyboardCoordinates,
+  rectSortingStrategy
+} from "@dnd-kit/sortable";
+
+import type { DragStartEvent, DragEndEvent, UniqueIdentifier } from "@dnd-kit/core";
+
+
+import { Module } from './types';
+
+import { modules, steps, module_types, empty_config } from './schema.json';
+import {
+  Stack,
+  Button,
+} from '@mui/material';
+import Grid from '@mui/material/Grid2';
+
+import { parseDocument, Document, YAMLSeq, YAMLMap, Scalar } from 'yaml'
+import StepCard from './StepCard';
+
+
+function FileDrop({ setYamlFile }: { setYamlFile: React.Dispatch<React.SetStateAction<Document>> }) {
+
+  const [showError, setShowError] = useState(false);
+  const [label, setLabel] = useState(<>Drag and drop your orchestration.yaml file here, or click to select a file.</>);
+  const wrapperRef = useRef(null);
+
+  function openYAMLFile(event: any) {
+    let file = event.target.files[0];
+    if (file.type.indexOf('yaml') === -1) {
+      setShowError(true);
+      setLabel(<>Invalid type, only YAML files are accepted.</>)
+      return;
+    }
+    let reader = new FileReader();
+    reader.onload = function (e) {
+      let contents = e.target ? e.target.result : '';
+      try {
+        let document = parseDocument(contents as string);
+        if (document.errors.length > 0) {
+          // not a valid yaml file
+          setShowError(true);
+          setLabel(<>Invalid file. Make sure your Orchestration is a valid YAML file with a 'steps' section in it.</>)
+          return;
+        } else {
+          setShowError(false);
+          setLabel(<>File loaded successfully.</>)
+        }
+        // do some basic validation of 'steps'
+        let steps = document.get('steps');
+        if (!steps) {
+          setShowError(true);
+          setLabel(<>Invalid file. Your orchestration file must have a 'steps' section in it.</>)
+          return;
+        }
+        const replacements = {
+          feeder: 'feeders',
+          formatter: 'formatters',
+          archivers: 'extractors',
+        };
+
+        let error = false;
+        for (let stepType of Object.keys(replacements)) {
+          if (steps.get(stepType) !== undefined) {
+            setShowError(true);
+            setLabel(<>Invalid file. Your orchestration file appears to be in the old (v0.12) format with a '{stepType}' section.<br/>You should manually update your orchestration file first (hint: {stepType} → {replacements[stepType]})</>);
+            error = true;
+            return;
+          }
+        };
+        setYamlFile(document);
+      } catch (e) {
+        console.error(e);
+      }
+    }
+    reader.readAsText(file);
+  }
+  return (
+    <>
+      <div
+      style={{
+        position: 'relative',
+        width: '100%',
+        border: 'dashed',
+        borderRadius:'5px',
+        textAlign: 'center',
+        borderWidth: '1px',
+        padding: '20px' }}
+      onDragEnter={(e) => {
+        e.currentTarget.style.backgroundColor = 'var(--mui-palette-LinearProgress-infoBg)';
+      }}
+      onDragLeave={(e) => {
+        e.currentTarget.style.backgroundColor = '';
+      }}
+      onDrop={(e) => {
+        e.currentTarget.style.backgroundColor = '';
+      }}
+      >
+        <FileUploadIcon style={{ fontSize: 50 }} />
+        <input style={{
+          opacity: 0,
+          position: 'absolute',
+          top: 0,
+          left: 0,
+          width: '100%',
+          height: '100%',
+          cursor: 'pointer',
+        }}
+        type="file" id="file"
+        accept=".yaml"
+        onChange={openYAMLFile} />
+        <Typography variant="body1" color={showError ? 'error' : ''} >
+          {label}
+        </Typography>
+      </div>
+    </>
+  );
+}
+
+function ModuleTypes({ stepType, setEnabledModules, enabledModules, configValues }: { stepType: string, setEnabledModules: any, enabledModules: any, configValues: any }) {
+  const [showError, setShowError] = useState<boolean>(false);
+  const [activeId, setActiveId] = useState<UniqueIdentifier>();
+  const [items, setItems] = useState<string[]>([]);
+
+  useEffect(() => {
+    setItems(enabledModules[stepType].map(([name, enabled]: [string, boolean]) => name));
+  }
+    , [enabledModules]);
+
+  const toggleModule = (event: any) => {
+    // make sure that 'feeder' and 'formatter' types only have one value
+    let name = event.target.id;
+    let checked = event.target.checked;
+    if (stepType === 'feeders' || stepType === 'formatters') {
+      // check how many modules of this type are enabled
+      const checkedModules = enabledModules[stepType].filter(([m, enabled]: [string, boolean]) => {
+        return (m !== name && enabled) || (checked && m === name)
+      });
+      if (checkedModules.length > 1) {
+        setShowError(true);
+      } else {
+        setShowError(false);
+      }
+    } else {
+      setShowError(false);
+    }
+    let newEnabledModules = { ...enabledModules };
+    newEnabledModules[stepType] = enabledModules[stepType].map(([m, enabled]: [string, boolean]) => {
+        return (m === name) ? [m, checked] : [m, enabled];
+      });
+    setEnabledModules(newEnabledModules);
+  }
+
+  const sensors = useSensors(
+    useSensor(PointerSensor),
+    useSensor(KeyboardSensor, {
+      coordinateGetter: sortableKeyboardCoordinates
+    })
+  );
+
+  const handleDragStart = (event: DragStartEvent) => {
+    setActiveId(event.active.id);
+  };
+
+  const handleDragEnd = (event: DragEndEvent) => {
+    setActiveId(undefined);
+    const { active, over } = event;
+
+    if (active.id !== over?.id) {
+      const oldIndex = items.indexOf(active.id as string);
+      const newIndex = items.indexOf(over?.id as string);
+
+      let newArray = arrayMove(items, oldIndex, newIndex);
+      // set it also on steps
+      let newEnabledModules = { ...enabledModules };
+      newEnabledModules[stepType] = enabledModules[stepType].sort((a, b) => {
+        return newArray.indexOf(a[0]) - newArray.indexOf(b[0]);
+      })
+      setEnabledModules(newEnabledModules);
+    }
+  };
+  return (
+    <>
+      <Box sx={{ my: 4 }}>
+        <Typography id={stepType} variant="h6" style={{ textTransform: 'capitalize' }} >
+          {stepType}
+        </Typography>
+        <Typography variant="body1" >
+          Select the <a href="<a href={`https://auto-archiver.readthedocs.io/en/latest/modules/${stepType.slice(0,-1)}.html`}" target="_blank">{stepType}</a> you wish to enable. Drag to reorder.
+        </Typography>
+      </Box>
+      {showError ? <Typography variant="body1" color="error" >Only one {stepType.slice(0,-1)} can be enabled at a time.</Typography> : null}
+
+      <DndContext
+        sensors={sensors}
+        collisionDetection={closestCenter}
+        onDragEnd={handleDragEnd}
+        onDragStart={handleDragStart}
+      >
+        <Grid container spacing={1} key={stepType}>
+          <SortableContext items={items} strategy={rectSortingStrategy}>
+            {items.map((name: string) => {
+              let m: Module = modules[name];
+              return (
+                <StepCard key={name} type={stepType} module={m} toggleModule={toggleModule} enabledModules={enabledModules} configValues={configValues} />
+              );
+            })}
+            <DragOverlay>
+              {activeId ? (
+                <div
+                  style={{
+                    width: "100%",
+                    height: "100%",
+                    backgroundColor: "grey",
+                    opacity: 0.1,
+                  }}
+                ></div>
+
+              ) : null}
+            </DragOverlay>
+          </SortableContext>
+        </Grid>
+      </DndContext>
+    </>
+  );
+}
+
+
+export default function App() {
+  const [yamlFile, setYamlFile] = useState<Document>(new Document());
+  const [enabledModules, setEnabledModules] = useState<{}>(Object.fromEntries(Object.keys(steps).map(type => [type, steps[type].map((name: string) => [name, false])])));
+  const [configValues, setConfigValues] = useState<{
+    [key: string]: {
+      [key: string
+      ]: any
+    }
+  }>(
+    Object.keys(modules).reduce((acc, module) => {
+      acc[module] = {};
+      return acc;
+    }, {})
+  );
+
+  const saveSettings = function (copy: boolean = false) {
+    // edit the yamlFile
+
+    // generate the steps config
+    let stepsConfig = enabledModules;
+
+    let finalYamlFile: Document = null;
+    if (!yamlFile || yamlFile.contents == null) {
+      // create the yaml file from 
+      finalYamlFile = parseDocument(empty_config as string);
+    } else {
+      finalYamlFile = yamlFile;
+    }
+
+    // set the steps
+    module_types.forEach((type: string) => {
+      let stepType = type + 's';
+      let existingSteps  = finalYamlFile.getIn(['steps', stepType]) as YAMLSeq;
+      stepsConfig[stepType].forEach(([name, enabled]: [string, boolean]) => {
+        let index = existingSteps.items.findIndex((item) => { 
+          return (item.value || item) === name
+        });
+        let stepItem = finalYamlFile.getIn(['steps', stepType], true) as YAMLSeq;
+
+        if (enabled && index === -1) {
+            finalYamlFile.addIn(['steps', stepType], name);
+            stepItem.commentBefore = stepItem.commentBefore?.replace("\n - " + name, '');
+            stepItem.comment = stepItem.comment?.replace("\n - " + name, '');
+        } else if (!enabled && index !== -1) {
+            // set the value to empty and add a comment before with the commented value
+            finalYamlFile.deleteIn(['steps', stepType, index]);
+            stepItem.commentBefore += "\n - " + name;
+            finalYamlFile.setIn(['steps', stepType], stepItem);
+        }
+      });
+      // sort the items
+      existingSteps.items.sort((a: Scalar | string, b: Scalar | string) => {
+        return (stepsConfig[stepType].findIndex((val: [string, boolean]) => {return val[0] === (a.value || a)}) -
+                stepsConfig[stepType].findIndex((val: [string, boolean]) => {return val[0] === (b.value || b)}))
+      });
+      existingSteps.flow = existingSteps.items.length ? false : true;
+    });
+
+    // set all other settings
+    // loop through each item that isn't 'steps' in the finalYamlFile and check if it exists in configValues
+
+        Object.keys(configValues).forEach((module_name: string) => {
+          // get an existing key
+          let existingConfig = finalYamlFile.get(module_name, true) as YAMLMap;
+          if (existingConfig) {
+            Object.keys(configValues[module_name]).forEach((config_name: string) => {
+              let existingConfigYAML = existingConfig.get(config_name, true) as Scalar;
+              if (existingConfigYAML) {
+                existingConfigYAML.value = configValues[module_name][config_name];
+                existingConfig.set(config_name, existingConfigYAML);
+              } else {
+                existingConfig.set(config_name, configValues[module_name][config_name]);
+              }
+            });
+            finalYamlFile.set(module_name, existingConfig);
+          } else {
+            if (configValues[module_name] && Object.keys(configValues[module_name]).length > 0) {
+              finalYamlFile.set(module_name, configValues[module_name]);
+            }
+          }
+        });
+
+    if (copy) {
+      navigator.clipboard.writeText(String(finalYamlFile)).then(() => {
+        alert("Settings copied to clipboard.");
+      });
+    } else {
+      // offer the file for download
+      const blob = new Blob([String(finalYamlFile)], { type: 'application/x-yaml' });
+      const url = URL.createObjectURL(blob);
+      const a = document.createElement('a');
+      a.href = url;
+      a.download = 'orchestration.yaml';
+      a.click();
+    }
+  }
+
+  useEffect(() => {
+    // load the configs, and set the default values if they exist
+    let newConfigValues = {};
+    Object.keys(modules).map((module: string) => {
+      let m = modules[module];
+      let configs = m.configs;
+      if (!configs) {
+        return;
+      }
+      newConfigValues[module] = {};
+      Object.keys(configs).map((config: string) => {
+        let config_args = configs[config];
+        if (config_args.default !== undefined) {
+          newConfigValues[module][config] = config_args.default;
+        }
+      });
+    })
+    setConfigValues(newConfigValues);
+  }, []);
+
+  useEffect(() => {
+    if (!yamlFile || yamlFile.contents == null) {
+      return;
+    }
+
+    let settings = yamlFile.toJS();
+    // make a deep copy of settings
+    let stepSettings = settings['steps'];
+   
+    let newEnabledModules = Object.fromEntries(Object.keys(steps).map((type: string) => {
+      return [type, steps[type].map((name: string) => {
+        return [name, stepSettings[type].indexOf(name) !== -1];
+      }).sort((a, b) => {
+        let aIndex = stepSettings[type].indexOf(a[0]);
+        let bIndex = stepSettings[type].indexOf(b[0]);
+        if (aIndex === -1 && bIndex === -1) {
+          return a - b;
+        }
+        if (bIndex === -1) {
+          return -1;
+        }
+        if (aIndex === -1) {
+          return 1;
+        }
+        return aIndex - bIndex;
+      })];
+    }).sort((a, b) => {
+      return module_types.indexOf(a[0]) - module_types.indexOf(b[0]);
+    }));
+    setEnabledModules(newEnabledModules);
+
+    // set the config values
+    let newConfigValues = settings;
+    delete newConfigValues['steps'];
+
+
+    setConfigValues(Object.keys(modules).reduce((acc, module) => {
+      acc[module] = newConfigValues[module] || {};
+      return acc;
+    }, {}));
+  }, [yamlFile]);
+
+
+
+  return (
+    <Container maxWidth="lg">
+      <Box sx={{ my: 4 }}>
+        <Box sx={{ my: 4 }}>
+          <Typography variant="h5" >
+            1. Select your orchestration.yaml settings file.
+          </Typography>
+          <Typography variant="body1">Or skip this step to start from scratch</Typography>
+          <FileDrop setYamlFile={setYamlFile} />
+        </Box>
+        <Box sx={{ my: 4 }}>
+          <Typography variant="h5" >
+            2. Choose the Modules you wish to enable/disable
+          </Typography>
+          {Object.keys(steps).map((stepType: string) => {
+            return (
+              <Box key={stepType} sx={{ my: 4 }}>
+                <ModuleTypes stepType={stepType} setEnabledModules={setEnabledModules} enabledModules={enabledModules} configValues={configValues} />
+              </Box>
+            );
+          })}
+        </Box>
+        <Box sx={{ my: 4 }}>
+          <Typography variant="h5" >
+            3. Configure your Enabled Modules
+          </Typography>
+          <Typography variant="body1" >
+            Next to each module you've enabled, you can click 'Configure' to set the module's settings.
+          </Typography>
+        </Box>
+        <Box sx={{ my: 4 }}>
+          <Typography variant="h5" >
+            4. Save your settings
+          </Typography>
+          <Stack direction="row" spacing={2} sx={{ my: 2 }}>
+            <Button variant="contained" color="primary" onClick={() => saveSettings(true)}>Copy Settings to Clipboard</Button>
+            <Button variant="contained" color="primary" onClick={() => saveSettings()}>Save Settings to File</Button>
+          </Stack>
+        </Box>
+      </Box>
+    </Container>
+  );
+}

scripts/settings/src/StepCard.tsx

@@ -0,0 +1,258 @@
+import { useState } from "react";
+import { useSortable } from "@dnd-kit/sortable";
+import ReactMarkdown from 'react-markdown';
+
+import { CSS } from "@dnd-kit/utilities";
+
+import {
+    Card,
+    CardActions,
+    CardHeader,
+    Button,
+    Dialog,
+    DialogTitle,
+    DialogContent,
+    Box,
+    IconButton,
+    Checkbox,
+    Select,
+    MenuItem,
+    FormControl,
+    FormControlLabel,
+    FormHelperText,
+    TextField,
+    Stack,
+    Typography,
+    InputAdornment,
+} from '@mui/material';
+import Grid from '@mui/material/Grid2';
+import DragIndicatorIcon from '@mui/icons-material/DragIndicator';
+import Visibility from '@mui/icons-material/Visibility';
+import VisibilityOff from '@mui/icons-material/VisibilityOff';
+import HelpIconOutlined from '@mui/icons-material/HelpOutline';
+import { Module, Config } from "./types";
+
+
+// adds 'capitalize' method to String prototype
+declare global {
+    interface String {
+        capitalize(): string;
+    }
+}
+String.prototype.capitalize = function (this: string) {
+    return this.charAt(0).toUpperCase() + this.slice(1);
+};
+
+const StepCard = ({
+    type,
+    module,
+    toggleModule,
+    enabledModules,
+    configValues
+}: {
+    type: string,
+    module: Module,
+    toggleModule: any,
+    enabledModules: any,
+    configValues: any
+}) => {
+    const {
+        attributes,
+        listeners,
+        setNodeRef,
+        transform,
+        transition,
+        isDragging
+    } = useSortable({ id: module.name });
+
+
+    const style = {
+        ...Card.style,
+        transform: CSS.Transform.toString(transform),
+        transition,
+        zIndex: isDragging ? "100" : "auto",
+        opacity: isDragging ? 0.3 : 1
+    };
+
+    let name = module.name;
+    const [helpOpen, setHelpOpen] = useState(false);
+    const [configOpen, setConfigOpen] = useState(false);
+    const enabled = enabledModules[type].find((m: any) => m[0] === name)[1];
+
+    return (
+        <Grid ref={setNodeRef} size={{ xs: 6, sm: 4, md: 3 }} style={style}>
+            <Card >
+                <CardHeader
+                    title={
+                        <FormControlLabel
+                            style={{paddingRight: '0 !important'}}
+                            control={<Checkbox title="Check to enable this module" sx={{paddingTop:0, paddingBottom:0}} id={name} onClick={toggleModule} checked={enabled} />}
+                            label={module.display_name} />
+                    }
+                />
+                <CardActions>
+                    <Box sx={{ justifyContent: 'space-between', display: 'flex', width: '100%' }}>
+                        <Box>
+                    <IconButton title="Module information" size="small" onClick={() => setHelpOpen(true)}>
+                        <HelpIconOutlined />
+                    </IconButton>
+                    {enabled && module.configs && name != 'cli_feeder' ? (
+                        <Button size="small" onClick={() => setConfigOpen(true)}>Configure</Button>
+                    ) : null}
+                    </Box>
+                    <IconButton size="small" title="Drag to reorder" sx={{ cursor: 'grab' }} {...listeners} {...attributes}>
+                        <DragIndicatorIcon/>
+                    </IconButton>
+                    </Box>
+                </CardActions>
+            </Card>
+            <Dialog
+                open={helpOpen}
+                onClose={() => setHelpOpen(false)}
+                maxWidth="lg"
+            >
+                <DialogTitle>
+                    {module.display_name}
+                </DialogTitle>
+                <DialogContent>
+                    <ReactMarkdown>
+                        {module.manifest.description.split("\n").map((line: string) => line.trim()).join("\n")}
+                    </ReactMarkdown>
+                </DialogContent>
+            </Dialog>
+            {module.configs && name != 'cli_feeder' && <ConfigPanel module={module} open={configOpen} setOpen={setConfigOpen} configValues={configValues} />}
+        </Grid>
+    )
+}
+
+function ConfigField({ config_value, module, configValues }: { config_value: any, module: Module, configValues: any }) {
+    const [showPassword, setShowPassword] = useState(false);
+    const handleClickShowPassword = () => setShowPassword((show) => !show);
+
+    const handleMouseDownPassword = (event: React.MouseEvent<HTMLButtonElement>) => {
+      event.preventDefault();
+    };
+  
+    const handleMouseUpPassword = (event: React.MouseEvent<HTMLButtonElement>) => {
+      event.preventDefault();
+    };
+
+    function setConfigValue(config: any, value: any) {
+        configValues[module.name][config] = value;
+    }
+    const config_args: Config = module.configs[config_value];
+    const config_name: string = config_value.replace(/_/g, " ");
+    const config_display_name = config_name.capitalize();
+    const value = configValues[module.name][config_value] || config_args.default;
+    
+
+    const config_value_lower = config_value.toLowerCase();
+    const is_password = config_value_lower.includes('password') ||
+                        config_value_lower.includes('secret') ||
+                        config_value_lower.includes('token') ||
+                        config_value_lower.includes('key') ||
+                        config_value_lower.includes('api_hash') ||
+                        config_args.type === 'password';
+
+    const text_input_type = is_password ? 'password' : (config_args.type === 'int' ? 'number' : 'text');
+
+    return (
+        <Box>
+            <Typography variant='body1' style={{ fontWeight: 'bold' }}>{config_display_name} {config_args.required && (`(required)`)} </Typography>
+            <FormControl size="small">
+                {config_args.type === 'bool' ?
+                    <FormControlLabel control={
+                        <Checkbox defaultChecked={value} size="small" id={`${module}.${config_value}`}
+                            onChange={(e) => {
+                                setConfigValue(config_value, e.target.checked);
+                            }}
+                        />} label={config_args.help.capitalize()}
+                    />
+                    :
+                    (
+                        config_args.choices !== undefined ?
+                            <Select size="small" id={`${module}.${config_value}`}
+                                defaultValue={config_args.default}
+                                value={value}
+                                onChange={(e) => {
+                                    setConfigValue(config_value, e.target.value);
+                                }}
+                            >
+                                {config_args.choices.map((choice: any) => {
+                                    return (
+                                        <MenuItem key={`${module}.${config_value}.${choice}`}
+                                            value={choice}>{choice}</MenuItem>
+                                    );
+                                })}
+                            </Select>
+                            :
+                            (config_args.type === 'json_loader' ?
+                                <TextField multiline size="small" id={`${module}.${config_value}`} defaultValue={JSON.stringify(value, null, 2)} rows={6} onChange={
+                                    (e) => {
+                                        try {
+                                            let val = JSON.parse(e.target.value);
+                                            setConfigValue(config_value, val);
+                                        } catch (e) {
+                                            console.log(e);
+                                        }
+                                    }
+                                } />
+                                :
+                                <TextField size="small" id={`${module}.${config_value}`} defaultValue={value} type={showPassword ? 'text' : text_input_type}
+                                    onChange={(e) => {
+                                        setConfigValue(config_value, e.target.value);
+                                    }}
+                                    required={config_args.required}
+                                    slotProps={ is_password ? {
+                                        input: { endAdornment: (
+                                            <InputAdornment position="end">
+                                                <IconButton
+                                                    aria-label="toggle password visibility"
+                                                    onClick={handleClickShowPassword}
+                                                    onMouseDown={handleMouseDownPassword}
+                                                    onMouseUp={handleMouseUpPassword}
+                                                >
+                                                    {showPassword ? <VisibilityOff /> : <Visibility />}
+                                                </IconButton>
+                                            </InputAdornment>
+                                        )}
+                                    } : {}}
+                                />
+                            )
+                    )
+                }
+                {config_args.type !== 'bool' && (
+                    <FormHelperText >{config_args.help.capitalize()}</FormHelperText>
+                )}
+            </FormControl>
+        </Box>
+    )
+}
+
+function ConfigPanel({ module, open, setOpen, configValues }: { module: Module, open: boolean, setOpen: any, configValues: any }) {
+
+    return (
+        <>
+            <Dialog
+                open={open}
+                onClose={() => setOpen(false)}
+                maxWidth="lg"
+            >
+                <DialogTitle>
+                    {module.display_name}
+                </DialogTitle>
+                <DialogContent>
+                    <Stack direction="column" spacing={1}>
+                        {Object.keys(module.configs).map((config_value: any) => {
+                            return (
+                                <ConfigField key={config_value} config_value={config_value} module={module} configValues={configValues} />
+                            );
+                        })}
+                    </Stack>
+                </DialogContent>
+            </Dialog>
+        </>
+    );
+}
+
+export default StepCard;

scripts/settings/src/main.tsx

@@ -0,0 +1,44 @@
+import * as React from 'react';
+import * as ReactDOM from 'react-dom/client';
+import { ThemeProvider } from '@mui/material/styles';
+import { CssBaseline } from '@mui/material';
+import App from './App';
+import { createTheme } from '@mui/material/styles';
+import { red } from '@mui/material/colors';
+import { useState, useEffect } from 'react';
+
+function RootApp() {
+  const [mode, setMode] = useState('light');
+
+useEffect(() => {
+    setMode(window.localStorage.getItem('theme') || 'light');
+}, []);
+
+var observer = new MutationObserver(function(mutations) {
+  setMode(window.localStorage.getItem('theme') || 'light');
+  
+})
+observer.observe(document.documentElement, {attributes: true, attributeFilter: ['data-theme']});
+
+// A custom theme for this app
+const theme = createTheme({
+  palette: {
+    mode: mode == 'light' ? 'light' : 'dark',
+  },
+  cssVariables: true
+});
+
+  return (
+    <ThemeProvider theme={theme}>
+    <CssBaseline />
+    <App  />
+  </ThemeProvider>
+  );
+}
+
+ReactDOM.createRoot(document.getElementById('root')!).render(
+  <React.StrictMode>
+    <RootApp />
+  </React.StrictMode>,
+);
+

scripts/settings/src/types.d.ts

@@ -0,0 +1,21 @@
+export interface Config {
+  name: string;
+  description: string;
+  type: string?;
+  default: any;
+  help: string;
+  choices: string[];
+  required: boolean;
+}
+
+interface Manifest {
+  description: string;
+}
+
+export interface Module {
+  name: string;
+  description: string;
+  configs: { [key: string]: Config };
+  manifest: Manifest;
+  display_name: string;
+}

scripts/settings/tsconfig.json

@@ -0,0 +1,21 @@
+{
+  "compilerOptions": {
+    "target": "ESNext",
+    "useDefineForClassFields": true,
+    "lib": ["DOM", "DOM.Iterable", "ESNext"],
+    "allowJs": false,
+    "skipLibCheck": true,
+    "esModuleInterop": false,
+    "allowSyntheticDefaultImports": true,
+    "strict": true,
+    "forceConsistentCasingInFileNames": true,
+    "module": "ESNext",
+    "moduleResolution": "Node",
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "noEmit": true,
+    "jsx": "react-jsx"
+  },
+  "include": ["src"],
+  "references": [{ "path": "./tsconfig.node.json" }]
+}

scripts/settings/tsconfig.node.json

@@ -0,0 +1,9 @@
+{
+  "compilerOptions": {
+    "composite": true,
+    "module": "ESNext",
+    "moduleResolution": "Node",
+    "allowSyntheticDefaultImports": true
+  },
+  "include": ["vite.config.ts"]
+}

scripts/settings/vite.config.ts

@@ -0,0 +1,12 @@
+import { defineConfig } from 'vite';
+import react from '@vitejs/plugin-react';
+import { viteSingleFile } from "vite-plugin-singlefile"
+
+// https://vite.dev/config/
+export default defineConfig({
+  plugins: [react(), viteSingleFile()],
+  build: {
+    minify: false,
+    sourcemap: true,
+  }
+});

src/auto_archiver/core/__init__.py

@@ -3,7 +3,7 @@
 """
 from .metadata import Metadata
 from .media import Media
-from .module import BaseModule
+from .base_module import BaseModule
 
 # cannot import ArchivingOrchestrator/Config to avoid circular dep
 # from .orchestrator import ArchivingOrchestrator

src/auto_archiver/core/base_module.py

@@ -1,13 +1,18 @@
 
-from urllib.parse import urlparse
-from typing import  Mapping, Any
+from __future__ import annotations
+
+from typing import  Mapping, Any, Type, TYPE_CHECKING
 from abc import ABC
 from copy import deepcopy, copy
 from tempfile import TemporaryDirectory
 from auto_archiver.utils import url as UrlUtil
+from auto_archiver.core.consts import MODULE_TYPES as CONF_MODULE_TYPES
 
 from loguru import logger
 
+if TYPE_CHECKING:
+    from .module import ModuleFactory
+
 class BaseModule(ABC):
 
     """
@@ -17,41 +22,24 @@ class BaseModule(ABC):
     however modules can have a .setup() method to run any setup code
     (e.g. logging in to a site, spinning up a browser etc.)
 
-    See BaseModule.MODULE_TYPES for the types of modules you can create, noting that
+    See consts.MODULE_TYPES for the types of modules you can create, noting that
     a subclass can be of multiple types. For example, a module that extracts data from
     a website and stores it in a database would be both an 'extractor' and a 'database' module.
 
     Each module is a python package, and should have a __manifest__.py file in the
     same directory as the module file. The __manifest__.py specifies the module information
-    like name, author, version, dependencies etc. See BaseModule._DEFAULT_MANIFEST for the
+    like name, author, version, dependencies etc. See DEFAULT_MANIFEST for the
     default manifest structure.
 
     """
 
-    MODULE_TYPES = [
-        'feeder',
-        'extractor',
-        'enricher',
-        'database',
-        'storage',
-        'formatter'
-    ]
-
-    _DEFAULT_MANIFEST = {
-    'name': '', # the display name of the module
-    'author': 'Bellingcat', # creator of the module, leave this as Bellingcat or set your own name!
-    'type': [], # the type of the module, can be one or more of BaseModule.MODULE_TYPES
-    'requires_setup': True, # whether or not this module requires additional setup such as setting API Keys or installing additional softare
-    'description': '', # a description of the module
-    'dependencies': {}, # external dependencies, e.g. python packages or binaries, in dictionary format
-    'entry_point': '', # the entry point for the module, in the format 'module_name::ClassName'. This can be left blank to use the default entry point of module_name::ModuleName
-    'version': '1.0', # the version of the module
-    'configs': {} # any configuration options this module has, these will be exposed to the user in the config file or via the command line
-}
+    MODULE_TYPES = CONF_MODULE_TYPES
 
+    # NOTE: these here are declard as class variables, but they are overridden by the instance variables in the __init__ method
     config: Mapping[str, Any]
     authentication: Mapping[str, Mapping[str, str]]
     name: str
+    module_factory: ModuleFactory
 
     # this is set by the orchestrator prior to archiving
     tmp_dir: TemporaryDirectory = None
@@ -62,8 +50,6 @@ class BaseModule(ABC):
 
     def config_setup(self, config: dict):
 
-        authentication = config.get('authentication', {})
-
         # this is important. Each instance is given its own deepcopied config, so modules cannot
         # change values to affect other modules
         config = deepcopy(config)
@@ -98,11 +84,13 @@ class BaseModule(ABC):
         * api_key: str - the API key to use for login\n
         * api_secret: str - the API secret to use for login\n
         * cookie: str - a cookie string to use for login (specific to this site)\n
+        * cookies_file: str - the path to a cookies file to use for login (specific to this site)\n
+        * cookies_from_browser: str - the name of the browser to extract cookies from (specitic for this site)\n
         """
         # TODO: think about if/how we can deal with sites that have multiple domains (main one is x.com/twitter.com)
         # for now the user must enter them both, like "x.com,twitter.com" in their config. Maybe we just hard-code?
 
-        site = UrlUtil.domain_for_url(site)
+        site = UrlUtil.domain_for_url(site).lstrip("www.")
         # add the 'www' version of the site to the list of sites to check
         authdict = {}
 
@@ -117,8 +105,8 @@ class BaseModule(ABC):
             for key in self.authentication.keys():
                 if key in site or site in key:
                     logger.debug(f"Could not find exact authentication information for site '{site}'. \
-                                    did find information for '{key}' which is close, is this what you meant? \
-                                    If so, edit your authentication settings to make sure it exactly matches.")
+did find information for '{key}' which is close, is this what you meant? \
+If so, edit your authentication settings to make sure it exactly matches.")
 
         def get_ytdlp_cookiejar(args):
             import yt_dlp
@@ -129,16 +117,29 @@ class BaseModule(ABC):
             ytdlp_opts = getattr(parse_options(args), 'ydl_opts')
             return yt_dlp.YoutubeDL(ytdlp_opts).cookiejar
 
-        # get the cookies jar, prefer the browser cookies than the file
-        if 'cookies_from_browser' in self.authentication:
+        get_cookiejar_options = None
+
+        # order of priority:
+        # 1. cookies_from_browser setting in site config
+        # 2. cookies_file setting in site config
+        # 3. cookies_from_browser setting in global config
+        # 4. cookies_file setting in global config
+
+        if 'cookies_from_browser' in authdict:
+            get_cookiejar_options = ['--cookies-from-browser', authdict['cookies_from_browser']]
+        elif 'cookies_file' in authdict:
+            get_cookiejar_options = ['--cookies', authdict['cookies_file']]
+        elif 'cookies_from_browser' in self.authentication:
             authdict['cookies_from_browser'] = self.authentication['cookies_from_browser']
-            if extract_cookies:
-                authdict['cookies_jar'] = get_ytdlp_cookiejar(['--cookies-from-browser', self.authentication['cookies_from_browser']])
+            get_cookiejar_options = ['--cookies-from-browser', self.authentication['cookies_from_browser']]
         elif 'cookies_file' in self.authentication:
             authdict['cookies_file'] = self.authentication['cookies_file']
-            if extract_cookies:
-                authdict['cookies_jar'] = get_ytdlp_cookiejar(['--cookies', self.authentication['cookies_file']])
+            get_cookiejar_options = ['--cookies', self.authentication['cookies_file']]
+
         
+        if get_cookiejar_options:
+            authdict['cookies_jar'] = get_ytdlp_cookiejar(get_cookiejar_options)
+
         return authdict
     
     def repr(self):

src/auto_archiver/core/config.py

@@ -7,21 +7,24 @@ flexible setup in various environments.
 
 import argparse
 from ruamel.yaml import YAML, CommentedMap, add_representer
+import json
 
 from loguru import logger
 
 from copy import deepcopy
-from .module import BaseModule
+from auto_archiver.core.consts import MODULE_TYPES
 
 from typing import Any, List, Type, Tuple
 
 _yaml: YAML = YAML()
 
+DEFAULT_CONFIG_FILE = "secrets/orchestration.yaml"
+
 EMPTY_CONFIG = _yaml.load("""
 # Auto Archiver Configuration
-# Steps are the modules that will be run in the order they are defined
 
-steps:""" + "".join([f"\n   {module}s: []" for module in BaseModule.MODULE_TYPES]) + \
+# Steps are the modules that will be run in the order they are defined
+steps:""" + "".join([f"\n   {module}s: []" for module in MODULE_TYPES]) + \
 """
 
 # Global configuration
@@ -52,6 +55,57 @@ logging:
 """)
 # note: 'logging' is explicitly added above in order to better format the config file
 
+
+# Arg Parse Actions/Classes
+class AuthenticationJsonParseAction(argparse.Action):
+    def __call__(self, parser, namespace, values, option_string=None):
+
+        try:
+            auth_dict = json.loads(values)
+            setattr(namespace, self.dest, auth_dict)
+        except json.JSONDecodeError as e:
+            raise argparse.ArgumentTypeError(f"Invalid JSON input for argument '{self.dest}': {e}")
+
+        def load_from_file(path):
+            try:
+                with open(path, 'r') as f:
+                    try:
+                        auth_dict = json.load(f)
+                    except json.JSONDecodeError:
+                        f.seek(0)
+                        # maybe it's yaml, try that
+                        auth_dict = _yaml.load(f)
+                    if auth_dict.get('authentication'):
+                        auth_dict = auth_dict['authentication']
+                    auth_dict['load_from_file']  = path
+                    return auth_dict
+            except:
+                return None
+
+        if isinstance(auth_dict, dict) and auth_dict.get('from_file'):
+            auth_dict = load_from_file(auth_dict['from_file'])
+        elif isinstance(auth_dict, str):
+            # if it's a string
+            auth_dict = load_from_file(auth_dict)
+        
+        if not isinstance(auth_dict, dict):
+            raise argparse.ArgumentTypeError("Authentication must be a dictionary of site names and their authentication methods")
+        global_options = ['cookies_from_browser', 'cookies_file', 'load_from_file']
+        for key, auth in auth_dict.items():
+            if key in global_options:
+                continue
+            if not isinstance(key, str) or not isinstance(auth, dict):
+                raise argparse.ArgumentTypeError(f"Authentication must be a dictionary of site names and their authentication methods. Valid global configs are {global_options}")
+
+        setattr(namespace, self.dest, auth_dict)
+
+
+class UniqueAppendAction(argparse.Action):
+    def __call__(self, parser, namespace, values, option_string=None):
+        for value in values:
+            if value not in getattr(namespace, self.dest):
+                getattr(namespace, self.dest).append(value)
+
 class DefaultValidatingParser(argparse.ArgumentParser):
 
     def error(self, message):
@@ -82,6 +136,7 @@ class DefaultValidatingParser(argparse.ArgumentParser):
 
         return super().parse_known_args(args, namespace)
 
+# Config Utils
 
 def to_dot_notation(yaml_conf: CommentedMap | dict) -> dict:
     dotdict = {}
@@ -153,8 +208,8 @@ def read_yaml(yaml_filename: str) -> CommentedMap:
         pass
 
     if not config:
-        config = EMPTY_CONFIG
-    
+        config = deepcopy(EMPTY_CONFIG)
+
     return config
 
 # TODO: make this tidier/find a way to notify of which keys should not be stored
@@ -170,4 +225,7 @@ def store_yaml(config: CommentedMap, yaml_filename: str) -> None:
 
     config_to_save.pop('urls', None)
     with open(yaml_filename, "w", encoding="utf-8") as outf:
-        _yaml.dump(config_to_save, outf)
+        _yaml.dump(config_to_save, outf)
+
+def is_valid_config(config: CommentedMap) -> bool:
+    return config and config != EMPTY_CONFIG

src/auto_archiver/core/consts.py

@@ -0,0 +1,23 @@
+
+MODULE_TYPES = [
+    'feeder',
+    'extractor',
+    'enricher',
+    'database',
+    'storage',
+    'formatter'
+]
+
+MANIFEST_FILE = "__manifest__.py"
+
+DEFAULT_MANIFEST = {
+    'name': '', # the display name of the module
+    'author': 'Bellingcat', # creator of the module, leave this as Bellingcat or set your own name!
+    'type': [], # the type of the module, can be one or more of MODULE_TYPES
+    'requires_setup': True, # whether or not this module requires additional setup such as setting API Keys or installing additional software
+    'description': '', # a description of the module
+    'dependencies': {}, # external dependencies, e.g. python packages or binaries, in dictionary format
+    'entry_point': '', # the entry point for the module, in the format 'module_name::ClassName'. This can be left blank to use the default entry point of module_name::ModuleName
+    'version': '1.0', # the version of the module
+    'configs': {} # any configuration options this module has, these will be exposed to the user in the config file or via the command line
+}

src/auto_archiver/core/enricher.py

@@ -13,7 +13,7 @@ from abc import abstractmethod
 from auto_archiver.core import Metadata, BaseModule
 
 class Enricher(BaseModule):
-    """Base classes and utilities for enrichers in the Auto-Archiver system.
+    """Base classes and utilities for enrichers in the Auto Archiver system.
     
     Enricher modules must implement the `enrich` method to define their behavior.
     """

src/auto_archiver/core/module.py

@@ -6,7 +6,7 @@ by handling user configuration, validating the steps properties, and implementin
 from __future__ import annotations
 
 from dataclasses import dataclass
-from typing import List
+from typing import List, TYPE_CHECKING
 import shutil
 import ast
 import copy
@@ -16,99 +16,116 @@ import os
 from os.path import join
 from loguru import logger
 import auto_archiver
-from .base_module import BaseModule
+from auto_archiver.core.consts import DEFAULT_MANIFEST, MANIFEST_FILE
 
-_LAZY_LOADED_MODULES = {}
-
-MANIFEST_FILE = "__manifest__.py"
+if TYPE_CHECKING:
+    from .base_module import BaseModule
 
 
-def setup_paths(paths: list[str]) -> None:
-    """
-    Sets up the paths for the modules to be loaded from
-    
-    This is necessary for the modules to be imported correctly
-    
-    """
-    for path in paths:
-        # check path exists, if it doesn't, log a warning
-        if not os.path.exists(path):
-            logger.warning(f"Path '{path}' does not exist. Skipping...")
-            continue
+HAS_SETUP_PATHS = False
 
-        # see odoo/module/module.py -> initialize_sys_path
-        if path not in auto_archiver.modules.__path__:
-                auto_archiver.modules.__path__.append(path)
+class ModuleFactory:
 
-    # sort based on the length of the path, so that the longest path is last in the list
-    auto_archiver.modules.__path__ = sorted(auto_archiver.modules.__path__, key=len, reverse=True)
+    def __init__(self):
+        self._lazy_modules = {}
 
-def get_module(module_name: str, config: dict) -> BaseModule:
-    """
-    Gets and sets up a module using the provided config
-    
-    This will actually load and instantiate the module, and load all its dependencies (i.e. not lazy)
-    
-    """
-    return get_module_lazy(module_name).load(config)
+    def setup_paths(self, paths: list[str]) -> None:
+        """
+        Sets up the paths for the modules to be loaded from
+        
+        This is necessary for the modules to be imported correctly
+        
+        """
+        global HAS_SETUP_PATHS
 
-def get_module_lazy(module_name: str, suppress_warnings: bool = False) -> LazyBaseModule:
-    """
-    Lazily loads a module, returning a LazyBaseModule
-    
-    This has all the information about the module, but does not load the module itself or its dependencies
-    
-    To load an actual module, call .setup() on a lazy module
-    
-    """
-    if module_name in _LAZY_LOADED_MODULES:
-        return _LAZY_LOADED_MODULES[module_name]
-
-    available = available_modules(limit_to_modules=[module_name], suppress_warnings=suppress_warnings)
-    if not available:
-        raise IndexError(f"Module '{module_name}' not found. Are you sure it's installed/exists?")
-    return available[0]
-
-def available_modules(with_manifest: bool=False, limit_to_modules: List[str]= [], suppress_warnings: bool = False) -> List[LazyBaseModule]:
-    
-    # search through all valid 'modules' paths. Default is 'modules' in the current directory
-
-    # see odoo/modules/module.py -> get_modules
-    def is_really_module(module_path):
-        if os.path.isfile(join(module_path, MANIFEST_FILE)):
-            return True
-
-    all_modules = []
-
-    for module_folder in auto_archiver.modules.__path__:
-        # walk through each module in module_folder and check if it has a valid manifest
-        try:
-            possible_modules = os.listdir(module_folder)
-        except FileNotFoundError:
-            logger.warning(f"Module folder {module_folder} does not exist")
-            continue
-
-        for possible_module in possible_modules:
-            if limit_to_modules and possible_module not in limit_to_modules:
+        for path in paths:
+            # check path exists, if it doesn't, log a warning
+            if not os.path.exists(path):
+                logger.warning(f"Path '{path}' does not exist. Skipping...")
                 continue
 
-            possible_module_path = join(module_folder, possible_module)
-            if not is_really_module(possible_module_path):
+            # see odoo/module/module.py -> initialize_sys_path
+            if path not in auto_archiver.modules.__path__:
+                    if HAS_SETUP_PATHS == True:
+                        logger.warning(f"You are attempting to re-initialise the module paths with: '{path}' for a 2nd time. \
+                                       This could lead to unexpected behaviour. It is recommended to only use a single modules path. \
+                                       If you wish to load modules from different paths then load a 2nd python interpreter (e.g. using multiprocessing).")
+                    auto_archiver.modules.__path__.append(path)
+
+        # sort based on the length of the path, so that the longest path is last in the list
+        auto_archiver.modules.__path__ = sorted(auto_archiver.modules.__path__, key=len, reverse=True)
+
+        HAS_SETUP_PATHS = True
+
+    def get_module(self, module_name: str, config: dict) -> BaseModule:
+        """
+        Gets and sets up a module using the provided config
+        
+        This will actually load and instantiate the module, and load all its dependencies (i.e. not lazy)
+        
+        """
+        return self.get_module_lazy(module_name).load(config)
+
+    def get_module_lazy(self, module_name: str, suppress_warnings: bool = False) -> LazyBaseModule:
+        """
+        Lazily loads a module, returning a LazyBaseModule
+        
+        This has all the information about the module, but does not load the module itself or its dependencies
+        
+        To load an actual module, call .setup() on a lazy module
+        
+        """
+        if module_name in self._lazy_modules:
+            return self._lazy_modules[module_name]
+
+        available = self.available_modules(limit_to_modules=[module_name], suppress_warnings=suppress_warnings)
+        if not available:
+            message = f"Module '{module_name}' not found. Are you sure it's installed/exists?"
+            if 'archiver' in module_name:
+                message += f" Did you mean {module_name.replace('archiver', 'extractor')}?"
+            raise IndexError(message)
+        return available[0]
+
+    def available_modules(self, limit_to_modules: List[str]= [], suppress_warnings: bool = False) -> List[LazyBaseModule]:
+        
+        # search through all valid 'modules' paths. Default is 'modules' in the current directory
+
+        # see odoo/modules/module.py -> get_modules
+        def is_really_module(module_path):
+            if os.path.isfile(join(module_path, MANIFEST_FILE)):
+                return True
+
+        all_modules = []
+
+        for module_folder in auto_archiver.modules.__path__:
+            # walk through each module in module_folder and check if it has a valid manifest
+            try:
+                possible_modules = os.listdir(module_folder)
+            except FileNotFoundError:
+                logger.warning(f"Module folder {module_folder} does not exist")
                 continue
-            if _LAZY_LOADED_MODULES.get(possible_module):
-                continue
-            lazy_module = LazyBaseModule(possible_module, possible_module_path)
 
-            _LAZY_LOADED_MODULES[possible_module] = lazy_module
+            for possible_module in possible_modules:
+                if limit_to_modules and possible_module not in limit_to_modules:
+                    continue
 
-            all_modules.append(lazy_module)
-    
-    if not suppress_warnings:
-        for module in limit_to_modules:
-            if not any(module == m.name for m in all_modules):
-                logger.warning(f"Module '{module}' not found. Are you sure it's installed?")
+                possible_module_path = join(module_folder, possible_module)
+                if not is_really_module(possible_module_path):
+                    continue
+                if self._lazy_modules.get(possible_module):
+                    continue
+                lazy_module = LazyBaseModule(possible_module, possible_module_path, factory=self)
 
-    return all_modules
+                self._lazy_modules[possible_module] = lazy_module
+
+                all_modules.append(lazy_module)
+        
+        if not suppress_warnings:
+            for module in limit_to_modules:
+                if not any(module == m.name for m in all_modules):
+                    logger.warning(f"Module '{module}' not found. Are you sure it's installed?")
+
+        return all_modules
 
 @dataclass
 class LazyBaseModule:
@@ -120,17 +137,22 @@ class LazyBaseModule:
 
     """
     name: str
-    type: list
     description: str
     path: str
+    module_factory: ModuleFactory
 
     _manifest: dict = None
     _instance: BaseModule = None
     _entry_point: str = None
 
-    def __init__(self, module_name, path):
+    def __init__(self, module_name, path, factory: ModuleFactory):
         self.name = module_name
         self.path = path
+        self.module_factory = factory
+
+    @property
+    def type(self):
+        return self.manifest['type']
 
     @property
     def entry_point(self):
@@ -161,16 +183,15 @@ class LazyBaseModule:
             return self._manifest
         # print(f"Loading manifest for module {module_path}")
         # load the manifest file
-        manifest = copy.deepcopy(BaseModule._DEFAULT_MANIFEST)
+        manifest = copy.deepcopy(DEFAULT_MANIFEST)
 
         with open(join(self.path, MANIFEST_FILE)) as f:
             try:
                 manifest.update(ast.literal_eval(f.read()))
             except (ValueError, TypeError, SyntaxError, MemoryError, RecursionError) as e:
-                logger.error(f"Error loading manifest from file {self.path}/{MANIFEST_FILE}: {e}")
+                raise ValueError(f"Error loading manifest from file {self.path}/{MANIFEST_FILE}: {e}")
             
         self._manifest = manifest
-        self.type = manifest['type']
         self._entry_point = manifest['entry_point']
         self.description = manifest['description']
         self.version = manifest['version']
@@ -189,13 +210,14 @@ class LazyBaseModule:
                     # clear out any empty strings that a user may have erroneously added
                     continue
                 if not check(dep):
-                    logger.error(f"Module '{self.name}' requires external dependency '{dep}' which is not available/setup. Have you installed the required dependencies for the '{self.name}' module? See the README for more information.")
+                    logger.error(f"Module '{self.name}' requires external dependency '{dep}' which is not available/setup. \
+                                 Have you installed the required dependencies for the '{self.name}' module? See the README for more information.")
                     exit(1)
 
         def check_python_dep(dep):
             # first check if it's a module:
             try:
-                m = get_module_lazy(dep, suppress_warnings=True)
+                m = self.module_factory.get_module_lazy(dep, suppress_warnings=True)
                 try:
                 # we must now load this module and set it up with the config
                     m.load(config)
@@ -230,19 +252,21 @@ class LazyBaseModule:
         __import__(f'{qualname}.{file_name}', fromlist=[self.entry_point])
         # finally, get the class instance
         instance: BaseModule = getattr(sys.modules[sub_qualname], class_name)()
-        if not getattr(instance, 'name', None):
-            instance.name = self.name
-
-        if not getattr(instance, 'display_name', None):
-            instance.display_name = self.display_name
-
-        self._instance = instance
 
+        # set the name, display name and module factory
+        instance.name = self.name
+        instance.display_name = self.display_name
+        instance.module_factory = self.module_factory
+        
         # merge the default config with the user config
-        default_config = dict((k, v['default']) for k, v in self.configs.items() if v.get('default'))
+        default_config = dict((k, v['default']) for k, v in self.configs.items() if 'default' in v)
+
         config[self.name] = default_config  | config.get(self.name, {})
         instance.config_setup(config)
         instance.setup()
+
+        # save the instance for future easy loading
+        self._instance = instance
         return instance
 
     def __repr__(self):

src/auto_archiver/core/orchestrator.py

@@ -5,96 +5,41 @@
 """
 
 from __future__ import annotations
-from typing import Generator, Union, List, Type
-from urllib.parse import urlparse
-from ipaddress import ip_address
-from copy import copy
+from typing import Generator, Union, List, Type, TYPE_CHECKING
 import argparse
 import os
 import sys
-import json
 from tempfile import TemporaryDirectory
 import traceback
+from copy import copy
 
 from rich_argparse import RichHelpFormatter
-
+from loguru import logger
+import requests
 
 from .metadata import Metadata, Media
 from auto_archiver.version import __version__
-from .config import _yaml, read_yaml, store_yaml, to_dot_notation, merge_dicts, EMPTY_CONFIG, DefaultValidatingParser
-from .module import available_modules, LazyBaseModule, get_module, setup_paths
+from .config import read_yaml, store_yaml, to_dot_notation, merge_dicts, is_valid_config, \
+    DefaultValidatingParser, UniqueAppendAction, AuthenticationJsonParseAction, DEFAULT_CONFIG_FILE
+from .module import ModuleFactory, LazyBaseModule
 from . import validators, Feeder, Extractor, Database, Storage, Formatter, Enricher
-from .module import BaseModule
-
-from loguru import logger
-
-
-DEFAULT_CONFIG_FILE = "orchestration.yaml"
-
-
-class JsonParseAction(argparse.Action):
-    def __call__(self, parser, namespace, values, option_string=None):
-        try:
-            setattr(namespace, self.dest, json.loads(values))
-        except json.JSONDecodeError as e:
-            raise argparse.ArgumentTypeError(f"Invalid JSON input for argument '{self.dest}': {e}")
-
-
-class AuthenticationJsonParseAction(JsonParseAction):
-    def __call__(self, parser, namespace, values, option_string=None):
-        super().__call__(parser, namespace, values, option_string)
-        auth_dict = getattr(namespace, self.dest)
-
-        def load_from_file(path):
-            try:
-                with open(path, 'r') as f:
-                    try:
-                        auth_dict = json.load(f)
-                    except json.JSONDecodeError:
-                        f.seek(0)
-                        # maybe it's yaml, try that
-                        auth_dict = _yaml.load(f)
-                    if auth_dict.get('authentication'):
-                        auth_dict = auth_dict['authentication']
-                    auth_dict['load_from_file']  = path
-                    return auth_dict
-            except:
-                return None
-
-        if isinstance(auth_dict, dict) and auth_dict.get('from_file'):
-            auth_dict = load_from_file(auth_dict['from_file'])
-        elif isinstance(auth_dict, str):
-            # if it's a string
-            auth_dict = load_from_file(auth_dict)
-        
-        if not isinstance(auth_dict, dict):
-            raise argparse.ArgumentTypeError("Authentication must be a dictionary of site names and their authentication methods")
-        global_options = ['cookies_from_browser', 'cookies_file', 'load_from_file']
-        for key, auth in auth_dict.items():
-            if key in global_options:
-                continue
-            if not isinstance(key, str) or not isinstance(auth, dict):
-                raise argparse.ArgumentTypeError(f"Authentication must be a dictionary of site names and their authentication methods. Valid global configs are {global_options}")
-        
-        # extract out concatenated sites
-        for key, val in copy(auth_dict).items():
-            if "," in key:
-                for site in key.split(","):
-                    auth_dict[site] = val
-                del auth_dict[key]
-
-        setattr(namespace, self.dest, auth_dict)
-
-
-class UniqueAppendAction(argparse.Action):
-    def __call__(self, parser, namespace, values, option_string=None):
-        for value in values:
-            if value not in getattr(namespace, self.dest):
-                getattr(namespace, self.dest).append(value)
+from .consts import MODULE_TYPES
+from auto_archiver.utils.url import check_url_or_raise
 
+if TYPE_CHECKING:
+    from .base_module import BaseModule
+    from .module import LazyBaseModule
 
+class SetupError(ValueError):
+    pass
 class ArchivingOrchestrator:
 
+    # instance variables
+    module_factory: ModuleFactory
+    setup_finished: bool
+    logger_id: int
+
+    # instance variables, used for convenience to access modules by step
     feeders: List[Type[Feeder]]
     extractors: List[Type[Extractor]]
     enrichers: List[Type[Enricher]]
@@ -102,6 +47,11 @@ class ArchivingOrchestrator:
     storages: List[Type[Storage]]
     formatters: List[Type[Formatter]]
 
+    def __init__(self):
+        self.module_factory = ModuleFactory()
+        self.setup_finished = False
+        self.logger_id = None
+
     def setup_basic_parser(self):
         parser = argparse.ArgumentParser(
             prog="auto-archiver",
@@ -123,17 +73,27 @@ class ArchivingOrchestrator:
 
         self.basic_parser = parser
         return parser
+    
+    def check_steps(self, config):
+        for module_type in MODULE_TYPES:
+            if not config['steps'].get(f"{module_type}s", []):
+                if module_type == 'feeder' or module_type == 'formatter' and config['steps'].get(f"{module_type}"):
+                    raise SetupError(f"It appears you have '{module_type}' set under 'steps' in your configuration file, but as of version 0.13.0 of Auto Archiver, you must use '{module_type}s'. Change this in your configuration file and try again. \
+Here's how that would look: \n\nsteps:\n  {module_type}s:\n  - [your_{module_type}_name_here]\n  {'extractors:...' if module_type == 'feeder' else '...'}\n")
+                if module_type == 'extractor' and config['steps'].get('archivers'):
+                    raise SetupError(f"As of version 0.13.0 of Auto Archiver, the 'archivers' step name has been changed to 'extractors'. Change this in your configuration file and try again. \
+Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_here]\n  enrichers:...\n")
+                raise SetupError(f"No {module_type}s were configured. Make sure to set at least one {module_type} in your configuration file or on the command line (using --{module_type}s)")
 
     def setup_complete_parser(self, basic_config: dict, yaml_config: dict, unused_args: list[str]) -> None:
 
-
         # modules parser to get the overridden 'steps' values
         modules_parser = argparse.ArgumentParser(
             add_help=False,
         )
         self.add_modules_args(modules_parser)
         cli_modules, unused_args = modules_parser.parse_known_args(unused_args)
-        for module_type in BaseModule.MODULE_TYPES:
+        for module_type in MODULE_TYPES:
             yaml_config['steps'][f"{module_type}s"] = getattr(cli_modules, f"{module_type}s", []) or yaml_config['steps'].get(f"{module_type}s", [])
 
         parser = DefaultValidatingParser(
@@ -150,33 +110,36 @@ class ArchivingOrchestrator:
         # TODO: BUG** - basic_config won't have steps in it, since these args aren't added to 'basic_parser'
         # but should we add them? Or should we just add them to the 'complete' parser?
 
-        if yaml_config != EMPTY_CONFIG:
+        if is_valid_config(yaml_config):
+            self.check_steps(yaml_config)
             # only load the modules enabled in config
             # TODO: if some steps are empty (e.g. 'feeders' is empty), should we default to the 'simple' ones? Or only if they are ALL empty?
             enabled_modules = []
             # first loads the modules from the config file, then from the command line
-            for module_type in BaseModule.MODULE_TYPES:
+            for module_type in MODULE_TYPES:
                 enabled_modules.extend(yaml_config['steps'].get(f"{module_type}s", []))
 
             # clear out duplicates, but keep the order
             enabled_modules = list(dict.fromkeys(enabled_modules))
-            avail_modules = available_modules(with_manifest=True, limit_to_modules=enabled_modules, suppress_warnings=True)
+            avail_modules = self.module_factory.available_modules(limit_to_modules=enabled_modules, suppress_warnings=True)
             self.add_individual_module_args(avail_modules, parser)
         elif basic_config.mode == 'simple':
-            simple_modules = [module for module in available_modules(with_manifest=True) if not module.requires_setup]
+            simple_modules = [module for module in self.module_factory.available_modules() if not module.requires_setup]
             self.add_individual_module_args(simple_modules, parser)
 
-            # for simple mode, we use the cli_feeder and any modules that don't require setup
-            if not yaml_config['steps']['feeders']:
-                yaml_config['steps']['feeders'] = ['cli_feeder']
-
             # add them to the config
             for module in simple_modules:
                 for module_type in module.type:
                     yaml_config['steps'].setdefault(f"{module_type}s", []).append(module.name)
         else:
             # load all modules, they're not using the 'simple' mode
-            self.add_individual_module_args(available_modules(with_manifest=True), parser)
+            all_modules = self.module_factory.available_modules()
+            # add all the modules to the steps
+            for module in all_modules:
+                for module_type in module.type:
+                    yaml_config['steps'].setdefault(f"{module_type}s", []).append(module.name)
+
+            self.add_individual_module_args(all_modules, parser)
         
         parser.set_defaults(**to_dot_notation(yaml_config))
 
@@ -185,6 +148,9 @@ class ArchivingOrchestrator:
         # merge the new config with the old one
         config = merge_dicts(vars(parsed), yaml_config)
 
+        # set up the authentication dict as needed
+        config = self.setup_authentication(config)
+
         # clean out args from the base_parser that we don't want in the config
         for key in vars(basic_config):
             config.pop(key, None)
@@ -206,16 +172,13 @@ class ArchivingOrchestrator:
             parser = self.parser
 
         # Module loading from the command line
-        for module_type in BaseModule.MODULE_TYPES:
+        for module_type in MODULE_TYPES:
             parser.add_argument(f'--{module_type}s', dest=f'{module_type}s', nargs='+', help=f'the {module_type}s to use', default=[], action=UniqueAppendAction)
 
     def add_additional_args(self, parser: argparse.ArgumentParser = None):
         if not parser:
             parser = self.parser
 
-        # allow passing URLs directly on the command line
-        parser.add_argument('urls', nargs='*', default=[], help='URL(s) to archive, either a single URL or a list of urls, should not come from config.yaml')
-
         parser.add_argument('--authentication', dest='authentication', help='A dictionary of sites and their authentication methods \
                                                                             (token, username etc.) that extractors can use to log into \
                                                                             a website. If passing this on the command line, use a JSON string. \
@@ -232,10 +195,14 @@ class ArchivingOrchestrator:
     def add_individual_module_args(self, modules: list[LazyBaseModule] = None, parser: argparse.ArgumentParser = None) -> None:
 
         if not modules:
-            modules = available_modules(with_manifest=True)
+            modules = self.module_factory.available_modules()
         
         for module in modules:
-
+            if module.name == 'cli_feeder':
+                # special case. For the CLI feeder, allow passing URLs directly on the command line without setting --cli_feeder.urls=
+                parser.add_argument('urls', nargs='*', default=[], help='URL(s) to archive, either a single URL or a list of urls, should not come from config.yaml')
+                continue
+                
             if not module.configs:
                 # this module has no configs, don't show anything in the help
                 # (TODO: do we want to show something about this module though, like a description?)
@@ -273,12 +240,25 @@ class ArchivingOrchestrator:
         self.basic_parser.exit()
 
     def setup_logging(self, config):
-        # setup loguru logging
-        logger.remove(0)  # remove the default logger
+
         logging_config = config['logging']
-        logger.add(sys.stderr, level=logging_config['level'])
-        if log_file := logging_config['file']:
-            logger.add(log_file) if not logging_config['rotation'] else logger.add(log_file, rotation=logging_config['rotation'])
+
+        if logging_config.get('enabled', True) is False:
+            # disabled logging settings, they're set on a higher level
+            logger.disable('auto_archiver')
+            return
+
+        # setup loguru logging
+        try:
+            logger.remove(0)  # remove the default logger
+        except ValueError:
+            pass
+
+        # add other logging info
+        if self.logger_id is None: # note - need direct comparison to None since need to consider falsy value 0
+            self.logger_id = logger.add(sys.stderr, level=logging_config['level'])
+            if log_file := logging_config['file']:
+                logger.add(log_file) if not logging_config['rotation'] else logger.add(log_file, rotation=logging_config['rotation'])
 
     def install_modules(self, modules_by_type):
         """
@@ -288,57 +268,36 @@ class ArchivingOrchestrator:
         """
 
         invalid_modules = []
-        for module_type in BaseModule.MODULE_TYPES:
+        for module_type in MODULE_TYPES:
 
             step_items = []
             modules_to_load = modules_by_type[f"{module_type}s"]
-            assert modules_to_load, f"No {module_type}s were configured. Make sure to set at least one {module_type} in your configuration file or on the command line (using --{module_type}s)"
+            if not modules_to_load:
+                raise SetupError(f"No {module_type}s were configured. Make sure to set at least one {module_type} in your configuration file or on the command line (using --{module_type}s)")
 
             def check_steps_ok():
                 if not len(step_items):
-                    logger.error(f"NO {module_type.upper()}S LOADED. Please check your configuration and try again.")
                     if len(modules_to_load):
-                        logger.error(f"Tried to load the following modules, but none were available: {modules_to_load}")
-                    exit()
+                        logger.error(f"Unable to load any {module_type}s. Tried the following, but none were available: {modules_to_load}")
+                    raise SetupError(f"NO {module_type.upper()}S LOADED. Please check your configuration and try again.")
+                
 
                 if (module_type == 'feeder' or module_type == 'formatter') and len(step_items) > 1:
-                    logger.error(f"Only one {module_type} is allowed, found {len(step_items)} {module_type}s. Please remove one of the following from your configuration file: {modules_to_load}")
-                    exit()
+                    raise SetupError(f"Only one {module_type} is allowed, found {len(step_items)} {module_type}s. Please remove one of the following from your configuration file: {modules_to_load}")
 
             for module in modules_to_load:
-                if module == 'cli_feeder':
-                    # pseudo module, don't load it
-                    urls = self.config['urls']
-                    if not urls:
-                        logger.error("No URLs provided. Please provide at least one URL via the command line, or set up an alternative feeder. Use --help for more information.")
-                        exit()
-                    # cli_feeder is a pseudo module, it just takes the command line args
-
-                    def feed(self) -> Generator[Metadata]:
-                        for url in urls:
-                            logger.debug(f"Processing URL: '{url}'")
-                            yield Metadata().set_url(url)
-
-                    pseudo_module = type('CLIFeeder', (Feeder,), {
-                        'name': 'cli_feeder',
-                        'display_name': 'CLI Feeder',
-                        '__iter__': feed
-
-                    })()
-
-                    pseudo_module.__iter__ = feed
-                    step_items.append(pseudo_module)
-                    continue
 
                 if module in invalid_modules:
                     continue
+
+                loaded_module = None
                 try:
-                    loaded_module: BaseModule = get_module(module, self.config)
+                    loaded_module: BaseModule = self.module_factory.get_module(module, self.config)
                 except (KeyboardInterrupt, Exception) as e:
                     logger.error(f"Error during setup of modules: {e}\n{traceback.format_exc()}")
-                    if module_type == 'extractor' and loaded_module.name == module:
+                    if loaded_module and module_type == 'extractor':
                         loaded_module.cleanup()
-                    exit()
+                    raise e
 
                 if not loaded_module:
                     invalid_modules.append(module)
@@ -352,21 +311,24 @@ class ArchivingOrchestrator:
     def load_config(self, config_file: str) -> dict:
         if not os.path.exists(config_file) and config_file != DEFAULT_CONFIG_FILE:
             logger.error(f"The configuration file {config_file} was  not found. Make sure the file exists and try again, or run without the --config file to use the default settings.")
-            exit()
+            raise FileNotFoundError(f"Configuration file {config_file} not found")
 
         return read_yaml(config_file)
     
     def setup_config(self, args: list) -> dict:
         """
         Sets up the configuration file, merging the default config with the user's config
+
+        This function should only ever be run once.
         """
+
         self.setup_basic_parser()
 
         # parse the known arguments for now (basically, we want the config file)
         basic_config, unused_args = self.basic_parser.parse_known_args(args)
 
         # setup any custom module paths, so they'll show in the help and for arg parsing
-        setup_paths(basic_config.module_paths)
+        self.module_factory.setup_paths(basic_config.module_paths)
 
         # if help flag was called, then show the help
         if basic_config.help:
@@ -375,19 +337,50 @@ class ArchivingOrchestrator:
         yaml_config = self.load_config(basic_config.config_file)
 
         return self.setup_complete_parser(basic_config, yaml_config, unused_args)
+    
+    def check_for_updates(self):
+        response = requests.get("https://pypi.org/pypi/auto-archiver/json").json()
+        latest_version = response['info']['version']
+        # check version compared to current version
+        if latest_version != __version__:
+            if os.environ.get('RUNNING_IN_DOCKER'):
+                update_cmd = "`docker pull bellingcat/auto-archiver:latest`"
+            else:
+                update_cmd = "`pip install --upgrade auto-archiver`"
+            logger.warning("")
+            logger.warning("********* IMPORTANT: UPDATE AVAILABLE ********")
+            logger.warning(f"A new version of auto-archiver is available (v{latest_version}, you have {__version__})")
+            logger.warning(f"Make sure to update to the latest version using: {update_cmd}")
+            logger.warning("")
 
+        
     def setup(self, args: list):
         """
-        Main entry point for the orchestrator, sets up the basic parser, loads the config file, and sets up the complete parser
+        Function to configure all setup of the orchestrator: setup configs and load modules.
+        
+        This method should only ever be called once
         """
+
+        self.check_for_updates()
+
+        if self.setup_finished:
+            logger.warning("The `setup_config()` function should only ever be run once. \
+                           If you need to re-run the setup, please re-instantiate a new instance of the orchestrator. \
+                           For code implementatations, you should call .setup_config() once then you may call .feed() \
+                           multiple times to archive multiple URLs.")
+            return
+
+        self.setup_basic_parser()
         self.config = self.setup_config(args)
 
         logger.info(f"======== Welcome to the AUTO ARCHIVER ({__version__}) ==========")
         self.install_modules(self.config['steps'])
 
         # log out the modules that were loaded
-        for module_type in BaseModule.MODULE_TYPES:
+        for module_type in MODULE_TYPES:
             logger.info(f"{module_type.upper()}S: " + ", ".join(m.display_name for m in getattr(self, f"{module_type}s")))
+        
+        self.setup_finished = True
 
     def _command_line_run(self, args: list) -> Generator[Metadata]:
         """
@@ -401,8 +394,12 @@ class ArchivingOrchestrator:
         If you wish to make code invocations yourself, you should use the 'setup' and 'feed' methods separately.
         To test configurations, without loading any modules you can also first call 'setup_configs'
         """
-        self.setup(args)
-        return self.feed()
+        try:
+            self.setup(args)
+            return self.feed()
+        except Exception as e:
+            logger.error(e)
+            exit(1)
 
     def cleanup(self) -> None:
         logger.info("Cleaning up")
@@ -467,8 +464,8 @@ class ArchivingOrchestrator:
 
         original_url = result.get_url().strip()
         try:
-            self.assert_valid_url(original_url)
-        except AssertionError as e:
+            check_url_or_raise(original_url)
+        except ValueError as e:
             logger.error(f"Error archiving URL {original_url}: {e}")
             raise e
 
@@ -528,26 +525,27 @@ class ArchivingOrchestrator:
                 logger.error(f"ERROR database {d.name}: {e}: {traceback.format_exc()}")
 
         return result
+    
 
-    def assert_valid_url(self, url: str) -> bool:
+    def setup_authentication(self, config: dict) -> dict:
         """
-        Blocks localhost, private, reserved, and link-local IPs and all non-http/https schemes.
+        Setup authentication for all modules that require it
+
+        Split up strings into multiple sites if they are comma separated
         """
-        assert url.startswith("http://") or url.startswith("https://"), f"Invalid URL scheme"
 
-        parsed = urlparse(url)
-        assert parsed.scheme in ["http", "https"], f"Invalid URL scheme"
-        assert parsed.hostname, f"Invalid URL hostname"
-        assert parsed.hostname != "localhost", f"Invalid URL"
+        authentication = config.get('authentication', {})
 
-        try:  # special rules for IP addresses
-            ip = ip_address(parsed.hostname)
-        except ValueError: pass
-        else:
-            assert ip.is_global, f"Invalid IP used"
-            assert not ip.is_reserved, f"Invalid IP used"
-            assert not ip.is_link_local, f"Invalid IP used"
-            assert not ip.is_private, f"Invalid IP used"
+        # extract out concatenated sites
+        for key, val in copy(authentication).items():
+            if "," in key:
+                for site in key.split(","):
+                    site = site.strip()
+                    authentication[site] = val
+                del authentication[key]
+        
+        config['authentication'] = authentication
+        return config
 
     # Helper Properties
 

src/auto_archiver/core/storage.py

@@ -14,7 +14,7 @@ from auto_archiver.utils.misc import random_str
 
 from auto_archiver.core import Media, BaseModule, Metadata
 from auto_archiver.modules.hash_enricher.hash_enricher import HashEnricher
-from auto_archiver.core.module import get_module
+
 class Storage(BaseModule):
     
     """
@@ -74,7 +74,7 @@ class Storage(BaseModule):
             filename = random_str(24)
         elif filename_generator == "static":
             # load the hash_enricher module
-            he = get_module(HashEnricher, self.config)
+            he = self.module_factory.get_module(HashEnricher, self.config)
             hd = he.calculate_hash(media.filename)
             filename = hd[:24]
         else:

src/auto_archiver/core/validators.py

@@ -1,6 +1,7 @@
 # used as validators for config values. Should raise an exception if the value is invalid.
 from pathlib import Path
 import argparse
+import json
 
 def example_validator(value):
     if "example" not in value:
@@ -16,4 +17,7 @@ def positive_number(value):
 def valid_file(value):
     if not Path(value).is_file():
         raise argparse.ArgumentTypeError(f"File '{value}' does not exist.")
-    return value
+    return value
+
+def json_loader(cli_val):
+    return json.loads(cli_val)

src/auto_archiver/modules/api_db/__manifest__.py

@@ -1,5 +1,5 @@
 {
-    "name": "Auto-Archiver API Database",
+    "name": "Auto Archiver API Database",
     "type": ["database"],
     "entry_point": "api_db::AAApiDb",
     "requires_setup": True,
@@ -39,7 +39,7 @@
         },
     },
     "description": """
-     Provides integration with the Auto-Archiver API for querying and storing archival data.
+     Provides integration with the Auto Archiver API for querying and storing archival data.
 
 ### Features
 - **API Integration**: Supports querying for existing archives and submitting results.
@@ -49,6 +49,6 @@
 - **Optional Storage**: Archives results conditionally based on configuration.
 
 ### Setup
-Requires access to an Auto-Archiver API instance and a valid API token.
+Requires access to an Auto Archiver API instance and a valid API token.
      """,
 }

src/auto_archiver/modules/atlos_db/__init__.py

@@ -1 +0,0 @@
-from atlos_db import AtlosDb

src/auto_archiver/modules/atlos_db/__manifest__.py

@@ -1,38 +0,0 @@
-{
-    "name": "Atlos Database",
-    "type": ["database"],
-    "entry_point": "atlos_db::AtlosDb",
-    "requires_setup": True,
-    "dependencies":
-        {"python": ["loguru",
-                    ""],
-         "bin": [""]},
-    "configs": {
-        "api_token": {
-            "default": None,
-            "help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/",
-            "required": True,
-            "type": "str",
-        },
-        "atlos_url": {
-            "default": "https://platform.atlos.org",
-            "help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.",
-            "type": "str"
-        },
-    },
-    "description": """
-Handles integration with the Atlos platform for managing archival results.
-
-### Features
-- Outputs archival results to the Atlos API for storage and tracking.
-- Updates failure status with error details when archiving fails.
-- Processes and formats metadata, including ISO formatting for datetime fields.
-- Skips processing for items without an Atlos ID.
-
-### Setup
-Required configs:
-- atlos_url: Base URL for the Atlos API.
-- api_token: Authentication token for API access.
-"""
-,
-}

src/auto_archiver/modules/atlos_db/atlos_db.py

@@ -1,66 +0,0 @@
-from typing import Union
-
-import requests
-from loguru import logger
-
-from auto_archiver.core import Database
-from auto_archiver.core import Metadata
-
-
-class AtlosDb(Database):
-    """
-    Outputs results to Atlos
-    """
-
-    def failed(self, item: Metadata, reason: str) -> None:
-        """Update DB accordingly for failure"""
-        # If the item has no Atlos ID, there's nothing for us to do
-        if not item.metadata.get("atlos_id"):
-            logger.info(f"Item {item.get_url()} has no Atlos ID, skipping")
-            return
-
-        requests.post(
-            f"{self.atlos_url}/api/v2/source_material/metadata/{item.metadata['atlos_id']}/auto_archiver",
-            headers={"Authorization": f"Bearer {self.api_token}"},
-            json={"metadata": {"processed": True, "status": "error", "error": reason}},
-        ).raise_for_status()
-        logger.info(
-            f"Stored failure for {item.get_url()} (ID {item.metadata['atlos_id']}) on Atlos: {reason}"
-        )
-
-    def fetch(self, item: Metadata) -> Union[Metadata, bool]:
-        """check and fetch if the given item has been archived already, each
-        database should handle its own caching, and configuration mechanisms"""
-        return False
-
-    def _process_metadata(self, item: Metadata) -> dict:
-        """Process metadata for storage on Atlos. Will convert any datetime
-        objects to ISO format."""
-
-        return {
-            k: v.isoformat() if hasattr(v, "isoformat") else v
-            for k, v in item.metadata.items()
-        }
-
-    def done(self, item: Metadata, cached: bool = False) -> None:
-        """archival result ready - should be saved to DB"""
-
-        if not item.metadata.get("atlos_id"):
-            logger.info(f"Item {item.get_url()} has no Atlos ID, skipping")
-            return
-
-        requests.post(
-            f"{self.atlos_url}/api/v2/source_material/metadata/{item.metadata['atlos_id']}/auto_archiver",
-            headers={"Authorization": f"Bearer {self.api_token}"},
-            json={
-                "metadata": dict(
-                    processed=True,
-                    status="success",
-                    results=self._process_metadata(item),
-                )
-            },
-        ).raise_for_status()
-
-        logger.info(
-            f"Stored success for {item.get_url()} (ID {item.metadata['atlos_id']}) on Atlos"
-        )

src/auto_archiver/modules/atlos_feeder/__init__.py

@@ -1 +0,0 @@
-from .atlos_feeder import AtlosFeeder

src/auto_archiver/modules/atlos_feeder/__manifest__.py

@@ -1,34 +0,0 @@
-{
-    "name": "Atlos Feeder",
-    "type": ["feeder"],
-    "requires_setup": True,
-    "dependencies": {
-        "python": ["loguru", "requests"],
-    },
-    "configs": {
-        "api_token": {
-            "type": "str",
-            "required": True,
-            "help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/",
-        },
-        "atlos_url": {
-            "default": "https://platform.atlos.org",
-            "help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.",
-            "type": "str"
-        },
-    },
-    "description": """
-    AtlosFeeder: A feeder module that integrates with the Atlos API to fetch source material URLs for archival.
-
-    ### Features
-    - Connects to the Atlos API to retrieve a list of source material URLs.
-    - Filters source materials based on visibility, processing status, and metadata.
-    - Converts filtered source materials into `Metadata` objects with the relevant `atlos_id` and URL.
-    - Iterates through paginated results using a cursor for efficient API interaction.
-
-    ### Notes
-    - Requires an Atlos API endpoint and a valid API token for authentication.
-    - Ensures only unprocessed, visible, and ready-to-archive URLs are returned.
-    - Handles pagination transparently when retrieving data from the Atlos API.
-    """
-}

src/auto_archiver/modules/atlos_feeder/atlos_feeder.py

@@ -1,42 +0,0 @@
-import requests
-from loguru import logger
-
-from auto_archiver.core import Feeder
-from auto_archiver.core import Metadata
-
-
-class AtlosFeeder(Feeder):
-
-    def __iter__(self) -> Metadata:
-        # Get all the urls from the Atlos API
-        count = 0
-        cursor = None
-        while True:
-            response = requests.get(
-                f"{self.atlos_url}/api/v2/source_material",
-                headers={"Authorization": f"Bearer {self.api_token}"},
-                params={"cursor": cursor},
-            )
-            data = response.json()
-            response.raise_for_status()
-            cursor = data["next"]
-
-            for item in data["results"]:
-                if (
-                    item["source_url"] not in [None, ""]
-                    and (
-                        item["metadata"]
-                        .get("auto_archiver", {})
-                        .get("processed", False)
-                        != True
-                    )
-                    and item["visibility"] == "visible"
-                    and item["status"] not in ["processing", "pending"]
-                ):
-                    yield Metadata().set_url(item["source_url"]).set(
-                        "atlos_id", item["id"]
-                    )
-                    count += 1
-
-            if len(data["results"]) == 0 or cursor is None:
-                break

src/auto_archiver/modules/atlos_feeder_db_storage/__init__.py

@@ -0,0 +1 @@
+from .atlos_feeder_db_storage import AtlosFeederDbStorage

src/auto_archiver/modules/atlos_feeder_db_storage/__manifest__.py

@@ -0,0 +1,46 @@
+{
+    "name": "Atlos Feeder Database Storage",
+    "type": ["feeder", "database", "storage"],
+"entry_point": "atlos_feeder_db_storage::AtlosFeederDbStorage",
+    "requires_setup": True,
+    "dependencies": {
+        "python": ["loguru", "requests"],
+    },
+    "configs": {
+        "api_token": {
+            "type": "str",
+            "required": True,
+            "help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/",
+        },
+        "atlos_url": {
+            "default": "https://platform.atlos.org",
+            "help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.",
+            "type": "str"
+        },
+    },
+    "description": """
+    A module that integrates with the Atlos API to fetch source material URLs for archival, uplaod extracted media,
+    
+    [Atlos](https://www.atlos.org/) is a visual investigation and archiving platform designed for investigative research, journalism, and open-source intelligence (OSINT). 
+    It helps users organize, analyze, and store media from various sources, making it easier to track and investigate digital evidence.
+    
+    To get started create a new project and obtain an API token from the settings page. You can group event's into Atlos's 'incidents'.
+    Here you can add 'source material' by URLn and the Atlos feeder will fetch these URLs for archival.
+    
+    You can use Atlos only as a 'feeder', however you can also implement the 'database' and 'storage' features to store the media files in Atlos which is recommended.
+    The Auto Archiver will retain the Atlos ID for each item, ensuring that the media and database outputs are uplaoded back into the relevant media item.
+    
+    
+    ### Features
+    - Connects to the Atlos API to retrieve a list of source material URLs.
+    - Iterates through the URLs from all source material items which are unprocessed, visible, and ready to archive.
+    - If the storage option is selected, it will store the media files alongside the original source material item in Atlos.
+    - Is the database option is selected it will output the results to the media item, as well as updating failure status with error details when archiving fails.
+    - Skips Storege/ database upload for items without an Atlos ID - restricting that you must use the Atlos feeder so that it has the Atlos ID to store the results with.
+
+    ### Notes
+    - Requires an Atlos account with a project and a valid API token for authentication.
+    - Ensures only unprocessed, visible, and ready-to-archive URLs are returned.
+    - Feches any media items within an Atlos project, regardless of separation into incidents.
+    """
+}

src/auto_archiver/modules/atlos_feeder_db_storage/atlos_feeder_db_storage.py

@@ -0,0 +1,153 @@
+import hashlib
+import os
+from typing import IO, Iterator, Optional, Union
+
+import requests
+from loguru import logger
+
+from auto_archiver.core import Database, Feeder, Media, Metadata, Storage
+from auto_archiver.utils import calculate_file_hash
+
+
+class AtlosFeederDbStorage(Feeder, Database, Storage):
+
+    def setup(self) -> requests.Session:
+        """create and return a persistent session."""
+        self.session = requests.Session()
+
+    def _get(self, endpoint: str, params: Optional[dict] = None) -> dict:
+        """Wrapper for GET requests to the Atlos API."""
+        url = f"{self.atlos_url}{endpoint}"
+        response = self.session.get(
+            url, headers={"Authorization": f"Bearer {self.api_token}"}, params=params
+        )
+        response.raise_for_status()
+        return response.json()
+
+    def _post(
+        self,
+        endpoint: str,
+        json: Optional[dict] = None,
+        params: Optional[dict] = None,
+        files: Optional[dict] = None,
+    ) -> dict:
+        """Wrapper for POST requests to the Atlos API."""
+        url = f"{self.atlos_url}{endpoint}"
+        response = self.session.post(
+            url,
+            headers={"Authorization": f"Bearer {self.api_token}"},
+            json=json,
+            params=params,
+            files=files,
+        )
+        response.raise_for_status()
+        return response.json()
+
+    # ! Atlos Module - Feeder Methods
+
+    def __iter__(self) -> Iterator[Metadata]:
+        """Iterate over unprocessed, visible source materials from Atlos."""
+        cursor = None
+        while True:
+            data = self._get("/api/v2/source_material", params={"cursor": cursor})
+            cursor = data.get("next")
+            results = data.get("results", [])
+            for item in results:
+                if (
+                    item.get("source_url") not in [None, ""]
+                    and not item.get("metadata", {}).get("auto_archiver", {}).get("processed", False)
+                    and item.get("visibility") == "visible"
+                    and item.get("status") not in ["processing", "pending"]
+                ):
+                    yield Metadata().set_url(item["source_url"]).set("atlos_id", item["id"])
+            if not results or cursor is None:
+                break
+
+    # ! Atlos Module - Database Methods
+
+    def failed(self, item: Metadata, reason: str) -> None:
+        """Mark an item as failed in Atlos, if the ID exists."""
+        atlos_id = item.metadata.get("atlos_id")
+        if not atlos_id:
+            logger.info(f"Item {item.get_url()} has no Atlos ID, skipping")
+            return
+        self._post(
+            f"/api/v2/source_material/metadata/{atlos_id}/auto_archiver",
+            json={"metadata": {"processed": True, "status": "error", "error": reason}},
+        )
+        logger.info(f"Stored failure for {item.get_url()} (ID {atlos_id}) on Atlos: {reason}")
+
+    def fetch(self, item: Metadata) -> Union[Metadata, bool]:
+        """check and fetch if the given item has been archived already, each
+        database should handle its own caching, and configuration mechanisms"""
+        return False
+
+    def _process_metadata(self, item: Metadata) -> dict:
+        """Process metadata for storage on Atlos. Will convert any datetime
+        objects to ISO format."""
+        return {
+            k: v.isoformat() if hasattr(v, "isoformat") else v
+            for k, v in item.metadata.items()
+        }
+
+    def done(self, item: Metadata, cached: bool = False) -> None:
+        """Mark an item as successfully archived in Atlos."""
+        atlos_id = item.metadata.get("atlos_id")
+        if not atlos_id:
+            logger.info(f"Item {item.get_url()} has no Atlos ID, skipping")
+            return
+        self._post(
+            f"/api/v2/source_material/metadata/{atlos_id}/auto_archiver",
+            json={
+                "metadata": {
+                    "processed": True,
+                    "status": "success",
+                    "results": self._process_metadata(item),
+                }
+            },
+        )
+        logger.info(f"Stored success for {item.get_url()} (ID {atlos_id}) on Atlos")
+
+    # ! Atlos Module - Storage Methods
+
+    def get_cdn_url(self, _media: Media) -> str:
+        """Return the base Atlos URL as the CDN URL."""
+        return self.atlos_url
+
+    def upload(self, media: Media, metadata: Optional[Metadata] = None, **_kwargs) -> bool:
+        """Upload a media file to Atlos if it has not been uploaded already."""
+        if metadata is None:
+            logger.error(f"No metadata provided for {media.filename}")
+            return False
+
+        atlos_id = metadata.get("atlos_id")
+        if not atlos_id:
+            logger.error(f"No Atlos ID found in metadata; can't store {media.filename} in Atlos.")
+            return False
+
+        media_hash = calculate_file_hash(media.filename, hash_algo=hashlib.sha256, chunksize=4096)
+
+        # Check whether the media has already been uploaded
+        source_material = self._get(f"/api/v2/source_material/{atlos_id}")["result"]
+        existing_media = [
+            artifact.get("file_hash_sha256")
+            for artifact in source_material.get("artifacts", [])
+        ]
+        if media_hash in existing_media:
+            logger.info(f"{media.filename} with SHA256 {media_hash} already uploaded to Atlos")
+            return True
+
+        # Upload the media to the Atlos API
+        with open(media.filename, "rb") as file_obj:
+            self._post(
+                f"/api/v2/source_material/upload/{atlos_id}",
+                params={"title": media.properties},
+                files={"file": (os.path.basename(media.filename), file_obj)},
+            )
+        logger.info(f"Uploaded {media.filename} to Atlos with ID {atlos_id} and title {media.key}")
+        return True
+
+    def uploadf(self, file: IO[bytes], key: str, **kwargs: dict) -> bool:
+        """Upload a file-like object; not implemented."""
+        pass
+

src/auto_archiver/modules/atlos_storage/__manifest__.py

@@ -1,32 +0,0 @@
-{
-    "name": "Atlos Storage",
-    "type": ["storage"],
-    "requires_setup": True,
-    "dependencies": {
-        "python": ["loguru", "boto3"],
-        "bin": []
-    },
-    "description": """
-    Stores media files in a [Atlos](https://www.atlos.org/).
-
-    ### Features
-    - Saves media files to Atlos, organizing them into folders based on the provided path structure.
-
-    ### Notes
-    - Requires setup with Atlos credentials.
-    - Files are uploaded to the specified `root_folder_id` and organized by the `media.key` structure.
-    """,
-    "configs": {
-        "api_token": {
-            "default": None,
-            "help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/",
-            "required": True,
-            "type": "str"
-        },
-        "atlos_url": {
-            "default": "https://platform.atlos.org",
-            "help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.",
-            "type": "str"
-        },
-    }
-}

src/auto_archiver/modules/atlos_storage/atlos_storage.py

@@ -1,66 +0,0 @@
-import hashlib
-import os
-from typing import IO, Optional
-
-import requests
-from loguru import logger
-
-from auto_archiver.core import Media, Metadata
-from auto_archiver.core import Storage
-
-
-class AtlosStorage(Storage):
-
-    def get_cdn_url(self, _media: Media) -> str:
-        # It's not always possible to provide an exact URL, because it's
-        # possible that the media once uploaded could have been copied to
-        # another project.
-        return self.atlos_url
-    
-    def _hash(self, media: Media) -> str:
-        # Hash the media file using sha-256. We don't use the existing auto archiver
-        # hash because there's no guarantee that the configuerer is using sha-256, which
-        # is how Atlos hashes files.
-
-        sha256 = hashlib.sha256()
-        with open(media.filename, "rb") as f:
-            while True:
-                buf = f.read(4096)
-                if not buf: break
-                sha256.update(buf)
-        return sha256.hexdigest()
-
-    def upload(self, media: Media, metadata: Optional[Metadata]=None, **_kwargs) -> bool:
-        atlos_id = metadata.get("atlos_id")
-        if atlos_id is None:
-            logger.error(f"No Atlos ID found in metadata; can't store {media.filename} on Atlos")
-            return False
-        
-        media_hash = self._hash(media)
-        
-        # Check whether the media has already been uploaded
-        source_material = requests.get(
-            f"{self.atlos_url}/api/v2/source_material/{atlos_id}",
-            headers={"Authorization": f"Bearer {self.api_token}"},
-        ).json()["result"]
-        existing_media = [x["file_hash_sha256"] for x in source_material.get("artifacts", [])]
-        if media_hash in existing_media:
-            logger.info(f"{media.filename} with SHA256 {media_hash} already uploaded to Atlos")
-            return True
-        
-        # Upload the media to the Atlos API
-        requests.post(
-            f"{self.atlos_url}/api/v2/source_material/upload/{atlos_id}",
-            headers={"Authorization": f"Bearer {self.api_token}"},
-            params={
-                "title": media.properties
-            },
-            files={"file": (os.path.basename(media.filename), open(media.filename, "rb"))},
-        ).raise_for_status()
-
-        logger.info(f"Uploaded {media.filename} to Atlos with ID {atlos_id} and title {media.key}")
-        
-        return True
-
-    # must be implemented even if unused
-    def uploadf(self, file: IO[bytes], key: str, **kwargs: dict) -> bool: pass

src/auto_archiver/modules/cli_feeder/__manifest__.py

@@ -0,0 +1,23 @@
+{
+    'name': 'Command Line Feeder',
+    'type': ['feeder'],
+    'entry_point': 'cli_feeder::CLIFeeder',
+    'requires_setup': False,
+    'description': 'Feeds URLs to orchestrator from the command line',
+    'configs': {
+        'urls': {
+            'default': None,
+            'help': 'URL(s) to archive, either a single URL or a list of urls, should not come from config.yaml',
+        },
+    },
+    'description': """
+The Command Line Feeder is the default enabled feeder for the Auto Archiver. It allows you to pass URLs directly to the orchestrator from the command line 
+without the need to specify any additional configuration or command line arguments:
+
+`auto-archiver --feeder cli_feeder -- "https://example.com/1/,https://example.com/2/"`
+
+You can pass multiple URLs by separating them with a space. The URLs will be processed in the order they are provided.
+
+`auto-archiver --feeder cli_feeder -- https://example.com/1/ https://example.com/2/`
+""",
+}

src/auto_archiver/modules/cli_feeder/cli_feeder.py

@@ -0,0 +1,21 @@
+from loguru import logger
+
+from auto_archiver.core.feeder import Feeder
+from auto_archiver.core.metadata import Metadata
+
+class CLIFeeder(Feeder):
+
+    def setup(self) -> None:
+        self.urls = self.config['urls']
+        if not self.urls:
+            raise ValueError("No URLs provided. Please provide at least one URL via the command line, or set up an alternative feeder. Use --help for more information.")
+
+    def __iter__(self) -> Metadata:
+        urls = self.config['urls']
+        for url in urls:
+            logger.debug(f"Processing {url}")
+            m = Metadata().set_url(url)
+            m.set_context("folder", "cli")
+            yield m
+
+        logger.success(f"Processed {len(urls)} URL(s)")

src/auto_archiver/modules/console_db/console_db.py

@@ -10,7 +10,7 @@ class ConsoleDb(Database):
     """
 
     def started(self, item: Metadata) -> None:
-        logger.warning(f"STARTED {item}")
+        logger.info(f"STARTED {item}")
 
     def failed(self, item: Metadata, reason:str) -> None:
         logger.error(f"FAILED {item}: {reason}")

src/auto_archiver/modules/csv_db/__manifest__.py

@@ -6,7 +6,7 @@
                               },
     'entry_point': 'csv_db::CSVDb',
     "configs": {
-            "csv_file": {"default": "db.csv", "help": "CSV file name"}
+            "csv_file": {"default": "db.csv", "help": "CSV file name to save metadata to"},
         },
     "description": """
 Handles exporting archival results to a CSV file.

src/auto_archiver/modules/generic_extractor/__manifest__.py

@@ -28,6 +28,13 @@ the broader archiving framework.
  metadata objects. Some dropins are included in this generic_archiver by default, but
 custom dropins can be created to handle additional websites and passed to the archiver
 via the command line using the `--dropins` option (TODO!).
+
+### Auto-Updates
+
+The Generic Extractor will also automatically check for updates to `yt-dlp` (every 5 days by default).
+This can be configured using the `ytdlp_update_interval` setting (or disabled by setting it to -1).
+If you are having issues with the extractor, you can review the version of `yt-dlp` being used with `yt-dlp --version`.
+
 """,
     "configs": {
         "subtitles": {"default": True, "help": "download subtitles if available", "type": "bool"},
@@ -64,5 +71,10 @@ via the command line using the `--dropins` option (TODO!).
             "default": "inf",
             "help": "Use to limit the number of videos to download when a channel or long page is being extracted. 'inf' means no limit.",
         },
+        "ytdlp_update_interval": {
+            "default": 5,
+            "help": "How often to check for yt-dlp updates (days). If positive, will check and update yt-dlp every [num] days. Set it to -1 to disable, or 0 to always update on every run.",
+            "type": "int",
+        },
     },
 }

src/auto_archiver/modules/generic_extractor/generic_extractor.py

@@ -1,7 +1,11 @@
-import datetime, os, yt_dlp, pysubs2
+import datetime, os
 import importlib
+import subprocess
 from typing import Generator, Type
+
+import yt_dlp
 from yt_dlp.extractor.common import InfoExtractor
+import pysubs2
 
 from loguru import logger
 
@@ -11,6 +15,44 @@ from auto_archiver.core import Metadata, Media
 class GenericExtractor(Extractor):
     _dropins = {}
 
+    def setup(self):
+        # check for file .ytdlp-update in the secrets folder
+        if self.ytdlp_update_interval < 0:
+            return
+        
+        use_secrets = os.path.exists('secrets')
+        path = os.path.join('secrets' if use_secrets else '', '.ytdlp-update')
+        next_update_check = None
+        if os.path.exists(path):
+            with open(path, "r") as f:
+                next_update_check = datetime.datetime.fromisoformat(f.read())
+        
+        if not next_update_check or next_update_check < datetime.datetime.now():
+            self.update_ytdlp()
+
+            next_update_check = datetime.datetime.now() + datetime.timedelta(days=self.ytdlp_update_interval)
+            with open(path, "w") as f:
+                f.write(next_update_check.isoformat())
+
+    def update_ytdlp(self):
+        logger.info("Checking and updating yt-dlp...")
+        logger.info(f"Tip: change the 'ytdlp_update_interval' setting to control how often yt-dlp is updated. Set to -1 to disable or 0 to enable on every run. Current setting: {self.ytdlp_update_interval}")
+        from importlib.metadata import version as get_version
+        old_version = get_version("yt-dlp")
+        try:
+            # try and update with pip (this works inside poetry environment and in a normal virtualenv)
+            result = subprocess.run(["pip", "install", "--upgrade", "yt-dlp"], check=True, capture_output=True)
+
+            if "Successfully installed yt-dlp" in result.stdout.decode():
+                new_version = importlib.metadata.version("yt-dlp")
+                logger.info(f"yt-dlp successfully (from {old_version} to {new_version})")
+                importlib.reload(yt_dlp)
+            else:
+                logger.info("yt-dlp already up to date")
+
+        except Exception as e:
+            logger.error(f"Error updating yt-dlp: {e}")
+
     def suitable_extractors(self, url: str) -> Generator[str, None, None]:
         """
         Returns a list of valid extractors for the given URL"""
@@ -86,7 +128,7 @@ class GenericExtractor(Extractor):
         # keep both 'title' and 'fulltitle', but prefer 'title', falling back to 'fulltitle' if it doesn't exist
         result.set_title(video_data.pop('title', video_data.pop('fulltitle', "")))
         result.set_url(url)
-
+        if "description" in video_data: result.set_content(video_data["description"])
         # extract comments if enabled
         if self.comments:
             result.set("comments", [{
@@ -280,7 +322,8 @@ class GenericExtractor(Extractor):
         
         # set up auth
         auth = self.auth_for_site(url, extract_cookies=False)
-        # order of importance: username/pasword -> api_key -> cookie -> cookie_from_browser -> cookies_file
+
+        # order of importance: username/pasword -> api_key -> cookie -> cookies_from_browser -> cookies_file
         if auth:
             if 'username' in auth and 'password' in auth:
                 logger.debug(f'Using provided auth username and password for {url}')
@@ -289,12 +332,12 @@ class GenericExtractor(Extractor):
             elif 'cookie' in auth:
                 logger.debug(f'Using provided auth cookie for {url}')
                 yt_dlp.utils.std_headers['cookie'] = auth['cookie']
-            elif 'cookie_from_browser' in auth:
-                logger.debug(f'Using extracted cookies from browser {self.cookies_from_browser} for {url}')
+            elif 'cookies_from_browser' in auth:
+                logger.debug(f'Using extracted cookies from browser {auth["cookies_from_browser"]} for {url}')
                 ydl_options['cookiesfrombrowser'] = auth['cookies_from_browser']
             elif 'cookies_file' in auth:
-                logger.debug(f'Using cookies from file {self.cookie_file} for {url}')
-                ydl_options['cookiesfile'] = auth['cookies_file']
+                logger.debug(f'Using cookies from file {auth["cookies_file"]} for {url}')
+                ydl_options['cookiefile'] = auth['cookies_file']
 
         ydl = yt_dlp.YoutubeDL(ydl_options) # allsubtitles and subtitleslangs not working as expected, so default lang is always "en"
 

src/auto_archiver/modules/gsheet_db/__init__.py

@@ -1 +0,0 @@
-from .gsheet_db import GsheetsDb

src/auto_archiver/modules/gsheet_db/__manifest__.py

@@ -1,38 +0,0 @@
-{
-    "name": "Google Sheets Database",
-    "type": ["database"],
-    "entry_point": "gsheet_db::GsheetsDb",
-    "requires_setup": True,
-    "dependencies": {
-        "python": ["loguru", "gspread", "slugify"],
-    },
-    "configs": {
-        "allow_worksheets": {
-            "default": set(),
-            "help": "(CSV) only worksheets whose name is included in allow are included (overrides worksheet_block), leave empty so all are allowed",
-        },
-        "block_worksheets": {
-            "default": set(),
-            "help": "(CSV) explicitly block some worksheets from being processed",
-        },
-        "use_sheet_names_in_stored_paths": {
-            "default": True,
-            "type": "bool",
-            "help": "if True the stored files path will include 'workbook_name/worksheet_name/...'",
-        }
-    },
-    "description": """
-    GsheetsDatabase:
-    Handles integration with Google Sheets for tracking archival tasks.
-
-### Features
-- Updates a Google Sheet with the status of the archived URLs, including in progress, success or failure, and method used.
-- Saves metadata such as title, text, timestamp, hashes, screenshots, and media URLs to designated columns.
-- Formats media-specific metadata, such as thumbnails and PDQ hashes for the sheet.
-- Skips redundant updates for empty or invalid data fields.
-
-### Notes
-- Currently works only with metadata provided by GsheetFeeder. 
-- Requires configuration of a linked Google Sheet and appropriate API credentials.
-    """
-}

src/auto_archiver/modules/gsheet_db/gsheet_db.py

@@ -1,114 +0,0 @@
-from typing import Union, Tuple
-from urllib.parse import quote
-
-from loguru import logger
-
-from auto_archiver.core import Database
-from auto_archiver.core import Metadata, Media
-from auto_archiver.modules.gsheet_feeder import GWorksheet
-from auto_archiver.utils.misc import get_current_timestamp
-
-
-class GsheetsDb(Database):
-    """
-    NB: only works if GsheetFeeder is used.
-    could be updated in the future to support non-GsheetFeeder metadata
-    """
-
-    def started(self, item: Metadata) -> None:
-        logger.warning(f"STARTED {item}")
-        gw, row = self._retrieve_gsheet(item)
-        gw.set_cell(row, "status", "Archive in progress")
-
-    def failed(self, item: Metadata, reason: str) -> None:
-        logger.error(f"FAILED {item}")
-        self._safe_status_update(item, f"Archive failed {reason}")
-
-    def aborted(self, item: Metadata) -> None:
-        logger.warning(f"ABORTED {item}")
-        self._safe_status_update(item, "")
-
-    def fetch(self, item: Metadata) -> Union[Metadata, bool]:
-        """check if the given item has been archived already"""
-        return False
-
-    def done(self, item: Metadata, cached: bool = False) -> None:
-        """archival result ready - should be saved to DB"""
-        logger.success(f"DONE {item.get_url()}")
-        gw, row = self._retrieve_gsheet(item)
-        # self._safe_status_update(item, 'done')
-
-        cell_updates = []
-        row_values = gw.get_row(row)
-
-        def batch_if_valid(col, val, final_value=None):
-            final_value = final_value or val
-            try:
-                if val and gw.col_exists(col) and gw.get_cell(row_values, col) == "":
-                    cell_updates.append((row, col, final_value))
-            except Exception as e:
-                logger.error(f"Unable to batch {col}={final_value} due to {e}")
-
-        status_message = item.status
-        if cached:
-            status_message = f"[cached] {status_message}"
-        cell_updates.append((row, "status", status_message))
-
-        media: Media = item.get_final_media()
-        if hasattr(media, "urls"):
-            batch_if_valid("archive", "\n".join(media.urls))
-        batch_if_valid("date", True, get_current_timestamp())
-        batch_if_valid("title", item.get_title())
-        batch_if_valid("text", item.get("content", ""))
-        batch_if_valid("timestamp", item.get_timestamp())
-        if media:
-            batch_if_valid("hash", media.get("hash", "not-calculated"))
-
-        # merge all pdq hashes into a single string, if present
-        pdq_hashes = []
-        all_media = item.get_all_media()
-        for m in all_media:
-            if pdq := m.get("pdq_hash"):
-                pdq_hashes.append(pdq)
-        if len(pdq_hashes):
-            batch_if_valid("pdq_hash", ",".join(pdq_hashes))
-
-        if (screenshot := item.get_media_by_id("screenshot")) and hasattr(
-            screenshot, "urls"
-        ):
-            batch_if_valid("screenshot", "\n".join(screenshot.urls))
-
-        if thumbnail := item.get_first_image("thumbnail"):
-            if hasattr(thumbnail, "urls"):
-                batch_if_valid("thumbnail", f'=IMAGE("{thumbnail.urls[0]}")')
-
-        if browsertrix := item.get_media_by_id("browsertrix"):
-            batch_if_valid("wacz", "\n".join(browsertrix.urls))
-            batch_if_valid(
-                "replaywebpage",
-                "\n".join(
-                    [
-                        f"https://replayweb.page/?source={quote(wacz)}#view=pages&url={quote(item.get_url())}"
-                        for wacz in browsertrix.urls
-                    ]
-                ),
-            )
-
-        gw.batch_set_cell(cell_updates)
-
-    def _safe_status_update(self, item: Metadata, new_status: str) -> None:
-        try:
-            gw, row = self._retrieve_gsheet(item)
-            gw.set_cell(row, "status", new_status)
-        except Exception as e:
-            logger.debug(f"Unable to update sheet: {e}")
-
-    def _retrieve_gsheet(self, item: Metadata) -> Tuple[GWorksheet, int]:
-
-        if gsheet := item.get_context("gsheet"):
-            gw: GWorksheet = gsheet.get("worksheet")
-            row: int = gsheet.get("row")
-        elif self.sheet_id:
-            logger.error(f"Unable to retrieve Gsheet for {item.get_url()}, GsheetDB must be used alongside GsheetFeeder.")
-
-        return gw, row

src/auto_archiver/modules/gsheet_feeder/__init__.py

@@ -1,2 +0,0 @@
-from .gworksheet import GWorksheet
-from .gsheet_feeder import GsheetsFeeder

src/auto_archiver/modules/gsheet_feeder/gsheet_feeder.py

@@ -1,96 +0,0 @@
-"""
-GsheetsFeeder: A Google Sheets-based feeder for the Auto Archiver.
-
-This reads data from Google Sheets and filters rows based on user-defined rules.
-The filtered rows are processed into `Metadata` objects.
-
-### Key properties
-- validates the sheet's structure and filters rows based on input configurations.
-- Ensures only rows with valid URLs and unprocessed statuses are included.
-"""
-import os
-import gspread
-
-from loguru import logger
-from slugify import slugify
-
-from auto_archiver.core import Feeder
-from auto_archiver.core import Metadata
-from . import GWorksheet
-
-
-class GsheetsFeeder(Feeder):
-
-    def setup(self) -> None:
-        self.gsheets_client = gspread.service_account(filename=self.service_account)
-        # TODO mv to validators
-        assert self.sheet or self.sheet_id, (
-            "You need to define either a 'sheet' name or a 'sheet_id' in your manifest."
-        )
-
-    def open_sheet(self):
-        if self.sheet:
-            return self.gsheets_client.open(self.sheet)
-        else:  # self.sheet_id
-            return self.gsheets_client.open_by_key(self.sheet_id)
-
-    def __iter__(self) -> Metadata:
-        sh = self.open_sheet()
-        for ii, worksheet in enumerate(sh.worksheets()):
-            if not self.should_process_sheet(worksheet.title):
-                logger.debug(f"SKIPPED worksheet '{worksheet.title}' due to allow/block rules")
-                continue
-            logger.info(f'Opening worksheet {ii=}: {worksheet.title=} header={self.header}')
-            gw = GWorksheet(worksheet, header_row=self.header, columns=self.columns)
-            if len(missing_cols := self.missing_required_columns(gw)):
-                logger.warning(f"SKIPPED worksheet '{worksheet.title}' due to missing required column(s) for {missing_cols}")
-                continue
-
-            # process and yield metadata here:
-            yield from self._process_rows(gw)
-            logger.success(f'Finished worksheet {worksheet.title}')
-
-    def _process_rows(self, gw: GWorksheet):
-        for row in range(1 + self.header, gw.count_rows() + 1):
-            url = gw.get_cell(row, 'url').strip()
-            if not len(url): continue
-            original_status = gw.get_cell(row, 'status')
-            status = gw.get_cell(row, 'status', fresh=original_status in ['', None])
-            # TODO: custom status parser(?) aka should_retry_from_status
-            if status not in ['', None]: continue
-
-            # All checks done - archival process starts here
-            m = Metadata().set_url(url)
-            self._set_context(m, gw, row)
-            yield m
-
-    def _set_context(self, m: Metadata, gw: GWorksheet, row: int) -> Metadata:
-        # TODO: Check folder value not being recognised
-        m.set_context("gsheet", {"row": row, "worksheet": gw})
-
-        if gw.get_cell_or_default(row, 'folder', "") is None:
-            folder = ''
-        else:
-            folder = slugify(gw.get_cell_or_default(row, 'folder', "").strip())
-        if len(folder):
-            if self.use_sheet_names_in_stored_paths:
-                m.set_context("folder", os.path.join(folder, slugify(self.sheet), slugify(gw.wks.title)))
-            else:
-                m.set_context("folder", folder)
-
-
-    def should_process_sheet(self, sheet_name: str) -> bool:
-        if len(self.allow_worksheets) and sheet_name not in self.allow_worksheets:
-            # ALLOW rules exist AND sheet name not explicitly allowed
-            return False
-        if len(self.block_worksheets) and sheet_name in self.block_worksheets:
-            # BLOCK rules exist AND sheet name is blocked
-            return False
-        return True
-
-    def missing_required_columns(self, gw: GWorksheet) -> list:
-        missing = []
-        for required_col in ['url', 'status']:
-            if not gw.col_exists(required_col):
-                missing.append(required_col)
-        return missing

src/auto_archiver/modules/gsheet_feeder_db/__init__.py

@@ -0,0 +1,2 @@
+from .gworksheet import GWorksheet
+from .gsheet_feeder_db import GsheetsFeederDB

src/auto_archiver/modules/gsheet_feeder_db/__manifest__.py

@@ -1,7 +1,7 @@
 {
-    "name": "Google Sheets Feeder",
-    "type": ["feeder"],
-    "entry_point": "gsheet_feeder::GsheetsFeeder",
+    "name": "Google Sheets Feeder Database",
+    "type": ["feeder", "database"],
+    "entry_point": "gsheet_feeder_db::GsheetsFeederDB",
     "requires_setup": True,
     "dependencies": {
         "python": ["loguru", "gspread", "slugify"],
@@ -12,10 +12,13 @@
             "default": None,
             "help": "the id of the sheet to archive (alternative to 'sheet' config)",
         },
-        "header": {"default": 1, "help": "index of the header row (starts at 1)", "type": "int"},
+        "header": {"default": 1,
+                   "type": "int",
+                   "help": "index of the header row (starts at 1)", "type": "int"},
         "service_account": {
             "default": "secrets/service_account.json",
-            "help": "service account JSON file path",
+            "help": "service account JSON file path. Learn how to create one: https://gspread.readthedocs.io/en/latest/oauth2.html",
+            "required": True,
         },
         "columns": {
             "default": {
@@ -34,8 +37,21 @@
                 "wacz": "wacz",
                 "replaywebpage": "replaywebpage",
             },
-            "help": "names of columns in the google sheet (stringified JSON object)",
-            "type": "auto_archiver.utils.json_loader",
+            "help": "Custom names for the columns in your Google sheet. If you don't want to use the default column names, change them with this setting",
+            "type": "json_loader",
+        },
+        "allow_worksheets": {
+            "default": set(),
+            "help": "A list of worksheet names that should be processed (overrides worksheet_block), leave empty so all are allowed",
+        },
+        "block_worksheets": {
+            "default": set(),
+            "help": "A list of worksheet names for worksheets that should be explicitly blocked from being processed",
+        },
+        "use_sheet_names_in_stored_paths": {
+            "default": True,
+            "help": "if True the stored files path will include 'workbook_name/worksheet_name/...'",
+            "type": "bool",
         },
         "allow_worksheets": {
             "default": set(),
@@ -47,13 +63,13 @@
         },
         "use_sheet_names_in_stored_paths": {
             "default": True,
-            "help": "if True the stored files path will include 'workbook_name/worksheet_name/...'",
             "type": "bool",
-        },
+            "help": "if True the stored files path will include 'workbook_name/worksheet_name/...'",
+        }
     },
     "description": """
-    GsheetsFeeder 
-    A Google Sheets-based feeder for the Auto Archiver.
+    GsheetsFeederDatabase
+    A Google Sheets-based feeder and optional database for the Auto Archiver.
 
     This reads data from Google Sheets and filters rows based on user-defined rules.
     The filtered rows are processed into `Metadata` objects.
@@ -63,9 +79,16 @@
     - Processes only worksheets allowed by the `allow_worksheets` and `block_worksheets` configurations.
     - Ensures only rows with valid URLs and unprocessed statuses are included for archival.
     - Supports organizing stored files into folder paths based on sheet and worksheet names.
+    - If the database is enabled, this updates the Google Sheet with the status of the archived URLs, including in progress, success or failure, and method used.
+    - Saves metadata such as title, text, timestamp, hashes, screenshots, and media URLs to designated columns.
+    - Formats media-specific metadata, such as thumbnails and PDQ hashes for the sheet.
+    - Skips redundant updates for empty or invalid data fields.
 
-    ### Notes
-    - Requires a Google Service Account JSON file for authentication. Suggested location is `secrets/gsheets_service_account.json`.
-    - Create the sheet using the template provided in the docs.
+    ### Setup
+    - Requires a Google Service Account JSON file for authentication, which should be stored in `secrets/gsheets_service_account.json`.
+    To set up a service account, follow the instructions [here](https://gspread.readthedocs.io/en/latest/oauth2.html).
+    - Define the `sheet` or `sheet_id` configuration to specify the sheet to archive.
+    - Customize the column names in your Google sheet using the `columns` configuration.
+    - The Google Sheet can be used soley as a feeder or as a feeder and database, but note you can't currently feed into the database from an alternate feeder.
     """,
 }

src/auto_archiver/modules/gsheet_feeder_db/gsheet_feeder_db.py

@@ -0,0 +1,196 @@
+"""
+GsheetsFeeder: A Google Sheets-based feeder for the Auto Archiver.
+
+This reads data from Google Sheets and filters rows based on user-defined rules.
+The filtered rows are processed into `Metadata` objects.
+
+### Key properties
+- validates the sheet's structure and filters rows based on input configurations.
+- Ensures only rows with valid URLs and unprocessed statuses are included.
+"""
+import os
+from typing import Tuple, Union
+from urllib.parse import quote
+
+import gspread
+from loguru import logger
+from slugify import slugify
+
+from auto_archiver.core import Feeder, Database, Media
+from auto_archiver.core import Metadata
+from auto_archiver.modules.gsheet_feeder_db import GWorksheet
+from auto_archiver.utils.misc import calculate_file_hash, get_current_timestamp
+
+
+class GsheetsFeederDB(Feeder, Database):
+
+    def setup(self) -> None:
+        self.gsheets_client = gspread.service_account(filename=self.service_account)
+        # TODO mv to validators
+        if not self.sheet and not self.sheet_id:
+            raise ValueError("You need to define either a 'sheet' name or a 'sheet_id' in your manifest.")
+
+    def open_sheet(self):
+        if self.sheet:
+            return self.gsheets_client.open(self.sheet)
+        else:  # self.sheet_id
+            return self.gsheets_client.open_by_key(self.sheet_id)
+
+    def __iter__(self) -> Metadata:
+        sh = self.open_sheet()
+        for ii, worksheet in enumerate(sh.worksheets()):
+            if not self.should_process_sheet(worksheet.title):
+                logger.debug(f"SKIPPED worksheet '{worksheet.title}' due to allow/block rules")
+                continue
+            logger.info(f'Opening worksheet {ii=}: {worksheet.title=} header={self.header}')
+            gw = GWorksheet(worksheet, header_row=self.header, columns=self.columns)
+            if len(missing_cols := self.missing_required_columns(gw)):
+                logger.warning(f"SKIPPED worksheet '{worksheet.title}' due to missing required column(s) for {missing_cols}")
+                continue
+
+            # process and yield metadata here:
+            yield from self._process_rows(gw)
+            logger.success(f'Finished worksheet {worksheet.title}')
+
+    def _process_rows(self, gw: GWorksheet):
+        for row in range(1 + self.header, gw.count_rows() + 1):
+            url = gw.get_cell(row, 'url').strip()
+            if not len(url): continue
+            original_status = gw.get_cell(row, 'status')
+            status = gw.get_cell(row, 'status', fresh=original_status in ['', None])
+            # TODO: custom status parser(?) aka should_retry_from_status
+            if status not in ['', None]: continue
+
+            # All checks done - archival process starts here
+            m = Metadata().set_url(url)
+            self._set_context(m, gw, row)
+            yield m
+
+    def _set_context(self, m: Metadata, gw: GWorksheet, row: int) -> Metadata:
+        # TODO: Check folder value not being recognised
+        m.set_context("gsheet", {"row": row, "worksheet": gw})
+
+        if gw.get_cell_or_default(row, 'folder', "") is None:
+            folder = ''
+        else:
+            folder = slugify(gw.get_cell_or_default(row, 'folder', "").strip())
+        if len(folder):
+            if self.use_sheet_names_in_stored_paths:
+                m.set_context("folder", os.path.join(folder, slugify(self.sheet), slugify(gw.wks.title)))
+            else:
+                m.set_context("folder", folder)
+
+    def should_process_sheet(self, sheet_name: str) -> bool:
+        if len(self.allow_worksheets) and sheet_name not in self.allow_worksheets:
+            # ALLOW rules exist AND sheet name not explicitly allowed
+            return False
+        if len(self.block_worksheets) and sheet_name in self.block_worksheets:
+            # BLOCK rules exist AND sheet name is blocked
+            return False
+        return True
+
+    def missing_required_columns(self, gw: GWorksheet) -> list:
+        missing = []
+        for required_col in ['url', 'status']:
+            if not gw.col_exists(required_col):
+                missing.append(required_col)
+        return missing
+
+
+    def started(self, item: Metadata) -> None:
+        logger.warning(f"STARTED {item}")
+        gw, row = self._retrieve_gsheet(item)
+        gw.set_cell(row, "status", "Archive in progress")
+
+    def failed(self, item: Metadata, reason: str) -> None:
+        logger.error(f"FAILED {item}")
+        self._safe_status_update(item, f"Archive failed {reason}")
+
+    def aborted(self, item: Metadata) -> None:
+        logger.warning(f"ABORTED {item}")
+        self._safe_status_update(item, "")
+
+    def fetch(self, item: Metadata) -> Union[Metadata, bool]:
+        """check if the given item has been archived already"""
+        return False
+
+    def done(self, item: Metadata, cached: bool = False) -> None:
+        """archival result ready - should be saved to DB"""
+        logger.success(f"DONE {item.get_url()}")
+        gw, row = self._retrieve_gsheet(item)
+        # self._safe_status_update(item, 'done')
+
+        cell_updates = []
+        row_values = gw.get_row(row)
+
+        def batch_if_valid(col, val, final_value=None):
+            final_value = final_value or val
+            try:
+                if val and gw.col_exists(col) and gw.get_cell(row_values, col) == "":
+                    cell_updates.append((row, col, final_value))
+            except Exception as e:
+                logger.error(f"Unable to batch {col}={final_value} due to {e}")
+
+        status_message = item.status
+        if cached:
+            status_message = f"[cached] {status_message}"
+        cell_updates.append((row, "status", status_message))
+
+        media: Media = item.get_final_media()
+        if hasattr(media, "urls"):
+            batch_if_valid("archive", "\n".join(media.urls))
+        batch_if_valid("date", True, get_current_timestamp())
+        batch_if_valid("title", item.get_title())
+        batch_if_valid("text", item.get("content", ""))
+        batch_if_valid("timestamp", item.get_timestamp())
+        if media:
+            batch_if_valid("hash", media.get("hash", "not-calculated"))
+
+        # merge all pdq hashes into a single string, if present
+        pdq_hashes = []
+        all_media = item.get_all_media()
+        for m in all_media:
+            if pdq := m.get("pdq_hash"):
+                pdq_hashes.append(pdq)
+        if len(pdq_hashes):
+            batch_if_valid("pdq_hash", ",".join(pdq_hashes))
+
+        if (screenshot := item.get_media_by_id("screenshot")) and hasattr(
+            screenshot, "urls"
+        ):
+            batch_if_valid("screenshot", "\n".join(screenshot.urls))
+
+        if thumbnail := item.get_first_image("thumbnail"):
+            if hasattr(thumbnail, "urls"):
+                batch_if_valid("thumbnail", f'=IMAGE("{thumbnail.urls[0]}")')
+
+        if browsertrix := item.get_media_by_id("browsertrix"):
+            batch_if_valid("wacz", "\n".join(browsertrix.urls))
+            batch_if_valid(
+                "replaywebpage",
+                "\n".join(
+                    [
+                        f"https://replayweb.page/?source={quote(wacz)}#view=pages&url={quote(item.get_url())}"
+                        for wacz in browsertrix.urls
+                    ]
+                ),
+            )
+
+        gw.batch_set_cell(cell_updates)
+
+    def _safe_status_update(self, item: Metadata, new_status: str) -> None:
+        try:
+            gw, row = self._retrieve_gsheet(item)
+            gw.set_cell(row, "status", new_status)
+        except Exception as e:
+            logger.debug(f"Unable to update sheet: {e}")
+
+    def _retrieve_gsheet(self, item: Metadata) -> Tuple[GWorksheet, int]:
+
+        if gsheet := item.get_context("gsheet"):
+            gw: GWorksheet = gsheet.get("worksheet")
+            row: int = gsheet.get("row")
+        elif self.sheet_id:
+            logger.error(f"Unable to retrieve Gsheet for {item.get_url()}, GsheetDB must be used alongside GsheetFeeder.")
+
+        return gw, row

src/auto_archiver/modules/gsheet_feeder_db/gworksheet.py

@@ -17,6 +17,7 @@ class GWorksheet:
         'thumbnail': 'thumbnail',
         'timestamp': 'upload timestamp',
         'title': 'upload title',
+        'text': 'text content',
         'screenshot': 'screenshot',
         'hash': 'hash',
         'pdq_hash': 'perceptual hashes',

src/auto_archiver/modules/html_formatter/__manifest__.py

@@ -7,7 +7,9 @@
                           "bin": [""]
     },
     "configs": {
-            "detect_thumbnails": {"default": True, "help": "if true will group by thumbnails generated by thumbnail enricher by id 'thumbnail_00'"}
+            "detect_thumbnails": {"default": True,
+                                  "help": "if true will group by thumbnails generated by thumbnail enricher by id 'thumbnail_00'",
+                                  "type": "bool"},
         },
     "description": """ """,
 }

src/auto_archiver/modules/html_formatter/html_formatter.py

@@ -10,7 +10,6 @@ from auto_archiver.version import __version__
 from auto_archiver.core import Metadata, Media
 from auto_archiver.core import Formatter
 from auto_archiver.utils.misc import random_str
-from auto_archiver.core.module import get_module
 
 class HtmlFormatter(Formatter):
     environment: Environment = None
@@ -50,7 +49,7 @@ class HtmlFormatter(Formatter):
         final_media = Media(filename=html_path, _mimetype="text/html")
 
         # get the already instantiated hash_enricher module
-        he = get_module('hash_enricher', self.config)
+        he = self.module_factory.get_module('hash_enricher', self.config)
         if len(hd := he.calculate_hash(final_media.filename)):
             final_media.set("hash", f"{he.algorithm}:{hd}")
 

src/auto_archiver/modules/instagram_extractor/__manifest__.py

@@ -10,25 +10,30 @@
     "requires_setup": True,
     "configs": {
         "username": {"required": True,
-                     "help": "a valid Instagram username"},
+                     "help": "A valid Instagram username."},
         "password": {
             "required": True,
-            "help": "the corresponding Instagram account password",
+            "help": "The corresponding Instagram account password.",
         },
         "download_folder": {
             "default": "instaloader",
-            "help": "name of a folder to temporarily download content to",
+            "help": "Name of a folder to temporarily download content to.",
         },
         "session_file": {
             "default": "secrets/instaloader.session",
-            "help": "path to the instagram session which saves session credentials",
+            "help": "Path to the instagram session file which saves session credentials. If one doesn't exist this gives the path to store a new one.",
         },
         # TODO: fine-grain
         # "download_stories": {"default": True, "help": "if the link is to a user profile: whether to get stories information"},
     },
     "description": """
-    Uses the [Instaloader library](https://instaloader.github.io/as-module.html) to download content from Instagram. This class handles both individual posts
-    and user profiles, downloading as much information as possible, including images, videos, text, stories,
+    Uses the [Instaloader library](https://instaloader.github.io/as-module.html) to download content from Instagram. 
+    
+      > ⚠️ **Warning**  
+      > This module is not actively maintained due to known issues with blocking.  
+      > Prioritise usage of the [Instagram Tbot Extractor](./instagram_tbot_extractor.md) and [Instagram API Extractor](./instagram_api_extractor.md)
+  
+    This class handles both individual posts and user profiles, downloading as much information as possible, including images, videos, text, stories,
     highlights, and tagged posts. 
     Authentication is required via username/password or a session file.
                     

src/auto_archiver/modules/instagram_extractor/instagram_extractor.py

@@ -3,7 +3,7 @@
     highlights, and tagged posts. Authentication is required via username/password or a session file.
 
 """
-import re, os, shutil, traceback
+import re, os, shutil
 import instaloader
 from loguru import logger
 
@@ -15,10 +15,9 @@ class InstagramExtractor(Extractor):
     """
     Uses Instaloader to download either a post (inc images, videos, text) or as much as possible from a profile (posts, stories, highlights, ...)
     """
+
     # NB: post regex should be tested before profile
-
     valid_url = re.compile(r"(?:(?:http|https):\/\/)?(?:www.)?(?:instagram.com|instagr.am|instagr.com)\/")
-
     # https://regex101.com/r/MGPquX/1
     post_pattern = re.compile(r"{valid_url}(?:p|reel)\/(\w+)".format(valid_url=valid_url))
     # https://regex101.com/r/6Wbsxa/1
@@ -28,19 +27,22 @@ class InstagramExtractor(Extractor):
     def setup(self) -> None:
 
         self.insta = instaloader.Instaloader(
-            download_geotags=True, download_comments=True, compress_json=False, dirname_pattern=self.download_folder, filename_pattern="{date_utc}_UTC_{target}__{typename}"
+            download_geotags=True,
+            download_comments=True,
+            compress_json=False,
+            dirname_pattern=self.download_folder,
+            filename_pattern="{date_utc}_UTC_{target}__{typename}"
         )
         try:
             self.insta.load_session_from_file(self.username, self.session_file)
         except Exception as e:
-            logger.error(f"Unable to login from session file: {e}\n{traceback.format_exc()}")
             try:
-                self.insta.login(self.username, config.instagram_self.password)
-                # TODO: wait for this issue to be fixed https://github.com/instaloader/instaloader/issues/1758
+                logger.debug(f"Session file failed", exc_info=True)
+                logger.info("No valid session file found - Attempting login with use and password.")
+                self.insta.login(self.username, self.password)
                 self.insta.save_session_to_file(self.session_file)
-            except Exception as e2:
-                logger.error(f"Unable to finish login (retrying from file): {e2}\n{traceback.format_exc()}")
-
+            except Exception as e:
+                logger.error(f"Failed to setup Instagram Extractor with Instagrapi. {e}")
 
 
     def download(self, item: Metadata) -> Metadata:

src/auto_archiver/modules/instagram_tbot_extractor/instagram_tbot_extractor.py

@@ -77,13 +77,14 @@ class InstagramTbotExtractor(Extractor):
             chat, since_id = self._send_url_to_bot(url)
             message = self._process_messages(chat, since_id, tmp_dir, result)
 
+            # This may be outdated and replaced by the below message, but keeping until confirmed
             if "You must enter a URL to a post" in message:
                 logger.debug(f"invalid link {url=} for {self.name}: {message}")
                 return False
-            # # TODO: It currently returns this as a success - is that intentional?
-            # if "Media not found or unavailable" in message:
-            #     logger.debug(f"invalid link {url=} for {self.name}: {message}")
-            #     return False
+
+            if "Media not found or unavailable" in message:
+                logger.debug(f"No media found for link {url=} for {self.name}: {message}")
+                return False
 
             if message:
                 result.set_content(message).set_title(message[:128])
@@ -103,7 +104,7 @@ class InstagramTbotExtractor(Extractor):
         message = ""
         time.sleep(3)
         # media is added before text by the bot so it can be used as a stop-logic mechanism
-        while attempts < (self.timeout - 3) and (not message or not len(seen_media)):
+        while attempts < max(self.timeout - 3, 3) and (not message or not len(seen_media)):
             attempts += 1
             time.sleep(1)
             for post in self.client.iter_messages(chat, min_id=since_id):

src/auto_archiver/modules/local_storage/__manifest__.py

@@ -17,7 +17,9 @@
             "choices": ["random", "static"],
         },
         "save_to": {"default": "./local_archive", "help": "folder where to save archived content"},
-        "save_absolute": {"default": False, "help": "whether the path to the stored file is absolute or relative in the output result inc. formatters (WARN: leaks the file structure)"},
+        "save_absolute": {"default": False, 
+                          "type": "bool",
+                          "help": "whether the path to the stored file is absolute or relative in the output result inc. formatters (WARN: leaks the file structure)"},
     },
     "description": """
     LocalStorage: A storage module for saving archived content locally on the filesystem.

src/auto_archiver/modules/screenshot_enricher/__manifest__.py

@@ -6,13 +6,25 @@
         "python": ["loguru", "selenium"],
     },
     "configs": {
-            "width": {"default": 1280, "help": "width of the screenshots"},
-            "height": {"default": 720, "help": "height of the screenshots"},
-            "timeout": {"default": 60, "help": "timeout for taking the screenshot"},
-            "sleep_before_screenshot": {"default": 4, "help": "seconds to wait for the pages to load before taking screenshot"},
+            "width": {"default": 1280,
+                      "type": "int",
+                      "help": "width of the screenshots"},
+            "height": {"default": 1024,
+                        "type": "int",
+                       "help": "height of the screenshots"},
+            "timeout": {"default": 60,
+                        "type": "int",
+                        "help": "timeout for taking the screenshot"},
+            "sleep_before_screenshot": {"default": 4,
+                                        "type": "int",
+                                        "help": "seconds to wait for the pages to load before taking screenshot"},
             "http_proxy": {"default": "", "help": "http proxy to use for the webdriver, eg http://proxy-user:password@proxy-ip:port"},
-            "save_to_pdf": {"default": False, "help": "save the page as pdf along with the screenshot. PDF saving options can be adjusted with the 'print_options' parameter"},
-            "print_options": {"default": {}, "help": "options to pass to the pdf printer"}
+            "save_to_pdf": {"default": False,
+                            "type": "bool",
+                            "help": "save the page as pdf along with the screenshot. PDF saving options can be adjusted with the 'print_options' parameter"},
+            "print_options": {"default": {},
+                              "help": "options to pass to the pdf printer, in JSON format. See https://www.selenium.dev/documentation/webdriver/interactions/print_page/ for more information",
+                              "type": "json_loader"},
         },
     "description": """
     Captures screenshots and optionally saves web pages as PDFs using a WebDriver.

src/auto_archiver/modules/screenshot_enricher/screenshot_enricher.py

@@ -11,6 +11,10 @@ from auto_archiver.core import Media, Metadata
 
 class ScreenshotEnricher(Enricher):
 
+    def __init__(self, webdriver_factory=None):
+        super().__init__()
+        self.webdriver_factory = webdriver_factory or Webdriver
+
     def enrich(self, to_enrich: Metadata) -> None:
         url = to_enrich.get_url()
 
@@ -20,7 +24,8 @@ class ScreenshotEnricher(Enricher):
 
         logger.debug(f"Enriching screenshot for {url=}")
         auth = self.auth_for_site(url)
-        with Webdriver(self.width, self.height, self.timeout, facebook_accept_cookies='facebook.com' in url,
+        with self.webdriver_factory(
+                self.width, self.height, self.timeout, facebook_accept_cookies='facebook.com' in url,
                        http_proxy=self.http_proxy, print_options=self.print_options, auth=auth) as driver:
             try:
                 driver.get(url)
@@ -38,3 +43,4 @@ class ScreenshotEnricher(Enricher):
                 logger.info("TimeoutException loading page for screenshot")
             except Exception as e:
                 logger.error(f"Got error while loading webdriver for screenshot enricher: {e}")
+

src/auto_archiver/modules/ssl_enricher/__manifest__.py

@@ -7,7 +7,9 @@
     },
     'entry_point': 'ssl_enricher::SSLEnricher',
     "configs": {
-        "skip_when_nothing_archived": {"default": True, "help": "if true, will skip enriching when no media is archived"},
+        "skip_when_nothing_archived": {"default": True,
+                                       "type": 'bool',
+                                       "help": "if true, will skip enriching when no media is archived"},
     },
     "description": """
     Retrieves SSL certificate information for a domain and stores it as a file.

src/auto_archiver/modules/telethon_extractor/__manifest__.py

@@ -14,11 +14,13 @@
             "api_hash": {"default": None, "help": "telegram API_HASH value, go to https://my.telegram.org/apps"},
             "bot_token": {"default": None, "help": "optional, but allows access to more content such as large videos, talk to @botfather"},
             "session_file": {"default": "secrets/anon", "help": "optional, records the telegram login session for future usage, '.session' will be appended to the provided value."},
-            "join_channels": {"default": True, "help": "disables the initial setup with channel_invites config, useful if you have a lot and get stuck"},
+            "join_channels": {"default": True,
+                              "type": "bool",
+                              "help": "disables the initial setup with channel_invites config, useful if you have a lot and get stuck"},
             "channel_invites": {
                 "default": {},
                 "help": "(JSON string) private channel invite links (format: t.me/joinchat/HASH OR t.me/+HASH) and (optional but important to avoid hanging for minutes on startup) channel id (format: CHANNEL_ID taken from a post url like https://t.me/c/CHANNEL_ID/1), the telegram account will join any new channels on setup",
-                "type": "auto_archiver.utils.json_loader",
+                "type": "json_loader",
             }
         },
     "description": """

src/auto_archiver/modules/thumbnail_enricher/__manifest__.py

@@ -7,8 +7,12 @@
         "bin": ["ffmpeg"]
     },
     "configs": {
-            "thumbnails_per_minute": {"default": 60, "help": "how many thumbnails to generate per minute of video, can be limited by max_thumbnails"},
-            "max_thumbnails": {"default": 16, "help": "limit the number of thumbnails to generate per video, 0 means no limit"},
+            "thumbnails_per_minute": {"default": 60,
+                                      "type": "int",
+                                      "help": "how many thumbnails to generate per minute of video, can be limited by max_thumbnails"},
+            "max_thumbnails": {"default": 16,
+                               "type": "int",
+                               "help": "limit the number of thumbnails to generate per video, 0 means no limit"},
         },
     "description": """
     Generates thumbnails for video files to provide visual previews.

src/auto_archiver/modules/thumbnail_enricher/thumbnail_enricher.py

@@ -42,7 +42,7 @@ class ThumbnailEnricher(Enricher):
                         logger.error(f"error getting duration of video {m.filename}: {e}")
                         return
 
-                num_thumbs = int(min(max(1, duration * self.thumbnails_per_minute), self.max_thumbnails))
+                num_thumbs = int(min(max(1, (duration / 60) * self.thumbnails_per_minute), self.max_thumbnails))
                 timestamps = [duration / (num_thumbs + 1) * i for i in range(1, num_thumbs + 1)]
 
                 thumbnails_media = []

src/auto_archiver/modules/wacz_enricher/__init__.py

@@ -1 +0,0 @@
-from .wacz_enricher import WaczExtractorEnricher

src/auto_archiver/modules/wacz_extractor_enricher/__init__.py

@@ -0,0 +1 @@
+from .wacz_extractor_enricher import WaczExtractorEnricher

src/auto_archiver/modules/wacz_extractor_enricher/__manifest__.py

@@ -1,7 +1,7 @@
 {
-    "name": "WACZ Enricher",
+    "name": "WACZ Enricher (and Extractor)",
     "type": ["enricher", "extractor"],
-    "entry_point": "wacz_enricher::WaczExtractorEnricher",
+    "entry_point": "wacz_extractor_enricher::WaczExtractorEnricher",
     "requires_setup": True,
     "dependencies": {
         "python": [
@@ -17,11 +17,19 @@
     "configs": {
             "profile": {"default": None, "help": "browsertrix-profile (for profile generation see https://github.com/webrecorder/browsertrix-crawler#creating-and-using-browser-profiles)."},
             "docker_commands": {"default": None, "help":"if a custom docker invocation is needed"},
-            "timeout": {"default": 120, "help": "timeout for WACZ generation in seconds"},
-            "extract_media": {"default": False, "help": "If enabled all the images/videos/audio present in the WACZ archive will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched."},
-            "extract_screenshot": {"default": True, "help": "If enabled the screenshot captured by browsertrix will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched."},
+            "timeout": {"default": 120,
+                        "type": "int",
+                        "help": "timeout for WACZ generation in seconds", "type": "int"},
+            "extract_media": {"default": False, 
+                              "type": 'bool',
+                              "help": "If enabled all the images/videos/audio present in the WACZ archive will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched."
+                              },
+            "extract_screenshot": {"default": True,
+                                    "type": 'bool',
+                                   "help": "If enabled the screenshot captured by browsertrix will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched."
+                                   },
             "socks_proxy_host": {"default": None, "help": "SOCKS proxy host for browsertrix-crawler, use in combination with socks_proxy_port. eg: user:password@host"},
-            "socks_proxy_port": {"default": None, "help": "SOCKS proxy port for browsertrix-crawler, use in combination with socks_proxy_host. eg 1234"},
+            "socks_proxy_port": {"default": None, "type":"int", "help": "SOCKS proxy port for browsertrix-crawler, use in combination with socks_proxy_host. eg 1234"},
             "proxy_server": {"default": None, "help": "SOCKS server proxy URL, in development"},
         },
     "description": """