From 529d8b60bf141afa78898a7304e33dd794f39bd5 Mon Sep 17 00:00:00 2001 From: Dave Mateer Date: Mon, 16 Jun 2025 14:37:21 +0100 Subject: [PATCH 01/14] Gitgnore to include launch.json and installtion docs to include build script. --- .gitignore | 4 +- docs/source/installation/installation.md | 266 +++++++++++++++++++++++ 2 files changed, 269 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index d14e3bb..3906b23 100644 --- a/.gitignore +++ b/.gitignore @@ -39,4 +39,6 @@ scripts/settings_page.html scripts/settings/src/schema.json .vite downloaded_files -latest_logs \ No newline at end of file +latest_logs +# for launch.json +.vscode \ No newline at end of file diff --git a/docs/source/installation/installation.md b/docs/source/installation/installation.md index 5f68dad..13c3131 100644 --- a/docs/source/installation/installation.md +++ b/docs/source/installation/installation.md @@ -55,6 +55,272 @@ If using the local installation method, you will also need to install the follow 3. (optional) [fonts-noto](https://fonts.google.com/noto) to deal with multiple unicode characters during selenium screenshots: `sudo apt install fonts-noto -y`. 4. [Browsertrix Crawler docker image](https://hub.docker.com/r/webrecorder/browsertrix-crawler) for the WACZ enricher/archiver +### Bash script for Ubuntu Server install + +This acts as a handy guide on all requirements. This is built and tested on the 29th of May 2025 on Ubuntu Server 24.04.2 LTS (which is the current latest LTS) + +```bash +#!/bin/sh + +# I usually run steps manually as logged in with the user: dave +# which the application runs under which makes debugging easier + +cd ~ + +# Clone only my latest branch +git clone -b v1-test --single-branch https://github.com/djhmateer/auto-archiver + +mkdir ~/auto-archiver/secrets +sudo chown -R dave ~/auto-archiver + +sudo apt update -y +sudo apt upgrade -y + +## Python 3.12.3 comes with Ubuntu 24.04.2 + +# Poetry install 2.1.3 on 2nd June 25 +curl -sSL https://install.python-poetry.org | python3 - + +# had to restart shell here.. neither of below worked +# source ~/.bashrc +# exec bash + +cd auto-archiver + +# C++ compiler so pdqhash will install next +sudo apt install build-essential python3-dev + +poetry install + +# FFMpeg +# 6.1.1-3ubuntu5 on 2nd June 25 +sudo apt install ffmpeg -y + +## Firefox +# 139.0+build2-0ubuntu0.24.04.1~mt1 on 2nd Jun 25 +# 16th Jun - don't need anymore as using Chrome in antibot +# cd ~ +# sudo add-apt-repository ppa:mozillateam/ppa -y + +# echo ' +# Package: * +# Pin: release o=LP-PPA-mozillateam +# Pin-Priority: 1001 +# ' | sudo tee /etc/apt/preferences.d/mozilla-firefox + +# echo 'Unattended-Upgrade::Allowed-Origins:: "LP-PPA-mozillateam:${distro_codename}";' | sudo tee /etc/apt/apt.conf.d/51unattended-upgrades-firefox + +# sudo apt install firefox -y + +wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb + +# Chrome +cd ~ +# got problems here - fixed below +# 137.0.7151.103 on 16th Jun 2025 +sudo dpkg -i google-chrome-stable_current_amd64.deb + +# fix dependencies on install above +sudo apt-get install -f + +# had to click a lot on UI to get going. +# to test +# google-chrome + +## Gecko driver +# check version numbers for new ones +# https://github.com/mozilla/geckodriver/releases/ +cd ~ +wget https://github.com/mozilla/geckodriver/releases/download/v0.35.0/geckodriver-v0.35.0-linux64.tar.gz +tar -xvzf geckodriver* +chmod +x geckodriver +sudo mv geckodriver /usr/local/bin/ +rm geckodriver* + +# Fonts +sudo apt install fonts-noto -y + +# Docker +# Add Docker's official GPG key: +sudo apt-get update -y +sudo apt-get install ca-certificates curl -y +sudo install -m 0755 -d /etc/apt/keyrings +sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc +sudo chmod a+r /etc/apt/keyrings/docker.asc + +# Add the repository to Apt sources: +echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ + $(. /etc/os-release && echo "${UBUNTU_CODENAME:-$VERSION_CODENAME}") stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + +sudo apt-get update -y + +sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -y + +# add dave user to docker group +sudo usermod -aG docker $USER + +# restart shell +# TODO try: source ~/.bashrc +# exec bash + +docker pull webrecorder/browsertrix-crawler:latest + +# exif +sudo apt install libimage-exiftool-perl -y + + +## CRON run every minute +# the cron job running as user dave will execute the shell script +sudo chmod +x ~/auto-archiver/scripts/cron_1.sh + +# don't want service to run until a reboot otherwise problems with Gecko driver +sudo service cron stop + +# runs the script every minute +# notice put in a # to disable so will have to manually start it. +cat <> run-auto-archive +#*/1 * * * * dave /home/dave/auto-archiver/scripts/cron_1.sh +EOT + +sudo mv run-auto-archive /etc/cron.d +sudo chown root /etc/cron.d/run-auto-archive +sudo chmod 600 /etc/cron.d/run-auto-archive + + +# secrets folder copy +# I run dev from: +# \\wsl.localhost\Ubuntu-24.04\home\dave\code\auto-archiver\secrets\ + +# orchestration.yaml - for aa config +# service_account - for google spreadsheet +# anon.session - for telethon so don't have to type in phone number +# vk_config.v2.json - so don't have to login to vk again +# profile.tar.gz - for wacz to have a logged in profile for facebook, x.com and instagram to get data + + +# Youtube - POT Tokens +# https://github.com/Brainicism/bgutil-ytdlp-pot-provider +docker run --name bgutil-provider --restart unless-stopped -d -p 4416:4416 brainicism/bgutil-ytdlp-pot-provider + + +# test run +cd ~/auto-archiver + +poetry run python src/auto_archiver --config secrets/orchestration-aa-demo-main.yaml + + + + + + +## HERE + +## OLD +sudo pip install pytest-playwright + +# x virtual frame buffer +# for playwright (screenshotter) to run in headed mode +sudo apt install xvfb -y + +sudo playwright install-deps + +sudo apt-get install libvpx7 -y + +TARGET_USER="dave" +sudo -i -u $TARGET_USER bash << EOF +playwright install +EOF + +#sudo apt-get install libgbm1 + +cat <> run-auto-archive +*/2 * * * * dave /home/dave/auto-archiver/infra/cron_pluro.sh +EOT + +sudo mv run-auto-archive /etc/cron.d + +sudo chown root /etc/cron.d/run-auto-archive +sudo chmod 600 /etc/cron.d/run-auto-archive + + +sudo reboot now + + +## DM 16th Oct 2024 +# am using playwright as a general screenshotter +# so need to install the dependencies for that + +sudo pip install pytest-playwright + +sudo apt install xvfb -y + +# playwright install + + + + + + + + +## +## FB Archiver from here down!!!! +## + +# specialised version of the archiver which runs on proxmox currently only +cat <> fb-run-auto-archive +#* * * * * dave /home/dave/auto-archiver/infra/cron_fb.sh +EOT + +# docker +# https://docs.docker.com/engine/install/ubuntu/ +sudo apt-get update -y +sudo apt-get install ca-certificates curl gnupg lsb-release -y + +sudo mkdir -p /etc/apt/keyrings +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg + +echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + +sudo apt-get update + +sudo chmod a+r /etc/apt/keyrings/docker.gpg +sudo apt-get update + + +sudo apt-get install docker-ce docker-ce-cli containerd.io docker-compose-plugin -y + +# docker as non sudo https://docs.docker.com/engine/install/linux-postinstall/#manage-docker-as-a-non-root-user +# cron runs as user dave +sudo usermod -aG docker dave + +sudo pip install pytest-playwright + +# x virtual frame buffer +# for playwright (screenshotter) to run in headed mode +sudo apt install xvfb -y + +# **need to run playwright install to download chrome** +# **NOT TESTED** +##sudo playwright install-deps +#sudo apt-get install libgbm1 + + +sudo reboot now + + + + +# MONITORING +# syslog in /var/log/syslog +# cron output is in /home/dave/log.txt + +# sudo service cron restart + + +``` + ## Developer Install From b236f2510d9612428f0a81e3a2dbe71fc628579b Mon Sep 17 00:00:00 2001 From: Dave Mateer Date: Mon, 16 Jun 2025 14:40:40 +0100 Subject: [PATCH 02/14] Updates to installation docs --- docs/source/installation/installation.md | 111 ----------------------- 1 file changed, 111 deletions(-) diff --git a/docs/source/installation/installation.md b/docs/source/installation/installation.md index 13c3131..fff7d26 100644 --- a/docs/source/installation/installation.md +++ b/docs/source/installation/installation.md @@ -208,117 +208,6 @@ docker run --name bgutil-provider --restart unless-stopped -d -p 4416:4416 brain cd ~/auto-archiver poetry run python src/auto_archiver --config secrets/orchestration-aa-demo-main.yaml - - - - - - -## HERE - -## OLD -sudo pip install pytest-playwright - -# x virtual frame buffer -# for playwright (screenshotter) to run in headed mode -sudo apt install xvfb -y - -sudo playwright install-deps - -sudo apt-get install libvpx7 -y - -TARGET_USER="dave" -sudo -i -u $TARGET_USER bash << EOF -playwright install -EOF - -#sudo apt-get install libgbm1 - -cat <> run-auto-archive -*/2 * * * * dave /home/dave/auto-archiver/infra/cron_pluro.sh -EOT - -sudo mv run-auto-archive /etc/cron.d - -sudo chown root /etc/cron.d/run-auto-archive -sudo chmod 600 /etc/cron.d/run-auto-archive - - -sudo reboot now - - -## DM 16th Oct 2024 -# am using playwright as a general screenshotter -# so need to install the dependencies for that - -sudo pip install pytest-playwright - -sudo apt install xvfb -y - -# playwright install - - - - - - - - -## -## FB Archiver from here down!!!! -## - -# specialised version of the archiver which runs on proxmox currently only -cat <> fb-run-auto-archive -#* * * * * dave /home/dave/auto-archiver/infra/cron_fb.sh -EOT - -# docker -# https://docs.docker.com/engine/install/ubuntu/ -sudo apt-get update -y -sudo apt-get install ca-certificates curl gnupg lsb-release -y - -sudo mkdir -p /etc/apt/keyrings -curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg - -echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null - -sudo apt-get update - -sudo chmod a+r /etc/apt/keyrings/docker.gpg -sudo apt-get update - - -sudo apt-get install docker-ce docker-ce-cli containerd.io docker-compose-plugin -y - -# docker as non sudo https://docs.docker.com/engine/install/linux-postinstall/#manage-docker-as-a-non-root-user -# cron runs as user dave -sudo usermod -aG docker dave - -sudo pip install pytest-playwright - -# x virtual frame buffer -# for playwright (screenshotter) to run in headed mode -sudo apt install xvfb -y - -# **need to run playwright install to download chrome** -# **NOT TESTED** -##sudo playwright install-deps -#sudo apt-get install libgbm1 - - -sudo reboot now - - - - -# MONITORING -# syslog in /var/log/syslog -# cron output is in /home/dave/log.txt - -# sudo service cron restart - - ``` From f07fdbc500b593c7d805c628f3750e8a783cb32d Mon Sep 17 00:00:00 2001 From: Dave Mateer Date: Mon, 16 Jun 2025 14:54:15 +0100 Subject: [PATCH 03/14] Custom local version comment in toml file --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index cdbb86b..2299350 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,6 +4,8 @@ build-backend = "poetry.core.masonry.api" [project] name = "auto-archiver" +# to add a custom version use local versioning eg 1.1.0+dm.1 +# then poetry install version = "1.1.0" description = "Automatically archive links to videos, images, and social media content from Google Sheets (and more)." From f2e80758a7bbac5c253624c610713ef1c61823a2 Mon Sep 17 00:00:00 2001 From: Dave Mateer Date: Mon, 16 Jun 2025 14:59:55 +0100 Subject: [PATCH 04/14] typo on authentication docs. Updated install docs. --- docs/source/installation/authentication.md | 2 +- docs/source/installation/installation.md | 37 ++++++++++++---------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/docs/source/installation/authentication.md b/docs/source/installation/authentication.md index 16e650f..a8b6ff5 100644 --- a/docs/source/installation/authentication.md +++ b/docs/source/installation/authentication.md @@ -34,7 +34,7 @@ You can save your authentication information directly inside your orchestration ```{note} -The Username & Password, and API settings only work with the Generic Extractor. Other modules (like the screenshot enricher) can only use the `cookies` options. Furthermore, many sites can still detect bots and block username/password logins. Twitter/X and YouTube are two prominent ones that block username/password logging. +The Username & Password, and API settings only work with the Generic Extractor. Other modules (like the screenshot enricher) can only use the `cookies` options. Furthermore, many sites can still detect bots and block username/password logins. Twitter/X and YouTube are two prominent ones that block username/password logins. One of the 'Cookies' options is recommended for the most robust archiving, but it still isn't guaranteed to work. ``` diff --git a/docs/source/installation/installation.md b/docs/source/installation/installation.md index fff7d26..38d955b 100644 --- a/docs/source/installation/installation.md +++ b/docs/source/installation/installation.md @@ -55,7 +55,7 @@ If using the local installation method, you will also need to install the follow 3. (optional) [fonts-noto](https://fonts.google.com/noto) to deal with multiple unicode characters during selenium screenshots: `sudo apt install fonts-noto -y`. 4. [Browsertrix Crawler docker image](https://hub.docker.com/r/webrecorder/browsertrix-crawler) for the WACZ enricher/archiver -### Bash script for Ubuntu Server install +### Bash script for Ubuntu 24 Server install This acts as a handy guide on all requirements. This is built and tested on the 29th of May 2025 on Ubuntu Server 24.04.2 LTS (which is the current latest LTS) @@ -66,6 +66,8 @@ This acts as a handy guide on all requirements. This is built and tested on the # which the application runs under which makes debugging easier cd ~ +sudo apt update -y +sudo apt upgrade -y # Clone only my latest branch git clone -b v1-test --single-branch https://github.com/djhmateer/auto-archiver @@ -81,14 +83,13 @@ sudo apt upgrade -y # Poetry install 2.1.3 on 2nd June 25 curl -sSL https://install.python-poetry.org | python3 - -# had to restart shell here.. neither of below worked -# source ~/.bashrc -# exec bash - -cd auto-archiver +# had to restart here.. +sudo reboot # C++ compiler so pdqhash will install next -sudo apt install build-essential python3-dev +sudo apt install build-essential python3-dev -y + +cd auto-archiver poetry install @@ -130,14 +131,13 @@ sudo apt-get install -f ## Gecko driver # check version numbers for new ones # https://github.com/mozilla/geckodriver/releases/ -cd ~ -wget https://github.com/mozilla/geckodriver/releases/download/v0.35.0/geckodriver-v0.35.0-linux64.tar.gz +wget https://github.com/mozilla/geckodriver/releases/download/v0.36.0/geckodriver-v0.36.0-linux64.tar.gz tar -xvzf geckodriver* chmod +x geckodriver sudo mv geckodriver /usr/local/bin/ rm geckodriver* -# Fonts +# Fonts so selenium via firefox can render other languages eg Burmese sudo apt install fonts-noto -y # Docker @@ -160,10 +160,11 @@ sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin # add dave user to docker group sudo usermod -aG docker $USER -# restart shell -# TODO try: source ~/.bashrc -# exec bash +# reboot otherwise can't pull images +# https://github.com/webrecorder/browsertrix-crawler +# https://hub.docker.com/r/webrecorder/browsertrix-crawler/tags +# 1.6.2 on 4th Jun 2025 docker pull webrecorder/browsertrix-crawler:latest # exif @@ -172,7 +173,9 @@ sudo apt install libimage-exiftool-perl -y ## CRON run every minute # the cron job running as user dave will execute the shell script -sudo chmod +x ~/auto-archiver/scripts/cron_1.sh +# I have many scripts running from cron_11 upwards. +# patch in the correct number +sudo chmod +x ~/auto-archiver/scripts/cron_15.sh # don't want service to run until a reboot otherwise problems with Gecko driver sudo service cron stop @@ -180,13 +183,15 @@ sudo service cron stop # runs the script every minute # notice put in a # to disable so will have to manually start it. cat <> run-auto-archive -#*/1 * * * * dave /home/dave/auto-archiver/scripts/cron_1.sh +#*/1 * * * * dave /home/dave/auto-archiver/scripts/cron_15.sh EOT sudo mv run-auto-archive /etc/cron.d sudo chown root /etc/cron.d/run-auto-archive sudo chmod 600 /etc/cron.d/run-auto-archive +# Helper alias 'c' to open the above file +echo "alias c='sudo vim /etc/cron.d/run-auto-archive'" >> ~/.bashrc # secrets folder copy # I run dev from: @@ -195,10 +200,8 @@ sudo chmod 600 /etc/cron.d/run-auto-archive # orchestration.yaml - for aa config # service_account - for google spreadsheet # anon.session - for telethon so don't have to type in phone number -# vk_config.v2.json - so don't have to login to vk again # profile.tar.gz - for wacz to have a logged in profile for facebook, x.com and instagram to get data - # Youtube - POT Tokens # https://github.com/Brainicism/bgutil-ytdlp-pot-provider docker run --name bgutil-provider --restart unless-stopped -d -p 4416:4416 brainicism/bgutil-ytdlp-pot-provider From a60d800b31fd0354858375841c492f363cdd0ec6 Mon Sep 17 00:00:00 2001 From: Dave Mateer Date: Mon, 16 Jun 2025 15:07:39 +0100 Subject: [PATCH 05/14] Changed log level for media --- src/auto_archiver/core/media.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/auto_archiver/core/media.py b/src/auto_archiver/core/media.py index 826b920..75fb366 100644 --- a/src/auto_archiver/core/media.py +++ b/src/auto_archiver/core/media.py @@ -116,7 +116,12 @@ class Media: # self.is_video() should be used together with this method try: streams = ffmpeg.probe(self.filename, select_streams="v")["streams"] - logger.warning(f"STREAMS FOR {self.filename} {streams}") + # DM 27th May 2025 + # https://x.com/dave_mateer/status/1524341442738638848 + # shows this warning for the gif file + # Have changed to debug for now to clean up logs + # logger.warning(f"STREAMS FOR {self.filename} {streams}") + logger.debug(f"STREAMS FOR {self.filename} {streams}") return any(s.get("duration_ts", 0) > 0 for s in streams) except Error: return False # ffmpeg errors when reading bad files From ba3f1a52e8b954b05a7c4650daec5f0724f202dc Mon Sep 17 00:00:00 2001 From: Dave Mateer Date: Mon, 16 Jun 2025 16:15:54 +0100 Subject: [PATCH 06/14] Logging each_level_in_separate_file feature --- poetry.lock | 283 ++++++++++++++----------- src/auto_archiver/core/orchestrator.py | 33 ++- 2 files changed, 191 insertions(+), 125 deletions(-) diff --git a/poetry.lock b/poetry.lock index 786ae5f..f641617 100644 --- a/poetry.lock +++ b/poetry.lock @@ -193,18 +193,18 @@ files = [ [[package]] name = "boto3" -version = "1.38.33" +version = "1.38.36" description = "The AWS SDK for Python" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "boto3-1.38.33-py3-none-any.whl", hash = "sha256:25d0717489c658f7ae6c3c7f0f7e1b4d611b30b2f08f0fcef6455ac6864a8768"}, - {file = "boto3-1.38.33.tar.gz", hash = "sha256:6467909c1ae01ff67981f021bb2568592211765ec8a9a1d2c4529191e46c3541"}, + {file = "boto3-1.38.36-py3-none-any.whl", hash = "sha256:34c27d7317cadb62c0e9856e5d5aa0271ef47202d340584831048bc7ac904136"}, + {file = "boto3-1.38.36.tar.gz", hash = "sha256:efe0aaa060f8fedd76e5c942055f051aee0432fc722d79d8830a9fd9db83593e"}, ] [package.dependencies] -botocore = ">=1.38.33,<1.39.0" +botocore = ">=1.38.36,<1.39.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.13.0,<0.14.0" @@ -213,14 +213,14 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.38.33" +version = "1.38.36" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "botocore-1.38.33-py3-none-any.whl", hash = "sha256:ad25233e93dcebe95809b1f9393c1f11a639696327793d166295fb78dd7bc597"}, - {file = "botocore-1.38.33.tar.gz", hash = "sha256:dbe8fea9d0426c644c89ef2018ead886ccbcc22901a02b377b4e65ce1cb69a2b"}, + {file = "botocore-1.38.36-py3-none-any.whl", hash = "sha256:b6a50b853f6d23af9edfed89a59800c6bc1687a947cdd3492879f7d64e002d30"}, + {file = "botocore-1.38.36.tar.gz", hash = "sha256:4a1ced1a4218bdff0ed5b46abb54570d473154ddefafa5d121a8d96e4b76ebc1"}, ] [package.dependencies] @@ -437,14 +437,14 @@ files = [ [[package]] name = "certifi" -version = "2025.4.26" +version = "2025.6.15" description = "Python package for providing Mozilla's CA Bundle." optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" groups = ["main", "docs"] files = [ - {file = "certifi-2025.4.26-py3-none-any.whl", hash = "sha256:30350364dfe371162649852c63336a15c70c6510c2ad5015b21c2345311805f3"}, - {file = "certifi-2025.4.26.tar.gz", hash = "sha256:0a816057ea3cdefcef70270d2c515e4506bbc954f417fa5ade2021213bb8f0c6"}, + {file = "certifi-2025.6.15-py3-none-any.whl", hash = "sha256:2e0c7ce7cb5d8f8634ca55d2ba7e6ec2689a2fd6537d8dec1296a477a4910057"}, + {file = "certifi-2025.6.15.tar.gz", hash = "sha256:d747aa5a8b9bbbb1bb8c22bb13e22bd1f18e9796defa16bab421f7f7a317323b"}, ] [[package]] @@ -941,14 +941,14 @@ files = [ [[package]] name = "google-api-core" -version = "2.25.0" +version = "2.25.1" description = "Google API client core library" optional = false python-versions = ">=3.7" groups = ["main"] files = [ - {file = "google_api_core-2.25.0-py3-none-any.whl", hash = "sha256:1db79d1281dcf9f3d10023283299ba38f3dc9f639ec41085968fd23e5bcf512e"}, - {file = "google_api_core-2.25.0.tar.gz", hash = "sha256:9b548e688702f82a34ed8409fb8a6961166f0b7795032f0be8f48308dff4333a"}, + {file = "google_api_core-2.25.1-py3-none-any.whl", hash = "sha256:8a2a56c1fef82987a524371f99f3bd0143702fecc670c72e600c1cda6bf8dbb7"}, + {file = "google_api_core-2.25.1.tar.gz", hash = "sha256:d2aaa0b13c78c61cb3f4282c464c046e45fbd75755683c9c525e6e8f7ed0a5e8"}, ] [package.dependencies] @@ -966,14 +966,14 @@ grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.0)"] [[package]] name = "google-api-python-client" -version = "2.171.0" +version = "2.172.0" description = "Google API Client Library for Python" optional = false python-versions = ">=3.7" groups = ["main"] files = [ - {file = "google_api_python_client-2.171.0-py3-none-any.whl", hash = "sha256:c9c9b76f561e9d9ac14e54a9e2c0842876201d5b96e69e48f967373f0784cbe9"}, - {file = "google_api_python_client-2.171.0.tar.gz", hash = "sha256:057a5c08d28463c6b9eb89746355de5f14b7ed27a65c11fdbf1d06c66bb66b23"}, + {file = "google_api_python_client-2.172.0-py3-none-any.whl", hash = "sha256:9f1b9a268d5dc1228207d246c673d3a09ee211b41a11521d38d9212aeaa43af7"}, + {file = "google_api_python_client-2.172.0.tar.gz", hash = "sha256:dcb3b7e067154b2aa41f1776cf86584a5739c0ac74e6ff46fc665790dca0e6a6"}, ] [package.dependencies] @@ -2164,64 +2164,70 @@ files = [ [[package]] name = "pyobjc-core" -version = "11.0" +version = "11.1" description = "Python<->ObjC Interoperability Module" optional = false python-versions = ">=3.8" groups = ["main"] markers = "platform_system == \"Darwin\"" files = [ - {file = "pyobjc_core-11.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:10866b3a734d47caf48e456eea0d4815c2c9b21856157db5917b61dee06893a1"}, - {file = "pyobjc_core-11.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:50675c0bb8696fe960a28466f9baf6943df2928a1fd85625d678fa2f428bd0bd"}, - {file = "pyobjc_core-11.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a03061d4955c62ddd7754224a80cdadfdf17b6b5f60df1d9169a3b1b02923f0b"}, - {file = "pyobjc_core-11.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:c338c1deb7ab2e9436d4175d1127da2eeed4a1b564b3d83b9f3ae4844ba97e86"}, - {file = "pyobjc_core-11.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:b4e9dc4296110f251a4033ff3f40320b35873ea7f876bd29a1c9705bb5e08c59"}, - {file = "pyobjc_core-11.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:02406ece449d0f41b31e579e47ca77ced3eb57533df955281bfcecc99da74fba"}, - {file = "pyobjc_core-11.0.tar.gz", hash = "sha256:63bced211cb8a8fb5c8ff46473603da30e51112861bd02c438fbbbc8578d9a70"}, + {file = "pyobjc_core-11.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:4c7536f3e94de0a3eae6bb382d75f1219280aa867cdf37beef39d9e7d580173c"}, + {file = "pyobjc_core-11.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ec36680b5c14e2f73d432b03ba7c1457dc6ca70fa59fd7daea1073f2b4157d33"}, + {file = "pyobjc_core-11.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:765b97dea6b87ec4612b3212258024d8496ea23517c95a1c5f0735f96b7fd529"}, + {file = "pyobjc_core-11.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:18986f83998fbd5d3f56d8a8428b2f3e0754fd15cef3ef786ca0d29619024f2c"}, + {file = "pyobjc_core-11.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:8849e78cfe6595c4911fbba29683decfb0bf57a350aed8a43316976ba6f659d2"}, + {file = "pyobjc_core-11.1-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:8cb9ed17a8d84a312a6e8b665dd22393d48336ea1d8277e7ad20c19a38edf731"}, + {file = "pyobjc_core-11.1-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:f2455683e807f8541f0d83fbba0f5d9a46128ab0d5cc83ea208f0bec759b7f96"}, + {file = "pyobjc_core-11.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:4a99e6558b48b8e47c092051e7b3be05df1c8d0617b62f6fa6a316c01902d157"}, + {file = "pyobjc_core-11.1.tar.gz", hash = "sha256:b63d4d90c5df7e762f34739b39cc55bc63dbcf9fb2fb3f2671e528488c7a87fe"}, ] [[package]] name = "pyobjc-framework-cocoa" -version = "11.0" +version = "11.1" description = "Wrappers for the Cocoa frameworks on macOS" optional = false python-versions = ">=3.9" groups = ["main"] markers = "platform_system == \"Darwin\"" files = [ - {file = "pyobjc_framework_Cocoa-11.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:fbc65f260d617d5463c7fb9dbaaffc23c9a4fabfe3b1a50b039b61870b8daefd"}, - {file = "pyobjc_framework_Cocoa-11.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3ea7be6e6dd801b297440de02d312ba3fa7fd3c322db747ae1cb237e975f5d33"}, - {file = "pyobjc_framework_Cocoa-11.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:280a577b83c68175a28b2b7138d1d2d3111f2b2b66c30e86f81a19c2b02eae71"}, - {file = "pyobjc_framework_Cocoa-11.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:15b2bd977ed340074f930f1330f03d42912d5882b697d78bd06f8ebe263ef92e"}, - {file = "pyobjc_framework_Cocoa-11.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:5750001db544e67f2b66f02067d8f0da96bb2ef71732bde104f01b8628f9d7ea"}, - {file = "pyobjc_framework_Cocoa-11.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ddff25b0755d59873d186e1e07d6aaddb19d55e3ae890d69ff2d9babf8627657"}, - {file = "pyobjc_framework_cocoa-11.0.tar.gz", hash = "sha256:00346a8cb81ad7b017b32ff7bf596000f9faa905807b1bd234644ebd47f692c5"}, + {file = "pyobjc_framework_cocoa-11.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b27a5bdb3ab6cdeb998443ff3fce194ffae5f518c6a079b832dbafc4426937f9"}, + {file = "pyobjc_framework_cocoa-11.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:7b9a9b8ba07f5bf84866399e3de2aa311ed1c34d5d2788a995bdbe82cc36cfa0"}, + {file = "pyobjc_framework_cocoa-11.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:806de56f06dfba8f301a244cce289d54877c36b4b19818e3b53150eb7c2424d0"}, + {file = "pyobjc_framework_cocoa-11.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:54e93e1d9b0fc41c032582a6f0834befe1d418d73893968f3f450281b11603da"}, + {file = "pyobjc_framework_cocoa-11.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:fd5245ee1997d93e78b72703be1289d75d88ff6490af94462b564892e9266350"}, + {file = "pyobjc_framework_cocoa-11.1-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:aede53a1afc5433e1e7d66568cc52acceeb171b0a6005407a42e8e82580b4fc0"}, + {file = "pyobjc_framework_cocoa-11.1-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:1b5de4e1757bb65689d6dc1f8d8717de9ec8587eb0c4831c134f13aba29f9b71"}, + {file = "pyobjc_framework_cocoa-11.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:bbee71eeb93b1b31ffbac8560b59a0524a8a4b90846a260d2c4f2188f3d4c721"}, + {file = "pyobjc_framework_cocoa-11.1.tar.gz", hash = "sha256:87df76b9b73e7ca699a828ff112564b59251bb9bbe72e610e670a4dc9940d038"}, ] [package.dependencies] -pyobjc-core = ">=11.0" +pyobjc-core = ">=11.1" [[package]] name = "pyobjc-framework-quartz" -version = "11.0" +version = "11.1" description = "Wrappers for the Quartz frameworks on macOS" optional = false python-versions = ">=3.9" groups = ["main"] markers = "platform_system == \"Darwin\"" files = [ - {file = "pyobjc_framework_Quartz-11.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:da3ab13c9f92361959b41b0ad4cdd41ae872f90a6d8c58a9ed699bc08ab1c45c"}, - {file = "pyobjc_framework_Quartz-11.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:d251696bfd8e8ef72fbc90eb29fec95cb9d1cc409008a183d5cc3246130ae8c2"}, - {file = "pyobjc_framework_Quartz-11.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:cb4a9f2d9d580ea15e25e6b270f47681afb5689cafc9e25712445ce715bcd18e"}, - {file = "pyobjc_framework_Quartz-11.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:973b4f9b8ab844574461a038bd5269f425a7368d6e677e3cc81fcc9b27b65498"}, - {file = "pyobjc_framework_Quartz-11.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:66ab58d65348863b8707e63b2ec5cdc54569ee8189d1af90d52f29f5fdf6272c"}, - {file = "pyobjc_framework_Quartz-11.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:1032f63f2a4ee98366764e69c249f1d93813821e17d224cf626cf11fb1801fc4"}, - {file = "pyobjc_framework_quartz-11.0.tar.gz", hash = "sha256:3205bf7795fb9ae34747f701486b3db6dfac71924894d1f372977c4d70c3c619"}, + {file = "pyobjc_framework_quartz-11.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:b5ef75c416b0209e25b2eb07a27bd7eedf14a8c6b2f968711969d45ceceb0f84"}, + {file = "pyobjc_framework_quartz-11.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2d501fe95ef15d8acf587cb7dc4ab4be3c5a84e2252017da8dbb7df1bbe7a72a"}, + {file = "pyobjc_framework_quartz-11.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:9ac806067541917d6119b98d90390a6944e7d9bd737f5c0a79884202327c9204"}, + {file = "pyobjc_framework_quartz-11.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:43a1138280571bbf44df27a7eef519184b5c4183a588598ebaaeb887b9e73e76"}, + {file = "pyobjc_framework_quartz-11.1-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:b23d81c30c564adf6336e00b357f355b35aad10075dd7e837cfd52a9912863e5"}, + {file = "pyobjc_framework_quartz-11.1-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:07cbda78b4a8fcf3a2d96e047a2ff01f44e3e1820f46f0f4b3b6d77ff6ece07c"}, + {file = "pyobjc_framework_quartz-11.1-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:39d02a3df4b5e3eee1e0da0fb150259476910d2a9aa638ab94153c24317a9561"}, + {file = "pyobjc_framework_quartz-11.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:9b1f451ddb5243d8d6316af55f240a02b0fffbfe165bff325628bf73f3df7f44"}, + {file = "pyobjc_framework_quartz-11.1.tar.gz", hash = "sha256:a57f35ccfc22ad48c87c5932818e583777ff7276605fef6afad0ac0741169f75"}, ] [package.dependencies] -pyobjc-core = ">=11.0" -pyobjc-framework-Cocoa = ">=11.0" +pyobjc-core = ">=11.1" +pyobjc-framework-Cocoa = ">=11.1" [[package]] name = "pyotp" @@ -3113,14 +3119,14 @@ websocket-client = ">=1.8.0,<1.9.0" [[package]] name = "seleniumbase" -version = "4.39.3" +version = "4.39.4" description = "A complete web automation framework for end-to-end testing." optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "seleniumbase-4.39.3-py3-none-any.whl", hash = "sha256:cbb94d7858a9ef3b0b4431a5879150649f4a73029afaa8ecfb7bda113f2565e1"}, - {file = "seleniumbase-4.39.3.tar.gz", hash = "sha256:b32978e685b1e4e2c7859b2dcb377ac14ba99bf748ea428548f9e450257b861b"}, + {file = "seleniumbase-4.39.4-py3-none-any.whl", hash = "sha256:15562b2550ce6f6fdcc524ff9bd87a1d7381a558767245f10ff63982f508c281"}, + {file = "seleniumbase-4.39.4.tar.gz", hash = "sha256:8880869b88fa5a48c649a776488bafa1ca97d786fb8a25f63e6d5b5b5fc47f44"}, ] [package.dependencies] @@ -3186,11 +3192,11 @@ wsproto = "1.2.0" [package.extras] allure = ["allure-behave (>=2.13.5)", "allure-pytest (>=2.13.5)", "allure-python-commons (>=2.13.5)"] -coverage = ["coverage (>=7.6.1) ; python_version < \"3.9\"", "coverage (>=7.8.2) ; python_version >= \"3.9\"", "pytest-cov (>=5.0.0) ; python_version < \"3.9\"", "pytest-cov (>=6.1.1) ; python_version >= \"3.9\""] +coverage = ["coverage (>=7.6.1) ; python_version < \"3.9\"", "coverage (>=7.9.0) ; python_version >= \"3.9\"", "pytest-cov (>=5.0.0) ; python_version < \"3.9\"", "pytest-cov (>=6.2.1) ; python_version >= \"3.9\""] flake8 = ["flake8 (==5.0.4) ; python_version < \"3.9\"", "flake8 (==7.2.0) ; python_version >= \"3.9\"", "mccabe (==0.7.0)", "pycodestyle (==2.13.0) ; python_version >= \"3.9\"", "pycodestyle (==2.9.1) ; python_version < \"3.9\"", "pyflakes (==2.5.0) ; python_version < \"3.9\"", "pyflakes (==3.3.2) ; python_version >= \"3.9\""] ipdb = ["ipdb (==0.13.13)", "ipython (==7.34.0)"] mss = ["mss (==10.0.0) ; python_version >= \"3.9\"", "mss (==9.0.2) ; python_version < \"3.9\""] -pdfminer = ["cffi (==1.17.1)", "cryptography (==39.0.2) ; python_version < \"3.9\"", "cryptography (==45.0.3) ; python_version >= \"3.9\"", "pdfminer.six (==20250324) ; python_version < \"3.9\"", "pdfminer.six (==20250506) ; python_version >= \"3.9\"", "pycparser (==2.22)"] +pdfminer = ["cffi (==1.17.1)", "cryptography (==39.0.2) ; python_version < \"3.9\"", "cryptography (==45.0.4) ; python_version >= \"3.9\"", "pdfminer.six (==20250324) ; python_version < \"3.9\"", "pdfminer.six (==20250506) ; python_version >= \"3.9\"", "pycparser (==2.22)"] pillow = ["Pillow (>=10.4.0) ; python_version < \"3.9\"", "Pillow (>=11.2.1) ; python_version >= \"3.9\""] pip-system-certs = ["pip-system-certs (==4.0) ; platform_system == \"Windows\""] proxy = ["proxy.py (==2.4.3)"] @@ -3895,83 +3901,118 @@ testing = ["hookdns", "httpbin (>=0.10.2)", "pytest", "pytest-cov", "requests", [[package]] name = "watchfiles" -version = "1.0.5" +version = "1.1.0" description = "Simple, modern and high performance file watching and code reload in python." optional = false python-versions = ">=3.9" groups = ["docs"] files = [ - {file = "watchfiles-1.0.5-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:5c40fe7dd9e5f81e0847b1ea64e1f5dd79dd61afbedb57759df06767ac719b40"}, - {file = "watchfiles-1.0.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8c0db396e6003d99bb2d7232c957b5f0b5634bbd1b24e381a5afcc880f7373fb"}, - {file = "watchfiles-1.0.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b551d4fb482fc57d852b4541f911ba28957d051c8776e79c3b4a51eb5e2a1b11"}, - {file = "watchfiles-1.0.5-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:830aa432ba5c491d52a15b51526c29e4a4b92bf4f92253787f9726fe01519487"}, - {file = "watchfiles-1.0.5-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a16512051a822a416b0d477d5f8c0e67b67c1a20d9acecb0aafa3aa4d6e7d256"}, - {file = "watchfiles-1.0.5-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bfe0cbc787770e52a96c6fda6726ace75be7f840cb327e1b08d7d54eadc3bc85"}, - {file = "watchfiles-1.0.5-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d363152c5e16b29d66cbde8fa614f9e313e6f94a8204eaab268db52231fe5358"}, - {file = "watchfiles-1.0.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ee32c9a9bee4d0b7bd7cbeb53cb185cf0b622ac761efaa2eba84006c3b3a614"}, - {file = "watchfiles-1.0.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:29c7fd632ccaf5517c16a5188e36f6612d6472ccf55382db6c7fe3fcccb7f59f"}, - {file = "watchfiles-1.0.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8e637810586e6fe380c8bc1b3910accd7f1d3a9a7262c8a78d4c8fb3ba6a2b3d"}, - {file = "watchfiles-1.0.5-cp310-cp310-win32.whl", hash = "sha256:cd47d063fbeabd4c6cae1d4bcaa38f0902f8dc5ed168072874ea11d0c7afc1ff"}, - {file = "watchfiles-1.0.5-cp310-cp310-win_amd64.whl", hash = "sha256:86c0df05b47a79d80351cd179893f2f9c1b1cae49d96e8b3290c7f4bd0ca0a92"}, - {file = "watchfiles-1.0.5-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:237f9be419e977a0f8f6b2e7b0475ababe78ff1ab06822df95d914a945eac827"}, - {file = "watchfiles-1.0.5-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e0da39ff917af8b27a4bdc5a97ac577552a38aac0d260a859c1517ea3dc1a7c4"}, - {file = "watchfiles-1.0.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2cfcb3952350e95603f232a7a15f6c5f86c5375e46f0bd4ae70d43e3e063c13d"}, - {file = "watchfiles-1.0.5-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:68b2dddba7a4e6151384e252a5632efcaa9bc5d1c4b567f3cb621306b2ca9f63"}, - {file = "watchfiles-1.0.5-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:95cf944fcfc394c5f9de794ce581914900f82ff1f855326f25ebcf24d5397418"}, - {file = "watchfiles-1.0.5-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ecf6cd9f83d7c023b1aba15d13f705ca7b7d38675c121f3cc4a6e25bd0857ee9"}, - {file = "watchfiles-1.0.5-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:852de68acd6212cd6d33edf21e6f9e56e5d98c6add46f48244bd479d97c967c6"}, - {file = "watchfiles-1.0.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d5730f3aa35e646103b53389d5bc77edfbf578ab6dab2e005142b5b80a35ef25"}, - {file = "watchfiles-1.0.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:18b3bd29954bc4abeeb4e9d9cf0b30227f0f206c86657674f544cb032296acd5"}, - {file = "watchfiles-1.0.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:ba5552a1b07c8edbf197055bc9d518b8f0d98a1c6a73a293bc0726dce068ed01"}, - {file = "watchfiles-1.0.5-cp311-cp311-win32.whl", hash = "sha256:2f1fefb2e90e89959447bc0420fddd1e76f625784340d64a2f7d5983ef9ad246"}, - {file = "watchfiles-1.0.5-cp311-cp311-win_amd64.whl", hash = "sha256:b6e76ceb1dd18c8e29c73f47d41866972e891fc4cc7ba014f487def72c1cf096"}, - {file = "watchfiles-1.0.5-cp311-cp311-win_arm64.whl", hash = "sha256:266710eb6fddc1f5e51843c70e3bebfb0f5e77cf4f27129278c70554104d19ed"}, - {file = "watchfiles-1.0.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:b5eb568c2aa6018e26da9e6c86f3ec3fd958cee7f0311b35c2630fa4217d17f2"}, - {file = "watchfiles-1.0.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0a04059f4923ce4e856b4b4e5e783a70f49d9663d22a4c3b3298165996d1377f"}, - {file = "watchfiles-1.0.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e380c89983ce6e6fe2dd1e1921b9952fb4e6da882931abd1824c092ed495dec"}, - {file = "watchfiles-1.0.5-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fe43139b2c0fdc4a14d4f8d5b5d967f7a2777fd3d38ecf5b1ec669b0d7e43c21"}, - {file = "watchfiles-1.0.5-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee0822ce1b8a14fe5a066f93edd20aada932acfe348bede8aa2149f1a4489512"}, - {file = "watchfiles-1.0.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a0dbcb1c2d8f2ab6e0a81c6699b236932bd264d4cef1ac475858d16c403de74d"}, - {file = "watchfiles-1.0.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a2014a2b18ad3ca53b1f6c23f8cd94a18ce930c1837bd891262c182640eb40a6"}, - {file = "watchfiles-1.0.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:10f6ae86d5cb647bf58f9f655fcf577f713915a5d69057a0371bc257e2553234"}, - {file = "watchfiles-1.0.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:1a7bac2bde1d661fb31f4d4e8e539e178774b76db3c2c17c4bb3e960a5de07a2"}, - {file = "watchfiles-1.0.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4ab626da2fc1ac277bbf752446470b367f84b50295264d2d313e28dc4405d663"}, - {file = "watchfiles-1.0.5-cp312-cp312-win32.whl", hash = "sha256:9f4571a783914feda92018ef3901dab8caf5b029325b5fe4558c074582815249"}, - {file = "watchfiles-1.0.5-cp312-cp312-win_amd64.whl", hash = "sha256:360a398c3a19672cf93527f7e8d8b60d8275119c5d900f2e184d32483117a705"}, - {file = "watchfiles-1.0.5-cp312-cp312-win_arm64.whl", hash = "sha256:1a2902ede862969077b97523987c38db28abbe09fb19866e711485d9fbf0d417"}, - {file = "watchfiles-1.0.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:0b289572c33a0deae62daa57e44a25b99b783e5f7aed81b314232b3d3c81a11d"}, - {file = "watchfiles-1.0.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a056c2f692d65bf1e99c41045e3bdcaea3cb9e6b5a53dcaf60a5f3bd95fc9763"}, - {file = "watchfiles-1.0.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b9dca99744991fc9850d18015c4f0438865414e50069670f5f7eee08340d8b40"}, - {file = "watchfiles-1.0.5-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:894342d61d355446d02cd3988a7326af344143eb33a2fd5d38482a92072d9563"}, - {file = "watchfiles-1.0.5-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ab44e1580924d1ffd7b3938e02716d5ad190441965138b4aa1d1f31ea0877f04"}, - {file = "watchfiles-1.0.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d6f9367b132078b2ceb8d066ff6c93a970a18c3029cea37bfd7b2d3dd2e5db8f"}, - {file = "watchfiles-1.0.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f2e55a9b162e06e3f862fb61e399fe9f05d908d019d87bf5b496a04ef18a970a"}, - {file = "watchfiles-1.0.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0125f91f70e0732a9f8ee01e49515c35d38ba48db507a50c5bdcad9503af5827"}, - {file = "watchfiles-1.0.5-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:13bb21f8ba3248386337c9fa51c528868e6c34a707f729ab041c846d52a0c69a"}, - {file = "watchfiles-1.0.5-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:839ebd0df4a18c5b3c1b890145b5a3f5f64063c2a0d02b13c76d78fe5de34936"}, - {file = "watchfiles-1.0.5-cp313-cp313-win32.whl", hash = "sha256:4a8ec1e4e16e2d5bafc9ba82f7aaecfeec990ca7cd27e84fb6f191804ed2fcfc"}, - {file = "watchfiles-1.0.5-cp313-cp313-win_amd64.whl", hash = "sha256:f436601594f15bf406518af922a89dcaab416568edb6f65c4e5bbbad1ea45c11"}, - {file = "watchfiles-1.0.5-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:2cfb371be97d4db374cba381b9f911dd35bb5f4c58faa7b8b7106c8853e5d225"}, - {file = "watchfiles-1.0.5-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a3904d88955fda461ea2531fcf6ef73584ca921415d5cfa44457a225f4a42bc1"}, - {file = "watchfiles-1.0.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2b7a21715fb12274a71d335cff6c71fe7f676b293d322722fe708a9ec81d91f5"}, - {file = "watchfiles-1.0.5-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:dfd6ae1c385ab481766b3c61c44aca2b3cd775f6f7c0fa93d979ddec853d29d5"}, - {file = "watchfiles-1.0.5-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b659576b950865fdad31fa491d31d37cf78b27113a7671d39f919828587b429b"}, - {file = "watchfiles-1.0.5-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1909e0a9cd95251b15bff4261de5dd7550885bd172e3536824bf1cf6b121e200"}, - {file = "watchfiles-1.0.5-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:832ccc221927c860e7286c55c9b6ebcc0265d5e072f49c7f6456c7798d2b39aa"}, - {file = "watchfiles-1.0.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:85fbb6102b3296926d0c62cfc9347f6237fb9400aecd0ba6bbda94cae15f2b3b"}, - {file = "watchfiles-1.0.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:15ac96dd567ad6c71c71f7b2c658cb22b7734901546cd50a475128ab557593ca"}, - {file = "watchfiles-1.0.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4b6227351e11c57ae997d222e13f5b6f1f0700d84b8c52304e8675d33a808382"}, - {file = "watchfiles-1.0.5-cp39-cp39-win32.whl", hash = "sha256:974866e0db748ebf1eccab17862bc0f0303807ed9cda465d1324625b81293a18"}, - {file = "watchfiles-1.0.5-cp39-cp39-win_amd64.whl", hash = "sha256:9848b21ae152fe79c10dd0197304ada8f7b586d3ebc3f27f43c506e5a52a863c"}, - {file = "watchfiles-1.0.5-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:f59b870db1f1ae5a9ac28245707d955c8721dd6565e7f411024fa374b5362d1d"}, - {file = "watchfiles-1.0.5-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:9475b0093767e1475095f2aeb1d219fb9664081d403d1dff81342df8cd707034"}, - {file = "watchfiles-1.0.5-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc533aa50664ebd6c628b2f30591956519462f5d27f951ed03d6c82b2dfd9965"}, - {file = "watchfiles-1.0.5-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fed1cd825158dcaae36acce7b2db33dcbfd12b30c34317a88b8ed80f0541cc57"}, - {file = "watchfiles-1.0.5-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:554389562c29c2c182e3908b149095051f81d28c2fec79ad6c8997d7d63e0009"}, - {file = "watchfiles-1.0.5-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:a74add8d7727e6404d5dc4dcd7fac65d4d82f95928bbee0cf5414c900e86773e"}, - {file = "watchfiles-1.0.5-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb1489f25b051a89fae574505cc26360c8e95e227a9500182a7fe0afcc500ce0"}, - {file = "watchfiles-1.0.5-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0901429650652d3f0da90bad42bdafc1f9143ff3605633c455c999a2d786cac"}, - {file = "watchfiles-1.0.5.tar.gz", hash = "sha256:b7529b5dcc114679d43827d8c35a07c493ad6f083633d573d81c660abc5979e9"}, + {file = "watchfiles-1.1.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:27f30e14aa1c1e91cb653f03a63445739919aef84c8d2517997a83155e7a2fcc"}, + {file = "watchfiles-1.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3366f56c272232860ab45c77c3ca7b74ee819c8e1f6f35a7125556b198bbc6df"}, + {file = "watchfiles-1.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8412eacef34cae2836d891836a7fff7b754d6bcac61f6c12ba5ca9bc7e427b68"}, + {file = "watchfiles-1.1.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:df670918eb7dd719642e05979fc84704af913d563fd17ed636f7c4783003fdcc"}, + {file = "watchfiles-1.1.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d7642b9bc4827b5518ebdb3b82698ada8c14c7661ddec5fe719f3e56ccd13c97"}, + {file = "watchfiles-1.1.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:199207b2d3eeaeb80ef4411875a6243d9ad8bc35b07fc42daa6b801cc39cc41c"}, + {file = "watchfiles-1.1.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a479466da6db5c1e8754caee6c262cd373e6e6c363172d74394f4bff3d84d7b5"}, + {file = "watchfiles-1.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:935f9edd022ec13e447e5723a7d14456c8af254544cefbc533f6dd276c9aa0d9"}, + {file = "watchfiles-1.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:8076a5769d6bdf5f673a19d51da05fc79e2bbf25e9fe755c47595785c06a8c72"}, + {file = "watchfiles-1.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:86b1e28d4c37e89220e924305cd9f82866bb0ace666943a6e4196c5df4d58dcc"}, + {file = "watchfiles-1.1.0-cp310-cp310-win32.whl", hash = "sha256:d1caf40c1c657b27858f9774d5c0e232089bca9cb8ee17ce7478c6e9264d2587"}, + {file = "watchfiles-1.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:a89c75a5b9bc329131115a409d0acc16e8da8dfd5867ba59f1dd66ae7ea8fa82"}, + {file = "watchfiles-1.1.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:c9649dfc57cc1f9835551deb17689e8d44666315f2e82d337b9f07bd76ae3aa2"}, + {file = "watchfiles-1.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:406520216186b99374cdb58bc48e34bb74535adec160c8459894884c983a149c"}, + {file = "watchfiles-1.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb45350fd1dc75cd68d3d72c47f5b513cb0578da716df5fba02fff31c69d5f2d"}, + {file = "watchfiles-1.1.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:11ee4444250fcbeb47459a877e5e80ed994ce8e8d20283857fc128be1715dac7"}, + {file = "watchfiles-1.1.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bda8136e6a80bdea23e5e74e09df0362744d24ffb8cd59c4a95a6ce3d142f79c"}, + {file = "watchfiles-1.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b915daeb2d8c1f5cee4b970f2e2c988ce6514aace3c9296e58dd64dc9aa5d575"}, + {file = "watchfiles-1.1.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ed8fc66786de8d0376f9f913c09e963c66e90ced9aa11997f93bdb30f7c872a8"}, + {file = "watchfiles-1.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe4371595edf78c41ef8ac8df20df3943e13defd0efcb732b2e393b5a8a7a71f"}, + {file = "watchfiles-1.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b7c5f6fe273291f4d414d55b2c80d33c457b8a42677ad14b4b47ff025d0893e4"}, + {file = "watchfiles-1.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:7738027989881e70e3723c75921f1efa45225084228788fc59ea8c6d732eb30d"}, + {file = "watchfiles-1.1.0-cp311-cp311-win32.whl", hash = "sha256:622d6b2c06be19f6e89b1d951485a232e3b59618def88dbeda575ed8f0d8dbf2"}, + {file = "watchfiles-1.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:48aa25e5992b61debc908a61ab4d3f216b64f44fdaa71eb082d8b2de846b7d12"}, + {file = "watchfiles-1.1.0-cp311-cp311-win_arm64.whl", hash = "sha256:00645eb79a3faa70d9cb15c8d4187bb72970b2470e938670240c7998dad9f13a"}, + {file = "watchfiles-1.1.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:9dc001c3e10de4725c749d4c2f2bdc6ae24de5a88a339c4bce32300a31ede179"}, + {file = "watchfiles-1.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d9ba68ec283153dead62cbe81872d28e053745f12335d037de9cbd14bd1877f5"}, + {file = "watchfiles-1.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:130fc497b8ee68dce163e4254d9b0356411d1490e868bd8790028bc46c5cc297"}, + {file = "watchfiles-1.1.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:50a51a90610d0845a5931a780d8e51d7bd7f309ebc25132ba975aca016b576a0"}, + {file = "watchfiles-1.1.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dc44678a72ac0910bac46fa6a0de6af9ba1355669b3dfaf1ce5f05ca7a74364e"}, + {file = "watchfiles-1.1.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a543492513a93b001975ae283a51f4b67973662a375a403ae82f420d2c7205ee"}, + {file = "watchfiles-1.1.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8ac164e20d17cc285f2b94dc31c384bc3aa3dd5e7490473b3db043dd70fbccfd"}, + {file = "watchfiles-1.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7590d5a455321e53857892ab8879dce62d1f4b04748769f5adf2e707afb9d4f"}, + {file = "watchfiles-1.1.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:37d3d3f7defb13f62ece99e9be912afe9dd8a0077b7c45ee5a57c74811d581a4"}, + {file = "watchfiles-1.1.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:7080c4bb3efd70a07b1cc2df99a7aa51d98685be56be6038c3169199d0a1c69f"}, + {file = "watchfiles-1.1.0-cp312-cp312-win32.whl", hash = "sha256:cbcf8630ef4afb05dc30107bfa17f16c0896bb30ee48fc24bf64c1f970f3b1fd"}, + {file = "watchfiles-1.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:cbd949bdd87567b0ad183d7676feb98136cde5bb9025403794a4c0db28ed3a47"}, + {file = "watchfiles-1.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:0a7d40b77f07be87c6faa93d0951a0fcd8cbca1ddff60a1b65d741bac6f3a9f6"}, + {file = "watchfiles-1.1.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:5007f860c7f1f8df471e4e04aaa8c43673429047d63205d1630880f7637bca30"}, + {file = "watchfiles-1.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:20ecc8abbd957046f1fe9562757903f5eaf57c3bce70929fda6c7711bb58074a"}, + {file = "watchfiles-1.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2f0498b7d2a3c072766dba3274fe22a183dbea1f99d188f1c6c72209a1063dc"}, + {file = "watchfiles-1.1.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:239736577e848678e13b201bba14e89718f5c2133dfd6b1f7846fa1b58a8532b"}, + {file = "watchfiles-1.1.0-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eff4b8d89f444f7e49136dc695599a591ff769300734446c0a86cba2eb2f9895"}, + {file = "watchfiles-1.1.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:12b0a02a91762c08f7264e2e79542f76870c3040bbc847fb67410ab81474932a"}, + {file = "watchfiles-1.1.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:29e7bc2eee15cbb339c68445959108803dc14ee0c7b4eea556400131a8de462b"}, + {file = "watchfiles-1.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d9481174d3ed982e269c090f780122fb59cee6c3796f74efe74e70f7780ed94c"}, + {file = "watchfiles-1.1.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:80f811146831c8c86ab17b640801c25dc0a88c630e855e2bef3568f30434d52b"}, + {file = "watchfiles-1.1.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:60022527e71d1d1fda67a33150ee42869042bce3d0fcc9cc49be009a9cded3fb"}, + {file = "watchfiles-1.1.0-cp313-cp313-win32.whl", hash = "sha256:32d6d4e583593cb8576e129879ea0991660b935177c0f93c6681359b3654bfa9"}, + {file = "watchfiles-1.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:f21af781a4a6fbad54f03c598ab620e3a77032c5878f3d780448421a6e1818c7"}, + {file = "watchfiles-1.1.0-cp313-cp313-win_arm64.whl", hash = "sha256:5366164391873ed76bfdf618818c82084c9db7fac82b64a20c44d335eec9ced5"}, + {file = "watchfiles-1.1.0-cp313-cp313t-macosx_10_12_x86_64.whl", hash = "sha256:17ab167cca6339c2b830b744eaf10803d2a5b6683be4d79d8475d88b4a8a4be1"}, + {file = "watchfiles-1.1.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:328dbc9bff7205c215a7807da7c18dce37da7da718e798356212d22696404339"}, + {file = "watchfiles-1.1.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f7208ab6e009c627b7557ce55c465c98967e8caa8b11833531fdf95799372633"}, + {file = "watchfiles-1.1.0-cp313-cp313t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a8f6f72974a19efead54195bc9bed4d850fc047bb7aa971268fd9a8387c89011"}, + {file = "watchfiles-1.1.0-cp313-cp313t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d181ef50923c29cf0450c3cd47e2f0557b62218c50b2ab8ce2ecaa02bd97e670"}, + {file = "watchfiles-1.1.0-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:adb4167043d3a78280d5d05ce0ba22055c266cf8655ce942f2fb881262ff3cdf"}, + {file = "watchfiles-1.1.0-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c5701dc474b041e2934a26d31d39f90fac8a3dee2322b39f7729867f932b1d4"}, + {file = "watchfiles-1.1.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b067915e3c3936966a8607f6fe5487df0c9c4afb85226613b520890049deea20"}, + {file = "watchfiles-1.1.0-cp313-cp313t-musllinux_1_1_aarch64.whl", hash = "sha256:9c733cda03b6d636b4219625a4acb5c6ffb10803338e437fb614fef9516825ef"}, + {file = "watchfiles-1.1.0-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:cc08ef8b90d78bfac66f0def80240b0197008e4852c9f285907377b2947ffdcb"}, + {file = "watchfiles-1.1.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:9974d2f7dc561cce3bb88dfa8eb309dab64c729de85fba32e98d75cf24b66297"}, + {file = "watchfiles-1.1.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c68e9f1fcb4d43798ad8814c4c1b61547b014b667216cb754e606bfade587018"}, + {file = "watchfiles-1.1.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:95ab1594377effac17110e1352989bdd7bdfca9ff0e5eeccd8c69c5389b826d0"}, + {file = "watchfiles-1.1.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fba9b62da882c1be1280a7584ec4515d0a6006a94d6e5819730ec2eab60ffe12"}, + {file = "watchfiles-1.1.0-cp314-cp314-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3434e401f3ce0ed6b42569128b3d1e3af773d7ec18751b918b89cd49c14eaafb"}, + {file = "watchfiles-1.1.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fa257a4d0d21fcbca5b5fcba9dca5a78011cb93c0323fb8855c6d2dfbc76eb77"}, + {file = "watchfiles-1.1.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7fd1b3879a578a8ec2076c7961076df540b9af317123f84569f5a9ddee64ce92"}, + {file = "watchfiles-1.1.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:62cc7a30eeb0e20ecc5f4bd113cd69dcdb745a07c68c0370cea919f373f65d9e"}, + {file = "watchfiles-1.1.0-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:891c69e027748b4a73847335d208e374ce54ca3c335907d381fde4e41661b13b"}, + {file = "watchfiles-1.1.0-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:12fe8eaffaf0faa7906895b4f8bb88264035b3f0243275e0bf24af0436b27259"}, + {file = "watchfiles-1.1.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:bfe3c517c283e484843cb2e357dd57ba009cff351edf45fb455b5fbd1f45b15f"}, + {file = "watchfiles-1.1.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:a9ccbf1f129480ed3044f540c0fdbc4ee556f7175e5ab40fe077ff6baf286d4e"}, + {file = "watchfiles-1.1.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba0e3255b0396cac3cc7bbace76404dd72b5438bf0d8e7cefa2f79a7f3649caa"}, + {file = "watchfiles-1.1.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4281cd9fce9fc0a9dbf0fc1217f39bf9cf2b4d315d9626ef1d4e87b84699e7e8"}, + {file = "watchfiles-1.1.0-cp314-cp314t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6d2404af8db1329f9a3c9b79ff63e0ae7131986446901582067d9304ae8aaf7f"}, + {file = "watchfiles-1.1.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e78b6ed8165996013165eeabd875c5dfc19d41b54f94b40e9fff0eb3193e5e8e"}, + {file = "watchfiles-1.1.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:249590eb75ccc117f488e2fabd1bfa33c580e24b96f00658ad88e38844a040bb"}, + {file = "watchfiles-1.1.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d05686b5487cfa2e2c28ff1aa370ea3e6c5accfe6435944ddea1e10d93872147"}, + {file = "watchfiles-1.1.0-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:d0e10e6f8f6dc5762adee7dece33b722282e1f59aa6a55da5d493a97282fedd8"}, + {file = "watchfiles-1.1.0-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:af06c863f152005c7592df1d6a7009c836a247c9d8adb78fef8575a5a98699db"}, + {file = "watchfiles-1.1.0-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:865c8e95713744cf5ae261f3067861e9da5f1370ba91fc536431e29b418676fa"}, + {file = "watchfiles-1.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:42f92befc848bb7a19658f21f3e7bae80d7d005d13891c62c2cd4d4d0abb3433"}, + {file = "watchfiles-1.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aa0cc8365ab29487eb4f9979fd41b22549853389e22d5de3f134a6796e1b05a4"}, + {file = "watchfiles-1.1.0-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:90ebb429e933645f3da534c89b29b665e285048973b4d2b6946526888c3eb2c7"}, + {file = "watchfiles-1.1.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c588c45da9b08ab3da81d08d7987dae6d2a3badd63acdb3e206a42dbfa7cb76f"}, + {file = "watchfiles-1.1.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7c55b0f9f68590115c25272b06e63f0824f03d4fc7d6deed43d8ad5660cabdbf"}, + {file = "watchfiles-1.1.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd17a1e489f02ce9117b0de3c0b1fab1c3e2eedc82311b299ee6b6faf6c23a29"}, + {file = "watchfiles-1.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:da71945c9ace018d8634822f16cbc2a78323ef6c876b1d34bbf5d5222fd6a72e"}, + {file = "watchfiles-1.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:51556d5004887045dba3acdd1fdf61dddea2be0a7e18048b5e853dcd37149b86"}, + {file = "watchfiles-1.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04e4ed5d1cd3eae68c89bcc1a485a109f39f2fd8de05f705e98af6b5f1861f1f"}, + {file = "watchfiles-1.1.0-cp39-cp39-win32.whl", hash = "sha256:c600e85f2ffd9f1035222b1a312aff85fd11ea39baff1d705b9b047aad2ce267"}, + {file = "watchfiles-1.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:3aba215958d88182e8d2acba0fdaf687745180974946609119953c0e112397dc"}, + {file = "watchfiles-1.1.0-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:3a6fd40bbb50d24976eb275ccb55cd1951dfb63dbc27cae3066a6ca5f4beabd5"}, + {file = "watchfiles-1.1.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:9f811079d2f9795b5d48b55a37aa7773680a5659afe34b54cc1d86590a51507d"}, + {file = "watchfiles-1.1.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a2726d7bfd9f76158c84c10a409b77a320426540df8c35be172444394b17f7ea"}, + {file = "watchfiles-1.1.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:df32d59cb9780f66d165a9a7a26f19df2c7d24e3bd58713108b41d0ff4f929c6"}, + {file = "watchfiles-1.1.0-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:0ece16b563b17ab26eaa2d52230c9a7ae46cf01759621f4fbbca280e438267b3"}, + {file = "watchfiles-1.1.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:51b81e55d40c4b4aa8658427a3ee7ea847c591ae9e8b81ef94a90b668999353c"}, + {file = "watchfiles-1.1.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2bcdc54ea267fe72bfc7d83c041e4eb58d7d8dc6f578dfddb52f037ce62f432"}, + {file = "watchfiles-1.1.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:923fec6e5461c42bd7e3fd5ec37492c6f3468be0499bc0707b4bbbc16ac21792"}, + {file = "watchfiles-1.1.0-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:7b3443f4ec3ba5aa00b0e9fa90cf31d98321cbff8b925a7c7b84161619870bc9"}, + {file = "watchfiles-1.1.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:7049e52167fc75fc3cc418fc13d39a8e520cbb60ca08b47f6cedb85e181d2f2a"}, + {file = "watchfiles-1.1.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54062ef956807ba806559b3c3d52105ae1827a0d4ab47b621b31132b6b7e2866"}, + {file = "watchfiles-1.1.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a7bd57a1bb02f9d5c398c0c1675384e7ab1dd39da0ca50b7f09af45fa435277"}, + {file = "watchfiles-1.1.0.tar.gz", hash = "sha256:693ed7ec72cbfcee399e92c895362b6e66d63dac6b91e2c11ae03d10d503e575"}, ] [package.dependencies] diff --git a/src/auto_archiver/core/orchestrator.py b/src/auto_archiver/core/orchestrator.py index 66073a7..00065e2 100644 --- a/src/auto_archiver/core/orchestrator.py +++ b/src/auto_archiver/core/orchestrator.py @@ -264,6 +264,14 @@ Here's how that would look: \n\nsteps:\n extractors:\n - [your_extractor_name_ default=None, ) + parser.add_argument( + "--logging.each_level_in_separate_file", + action="store", + dest="logging.each_level_in_separate_file", + help="whether to write each logging level to a separate file", + default=False, + ) + def add_individual_module_args( self, modules: list[LazyBaseModule] = None, parser: argparse.ArgumentParser = None ) -> None: @@ -334,10 +342,27 @@ Here's how that would look: \n\nsteps:\n extractors:\n - [your_extractor_name_ # add other logging info if self.logger_id is None: # note - need direct comparison to None since need to consider falsy value 0 self.logger_id = logger.add(sys.stderr, level=logging_config["level"]) - if log_file := logging_config["file"]: - logger.add(log_file) if not logging_config["rotation"] else logger.add( - log_file, rotation=logging_config["rotation"] - ) + + # Default to False (above in parser code) + separate_file = logging_config["each_level_in_separate_file"] + + # Default to None if not set + rotation=logging_config["rotation"] + + if separate_file: + logger.add("logs/1debug.log", level="DEBUG", rotation=rotation) + logger.add("logs/2info.log", level="INFO", rotation=rotation) + logger.add("logs/3success.log", level="SUCCESS", rotation=rotation) + logger.add("logs/4warning.log", level="WARNING", rotation=rotation) + logger.add("logs/5error.log", level="ERROR", rotation=rotation) + else: + log_file = logging_config["file"] + logger.add(log_file, rotation=rotation) + + # if log_file := logging_config["file"]: + # logger.add(log_file) if not logging_config["rotation"] else logger.add( + # log_file, rotation=logging_config["rotation"] + # ) def install_modules(self, modules_by_type): """ From b3adc5603a44ad29af12c91b78cbec53cfa330c3 Mon Sep 17 00:00:00 2001 From: Dave Mateer Date: Tue, 17 Jun 2025 09:51:19 +0100 Subject: [PATCH 07/14] metadata.json hardcode in storage. add new metadata_json_enricher. log level change in orchestrator --- src/auto_archiver/core/orchestrator.py | 2 +- src/auto_archiver/core/storage.py | 7 +++- .../metadata_json_enricher/__init__.py | 1 + .../metadata_json_enricher/__manifest__.py | 37 +++++++++++++++++++ .../metadata_json_enricher.py | 21 +++++++++++ 5 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 src/auto_archiver/modules/metadata_json_enricher/__init__.py create mode 100644 src/auto_archiver/modules/metadata_json_enricher/__manifest__.py create mode 100644 src/auto_archiver/modules/metadata_json_enricher/metadata_json_enricher.py diff --git a/src/auto_archiver/core/orchestrator.py b/src/auto_archiver/core/orchestrator.py index 00065e2..b45afc0 100644 --- a/src/auto_archiver/core/orchestrator.py +++ b/src/auto_archiver/core/orchestrator.py @@ -541,7 +541,7 @@ Here's how that would look: \n\nsteps:\n extractors:\n - [your_extractor_name_ yield self.feed_item(item) url_count += 1 - logger.success(f"Processed {url_count} URL(s)") + logger.info(f"Processed {url_count} URL(s)") self.cleanup() def feed_item(self, item: Metadata) -> Metadata: diff --git a/src/auto_archiver/core/storage.py b/src/auto_archiver/core/storage.py index 3205f5a..feb3d5f 100644 --- a/src/auto_archiver/core/storage.py +++ b/src/auto_archiver/core/storage.py @@ -100,7 +100,12 @@ class Storage(BaseModule): # Handle filename_generator logic filename_generator = self.filename_generator - if filename_generator == "random": + # DM 9th Jun 25 - special case for metadata.json file in metadata_json_enricher + # where we want the filename to remain metadata.json + # TODO - should this be a config option to keep the original filename? Is it useful anywhere else? + if filename.endswith('metadata'): + filename = 'metadata' + elif filename_generator == "random": filename = random_str(24) elif filename_generator == "static": # load the hash_enricher module diff --git a/src/auto_archiver/modules/metadata_json_enricher/__init__.py b/src/auto_archiver/modules/metadata_json_enricher/__init__.py new file mode 100644 index 0000000..4eed90b --- /dev/null +++ b/src/auto_archiver/modules/metadata_json_enricher/__init__.py @@ -0,0 +1 @@ +from .metadata_json_enricher import MetadataJsonEnricher \ No newline at end of file diff --git a/src/auto_archiver/modules/metadata_json_enricher/__manifest__.py b/src/auto_archiver/modules/metadata_json_enricher/__manifest__.py new file mode 100644 index 0000000..b737b16 --- /dev/null +++ b/src/auto_archiver/modules/metadata_json_enricher/__manifest__.py @@ -0,0 +1,37 @@ +{ + "name": "Metadata JSON Enricher", + "type": ["enricher"], + "requires_setup": True, + "dependencies": { + "python": ["loguru"], + }, + "configs": { + # "width": {"default": 1280, "type": "int", "help": "width of the screenshots"}, + # "height": {"default": 1024, "type": "int", "help": "height of the screenshots"}, + # "timeout": {"default": 60, "type": "int", "help": "timeout for taking the screenshot"}, + # "sleep_before_screenshot": { + # "default": 4, + # "type": "int", + # "help": "seconds to wait for the pages to load before taking screenshot", + # }, + # "http_proxy": { + # "default": "", + # "help": "http proxy to use for the webdriver, eg http://proxy-user:password@proxy-ip:port", + # }, + # "save_to_pdf": { + # "default": False, + # "type": "bool", + # "help": "save the page as pdf along with the screenshot. PDF saving options can be adjusted with the 'print_options' parameter", + # }, + # "print_options": { + # "default": {}, + # "help": "options to pass to the pdf printer, in JSON format. See https://www.selenium.dev/documentation/webdriver/interactions/print_page/ for more information", + # "type": "json_loader", + # }, + }, + "description": """ + + Writes all the metadata to a json file so can be parsed by other tools. + + """, +} diff --git a/src/auto_archiver/modules/metadata_json_enricher/metadata_json_enricher.py b/src/auto_archiver/modules/metadata_json_enricher/metadata_json_enricher.py new file mode 100644 index 0000000..312f922 --- /dev/null +++ b/src/auto_archiver/modules/metadata_json_enricher/metadata_json_enricher.py @@ -0,0 +1,21 @@ +import json +from loguru import logger +import os + +from auto_archiver.core import Enricher +from auto_archiver.core import Media, Metadata + +class MetadataJsonEnricher(Enricher): + def __init__(self): + super().__init__() + + def enrich(self, to_enrich: Metadata) -> None: + url = to_enrich.get_url() + + logger.debug(f"Metadata JSON Enricher for {url=}") + + item_path = os.path.join(self.tmp_dir, f"metadata.json") + with open(item_path, mode="w", encoding="utf-8") as outf: + json.dump(to_enrich.to_dict(), outf, indent=4, default=str) + + to_enrich.add_media(Media(filename=item_path), id="metadata_json") \ No newline at end of file From 8067da0f608fc7bddfa18abd08e2f3b239113a6b Mon Sep 17 00:00:00 2001 From: msramalho <19508417+msramalho@users.noreply.github.com> Date: Tue, 17 Jun 2025 13:15:13 +0100 Subject: [PATCH 08/14] custom user to its own file --- .../ubuntu_24_server_install.md | 160 ++++++++++++++++++ docs/source/installation/installation.md | 159 +---------------- 2 files changed, 162 insertions(+), 157 deletions(-) create mode 100644 docs/source/installation/example_scripts/ubuntu_24_server_install.md diff --git a/docs/source/installation/example_scripts/ubuntu_24_server_install.md b/docs/source/installation/example_scripts/ubuntu_24_server_install.md new file mode 100644 index 0000000..ea31519 --- /dev/null +++ b/docs/source/installation/example_scripts/ubuntu_24_server_install.md @@ -0,0 +1,160 @@ + +### Bash script for Ubuntu 24 Server install + +> NOTE: this script has not been tested by the maintainers and results from the personal experience of a user. It is meant as a guide and not an out of the box script, as you will see it's aimed at a custom branches, users, and features like the Geckodriver which are removed as of version 1.0.2. + +This acts as a handy guide on all requirements. This is built and tested on the 29th of May 2025 on Ubuntu Server 24.04.2 LTS (which is the current latest LTS) + +```bash +#!/bin/sh + +# I usually run steps manually as logged in with the user: dave +# which the application runs under which makes debugging easier + +cd ~ +sudo apt update -y +sudo apt upgrade -y + +# Clone only my latest branch +git clone -b v1-test --single-branch https://github.com/djhmateer/auto-archiver + +mkdir ~/auto-archiver/secrets +sudo chown -R dave ~/auto-archiver + +sudo apt update -y +sudo apt upgrade -y + +## Python 3.12.3 comes with Ubuntu 24.04.2 + +# Poetry install 2.1.3 on 2nd June 25 +curl -sSL https://install.python-poetry.org | python3 - + +# had to restart here.. +sudo reboot + +# C++ compiler so pdqhash will install next +sudo apt install build-essential python3-dev -y + +cd auto-archiver + +poetry install + +# FFMpeg +# 6.1.1-3ubuntu5 on 2nd June 25 +sudo apt install ffmpeg -y + +## Firefox +# 139.0+build2-0ubuntu0.24.04.1~mt1 on 2nd Jun 25 +# 16th Jun - don't need anymore as using Chrome in antibot +# cd ~ +# sudo add-apt-repository ppa:mozillateam/ppa -y + +# echo ' +# Package: * +# Pin: release o=LP-PPA-mozillateam +# Pin-Priority: 1001 +# ' | sudo tee /etc/apt/preferences.d/mozilla-firefox + +# echo 'Unattended-Upgrade::Allowed-Origins:: "LP-PPA-mozillateam:${distro_codename}";' | sudo tee /etc/apt/apt.conf.d/51unattended-upgrades-firefox + +# sudo apt install firefox -y + +wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb + +# Chrome +cd ~ +# got problems here - fixed below +# 137.0.7151.103 on 16th Jun 2025 +sudo dpkg -i google-chrome-stable_current_amd64.deb + +# fix dependencies on install above +sudo apt-get install -f + +# had to click a lot on UI to get going. +# to test +# google-chrome + +## Gecko driver +# check version numbers for new ones +# https://github.com/mozilla/geckodriver/releases/ +wget https://github.com/mozilla/geckodriver/releases/download/v0.36.0/geckodriver-v0.36.0-linux64.tar.gz +tar -xvzf geckodriver* +chmod +x geckodriver +sudo mv geckodriver /usr/local/bin/ +rm geckodriver* + +# Fonts so selenium via firefox can render other languages eg Burmese +sudo apt install fonts-noto -y + +# Docker +# Add Docker's official GPG key: +sudo apt-get update -y +sudo apt-get install ca-certificates curl -y +sudo install -m 0755 -d /etc/apt/keyrings +sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc +sudo chmod a+r /etc/apt/keyrings/docker.asc + +# Add the repository to Apt sources: +echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ + $(. /etc/os-release && echo "${UBUNTU_CODENAME:-$VERSION_CODENAME}") stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + +sudo apt-get update -y + +sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -y + +# add dave user to docker group +sudo usermod -aG docker $USER + +# reboot otherwise can't pull images + +# https://github.com/webrecorder/browsertrix-crawler +# https://hub.docker.com/r/webrecorder/browsertrix-crawler/tags +# 1.6.2 on 4th Jun 2025 +docker pull webrecorder/browsertrix-crawler:latest + +# exif +sudo apt install libimage-exiftool-perl -y + + +## CRON run every minute +# the cron job running as user dave will execute the shell script +# I have many scripts running from cron_11 upwards. +# patch in the correct number +sudo chmod +x ~/auto-archiver/scripts/cron_15.sh + +# don't want service to run until a reboot otherwise problems with Gecko driver +sudo service cron stop + +# runs the script every minute +# notice put in a # to disable so will have to manually start it. +cat <> run-auto-archive +#*/1 * * * * dave /home/dave/auto-archiver/scripts/cron_15.sh +EOT + +sudo mv run-auto-archive /etc/cron.d +sudo chown root /etc/cron.d/run-auto-archive +sudo chmod 600 /etc/cron.d/run-auto-archive + +# Helper alias 'c' to open the above file +echo "alias c='sudo vim /etc/cron.d/run-auto-archive'" >> ~/.bashrc + +# secrets folder copy +# I run dev from: +# \\wsl.localhost\Ubuntu-24.04\home\dave\code\auto-archiver\secrets\ + +# orchestration.yaml - for aa config +# service_account - for google spreadsheet +# anon.session - for telethon so don't have to type in phone number +# profile.tar.gz - for wacz to have a logged in profile for facebook, x.com and instagram to get data + +# Youtube - POT Tokens +# https://github.com/Brainicism/bgutil-ytdlp-pot-provider +docker run --name bgutil-provider --restart unless-stopped -d -p 4416:4416 brainicism/bgutil-ytdlp-pot-provider + + +# test run +cd ~/auto-archiver + +poetry run python src/auto_archiver --config secrets/orchestration-aa-demo-main.yaml +``` diff --git a/docs/source/installation/installation.md b/docs/source/installation/installation.md index 38d955b..1bc67a4 100644 --- a/docs/source/installation/installation.md +++ b/docs/source/installation/installation.md @@ -55,164 +55,9 @@ If using the local installation method, you will also need to install the follow 3. (optional) [fonts-noto](https://fonts.google.com/noto) to deal with multiple unicode characters during selenium screenshots: `sudo apt install fonts-noto -y`. 4. [Browsertrix Crawler docker image](https://hub.docker.com/r/webrecorder/browsertrix-crawler) for the WACZ enricher/archiver -### Bash script for Ubuntu 24 Server install - -This acts as a handy guide on all requirements. This is built and tested on the 29th of May 2025 on Ubuntu Server 24.04.2 LTS (which is the current latest LTS) - -```bash -#!/bin/sh - -# I usually run steps manually as logged in with the user: dave -# which the application runs under which makes debugging easier - -cd ~ -sudo apt update -y -sudo apt upgrade -y - -# Clone only my latest branch -git clone -b v1-test --single-branch https://github.com/djhmateer/auto-archiver - -mkdir ~/auto-archiver/secrets -sudo chown -R dave ~/auto-archiver - -sudo apt update -y -sudo apt upgrade -y - -## Python 3.12.3 comes with Ubuntu 24.04.2 - -# Poetry install 2.1.3 on 2nd June 25 -curl -sSL https://install.python-poetry.org | python3 - - -# had to restart here.. -sudo reboot - -# C++ compiler so pdqhash will install next -sudo apt install build-essential python3-dev -y - -cd auto-archiver - -poetry install - -# FFMpeg -# 6.1.1-3ubuntu5 on 2nd June 25 -sudo apt install ffmpeg -y - -## Firefox -# 139.0+build2-0ubuntu0.24.04.1~mt1 on 2nd Jun 25 -# 16th Jun - don't need anymore as using Chrome in antibot -# cd ~ -# sudo add-apt-repository ppa:mozillateam/ppa -y - -# echo ' -# Package: * -# Pin: release o=LP-PPA-mozillateam -# Pin-Priority: 1001 -# ' | sudo tee /etc/apt/preferences.d/mozilla-firefox - -# echo 'Unattended-Upgrade::Allowed-Origins:: "LP-PPA-mozillateam:${distro_codename}";' | sudo tee /etc/apt/apt.conf.d/51unattended-upgrades-firefox - -# sudo apt install firefox -y - -wget https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb - -# Chrome -cd ~ -# got problems here - fixed below -# 137.0.7151.103 on 16th Jun 2025 -sudo dpkg -i google-chrome-stable_current_amd64.deb - -# fix dependencies on install above -sudo apt-get install -f - -# had to click a lot on UI to get going. -# to test -# google-chrome - -## Gecko driver -# check version numbers for new ones -# https://github.com/mozilla/geckodriver/releases/ -wget https://github.com/mozilla/geckodriver/releases/download/v0.36.0/geckodriver-v0.36.0-linux64.tar.gz -tar -xvzf geckodriver* -chmod +x geckodriver -sudo mv geckodriver /usr/local/bin/ -rm geckodriver* - -# Fonts so selenium via firefox can render other languages eg Burmese -sudo apt install fonts-noto -y - -# Docker -# Add Docker's official GPG key: -sudo apt-get update -y -sudo apt-get install ca-certificates curl -y -sudo install -m 0755 -d /etc/apt/keyrings -sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc -sudo chmod a+r /etc/apt/keyrings/docker.asc - -# Add the repository to Apt sources: -echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu \ - $(. /etc/os-release && echo "${UBUNTU_CODENAME:-$VERSION_CODENAME}") stable" | \ - sudo tee /etc/apt/sources.list.d/docker.list > /dev/null - -sudo apt-get update -y - -sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin -y - -# add dave user to docker group -sudo usermod -aG docker $USER - -# reboot otherwise can't pull images - -# https://github.com/webrecorder/browsertrix-crawler -# https://hub.docker.com/r/webrecorder/browsertrix-crawler/tags -# 1.6.2 on 4th Jun 2025 -docker pull webrecorder/browsertrix-crawler:latest - -# exif -sudo apt install libimage-exiftool-perl -y - - -## CRON run every minute -# the cron job running as user dave will execute the shell script -# I have many scripts running from cron_11 upwards. -# patch in the correct number -sudo chmod +x ~/auto-archiver/scripts/cron_15.sh - -# don't want service to run until a reboot otherwise problems with Gecko driver -sudo service cron stop - -# runs the script every minute -# notice put in a # to disable so will have to manually start it. -cat <> run-auto-archive -#*/1 * * * * dave /home/dave/auto-archiver/scripts/cron_15.sh -EOT - -sudo mv run-auto-archive /etc/cron.d -sudo chown root /etc/cron.d/run-auto-archive -sudo chmod 600 /etc/cron.d/run-auto-archive - -# Helper alias 'c' to open the above file -echo "alias c='sudo vim /etc/cron.d/run-auto-archive'" >> ~/.bashrc - -# secrets folder copy -# I run dev from: -# \\wsl.localhost\Ubuntu-24.04\home\dave\code\auto-archiver\secrets\ - -# orchestration.yaml - for aa config -# service_account - for google spreadsheet -# anon.session - for telethon so don't have to type in phone number -# profile.tar.gz - for wacz to have a logged in profile for facebook, x.com and instagram to get data - -# Youtube - POT Tokens -# https://github.com/Brainicism/bgutil-ytdlp-pot-provider -docker run --name bgutil-provider --restart unless-stopped -d -p 4416:4416 brainicism/bgutil-ytdlp-pot-provider - - -# test run -cd ~/auto-archiver - -poetry run python src/auto_archiver --config secrets/orchestration-aa-demo-main.yaml -``` +### Custom installation scripts +- [Ubuntu 24 Server Install by @djhmateer](example_scripts/ubuntu_24_server_install.md) - a WYSIWYG example script from a user who set up the Auto Archiver on a fresh Ubuntu 24 server. ## Developer Install From f0b876e67c5499168bf374d1e17b0d37af894c0e Mon Sep 17 00:00:00 2001 From: msramalho <19508417+msramalho@users.noreply.github.com> Date: Tue, 17 Jun 2025 13:16:36 +0100 Subject: [PATCH 09/14] removes dev specific instructions --- pyproject.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2299350..cdbb86b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,8 +4,6 @@ build-backend = "poetry.core.masonry.api" [project] name = "auto-archiver" -# to add a custom version use local versioning eg 1.1.0+dm.1 -# then poetry install version = "1.1.0" description = "Automatically archive links to videos, images, and social media content from Google Sheets (and more)." From 1b260788de5f4ba40f55764fa4d2dee04178673c Mon Sep 17 00:00:00 2001 From: msramalho <19508417+msramalho@users.noreply.github.com> Date: Tue, 17 Jun 2025 13:18:12 +0100 Subject: [PATCH 10/14] do not add commit comments to code --- src/auto_archiver/core/media.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/auto_archiver/core/media.py b/src/auto_archiver/core/media.py index 75fb366..2fad0ec 100644 --- a/src/auto_archiver/core/media.py +++ b/src/auto_archiver/core/media.py @@ -116,11 +116,6 @@ class Media: # self.is_video() should be used together with this method try: streams = ffmpeg.probe(self.filename, select_streams="v")["streams"] - # DM 27th May 2025 - # https://x.com/dave_mateer/status/1524341442738638848 - # shows this warning for the gif file - # Have changed to debug for now to clean up logs - # logger.warning(f"STREAMS FOR {self.filename} {streams}") logger.debug(f"STREAMS FOR {self.filename} {streams}") return any(s.get("duration_ts", 0) > 0 for s in streams) except Error: From 664ee8d037399431bfeef159d43e781ebabe47da Mon Sep 17 00:00:00 2001 From: msramalho <19508417+msramalho@users.noreply.github.com> Date: Tue, 17 Jun 2025 14:10:46 +0100 Subject: [PATCH 11/14] fixes bugs and limited configuration of multi-level logs --- docs/source/how_to/03_logging.md | 34 ++++++++++++++++------ src/auto_archiver/core/orchestrator.py | 40 ++++++++++++-------------- 2 files changed, 44 insertions(+), 30 deletions(-) diff --git a/docs/source/how_to/03_logging.md b/docs/source/how_to/03_logging.md index d88882d..c283b77 100644 --- a/docs/source/how_to/03_logging.md +++ b/docs/source/how_to/03_logging.md @@ -1,6 +1,6 @@ # Keeping Logs -Auto Archiver's logs can be helpful for debugging problematic archiving processes. This guide shows you how to use the logs to +Auto Archiver's logs can be helpful for debugging problematic archiving processes. This guide shows you how to use the logs configuration. ## Setting up logging @@ -8,10 +8,10 @@ Logging settings can be set on the command line or using the orchestration confi #### Enabling or Disabling Logging -Logging to the console is enabled by default. If you want to globally disable Auto Archiver's logging, then you can set `enabled: false` in your `logging` config: +Logging to the console is enabled by default. If you want to globally disable Auto Archiver's logging, then you can set `enabled: false` in your `logging` config file: ```{code} yaml - +:caption: orchestration.yaml ... logging: enabled: false @@ -24,7 +24,7 @@ This will disable all logs from Auto Archiver, but it does not disable logs for #### Logging Level -There are 7 logging levels in total, with 4 commonly used levels. They are: `DEBUG`, `INFO`, `WARNING` and `ERROR`. +There are 7 logging levels in total, with 5 of them used in this tool. They are: `DEBUG`, `INFO`, `SUCCESS`, `WARNING` and `ERROR`. Change the warning level by setting the value in your orchestration config file: @@ -44,7 +44,7 @@ For normal usage, it is recommended to use the `INFO` level, or if you prefer qu ### Logging to a file -As default, auto-archiver will log to the console. But if you wish to store your logs for future reference, or you are running the auto-archiver from within code a implementation, then you may with to enable file logging. This can be done by setting the `file:` config value in the logging settings. +As default, auto-archiver will log to the console. But if you wish to store your logs for future reference, or you are running the auto-archiver from within code a implementation, then you may wish to enable file logging. This can be done by setting the `file:` config value in the logging settings. **Rotation:** For file logging, you can choose to 'rotate' your log files (creating new log files) so they do not get too large. Change this by setting the 'rotation' option in your logging settings. For a full list of rotation options, see the [loguru docs](https://loguru.readthedocs.io/en/stable/overview.html#easier-file-logging-with-rotation-retention-compression). @@ -57,15 +57,33 @@ logging: rotation: 1 day ``` -### Full logging example +### Logging each level to a different file +If you want to log each level to a different file, you can do this by setting the `each_level_in_separate_file:` option to `true` and also setting your `file:` name, a new file will be created for each of the 5 levels used, by appending the `0_level` name to the file like so `your_file.log.1_error`. In this case the `level:` option is ignored, and all levels will be logged. -The below example logs only `WARNING` logs to the console and to the file `/my/file.log`, rotating that file once per week: ```{code} yaml :caption: orchestration.yaml logging: - level: WARNING + each_level_in_separate_file: true + file: /my/logs/file.log +``` +This will create the following files: +- `/my/logs/file.log.1_debug` +- `/my/logs/file.log.2_info` +- `/my/logs/file.log.3_success` +- `/my/logs/file.log.4_warning` +- `/my/logs/file.log.5_error` + +### Full logging example + +The below example logs only `DEBUG` logs to the console and to the file `/my/file.log`, rotating that file once per week: + +```{code} yaml +:caption: orchestration.yaml + +logging: + level: DEBUG file: /my/file.log rotation: 1 week ``` \ No newline at end of file diff --git a/src/auto_archiver/core/orchestrator.py b/src/auto_archiver/core/orchestrator.py index b45afc0..a028ac7 100644 --- a/src/auto_archiver/core/orchestrator.py +++ b/src/auto_archiver/core/orchestrator.py @@ -249,7 +249,7 @@ Here's how that would look: \n\nsteps:\n extractors:\n - [your_extractor_name_ action="store", dest="logging.level", choices=["INFO", "DEBUG", "ERROR", "WARNING"], - help="the logging level to use", + help="the logging level to use for the standard output and file logging", default="INFO", type=str.upper, ) @@ -268,7 +268,7 @@ Here's how that would look: \n\nsteps:\n extractors:\n - [your_extractor_name_ "--logging.each_level_in_separate_file", action="store", dest="logging.each_level_in_separate_file", - help="whether to write each logging level to a separate file", + help="if set, writes each logging level to a separate file (ignores --logging.level), you must also set --logging.file. Each level will have a dedicate logs file matching your .debug, .info, etc.", default=False, ) @@ -341,28 +341,24 @@ Here's how that would look: \n\nsteps:\n extractors:\n - [your_extractor_name_ # add other logging info if self.logger_id is None: # note - need direct comparison to None since need to consider falsy value 0 - self.logger_id = logger.add(sys.stderr, level=logging_config["level"]) + use_level = logging_config["level"] + self.logger_id = logger.add(sys.stderr, level=use_level) - # Default to False (above in parser code) - separate_file = logging_config["each_level_in_separate_file"] + rotation = logging_config["rotation"] + log_file = logging_config["file"] - # Default to None if not set - rotation=logging_config["rotation"] - - if separate_file: - logger.add("logs/1debug.log", level="DEBUG", rotation=rotation) - logger.add("logs/2info.log", level="INFO", rotation=rotation) - logger.add("logs/3success.log", level="SUCCESS", rotation=rotation) - logger.add("logs/4warning.log", level="WARNING", rotation=rotation) - logger.add("logs/5error.log", level="ERROR", rotation=rotation) - else: - log_file = logging_config["file"] - logger.add(log_file, rotation=rotation) - - # if log_file := logging_config["file"]: - # logger.add(log_file) if not logging_config["rotation"] else logger.add( - # log_file, rotation=logging_config["rotation"] - # ) + if logging_config.get("each_level_in_separate_file"): + assert logging_config["file"], ( + "You must set --logging.file if you want to use --logging.each_level_in_separate_file" + ) + for i, level in enumerate(["DEBUG", "INFO", "SUCCESS", "WARNING", "ERROR"], start=1): + logger.add( + f"{log_file}.{i}_{level.lower()}", + filter=lambda rec, lvl=level: rec["level"].name == lvl, + rotation=rotation, + ) + elif log_file: + logger.add(log_file, rotation=rotation, level=use_level) def install_modules(self, modules_by_type): """ From 2f1a07abbf3a5ab3cba763f8c05dc51bd8d5bd66 Mon Sep 17 00:00:00 2001 From: msramalho <19508417+msramalho@users.noreply.github.com> Date: Tue, 17 Jun 2025 16:06:04 +0100 Subject: [PATCH 12/14] renaming and code improvements to json_e richer --- .../modules/json_enricher/__init__.py | 1 + .../modules/json_enricher/__manifest__.py | 16 ++++++++ .../json_enricher.py} | 14 +++---- .../metadata_json_enricher/__init__.py | 1 - .../metadata_json_enricher/__manifest__.py | 37 ------------------- 5 files changed, 23 insertions(+), 46 deletions(-) create mode 100644 src/auto_archiver/modules/json_enricher/__init__.py create mode 100644 src/auto_archiver/modules/json_enricher/__manifest__.py rename src/auto_archiver/modules/{metadata_json_enricher/metadata_json_enricher.py => json_enricher/json_enricher.py} (63%) delete mode 100644 src/auto_archiver/modules/metadata_json_enricher/__init__.py delete mode 100644 src/auto_archiver/modules/metadata_json_enricher/__manifest__.py diff --git a/src/auto_archiver/modules/json_enricher/__init__.py b/src/auto_archiver/modules/json_enricher/__init__.py new file mode 100644 index 0000000..5e0a04f --- /dev/null +++ b/src/auto_archiver/modules/json_enricher/__init__.py @@ -0,0 +1 @@ +from .json_enricher import JsonEnricher diff --git a/src/auto_archiver/modules/json_enricher/__manifest__.py b/src/auto_archiver/modules/json_enricher/__manifest__.py new file mode 100644 index 0000000..b80f493 --- /dev/null +++ b/src/auto_archiver/modules/json_enricher/__manifest__.py @@ -0,0 +1,16 @@ +{ + "name": "JSON Enricher", + "type": ["enricher"], + "requires_setup": True, + "dependencies": { + "python": ["loguru"], + }, + "configs": {}, + "description": """ + + Writes all archiving process metadata to a JSON file so it can be parsed by other tools. As this is an Enricher, it will not contain the final stored URLs. + + WARNING: The resulting JSON may reveal sensitive information about the computer and settings in which the archiving process was run. + + """, +} diff --git a/src/auto_archiver/modules/metadata_json_enricher/metadata_json_enricher.py b/src/auto_archiver/modules/json_enricher/json_enricher.py similarity index 63% rename from src/auto_archiver/modules/metadata_json_enricher/metadata_json_enricher.py rename to src/auto_archiver/modules/json_enricher/json_enricher.py index 312f922..b0900b6 100644 --- a/src/auto_archiver/modules/metadata_json_enricher/metadata_json_enricher.py +++ b/src/auto_archiver/modules/json_enricher/json_enricher.py @@ -5,17 +5,15 @@ import os from auto_archiver.core import Enricher from auto_archiver.core import Media, Metadata -class MetadataJsonEnricher(Enricher): - def __init__(self): - super().__init__() +class JsonEnricher(Enricher): def enrich(self, to_enrich: Metadata) -> None: url = to_enrich.get_url() - logger.debug(f"Metadata JSON Enricher for {url=}") + logger.debug(f"JSON Enricher for {url=}") - item_path = os.path.join(self.tmp_dir, f"metadata.json") + item_path = os.path.join(self.tmp_dir, "metadata.json") with open(item_path, mode="w", encoding="utf-8") as outf: - json.dump(to_enrich.to_dict(), outf, indent=4, default=str) - - to_enrich.add_media(Media(filename=item_path), id="metadata_json") \ No newline at end of file + json.dump(to_enrich.to_dict(), outf, indent=4, default=str, ensure_ascii=False) + + to_enrich.add_media(Media(filename=item_path), id="metadata_json") diff --git a/src/auto_archiver/modules/metadata_json_enricher/__init__.py b/src/auto_archiver/modules/metadata_json_enricher/__init__.py deleted file mode 100644 index 4eed90b..0000000 --- a/src/auto_archiver/modules/metadata_json_enricher/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .metadata_json_enricher import MetadataJsonEnricher \ No newline at end of file diff --git a/src/auto_archiver/modules/metadata_json_enricher/__manifest__.py b/src/auto_archiver/modules/metadata_json_enricher/__manifest__.py deleted file mode 100644 index b737b16..0000000 --- a/src/auto_archiver/modules/metadata_json_enricher/__manifest__.py +++ /dev/null @@ -1,37 +0,0 @@ -{ - "name": "Metadata JSON Enricher", - "type": ["enricher"], - "requires_setup": True, - "dependencies": { - "python": ["loguru"], - }, - "configs": { - # "width": {"default": 1280, "type": "int", "help": "width of the screenshots"}, - # "height": {"default": 1024, "type": "int", "help": "height of the screenshots"}, - # "timeout": {"default": 60, "type": "int", "help": "timeout for taking the screenshot"}, - # "sleep_before_screenshot": { - # "default": 4, - # "type": "int", - # "help": "seconds to wait for the pages to load before taking screenshot", - # }, - # "http_proxy": { - # "default": "", - # "help": "http proxy to use for the webdriver, eg http://proxy-user:password@proxy-ip:port", - # }, - # "save_to_pdf": { - # "default": False, - # "type": "bool", - # "help": "save the page as pdf along with the screenshot. PDF saving options can be adjusted with the 'print_options' parameter", - # }, - # "print_options": { - # "default": {}, - # "help": "options to pass to the pdf printer, in JSON format. See https://www.selenium.dev/documentation/webdriver/interactions/print_page/ for more information", - # "type": "json_loader", - # }, - }, - "description": """ - - Writes all the metadata to a json file so can be parsed by other tools. - - """, -} From 33cca734d9ea4b9b953e0841470f9c2b65617613 Mon Sep 17 00:00:00 2001 From: msramalho <19508417+msramalho@users.noreply.github.com> Date: Tue, 17 Jun 2025 16:06:25 +0100 Subject: [PATCH 13/14] original_url changes still constitute empty result --- src/auto_archiver/core/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/auto_archiver/core/metadata.py b/src/auto_archiver/core/metadata.py index bbb124d..370af78 100644 --- a/src/auto_archiver/core/metadata.py +++ b/src/auto_archiver/core/metadata.py @@ -96,7 +96,7 @@ class Metadata: def is_empty(self) -> bool: meaningfull_ids = set(self.metadata.keys()) - set( - ["_processed_at", "url", "total_bytes", "total_size", "archive_duration_seconds"] + ["_processed_at", "url", "original_url", "total_bytes", "total_size", "archive_duration_seconds"] ) return not self.is_success() and len(self.media) == 0 and len(meaningfull_ids) == 0 From 6085a66c584726769ea1d47cd2289924a7f569a9 Mon Sep 17 00:00:00 2001 From: msramalho <19508417+msramalho@users.noreply.github.com> Date: Tue, 17 Jun 2025 16:10:24 +0100 Subject: [PATCH 14/14] revert metadata json renaming --- src/auto_archiver/core/storage.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/auto_archiver/core/storage.py b/src/auto_archiver/core/storage.py index feb3d5f..3205f5a 100644 --- a/src/auto_archiver/core/storage.py +++ b/src/auto_archiver/core/storage.py @@ -100,12 +100,7 @@ class Storage(BaseModule): # Handle filename_generator logic filename_generator = self.filename_generator - # DM 9th Jun 25 - special case for metadata.json file in metadata_json_enricher - # where we want the filename to remain metadata.json - # TODO - should this be a config option to keep the original filename? Is it useful anywhere else? - if filename.endswith('metadata'): - filename = 'metadata' - elif filename_generator == "random": + if filename_generator == "random": filename = random_str(24) elif filename_generator == "static": # load the hash_enricher module