From 2d879350420b106db68087ac2f21ed50d8ee8dbf Mon Sep 17 00:00:00 2001
From: Patrick Robertson <robertson.patrick@gmail.com>
Date: Tue, 11 Feb 2025 14:54:46 +0000
Subject: [PATCH 01/55] Start on opentimestamps enricher

---
 .../opentimestamps_enricher/__manifest__.py   | 50 +++++++++++++++++++
 .../opentimestamps_enricher.py                |  0
 2 files changed, 50 insertions(+)
 create mode 100644 src/auto_archiver/modules/opentimestamps_enricher/__manifest__.py
 create mode 100644 src/auto_archiver/modules/opentimestamps_enricher/opentimestamps_enricher.py

diff --git a/src/auto_archiver/modules/opentimestamps_enricher/__manifest__.py b/src/auto_archiver/modules/opentimestamps_enricher/__manifest__.py
new file mode 100644
index 0000000..cfed1fb
--- /dev/null
+++ b/src/auto_archiver/modules/opentimestamps_enricher/__manifest__.py
@@ -0,0 +1,50 @@
+{
+    "name": "Opentimestamps Enricher",
+    "type": ["enricher"],
+    "requires_setup": False,
+    "dependencies": {
+        "python": [
+            "loguru",
+            "opentimestamps",
+        ],
+    },
+    "configs": {
+        "tsa_urls": {
+            "default": [
+                    # [Adobe Approved Trust List] and [Windows Cert Store]
+                    "http://timestamp.digicert.com",
+                    "http://timestamp.identrust.com",
+                    # "https://timestamp.entrust.net/TSS/RFC3161sha2TS", # not valid for timestamping
+                    # "https://timestamp.sectigo.com", # wait 15 seconds between each request.
+
+                    # [Adobe: European Union Trusted Lists].
+                    # "https://timestamp.sectigo.com/qualified", # wait 15 seconds between each request.
+
+                    # [Windows Cert Store]
+                    "http://timestamp.globalsign.com/tsa/r6advanced1",
+                    # [Adobe: European Union Trusted Lists] and [Windows Cert Store]
+                    # "http://ts.quovadisglobal.com/eu", # not valid for timestamping
+                    # "http://tsa.belgium.be/connect", # self-signed certificate in certificate chain
+                    # "https://timestamp.aped.gov.gr/qtss", # self-signed certificate in certificate chain
+                    # "http://tsa.sep.bg", # self-signed certificate in certificate chain
+                    # "http://tsa.izenpe.com", #unable to get local issuer certificate
+                    # "http://kstamp.keynectis.com/KSign", # unable to get local issuer certificate
+                    "http://tss.accv.es:8318/tsa",
+                ],
+            "help": "List of RFC3161 Time Stamp Authorities to use, separate with commas if passed via the command line.",
+        }
+    },
+    "description": """
+    Generates RFC3161-compliant timestamp tokens using Time Stamp Authorities (TSA) for archived files.
+
+    ### Features
+    - Creates timestamp tokens to prove the existence of files at a specific time, useful for legal and authenticity purposes.
+    - Aggregates file hashes into a text file and timestamps the concatenated data.
+    - Uses multiple Time Stamp Authorities (TSAs) to ensure reliability and redundancy.
+    - Validates timestamping certificates against trusted Certificate Authorities (CAs) using the `certifi` trust store.
+
+    ### Notes
+    - Should be run after the `hash_enricher` to ensure file hashes are available.
+    - Requires internet access to interact with the configured TSAs.
+    """
+}
diff --git a/src/auto_archiver/modules/opentimestamps_enricher/opentimestamps_enricher.py b/src/auto_archiver/modules/opentimestamps_enricher/opentimestamps_enricher.py
new file mode 100644
index 0000000..e69de29

From 1a2d9de819e56850a88fb1c5b016d639e94a74fb Mon Sep 17 00:00:00 2001
From: erinhmclark <erinhannahmary.clark@gmail.com>
Date: Fri, 7 Mar 2025 17:33:32 +0000
Subject: [PATCH 02/55] Update the release process docs and the latest version
 in pyproject.toml

---
 docs/source/development/release.md |  22 +++-
 poetry.lock                        | 188 ++++++++++++++---------------
 pyproject.toml                     |   2 +-
 3 files changed, 113 insertions(+), 99 deletions(-)

diff --git a/docs/source/development/release.md b/docs/source/development/release.md
index 694af78..0d9407c 100644
--- a/docs/source/development/release.md
+++ b/docs/source/development/release.md
@@ -3,11 +3,25 @@
 ```{note} This is a work in progress.
 ```
 
-1. Update the version number in [version.py](src/auto_archiver/version.py)
-2. Go to github releases > new release > use `vx.y.z` for matching version notation
-   1. package is automatically updated in pypi
-   2. docker image is automatically pushed to dockerhub
+1. Update the version number in the project file: [pyproject.toml](../../pyproject.toml) following SemVer:
+```toml
+[project]
+name = "auto-archiver"
+version = "0.1.1"
+```
+Then commit and push the changes.
 
+2. Next add a new git tag with the version number:
+```shell
+git tag -a v0.1.1
+git push origin v0.1.1
+```
+
+* The package version is automatically updated in PyPi using the workflow [python-publish.yml](../../.github/workflows/python-publish.yml)
+* A Docker image is automatically pushed with the git tag to dockerhub using the workflow [docker-publish.yml](../../.github/workflows/docker-publish.yml)
+
+
+3. Go to GitHub releases > new release > and select the tag you just created. 
 
 
 manual release to docker hub
diff --git a/poetry.lock b/poetry.lock
index 16ec2f8..f59d5c9 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -172,18 +172,18 @@ lxml = ["lxml"]
 
 [[package]]
 name = "boto3"
-version = "1.37.5"
+version = "1.37.8"
 description = "The AWS SDK for Python"
 optional = false
 python-versions = ">=3.8"
 groups = ["main"]
 files = [
-    {file = "boto3-1.37.5-py3-none-any.whl", hash = "sha256:12166353519aca0cc8d9dcfbbb0d38f8915955a5912b8cb241b2b2314f0dbc14"},
-    {file = "boto3-1.37.5.tar.gz", hash = "sha256:ae6e7048beeaa4478368e554a4b290e3928beb0ae8d8767d108d72381a81af30"},
+    {file = "boto3-1.37.8-py3-none-any.whl", hash = "sha256:b9f506e08c9f54687d6c073ef1c550a24a62cc2d1e0bc7cda9f13112a38818bf"},
+    {file = "boto3-1.37.8.tar.gz", hash = "sha256:9448f4a079189e19c3253cfdc5b8ef6dc51a3b82431e8347a51f4c1b2d9dab42"},
 ]
 
 [package.dependencies]
-botocore = ">=1.37.5,<1.38.0"
+botocore = ">=1.37.8,<1.38.0"
 jmespath = ">=0.7.1,<2.0.0"
 s3transfer = ">=0.11.0,<0.12.0"
 
@@ -192,14 +192,14 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"]
 
 [[package]]
 name = "botocore"
-version = "1.37.5"
+version = "1.37.8"
 description = "Low-level, data-driven core of boto 3."
 optional = false
 python-versions = ">=3.8"
 groups = ["main"]
 files = [
-    {file = "botocore-1.37.5-py3-none-any.whl", hash = "sha256:e5cfbb8026d5b4fadd9b3a18b61d238a41a8b8f620ab75873dc1467d456150d6"},
-    {file = "botocore-1.37.5.tar.gz", hash = "sha256:f8f526d33ae74d242c577e0440b57b9ec7d53edd41db211155ec8087fe7a5a21"},
+    {file = "botocore-1.37.8-py3-none-any.whl", hash = "sha256:a6c94f33de12f4b10b10684019e554c980469b8394c6d82448a738cbd8452cef"},
+    {file = "botocore-1.37.8.tar.gz", hash = "sha256:b5825e08dd3e25642aa22a0d7d92bf81fef1ef857117e4155f923bbccf5aba63"},
 ]
 
 [package.dependencies]
@@ -781,14 +781,14 @@ grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"]
 
 [[package]]
 name = "google-api-python-client"
-version = "2.162.0"
+version = "2.163.0"
 description = "Google API Client Library for Python"
 optional = false
 python-versions = ">=3.7"
 groups = ["main"]
 files = [
-    {file = "google_api_python_client-2.162.0-py2.py3-none-any.whl", hash = "sha256:49365fa4f7795fe81a747f5544d6528ea94314fa59664e0ea1005f603facf1ec"},
-    {file = "google_api_python_client-2.162.0.tar.gz", hash = "sha256:5f8bc934a5b6eea73a7d12d999e6585c1823179f48340234acb385e2502e735a"},
+    {file = "google_api_python_client-2.163.0-py2.py3-none-any.whl", hash = "sha256:080e8bc0669cb4c1fb8efb8da2f5b91a2625d8f0e7796cfad978f33f7016c6c4"},
+    {file = "google_api_python_client-2.163.0.tar.gz", hash = "sha256:88dee87553a2d82176e2224648bf89272d536c8f04dcdda37ef0a71473886dd7"},
 ]
 
 [package.dependencies]
@@ -860,14 +860,14 @@ tool = ["click (>=6.0.0)"]
 
 [[package]]
 name = "googleapis-common-protos"
-version = "1.69.0"
+version = "1.69.1"
 description = "Common protobufs used in Google APIs"
 optional = false
 python-versions = ">=3.7"
 groups = ["main"]
 files = [
-    {file = "googleapis_common_protos-1.69.0-py2.py3-none-any.whl", hash = "sha256:17835fdc4fa8da1d61cfe2d4d5d57becf7c61d4112f8d81c67eaa9d7ce43042d"},
-    {file = "googleapis_common_protos-1.69.0.tar.gz", hash = "sha256:5a46d58af72846f59009b9c4710425b9af2139555c71837081706b213b298187"},
+    {file = "googleapis_common_protos-1.69.1-py2.py3-none-any.whl", hash = "sha256:4077f27a6900d5946ee5a369fab9c8ded4c0ef1c6e880458ea2f70c14f7b70d5"},
+    {file = "googleapis_common_protos-1.69.1.tar.gz", hash = "sha256:e20d2d8dda87da6fe7340afbbdf4f0bcb4c8fae7e6cadf55926c31f946b0b9b1"},
 ]
 
 [package.dependencies]
@@ -978,14 +978,14 @@ browser-cookie3 = ["browser_cookie3 (>=0.19.1)"]
 
 [[package]]
 name = "jinja2"
-version = "3.1.5"
+version = "3.1.6"
 description = "A very fast and expressive template engine."
 optional = false
 python-versions = ">=3.7"
 groups = ["main", "docs"]
 files = [
-    {file = "jinja2-3.1.5-py3-none-any.whl", hash = "sha256:aba0f4dc9ed8013c424088f68a5c226f7d6097ed89b246d7749c2ec4175c6adb"},
-    {file = "jinja2-3.1.5.tar.gz", hash = "sha256:8fefff8dc3034e27bb80d67c671eb8a9bc424c0ef4c0826edbff304cceff43bb"},
+    {file = "jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67"},
+    {file = "jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d"},
 ]
 
 [package.dependencies]
@@ -2248,21 +2248,21 @@ files = [
 
 [[package]]
 name = "s3transfer"
-version = "0.11.3"
+version = "0.11.4"
 description = "An Amazon S3 Transfer Manager"
 optional = false
 python-versions = ">=3.8"
 groups = ["main"]
 files = [
-    {file = "s3transfer-0.11.3-py3-none-any.whl", hash = "sha256:ca855bdeb885174b5ffa95b9913622459d4ad8e331fc98eb01e6d5eb6a30655d"},
-    {file = "s3transfer-0.11.3.tar.gz", hash = "sha256:edae4977e3a122445660c7c114bba949f9d191bae3b34a096f18a1c8c354527a"},
+    {file = "s3transfer-0.11.4-py3-none-any.whl", hash = "sha256:ac265fa68318763a03bf2dc4f39d5cbd6a9e178d81cc9483ad27da33637e320d"},
+    {file = "s3transfer-0.11.4.tar.gz", hash = "sha256:559f161658e1cf0a911f45940552c696735f5c74e64362e515f333ebed87d679"},
 ]
 
 [package.dependencies]
-botocore = ">=1.36.0,<2.0a.0"
+botocore = ">=1.37.4,<2.0a.0"
 
 [package.extras]
-crt = ["botocore[crt] (>=1.36.0,<2.0a.0)"]
+crt = ["botocore[crt] (>=1.37.4,<2.0a.0)"]
 
 [[package]]
 name = "selenium"
@@ -2799,14 +2799,14 @@ files = [
 
 [[package]]
 name = "tzlocal"
-version = "5.3"
+version = "5.3.1"
 description = "tzinfo object for the local timezone"
 optional = false
 python-versions = ">=3.9"
 groups = ["main"]
 files = [
-    {file = "tzlocal-5.3-py3-none-any.whl", hash = "sha256:3814135a1bb29763c6e4f08fd6e41dbb435c7a60bfbb03270211bcc537187d8c"},
-    {file = "tzlocal-5.3.tar.gz", hash = "sha256:2fafbfc07e9d8b49ade18f898d6bcd37ae88ce3ad6486842a2e4f03af68323d2"},
+    {file = "tzlocal-5.3.1-py3-none-any.whl", hash = "sha256:eb1a66c3ef5847adf7a834f1be0800581b683b5608e74f86ecbcef8ab91bb85d"},
+    {file = "tzlocal-5.3.1.tar.gz", hash = "sha256:cceffc7edecefea1f595541dbd6e990cb1ea3d19bf01b2809f362a03dd7921fd"},
 ]
 
 [package.dependencies]
@@ -3052,81 +3052,81 @@ test = ["websockets"]
 
 [[package]]
 name = "websockets"
-version = "15.0"
+version = "15.0.1"
 description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)"
 optional = false
 python-versions = ">=3.9"
 groups = ["main", "docs"]
 files = [
-    {file = "websockets-15.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5e6ee18a53dd5743e6155b8ff7e8e477c25b29b440f87f65be8165275c87fef0"},
-    {file = "websockets-15.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ee06405ea2e67366a661ed313e14cf2a86e84142a3462852eb96348f7219cee3"},
-    {file = "websockets-15.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8711682a629bbcaf492f5e0af72d378e976ea1d127a2d47584fa1c2c080b436b"},
-    {file = "websockets-15.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94c4a9b01eede952442c088d415861b0cf2053cbd696b863f6d5022d4e4e2453"},
-    {file = "websockets-15.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:45535fead66e873f411c1d3cf0d3e175e66f4dd83c4f59d707d5b3e4c56541c4"},
-    {file = "websockets-15.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e389efe46ccb25a1f93d08c7a74e8123a2517f7b7458f043bd7529d1a63ffeb"},
-    {file = "websockets-15.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:67a04754d121ea5ca39ddedc3f77071651fb5b0bc6b973c71c515415b44ed9c5"},
-    {file = "websockets-15.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:bd66b4865c8b853b8cca7379afb692fc7f52cf898786537dfb5e5e2d64f0a47f"},
-    {file = "websockets-15.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:a4cc73a6ae0a6751b76e69cece9d0311f054da9b22df6a12f2c53111735657c8"},
-    {file = "websockets-15.0-cp310-cp310-win32.whl", hash = "sha256:89da58e4005e153b03fe8b8794330e3f6a9774ee9e1c3bd5bc52eb098c3b0c4f"},
-    {file = "websockets-15.0-cp310-cp310-win_amd64.whl", hash = "sha256:4ff380aabd7a74a42a760ee76c68826a8f417ceb6ea415bd574a035a111fd133"},
-    {file = "websockets-15.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:dd24c4d256558429aeeb8d6c24ebad4e982ac52c50bc3670ae8646c181263965"},
-    {file = "websockets-15.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f83eca8cbfd168e424dfa3b3b5c955d6c281e8fc09feb9d870886ff8d03683c7"},
-    {file = "websockets-15.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4095a1f2093002c2208becf6f9a178b336b7572512ee0a1179731acb7788e8ad"},
-    {file = "websockets-15.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb915101dfbf318486364ce85662bb7b020840f68138014972c08331458d41f3"},
-    {file = "websockets-15.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:45d464622314973d78f364689d5dbb9144e559f93dca11b11af3f2480b5034e1"},
-    {file = "websockets-15.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ace960769d60037ca9625b4c578a6f28a14301bd2a1ff13bb00e824ac9f73e55"},
-    {file = "websockets-15.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c7cd4b1015d2f60dfe539ee6c95bc968d5d5fad92ab01bb5501a77393da4f596"},
-    {file = "websockets-15.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:4f7290295794b5dec470867c7baa4a14182b9732603fd0caf2a5bf1dc3ccabf3"},
-    {file = "websockets-15.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3abd670ca7ce230d5a624fd3d55e055215d8d9b723adee0a348352f5d8d12ff4"},
-    {file = "websockets-15.0-cp311-cp311-win32.whl", hash = "sha256:110a847085246ab8d4d119632145224d6b49e406c64f1bbeed45c6f05097b680"},
-    {file = "websockets-15.0-cp311-cp311-win_amd64.whl", hash = "sha256:8d7bbbe2cd6ed80aceef2a14e9f1c1b61683194c216472ed5ff33b700e784e37"},
-    {file = "websockets-15.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:cccc18077acd34c8072578394ec79563664b1c205f7a86a62e94fafc7b59001f"},
-    {file = "websockets-15.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d4c22992e24f12de340ca5f824121a5b3e1a37ad4360b4e1aaf15e9d1c42582d"},
-    {file = "websockets-15.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1206432cc6c644f6fc03374b264c5ff805d980311563202ed7fef91a38906276"},
-    {file = "websockets-15.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d3cc75ef3e17490042c47e0523aee1bcc4eacd2482796107fd59dd1100a44bc"},
-    {file = "websockets-15.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b89504227a5311610e4be16071465885a0a3d6b0e82e305ef46d9b064ce5fb72"},
-    {file = "websockets-15.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:56e3efe356416bc67a8e093607315951d76910f03d2b3ad49c4ade9207bf710d"},
-    {file = "websockets-15.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0f2205cdb444a42a7919690238fb5979a05439b9dbb73dd47c863d39640d85ab"},
-    {file = "websockets-15.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:aea01f40995fa0945c020228ab919b8dfc93fc8a9f2d3d705ab5b793f32d9e99"},
-    {file = "websockets-15.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a9f8e33747b1332db11cf7fcf4a9512bef9748cb5eb4d3f7fbc8c30d75dc6ffc"},
-    {file = "websockets-15.0-cp312-cp312-win32.whl", hash = "sha256:32e02a2d83f4954aa8c17e03fe8ec6962432c39aca4be7e8ee346b05a3476904"},
-    {file = "websockets-15.0-cp312-cp312-win_amd64.whl", hash = "sha256:ffc02b159b65c05f2ed9ec176b715b66918a674bd4daed48a9a7a590dd4be1aa"},
-    {file = "websockets-15.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:d2244d8ab24374bed366f9ff206e2619345f9cd7fe79aad5225f53faac28b6b1"},
-    {file = "websockets-15.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:3a302241fbe825a3e4fe07666a2ab513edfdc6d43ce24b79691b45115273b5e7"},
-    {file = "websockets-15.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:10552fed076757a70ba2c18edcbc601c7637b30cdfe8c24b65171e824c7d6081"},
-    {file = "websockets-15.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c53f97032b87a406044a1c33d1e9290cc38b117a8062e8a8b285175d7e2f99c9"},
-    {file = "websockets-15.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1caf951110ca757b8ad9c4974f5cac7b8413004d2f29707e4d03a65d54cedf2b"},
-    {file = "websockets-15.0-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8bf1ab71f9f23b0a1d52ec1682a3907e0c208c12fef9c3e99d2b80166b17905f"},
-    {file = "websockets-15.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bfcd3acc1a81f106abac6afd42327d2cf1e77ec905ae11dc1d9142a006a496b6"},
-    {file = "websockets-15.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:c8c5c8e1bac05ef3c23722e591ef4f688f528235e2480f157a9cfe0a19081375"},
-    {file = "websockets-15.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:86bfb52a9cfbcc09aba2b71388b0a20ea5c52b6517c0b2e316222435a8cdab72"},
-    {file = "websockets-15.0-cp313-cp313-win32.whl", hash = "sha256:26ba70fed190708551c19a360f9d7eca8e8c0f615d19a574292b7229e0ae324c"},
-    {file = "websockets-15.0-cp313-cp313-win_amd64.whl", hash = "sha256:ae721bcc8e69846af00b7a77a220614d9b2ec57d25017a6bbde3a99473e41ce8"},
-    {file = "websockets-15.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c348abc5924caa02a62896300e32ea80a81521f91d6db2e853e6b1994017c9f6"},
-    {file = "websockets-15.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5294fcb410ed0a45d5d1cdedc4e51a60aab5b2b3193999028ea94afc2f554b05"},
-    {file = "websockets-15.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c24ba103ecf45861e2e1f933d40b2d93f5d52d8228870c3e7bf1299cd1cb8ff1"},
-    {file = "websockets-15.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc8821a03bcfb36e4e4705316f6b66af28450357af8a575dc8f4b09bf02a3dee"},
-    {file = "websockets-15.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffc5ae23ada6515f31604f700009e2df90b091b67d463a8401c1d8a37f76c1d7"},
-    {file = "websockets-15.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ac67b542505186b3bbdaffbc303292e1ee9c8729e5d5df243c1f20f4bb9057e"},
-    {file = "websockets-15.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:c86dc2068f1c5ca2065aca34f257bbf4f78caf566eb230f692ad347da191f0a1"},
-    {file = "websockets-15.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:30cff3ef329682b6182c01c568f551481774c476722020b8f7d0daacbed07a17"},
-    {file = "websockets-15.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:98dcf978d4c6048965d1762abd534c9d53bae981a035bfe486690ba11f49bbbb"},
-    {file = "websockets-15.0-cp39-cp39-win32.whl", hash = "sha256:37d66646f929ae7c22c79bc73ec4074d6db45e6384500ee3e0d476daf55482a9"},
-    {file = "websockets-15.0-cp39-cp39-win_amd64.whl", hash = "sha256:24d5333a9b2343330f0f4eb88546e2c32a7f5c280f8dd7d3cc079beb0901781b"},
-    {file = "websockets-15.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:b499caef4bca9cbd0bd23cd3386f5113ee7378094a3cb613a2fa543260fe9506"},
-    {file = "websockets-15.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:17f2854c6bd9ee008c4b270f7010fe2da6c16eac5724a175e75010aacd905b31"},
-    {file = "websockets-15.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:89f72524033abbfde880ad338fd3c2c16e31ae232323ebdfbc745cbb1b3dcc03"},
-    {file = "websockets-15.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1657a9eecb29d7838e3b415458cc494e6d1b194f7ac73a34aa55c6fb6c72d1f3"},
-    {file = "websockets-15.0-pp310-pypy310_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e413352a921f5ad5d66f9e2869b977e88d5103fc528b6deb8423028a2befd842"},
-    {file = "websockets-15.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:8561c48b0090993e3b2a54db480cab1d23eb2c5735067213bb90f402806339f5"},
-    {file = "websockets-15.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:190bc6ef8690cd88232a038d1b15714c258f79653abad62f7048249b09438af3"},
-    {file = "websockets-15.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:327adab7671f3726b0ba69be9e865bba23b37a605b585e65895c428f6e47e766"},
-    {file = "websockets-15.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2bd8ef197c87afe0a9009f7a28b5dc613bfc585d329f80b7af404e766aa9e8c7"},
-    {file = "websockets-15.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:789c43bf4a10cd067c24c321238e800b8b2716c863ddb2294d2fed886fa5a689"},
-    {file = "websockets-15.0-pp39-pypy39_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7394c0b7d460569c9285fa089a429f58465db930012566c03046f9e3ab0ed181"},
-    {file = "websockets-15.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:2ea4f210422b912ebe58ef0ad33088bc8e5c5ff9655a8822500690abc3b1232d"},
-    {file = "websockets-15.0-py3-none-any.whl", hash = "sha256:51ffd53c53c4442415b613497a34ba0aa7b99ac07f1e4a62db5dcd640ae6c3c3"},
-    {file = "websockets-15.0.tar.gz", hash = "sha256:ca36151289a15b39d8d683fd8b7abbe26fc50be311066c5f8dcf3cb8cee107ab"},
+    {file = "websockets-15.0.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d63efaa0cd96cf0c5fe4d581521d9fa87744540d4bc999ae6e08595a1014b45b"},
+    {file = "websockets-15.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ac60e3b188ec7574cb761b08d50fcedf9d77f1530352db4eef1707fe9dee7205"},
+    {file = "websockets-15.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5756779642579d902eed757b21b0164cd6fe338506a8083eb58af5c372e39d9a"},
+    {file = "websockets-15.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0fdfe3e2a29e4db3659dbd5bbf04560cea53dd9610273917799f1cde46aa725e"},
+    {file = "websockets-15.0.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c2529b320eb9e35af0fa3016c187dffb84a3ecc572bcee7c3ce302bfeba52bf"},
+    {file = "websockets-15.0.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac1e5c9054fe23226fb11e05a6e630837f074174c4c2f0fe442996112a6de4fb"},
+    {file = "websockets-15.0.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5df592cd503496351d6dc14f7cdad49f268d8e618f80dce0cd5a36b93c3fc08d"},
+    {file = "websockets-15.0.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:0a34631031a8f05657e8e90903e656959234f3a04552259458aac0b0f9ae6fd9"},
+    {file = "websockets-15.0.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3d00075aa65772e7ce9e990cab3ff1de702aa09be3940d1dc88d5abf1ab8a09c"},
+    {file = "websockets-15.0.1-cp310-cp310-win32.whl", hash = "sha256:1234d4ef35db82f5446dca8e35a7da7964d02c127b095e172e54397fb6a6c256"},
+    {file = "websockets-15.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:39c1fec2c11dc8d89bba6b2bf1556af381611a173ac2b511cf7231622058af41"},
+    {file = "websockets-15.0.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:823c248b690b2fd9303ba00c4f66cd5e2d8c3ba4aa968b2779be9532a4dad431"},
+    {file = "websockets-15.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678999709e68425ae2593acf2e3ebcbcf2e69885a5ee78f9eb80e6e371f1bf57"},
+    {file = "websockets-15.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d50fd1ee42388dcfb2b3676132c78116490976f1300da28eb629272d5d93e905"},
+    {file = "websockets-15.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d99e5546bf73dbad5bf3547174cd6cb8ba7273062a23808ffea025ecb1cf8562"},
+    {file = "websockets-15.0.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:66dd88c918e3287efc22409d426c8f729688d89a0c587c88971a0faa2c2f3792"},
+    {file = "websockets-15.0.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8dd8327c795b3e3f219760fa603dcae1dcc148172290a8ab15158cf85a953413"},
+    {file = "websockets-15.0.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8fdc51055e6ff4adeb88d58a11042ec9a5eae317a0a53d12c062c8a8865909e8"},
+    {file = "websockets-15.0.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:693f0192126df6c2327cce3baa7c06f2a117575e32ab2308f7f8216c29d9e2e3"},
+    {file = "websockets-15.0.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:54479983bd5fb469c38f2f5c7e3a24f9a4e70594cd68cd1fa6b9340dadaff7cf"},
+    {file = "websockets-15.0.1-cp311-cp311-win32.whl", hash = "sha256:16b6c1b3e57799b9d38427dda63edcbe4926352c47cf88588c0be4ace18dac85"},
+    {file = "websockets-15.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:27ccee0071a0e75d22cb35849b1db43f2ecd3e161041ac1ee9d2352ddf72f065"},
+    {file = "websockets-15.0.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:3e90baa811a5d73f3ca0bcbf32064d663ed81318ab225ee4f427ad4e26e5aff3"},
+    {file = "websockets-15.0.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:592f1a9fe869c778694f0aa806ba0374e97648ab57936f092fd9d87f8bc03665"},
+    {file = "websockets-15.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0701bc3cfcb9164d04a14b149fd74be7347a530ad3bbf15ab2c678a2cd3dd9a2"},
+    {file = "websockets-15.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8b56bdcdb4505c8078cb6c7157d9811a85790f2f2b3632c7d1462ab5783d215"},
+    {file = "websockets-15.0.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0af68c55afbd5f07986df82831c7bff04846928ea8d1fd7f30052638788bc9b5"},
+    {file = "websockets-15.0.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64dee438fed052b52e4f98f76c5790513235efaa1ef7f3f2192c392cd7c91b65"},
+    {file = "websockets-15.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d5f6b181bb38171a8ad1d6aa58a67a6aa9d4b38d0f8c5f496b9e42561dfc62fe"},
+    {file = "websockets-15.0.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:5d54b09eba2bada6011aea5375542a157637b91029687eb4fdb2dab11059c1b4"},
+    {file = "websockets-15.0.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3be571a8b5afed347da347bfcf27ba12b069d9d7f42cb8c7028b5e98bbb12597"},
+    {file = "websockets-15.0.1-cp312-cp312-win32.whl", hash = "sha256:c338ffa0520bdb12fbc527265235639fb76e7bc7faafbb93f6ba80d9c06578a9"},
+    {file = "websockets-15.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:fcd5cf9e305d7b8338754470cf69cf81f420459dbae8a3b40cee57417f4614a7"},
+    {file = "websockets-15.0.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ee443ef070bb3b6ed74514f5efaa37a252af57c90eb33b956d35c8e9c10a1931"},
+    {file = "websockets-15.0.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5a939de6b7b4e18ca683218320fc67ea886038265fd1ed30173f5ce3f8e85675"},
+    {file = "websockets-15.0.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:746ee8dba912cd6fc889a8147168991d50ed70447bf18bcda7039f7d2e3d9151"},
+    {file = "websockets-15.0.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:595b6c3969023ecf9041b2936ac3827e4623bfa3ccf007575f04c5a6aa318c22"},
+    {file = "websockets-15.0.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c714d2fc58b5ca3e285461a4cc0c9a66bd0e24c5da9911e30158286c9b5be7f"},
+    {file = "websockets-15.0.1-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f3c1e2ab208db911594ae5b4f79addeb3501604a165019dd221c0bdcabe4db8"},
+    {file = "websockets-15.0.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:229cf1d3ca6c1804400b0a9790dc66528e08a6a1feec0d5040e8b9eb14422375"},
+    {file = "websockets-15.0.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:756c56e867a90fb00177d530dca4b097dd753cde348448a1012ed6c5131f8b7d"},
+    {file = "websockets-15.0.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:558d023b3df0bffe50a04e710bc87742de35060580a293c2a984299ed83bc4e4"},
+    {file = "websockets-15.0.1-cp313-cp313-win32.whl", hash = "sha256:ba9e56e8ceeeedb2e080147ba85ffcd5cd0711b89576b83784d8605a7df455fa"},
+    {file = "websockets-15.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:e09473f095a819042ecb2ab9465aee615bd9c2028e4ef7d933600a8401c79561"},
+    {file = "websockets-15.0.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5f4c04ead5aed67c8a1a20491d54cdfba5884507a48dd798ecaf13c74c4489f5"},
+    {file = "websockets-15.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:abdc0c6c8c648b4805c5eacd131910d2a7f6455dfd3becab248ef108e89ab16a"},
+    {file = "websockets-15.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a625e06551975f4b7ea7102bc43895b90742746797e2e14b70ed61c43a90f09b"},
+    {file = "websockets-15.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d591f8de75824cbb7acad4e05d2d710484f15f29d4a915092675ad3456f11770"},
+    {file = "websockets-15.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:47819cea040f31d670cc8d324bb6435c6f133b8c7a19ec3d61634e62f8d8f9eb"},
+    {file = "websockets-15.0.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac017dd64572e5c3bd01939121e4d16cf30e5d7e110a119399cf3133b63ad054"},
+    {file = "websockets-15.0.1-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:4a9fac8e469d04ce6c25bb2610dc535235bd4aa14996b4e6dbebf5e007eba5ee"},
+    {file = "websockets-15.0.1-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:363c6f671b761efcb30608d24925a382497c12c506b51661883c3e22337265ed"},
+    {file = "websockets-15.0.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:2034693ad3097d5355bfdacfffcbd3ef5694f9718ab7f29c29689a9eae841880"},
+    {file = "websockets-15.0.1-cp39-cp39-win32.whl", hash = "sha256:3b1ac0d3e594bf121308112697cf4b32be538fb1444468fb0a6ae4feebc83411"},
+    {file = "websockets-15.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:b7643a03db5c95c799b89b31c036d5f27eeb4d259c798e878d6937d71832b1e4"},
+    {file = "websockets-15.0.1-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0c9e74d766f2818bb95f84c25be4dea09841ac0f734d1966f415e4edfc4ef1c3"},
+    {file = "websockets-15.0.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:1009ee0c7739c08a0cd59de430d6de452a55e42d6b522de7aa15e6f67db0b8e1"},
+    {file = "websockets-15.0.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76d1f20b1c7a2fa82367e04982e708723ba0e7b8d43aa643d3dcd404d74f1475"},
+    {file = "websockets-15.0.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f29d80eb9a9263b8d109135351caf568cc3f80b9928bccde535c235de55c22d9"},
+    {file = "websockets-15.0.1-pp310-pypy310_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b359ed09954d7c18bbc1680f380c7301f92c60bf924171629c5db97febb12f04"},
+    {file = "websockets-15.0.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:cad21560da69f4ce7658ca2cb83138fb4cf695a2ba3e475e0559e05991aa8122"},
+    {file = "websockets-15.0.1-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7f493881579c90fc262d9cdbaa05a6b54b3811c2f300766748db79f098db9940"},
+    {file = "websockets-15.0.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:47b099e1f4fbc95b701b6e85768e1fcdaf1630f3cbe4765fa216596f12310e2e"},
+    {file = "websockets-15.0.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:67f2b6de947f8c757db2db9c71527933ad0019737ec374a8a6be9a956786aaf9"},
+    {file = "websockets-15.0.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d08eb4c2b7d6c41da6ca0600c077e93f5adcfd979cd777d747e9ee624556da4b"},
+    {file = "websockets-15.0.1-pp39-pypy39_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b826973a4a2ae47ba357e4e82fa44a463b8f168e1ca775ac64521442b19e87f"},
+    {file = "websockets-15.0.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:21c1fa28a6a7e3cbdc171c694398b6df4744613ce9b36b1a498e816787e28123"},
+    {file = "websockets-15.0.1-py3-none-any.whl", hash = "sha256:f7a866fbc1e97b5c617ee4116daaa09b722101d4a3c170c787450ba409f9736f"},
+    {file = "websockets-15.0.1.tar.gz", hash = "sha256:82544de02076bafba038ce055ee6412d68da13ab47f0c60cab827346de828dee"},
 ]
 
 [[package]]
diff --git a/pyproject.toml b/pyproject.toml
index 3c64eae..30693d5 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [project]
 name = "auto-archiver"
-version = "0.13.4"
+version = "0.13.5"
 description = "Automatically archive links to videos, images, and social media content from Google Sheets (and more)."
 
 requires-python = ">=3.10,<3.13"

From b386ae6287030bf5bd8c9750ff88fec04668f543 Mon Sep 17 00:00:00 2001
From: erinhmclark <erinhannahmary.clark@gmail.com>
Date: Fri, 7 Mar 2025 18:01:08 +0000
Subject: [PATCH 03/55] Add poetry.lock and pyproject.toml paths to trigger
 tests.

---
 .github/workflows/tests-core.yaml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/tests-core.yaml b/.github/workflows/tests-core.yaml
index 917cfbb..57028bd 100644
--- a/.github/workflows/tests-core.yaml
+++ b/.github/workflows/tests-core.yaml
@@ -5,9 +5,13 @@ on:
     branches: [ main ]
     paths:
       - src/**
+      - poetry.lock
+      - pyproject.toml
   pull_request:
     paths:
       - src/**
+      - poetry.lock
+      - pyproject.toml
 
 jobs:
   tests:

From 7e10040bbd7f51efa9359924f247c57aa502e89c Mon Sep 17 00:00:00 2001
From: erinhmclark <erinhannahmary.clark@gmail.com>
Date: Fri, 7 Mar 2025 18:04:51 +0000
Subject: [PATCH 04/55] Update the release description to tag on release

---
 docs/source/development/release.md | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/docs/source/development/release.md b/docs/source/development/release.md
index 0d9407c..a2ed4c6 100644
--- a/docs/source/development/release.md
+++ b/docs/source/development/release.md
@@ -2,8 +2,9 @@
 
 ```{note} This is a work in progress.
 ```
+### Update the project version
 
-1. Update the version number in the project file: [pyproject.toml](../../pyproject.toml) following SemVer:
+Update the version number in the project file: [pyproject.toml](../../pyproject.toml) following SemVer:
 ```toml
 [project]
 name = "auto-archiver"
@@ -11,17 +12,15 @@ version = "0.1.1"
 ```
 Then commit and push the changes.
 
-2. Next add a new git tag with the version number:
-```shell
-git tag -a v0.1.1
-git push origin v0.1.1
-```
-
 * The package version is automatically updated in PyPi using the workflow [python-publish.yml](../../.github/workflows/python-publish.yml)
 * A Docker image is automatically pushed with the git tag to dockerhub using the workflow [docker-publish.yml](../../.github/workflows/docker-publish.yml)
 
+### Create the release on Git
 
-3. Go to GitHub releases > new release > and select the tag you just created. 
+The release needs a git tag which should match the project version number, prefixed with a 'v'. For example, if the project version is `0.1.1`, the git tag should be `v0.1.1`.
+This can be done the usual way, or created within the Github UI when you create the release.
+
+Go to GitHub releases > new release > create the release with the new tag and the release notes.
 
 
 manual release to docker hub

From e89a8da3b49670e380a2b01d03eba48090b575e0 Mon Sep 17 00:00:00 2001
From: Patrick Robertson <robertson.patrick@gmail.com>
Date: Mon, 10 Mar 2025 11:30:15 +0000
Subject: [PATCH 05/55] Unit tests for storage types + fix storage too long
 issues for local storage

---
 src/auto_archiver/core/media.py               |  4 +-
 src/auto_archiver/core/storage.py             | 34 ++++++--
 .../modules/local_storage/local_storage.py    | 22 +++++
 .../modules/s3_storage/s3_storage.py          |  4 +
 tests/storages/test_local_storage.py          |  9 ++-
 tests/storages/test_storage_base.py           | 80 ++++++++++++++++++-
 6 files changed, 142 insertions(+), 11 deletions(-)

diff --git a/src/auto_archiver/core/media.py b/src/auto_archiver/core/media.py
index b6820ab..0370c00 100644
--- a/src/auto_archiver/core/media.py
+++ b/src/auto_archiver/core/media.py
@@ -6,7 +6,7 @@ nested media retrieval, and type validation.
 from __future__ import annotations
 import os
 import traceback
-from typing import Any, List
+from typing import Any, List, Iterator
 from dataclasses import dataclass, field
 from dataclasses_json import dataclass_json, config
 import mimetypes
@@ -47,7 +47,7 @@ class Media:
             for any_media in self.all_inner_media(include_self=True):
                 s.store(any_media, url, metadata=metadata)
 
-    def all_inner_media(self, include_self=False):
+    def all_inner_media(self, include_self=False) -> Iterator[Media]:
         """Retrieves all media, including nested media within properties or transformations on original media.
         This function returns a generator for all the inner media.
 
diff --git a/src/auto_archiver/core/storage.py b/src/auto_archiver/core/storage.py
index 1535eab..2a14073 100644
--- a/src/auto_archiver/core/storage.py
+++ b/src/auto_archiver/core/storage.py
@@ -27,6 +27,7 @@ class Storage(BaseModule):
         if media.is_stored(in_storage=self): 
             logger.debug(f"{media.key} already stored, skipping")
             return
+
         self.set_key(media, url, metadata)
         self.upload(media, metadata=metadata)
         media.add_url(self.get_cdn_url(media))
@@ -50,34 +51,55 @@ class Storage(BaseModule):
         with open(media.filename, 'rb') as f:
             return self.uploadf(f, media, **kwargs)
 
-    def set_key(self, media: Media, url, metadata: Metadata) -> None:
+    def set_key(self, media: Media, url: str, metadata: Metadata) -> None:
         """takes the media and optionally item info and generates a key"""
         if media.key is not None and len(media.key) > 0: return
         folder = metadata.get_context('folder', '')
         filename, ext = os.path.splitext(media.filename)
 
         # Handle path_generator logic
-        path_generator = self.config.get("path_generator", "url")
+        path_generator = self.path_generator
         if path_generator == "flat":
             path = ""
+            # TODO: this is never used
             filename = slugify(filename)  # Ensure filename is slugified
         elif path_generator == "url":
             path = slugify(url)
         elif path_generator == "random":
-            path = self.config.get("random_path", random_str(24), True)
+            path = random_str(24)
         else:
             raise ValueError(f"Invalid path_generator: {path_generator}")
 
         # Handle filename_generator logic
-        filename_generator = self.config.get("filename_generator", "random")
+        filename_generator = self.filename_generator
         if filename_generator == "random":
             filename = random_str(24)
         elif filename_generator == "static":
             # load the hash_enricher module
-            he = self.module_factory.get_module(HashEnricher, self.config)
+            he = self.module_factory.get_module("hash_enricher", self.config)
             hd = he.calculate_hash(media.filename)
             filename = hd[:24]
         else:
             raise ValueError(f"Invalid filename_generator: {filename_generator}")
 
-        media.key = os.path.join(folder, path, f"{filename}{ext}")
+        key = os.path.join(folder, path, f"{filename}{ext}")
+        if len(key) > self.max_file_length():
+            # truncate the path
+            max_path_length = self.max_file_length() - len(filename) - len(ext) - len(folder) - 1
+            path = path[:max_path_length]
+            logger.warning(f'Filename too long ({len(key)} characters), truncating to {self.max_file_length()} characters')
+            key = os.path.join(folder, path, f"{filename}{ext}")
+
+
+        media.key = key
+
+
+    def max_file_length(self) -> int:
+        """
+        Returns the maximum length of a file name that can be stored in the storage service.
+
+        Files are truncated if they exceed this length.
+        Override this method in subclasses if the storage service has a different maximum file length.
+        """
+        return 255 # safe max file length for most filesystems (macOS, Windows, Linux)
+
diff --git a/src/auto_archiver/modules/local_storage/local_storage.py b/src/auto_archiver/modules/local_storage/local_storage.py
index b995577..c07f682 100644
--- a/src/auto_archiver/modules/local_storage/local_storage.py
+++ b/src/auto_archiver/modules/local_storage/local_storage.py
@@ -10,6 +10,8 @@ from auto_archiver.core import Storage
 
 class LocalStorage(Storage):
 
+    MAX_FILE_LENGTH = 255
+
     def get_cdn_url(self, media: Media) -> str:
         # TODO: is this viable with Storage.configs on path/filename?
         dest = os.path.join(self.save_to, media.key)
@@ -20,11 +22,31 @@ class LocalStorage(Storage):
     def upload(self, media: Media, **kwargs) -> bool:
         # override parent so that we can use shutil.copy2 and keep metadata
         dest = os.path.join(self.save_to, media.key)
+
+        if len(dest) > self.max_file_length():
+            old_dest_length = len(dest)
+            filename, ext = os.path.splitext(media.key)
+            dir, filename = os.path.split(filename)
+            # see whether we should truncate filename or dir
+            if len(dir) > len(filename):
+                dir = dir[:self.MAX_FILE_LENGTH - len(self.save_to) - len(ext) - len(filename) - 1]
+            else:
+                filename = filename[:self.MAX_FILE_LENGTH - len(self.save_to) - len(ext) - len(filename) - 1]
+
+            # override media.key
+            media.key = os.path.join(dir, f"{filename}{ext}")
+            dest = os.path.join(self.save_to, dir, f"{filename}{ext}")
+            logger.warning(f'Filename too long ({old_dest_length} characters), truncating to {len(dest)} characters')
+
         os.makedirs(os.path.dirname(dest), exist_ok=True)
         logger.debug(f'[{self.__class__.__name__}] storing file {media.filename} with key {media.key} to {dest}')
+
         res = shutil.copy2(media.filename, dest)
         logger.info(res)
         return True
 
     # must be implemented even if unused
     def uploadf(self, file: IO[bytes], key: str, **kwargs: dict) -> bool: pass
+
+    def max_file_length(self):
+        return self.MAX_FILE_LENGTH
\ No newline at end of file
diff --git a/src/auto_archiver/modules/s3_storage/s3_storage.py b/src/auto_archiver/modules/s3_storage/s3_storage.py
index 6590ac9..c8569b6 100644
--- a/src/auto_archiver/modules/s3_storage/s3_storage.py
+++ b/src/auto_archiver/modules/s3_storage/s3_storage.py
@@ -67,4 +67,8 @@ class S3Storage(Storage):
         if 'Contents' in resp:
             return resp['Contents'][0]['Key']
         return False
+    
+    def max_file_length(self):
+        # Amazon AWS max file length is 1024, but we will use 1000 to be safe
+        return 1000
 
diff --git a/tests/storages/test_local_storage.py b/tests/storages/test_local_storage.py
index 7617867..7282e03 100644
--- a/tests/storages/test_local_storage.py
+++ b/tests/storages/test_local_storage.py
@@ -11,6 +11,7 @@ from auto_archiver.modules.local_storage import LocalStorage
 @pytest.fixture
 def local_storage(setup_module, tmp_path) -> LocalStorage:
     save_to = tmp_path / "local_archive"
+    save_to.mkdir()
     configs: dict = {
         "path_generator": "flat",
         "filename_generator": "static",
@@ -19,7 +20,6 @@ def local_storage(setup_module, tmp_path) -> LocalStorage:
     }
     return setup_module("local_storage", configs)
 
-
 @pytest.fixture
 def sample_media(tmp_path) -> Media:
     """Fixture creating a Media object with temporary source file"""
@@ -27,6 +27,13 @@ def sample_media(tmp_path) -> Media:
     src_file.write_text("test content")
     return Media(key="subdir/test.txt", filename=str(src_file))
 
+def test_really_long_website_url_save(local_storage, tmp_path):
+    long_filename = os.path.join(local_storage.save_to, "file"*100 + ".txt")
+    src_file = tmp_path / "source.txt"
+    src_file.write_text("test content")
+    media = Media(key=long_filename, filename=str(src_file))
+    assert local_storage.upload(media) is True
+    assert src_file.read_text() == Path(local_storage.get_cdn_url(media)).read_text()
 
 def test_get_cdn_url_relative(local_storage):
     media = Media(key="test.txt", filename="dummy.txt")
diff --git a/tests/storages/test_storage_base.py b/tests/storages/test_storage_base.py
index 7578acd..fd07c32 100644
--- a/tests/storages/test_storage_base.py
+++ b/tests/storages/test_storage_base.py
@@ -2,9 +2,9 @@ from typing import Type
 
 import pytest
 
-from auto_archiver.core.metadata import Metadata
+from auto_archiver.core.metadata import Metadata, Media
 from auto_archiver.core.storage import Storage
-
+from auto_archiver.core.module import ModuleFactory
 
 class TestStorageBase(object):
 
@@ -20,3 +20,79 @@ class TestStorageBase(object):
         self.storage: Type[Storage] = setup_module(
             self.module_name, self.config
         )
+
+
+class TestBaseStorage(Storage):
+
+    name = "test_storage"
+
+    def get_cdn_url(self, media):
+        return "cdn_url"
+    
+    def uploadf(self, file, key, **kwargs):
+        return True
+
+@pytest.fixture
+def storage_base():
+    def _storage_base(config):
+        storage_base = TestBaseStorage()
+        storage_base.config_setup({TestBaseStorage.name : config})
+        storage_base.module_factory = ModuleFactory()
+        return storage_base
+    
+    return _storage_base
+
+@pytest.mark.parametrize(
+    "path_generator, filename_generator, url, expected_key",
+    [
+        ("flat", "static", "https://example.com/file/", "folder/6ae8a75555209fd6c44157c0.txt"),
+        ("flat", "random", "https://example.com/file/", "folder/pretend-random.txt"),
+        ("url", "static", "https://example.com/file/", "folder/https-example-com-file/6ae8a75555209fd6c44157c0.txt"),
+        ("url", "random", "https://example.com/file/", "folder/https-example-com-file/pretend-random.txt"),
+        ("random", "static", "https://example.com/file/", "folder/pretend-random/6ae8a75555209fd6c44157c0.txt"),
+        ("random", "random", "https://example.com/file/", "folder/pretend-random/pretend-random.txt"),
+
+    ],
+)
+def test_storage_setup(storage_base, path_generator, filename_generator, url, expected_key, mocker):
+    mock_random = mocker.patch("auto_archiver.core.storage.random_str")
+    mock_random.return_value = "pretend-random"
+
+    # create dummy.txt file
+    with open("dummy.txt", "w") as f:
+        f.write("test content")
+
+    config: dict = {
+        "path_generator": path_generator,
+        "filename_generator": filename_generator,
+    }
+    storage: Storage = storage_base(config)
+    assert storage.path_generator == path_generator
+    assert storage.filename_generator == filename_generator
+
+    metadata = Metadata()
+    metadata.set_context("folder", "folder")
+    media = Media(filename="dummy.txt")
+    storage.set_key(media, url, metadata)
+    print(media.key)
+    assert media.key == expected_key
+
+
+def test_really_long_name(storage_base):
+    config: dict = {
+        "path_generator": "url",
+        "filename_generator": "static",
+    }
+    storage: Storage = storage_base(config)
+
+    # create dummy.txt file
+    with open("dummy.txt", "w") as f:
+        f.write("test content")
+
+    url = f"https://example.com/{'file'*100}"
+    media = Media(filename="dummy.txt")
+    storage.set_key(media, url, Metadata())
+    assert len(media.key) <= storage.max_file_length()
+    assert media.key == "https-example-com-filefilefilefilefilefilefilefilefilefilefilefile\
+filefilefilefilefilefilefilefilefilefilefilefilefilefilefilefilefilefilefilefilefilefilefile\
+filefilefilefilefilefilefilefilefilefilefilefilefilefilefilefilefile/6ae8a75555209fd6c44157c0.txt"
\ No newline at end of file

From 58bd38e29202b1f9d0603e1cfcbc98b24f03dee8 Mon Sep 17 00:00:00 2001
From: Miguel Sozinho Ramalho <19508417+msramalho@users.noreply.github.com>
Date: Mon, 10 Mar 2025 17:56:45 +0600
Subject: [PATCH 06/55] Adds new extractor for tiktok via unofficial API (#237)

* minor update to defaults in api_db

* readme typo

* adds and tests new tikwm tiktok downloader

* addresses PR comments
---
 README.md                                     |   2 +-
 .../modules/api_db/__manifest__.py            |   4 +-
 .../tiktok_tikwm_extractor/__init__.py        |   1 +
 .../tiktok_tikwm_extractor/__manifest__.py    |  23 +++
 .../tiktok_tikwm_extractor.py                 |  75 +++++++++
 .../extractors/test_tiktok_tikwm_extractor.py | 154 ++++++++++++++++++
 6 files changed, 256 insertions(+), 3 deletions(-)
 create mode 100644 src/auto_archiver/modules/tiktok_tikwm_extractor/__init__.py
 create mode 100644 src/auto_archiver/modules/tiktok_tikwm_extractor/__manifest__.py
 create mode 100644 src/auto_archiver/modules/tiktok_tikwm_extractor/tiktok_tikwm_extractor.py
 create mode 100644 tests/extractors/test_tiktok_tikwm_extractor.py

diff --git a/README.md b/README.md
index b39038e..8baa722 100644
--- a/README.md
+++ b/README.md
@@ -10,7 +10,7 @@
 
 
 
-Auto Archiver is a Python tool to automatically archive content on the web in a secure and verifiable way. It takes URLs from different sources (e.g. a CSV file, Google Sheets, command line etc.) and archives the content of each one. It can archive social media posts, videos, images and webpages. Content can enriched, then saved either locally or remotely (S3 bucket, Google Drive). The status of the archiving process can be appended to a CSV report, or if using Google Sheets – back to the original sheet.
+Auto Archiver is a Python tool to automatically archive content on the web in a secure and verifiable way. It takes URLs from different sources (e.g. a CSV file, Google Sheets, command line etc.) and archives the content of each one. It can archive social media posts, videos, images and webpages. Content can be enriched, then saved either locally or remotely (S3 bucket, Google Drive). The status of the archiving process can be appended to a CSV report, or if using Google Sheets – back to the original sheet.
 
 <div class="hidden_rtd">
   
diff --git a/src/auto_archiver/modules/api_db/__manifest__.py b/src/auto_archiver/modules/api_db/__manifest__.py
index e67b31a..c9ac461 100644
--- a/src/auto_archiver/modules/api_db/__manifest__.py
+++ b/src/auto_archiver/modules/api_db/__manifest__.py
@@ -24,9 +24,9 @@
             "help": "which group of users have access to the archive in case public=false as author",
         },
         "use_api_cache": {
-            "default": True,
+            "default": False,
             "type": "bool",
-            "help": "if False then the API database will be queried prior to any archiving operations and stop if the link has already been archived",
+            "help": "if True then the API database will be queried prior to any archiving operations and stop if the link has already been archived",
         },
         "store_results": {
             "default": True,
diff --git a/src/auto_archiver/modules/tiktok_tikwm_extractor/__init__.py b/src/auto_archiver/modules/tiktok_tikwm_extractor/__init__.py
new file mode 100644
index 0000000..25a20f5
--- /dev/null
+++ b/src/auto_archiver/modules/tiktok_tikwm_extractor/__init__.py
@@ -0,0 +1 @@
+from .tiktok_tikwm_extractor import TiktokTikwmExtractor
\ No newline at end of file
diff --git a/src/auto_archiver/modules/tiktok_tikwm_extractor/__manifest__.py b/src/auto_archiver/modules/tiktok_tikwm_extractor/__manifest__.py
new file mode 100644
index 0000000..56d8e3e
--- /dev/null
+++ b/src/auto_archiver/modules/tiktok_tikwm_extractor/__manifest__.py
@@ -0,0 +1,23 @@
+{
+    "name": "Tiktok Tikwm Extractor",
+    "type": ["extractor"],
+    "requires_setup": False,
+    "dependencies": {
+        "python": ["loguru", "requests"],
+        "bin": []
+    },
+    "description": """
+    Uses an unofficial TikTok video download platform's API to download videos: https://tikwm.com/
+	
+	This extractor complements the generic_extractor which can already get TikTok videos, but this one can extract special videos like those marked as sensitive.
+
+    ### Features
+    - Downloads the video and, if possible, also the video cover.
+	- Stores extra metadata about the post like author information, and more as returned by tikwm.com. 
+
+    ### Notes
+    - If tikwm.com is down, this extractor will not work.
+	- If tikwm.com changes their API, this extractor may break.
+	- If no video is found, this extractor will consider the extraction failed.
+    """
+}
diff --git a/src/auto_archiver/modules/tiktok_tikwm_extractor/tiktok_tikwm_extractor.py b/src/auto_archiver/modules/tiktok_tikwm_extractor/tiktok_tikwm_extractor.py
new file mode 100644
index 0000000..8b07775
--- /dev/null
+++ b/src/auto_archiver/modules/tiktok_tikwm_extractor/tiktok_tikwm_extractor.py
@@ -0,0 +1,75 @@
+import re
+import requests
+from loguru import logger
+from datetime import datetime, timezone
+from yt_dlp.extractor.tiktok import TikTokIE
+
+from auto_archiver.core import Extractor
+from auto_archiver.core import Metadata, Media
+
+
+class TiktokTikwmExtractor(Extractor):
+    """
+    Extractor for TikTok that uses an unofficial API and can capture content that requires a login, like sensitive content.
+    """
+    TIKWM_ENDPOINT = "https://www.tikwm.com/api/?url={url}"
+
+    def download(self, item: Metadata) -> Metadata:
+        url = item.get_url()
+        
+        if not re.match(TikTokIE._VALID_URL, url):
+            return False
+
+        endpoint = TiktokTikwmExtractor.TIKWM_ENDPOINT.format(url=url)
+
+        r = requests.get(endpoint)
+        if r.status_code != 200:
+            logger.error(f"unexpected status code '{r.status_code}' from tikwm.com for {url=}:")
+            return False
+
+        try:
+            json_response = r.json()
+        except ValueError:
+            logger.error(f"failed to parse JSON response from tikwm.com for {url=}")
+            return False
+
+        if not json_response.get('msg') == 'success' or not (api_data := json_response.get('data', {})):
+            logger.error(f"failed to get a valid response from tikwm.com for {url=}: {json_response}")
+            return False
+
+        # tries to get the non-watermarked version first
+        video_url = api_data.pop("play", api_data.pop("wmplay", None))
+        if not video_url:
+            logger.error(f"no valid video URL found in response from tikwm.com for {url=}")
+            return False
+
+        # prepare result, start by downloading video
+        result = Metadata()
+
+        # get the cover if possible
+        cover_url = api_data.pop("origin_cover", api_data.pop("cover", api_data.pop("ai_dynamic_cover", None)))
+        if cover_url and (cover_downloaded := self.download_from_url(cover_url)):
+            result.add_media(Media(cover_downloaded))
+
+        # get the video or fail
+        video_downloaded = self.download_from_url(video_url, f"vid_{api_data.get('id', '')}")
+        if not video_downloaded:
+            logger.error(f"failed to download video from {video_url}")
+            return False
+        video_media = Media(video_downloaded)
+        if duration := api_data.pop("duration", None):
+            video_media.set("duration", duration)
+        result.add_media(video_media)
+
+        # add remaining metadata
+        result.set_title(api_data.pop("title", ""))
+
+        if created_at := api_data.pop("create_time", None):
+            result.set_timestamp(datetime.fromtimestamp(created_at, tz=timezone.utc))
+
+        if (author := api_data.pop("author", None)):
+            result.set("author", author)
+
+        result.set("api_data", api_data)
+
+        return result.success("tikwm")
diff --git a/tests/extractors/test_tiktok_tikwm_extractor.py b/tests/extractors/test_tiktok_tikwm_extractor.py
new file mode 100644
index 0000000..e8ad8df
--- /dev/null
+++ b/tests/extractors/test_tiktok_tikwm_extractor.py
@@ -0,0 +1,154 @@
+from datetime import datetime, timezone
+import time
+import pytest
+
+from auto_archiver.modules.tiktok_tikwm_extractor.tiktok_tikwm_extractor import TiktokTikwmExtractor
+from .test_extractor_base import TestExtractorBase
+
+
+class TestTiktokTikwmExtractor(TestExtractorBase):
+    """
+    Test suite for TestTiktokTikwmExtractor.
+    """
+
+    extractor_module = "tiktok_tikwm_extractor"
+    extractor: TiktokTikwmExtractor
+
+    config = {}
+
+    VALID_EXAMPLE_URL = "https://www.tiktok.com/@example/video/1234"
+
+    @staticmethod
+    def get_mockers(mocker):
+        mock_get = mocker.patch("auto_archiver.modules.tiktok_tikwm_extractor.tiktok_tikwm_extractor.requests.get")
+        mock_logger = mocker.patch("auto_archiver.modules.tiktok_tikwm_extractor.tiktok_tikwm_extractor.logger")
+        return mock_get, mock_logger
+
+    @pytest.mark.parametrize("url,valid_url", [
+        ("https://bellingcat.com", False),
+        ("https://youtube.com", False),
+        ("https://tiktok.co/", False),
+        ("https://tiktok.com/", False),
+        ("https://www.tiktok.com/", False),
+        ("https://api.cool.tiktok.com/", False),
+        (VALID_EXAMPLE_URL, True),
+        ("https://www.tiktok.com/@bbcnews/video/7478038212070411542", True),
+        ("https://www.tiktok.com/@ggs68taiwan.official/video/7441821351142362375", True),
+    ])
+    def test_valid_urls(self, mocker, make_item, url, valid_url):
+        mock_get, mock_logger = self.get_mockers(mocker)
+        if valid_url:
+            mock_get.return_value.status_code = 404
+        assert self.extractor.download(make_item(url)) == False
+        assert mock_get.call_count == int(valid_url)
+        assert mock_logger.error.call_count == int(valid_url)
+
+    def test_invalid_json_responses(self, mocker, make_item):
+        mock_get, mock_logger = self.get_mockers(mocker)
+        mock_get.return_value.status_code = 200
+        mock_get.return_value.json.side_effect = ValueError
+        assert self.extractor.download(make_item(self.VALID_EXAMPLE_URL)) == False
+        mock_get.assert_called_once()
+        mock_get.return_value.json.assert_called_once()
+        mock_logger.error.assert_called_once()
+        assert mock_logger.error.call_args[0][0].startswith("failed to parse JSON response")
+
+        mock_get.return_value.json.side_effect = Exception
+        with pytest.raises(Exception):
+            self.extractor.download(make_item(self.VALID_EXAMPLE_URL))
+        mock_get.assert_called()
+        assert mock_get.call_count == 2
+        assert mock_get.return_value.json.call_count == 2
+
+    @pytest.mark.parametrize("response", [
+        ({"msg": "failure"}),
+        ({"msg": "success"}),
+    ])
+    def test_unsuccessful_responses(self, mocker, make_item, response):
+        mock_get, mock_logger = self.get_mockers(mocker)
+        mock_get.return_value.status_code = 200
+        mock_get.return_value.json.return_value = response
+        assert self.extractor.download(make_item(self.VALID_EXAMPLE_URL)) == False
+        mock_get.assert_called_once()
+        mock_get.return_value.json.assert_called_once()
+        mock_logger.error.assert_called_once()
+        assert mock_logger.error.call_args[0][0].startswith("failed to get a valid response")
+
+    @pytest.mark.parametrize("response,has_vid", [
+        ({"data": {"id": 123}}, False),
+        ({"data": {"wmplay": "url"}}, True),
+        ({"data": {"play": "url"}}, True),
+    ])
+    def test_correct_extraction(self, mocker, make_item, response, has_vid):
+        mock_get, mock_logger = self.get_mockers(mocker)
+        mock_get.return_value.status_code = 200
+        mock_get.return_value.json.return_value = {"msg": "success", **response}
+
+        result = self.extractor.download(make_item(self.VALID_EXAMPLE_URL))
+        if not has_vid:
+            assert result == False
+        else:
+            assert result.is_success()
+            assert len(result.media) == 1
+        mock_get.assert_called()
+        assert mock_get.call_count == 1 + int(has_vid)
+        mock_get.return_value.json.assert_called_once()
+        if not has_vid:
+            mock_logger.error.assert_called_once()
+            assert mock_logger.error.call_args[0][0].startswith("no valid video URL found")
+        else:
+            mock_logger.error.assert_not_called()
+
+    def test_correct_extraction(self, mocker, make_item):
+        mock_get, _ = self.get_mockers(mocker)
+        mock_get.return_value.status_code = 200
+        mock_get.return_value.json.return_value = {"msg": "success", "data": {
+            "wmplay": "url",
+            "origin_cover": "cover.jpg",
+            "title": "Title",
+            "id": 123,
+            "duration": 60,
+            "create_time": 1736301699,
+            "author": "Author",
+            "other": "data"
+        }}
+
+        result = self.extractor.download(make_item(self.VALID_EXAMPLE_URL))
+        assert result.is_success()
+        assert len(result.media) == 2
+        assert result.get_title() == "Title"
+        assert result.get("author") == "Author"
+        assert result.get("api_data") == {"other": "data", "id": 123}
+        assert result.media[1].get("duration") == 60
+        assert result.get("timestamp") == datetime.fromtimestamp(1736301699, tz=timezone.utc)
+
+    @pytest.mark.download
+    def test_download_video(self, make_item):
+        url = "https://www.tiktok.com/@bbcnews/video/7478038212070411542"
+
+        result = self.extractor.download(make_item(url))
+        assert result.is_success()
+        assert len(result.media) == 2
+        assert result.get_title() == "The A23a iceberg is one of the world's oldest and it's so big you can see it from space. #Iceberg  #A23a  #Antarctica  #Ice  #ClimateChange  #DavidAttenborough  #Ocean  #Sea  #SouthGeorgia  #BBCNews "
+        assert result.get("author").get("unique_id") == "bbcnews"
+        assert result.get("api_data").get("id") == '7478038212070411542'
+        assert result.media[1].get("duration") == 59
+        assert result.get("timestamp") == datetime.fromtimestamp(1741122000, tz=timezone.utc)
+
+    @pytest.mark.download
+    def test_download_sensitive_video(self, make_item, mock_sleep):
+        # sleep is needed because of the rate limit
+        mock_sleep.stop()
+        time.sleep(1.1)
+        mock_sleep.start()
+
+        url = "https://www.tiktok.com/@ggs68taiwan.official/video/7441821351142362375"
+
+        result = self.extractor.download(make_item(url))
+        assert result.is_success()
+        assert len(result.media) == 2
+        assert result.get_title() == "Căng nhất lúc này #ggs68 #ggs68taiwan #taiwan #dailoan #tiktoknews"
+        assert result.get("author").get("id") == "7197400619475649562"
+        assert result.get("api_data").get("id") == '7441821351142362375'
+        assert result.media[1].get("duration") == 34
+        assert result.get("timestamp") == datetime.fromtimestamp(1732684060, tz=timezone.utc)

From f4f2424eb559d42379a942a3aac77e1d1e7ec5d5 Mon Sep 17 00:00:00 2001
From: erinhmclark <erinhannahmary.clark@gmail.com>
Date: Mon, 10 Mar 2025 13:15:11 +0000
Subject: [PATCH 07/55] Add black and flake8

---
 poetry.lock    | 128 +++++++++++++++++++++++++++++++++++++++++++++++--
 pyproject.toml |   2 +
 2 files changed, 125 insertions(+), 5 deletions(-)

diff --git a/poetry.lock b/poetry.lock
index f59d5c9..bc3ffc9 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -170,6 +170,53 @@ charset-normalizer = ["charset-normalizer"]
 html5lib = ["html5lib"]
 lxml = ["lxml"]
 
+[[package]]
+name = "black"
+version = "25.1.0"
+description = "The uncompromising code formatter."
+optional = false
+python-versions = ">=3.9"
+groups = ["dev"]
+files = [
+    {file = "black-25.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:759e7ec1e050a15f89b770cefbf91ebee8917aac5c20483bc2d80a6c3a04df32"},
+    {file = "black-25.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0e519ecf93120f34243e6b0054db49c00a35f84f195d5bce7e9f5cfc578fc2da"},
+    {file = "black-25.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:055e59b198df7ac0b7efca5ad7ff2516bca343276c466be72eb04a3bcc1f82d7"},
+    {file = "black-25.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:db8ea9917d6f8fc62abd90d944920d95e73c83a5ee3383493e35d271aca872e9"},
+    {file = "black-25.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a39337598244de4bae26475f77dda852ea00a93bd4c728e09eacd827ec929df0"},
+    {file = "black-25.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:96c1c7cd856bba8e20094e36e0f948718dc688dba4a9d78c3adde52b9e6c2299"},
+    {file = "black-25.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bce2e264d59c91e52d8000d507eb20a9aca4a778731a08cfff7e5ac4a4bb7096"},
+    {file = "black-25.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:172b1dbff09f86ce6f4eb8edf9dede08b1fce58ba194c87d7a4f1a5aa2f5b3c2"},
+    {file = "black-25.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4b60580e829091e6f9238c848ea6750efed72140b91b048770b64e74fe04908b"},
+    {file = "black-25.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1e2978f6df243b155ef5fa7e558a43037c3079093ed5d10fd84c43900f2d8ecc"},
+    {file = "black-25.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3b48735872ec535027d979e8dcb20bf4f70b5ac75a8ea99f127c106a7d7aba9f"},
+    {file = "black-25.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:ea0213189960bda9cf99be5b8c8ce66bb054af5e9e861249cd23471bd7b0b3ba"},
+    {file = "black-25.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8f0b18a02996a836cc9c9c78e5babec10930862827b1b724ddfe98ccf2f2fe4f"},
+    {file = "black-25.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:afebb7098bfbc70037a053b91ae8437c3857482d3a690fefc03e9ff7aa9a5fd3"},
+    {file = "black-25.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:030b9759066a4ee5e5aca28c3c77f9c64789cdd4de8ac1df642c40b708be6171"},
+    {file = "black-25.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:a22f402b410566e2d1c950708c77ebf5ebd5d0d88a6a2e87c86d9fb48afa0d18"},
+    {file = "black-25.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a1ee0a0c330f7b5130ce0caed9936a904793576ef4d2b98c40835d6a65afa6a0"},
+    {file = "black-25.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f3df5f1bf91d36002b0a75389ca8663510cf0531cca8aa5c1ef695b46d98655f"},
+    {file = "black-25.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d9e6827d563a2c820772b32ce8a42828dc6790f095f441beef18f96aa6f8294e"},
+    {file = "black-25.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:bacabb307dca5ebaf9c118d2d2f6903da0d62c9faa82bd21a33eecc319559355"},
+    {file = "black-25.1.0-py3-none-any.whl", hash = "sha256:95e8176dae143ba9097f351d174fdaf0ccd29efb414b362ae3fd72bf0f710717"},
+    {file = "black-25.1.0.tar.gz", hash = "sha256:33496d5cd1222ad73391352b4ae8da15253c5de89b93a80b3e2c8d9a19ec2666"},
+]
+
+[package.dependencies]
+click = ">=8.0.0"
+mypy-extensions = ">=0.4.3"
+packaging = ">=22.0"
+pathspec = ">=0.9.0"
+platformdirs = ">=2"
+tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""}
+typing-extensions = {version = ">=4.0.1", markers = "python_version < \"3.11\""}
+
+[package.extras]
+colorama = ["colorama (>=0.4.3)"]
+d = ["aiohttp (>=3.10)"]
+jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"]
+uvloop = ["uvloop (>=0.15.2)"]
+
 [[package]]
 name = "boto3"
 version = "1.37.8"
@@ -589,7 +636,7 @@ version = "8.1.8"
 description = "Composable command line interface toolkit"
 optional = false
 python-versions = ">=3.7"
-groups = ["docs"]
+groups = ["dev", "docs"]
 files = [
     {file = "click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2"},
     {file = "click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a"},
@@ -609,7 +656,7 @@ files = [
     {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"},
     {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"},
 ]
-markers = {main = "sys_platform == \"win32\" or platform_system == \"Windows\"", dev = "sys_platform == \"win32\""}
+markers = {main = "sys_platform == \"win32\" or platform_system == \"Windows\"", dev = "sys_platform == \"win32\" or platform_system == \"Windows\""}
 
 [[package]]
 name = "cryptography"
@@ -742,6 +789,23 @@ future = "*"
 [package.extras]
 dev = ["Sphinx (==2.1.0)", "future (==0.17.1)", "numpy (==1.16.4)", "pytest (==4.6.1)", "pytest-mock (==1.10.4)", "tox (==3.12.1)"]
 
+[[package]]
+name = "flake8"
+version = "7.1.2"
+description = "the modular source code checker: pep8 pyflakes and co"
+optional = false
+python-versions = ">=3.8.1"
+groups = ["dev"]
+files = [
+    {file = "flake8-7.1.2-py2.py3-none-any.whl", hash = "sha256:1cbc62e65536f65e6d754dfe6f1bada7f5cf392d6f5db3c2b85892466c3e7c1a"},
+    {file = "flake8-7.1.2.tar.gz", hash = "sha256:c586ffd0b41540951ae41af572e6790dbd49fc12b3aa2541685d253d9bd504bd"},
+]
+
+[package.dependencies]
+mccabe = ">=0.7.0,<0.8.0"
+pycodestyle = ">=2.12.0,<2.13.0"
+pyflakes = ">=3.2.0,<3.3.0"
+
 [[package]]
 name = "future"
 version = "1.0.0"
@@ -1177,6 +1241,18 @@ dev = ["marshmallow[tests]", "pre-commit (>=3.5,<5.0)", "tox"]
 docs = ["autodocsumm (==0.2.14)", "furo (==2024.8.6)", "sphinx (==8.1.3)", "sphinx-copybutton (==0.5.2)", "sphinx-issues (==5.0.0)", "sphinxext-opengraph (==0.9.1)"]
 tests = ["pytest", "simplejson"]
 
+[[package]]
+name = "mccabe"
+version = "0.7.0"
+description = "McCabe checker, plugin for flake8"
+optional = false
+python-versions = ">=3.6"
+groups = ["dev"]
+files = [
+    {file = "mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"},
+    {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"},
+]
+
 [[package]]
 name = "mdit-py-plugins"
 version = "0.4.2"
@@ -1227,7 +1303,7 @@ version = "1.0.0"
 description = "Type system extensions for programs checked with the mypy type checker."
 optional = false
 python-versions = ">=3.5"
-groups = ["main"]
+groups = ["main", "dev"]
 files = [
     {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"},
     {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"},
@@ -1403,6 +1479,18 @@ files = [
     {file = "packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f"},
 ]
 
+[[package]]
+name = "pathspec"
+version = "0.12.1"
+description = "Utility library for gitignore style pattern matching of file paths."
+optional = false
+python-versions = ">=3.8"
+groups = ["dev"]
+files = [
+    {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"},
+    {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"},
+]
+
 [[package]]
 name = "pdqhash"
 version = "0.2.7"
@@ -1513,6 +1601,23 @@ tests = ["check-manifest", "coverage (>=7.4.2)", "defusedxml", "markdown2", "ole
 typing = ["typing-extensions"]
 xmp = ["defusedxml"]
 
+[[package]]
+name = "platformdirs"
+version = "4.3.6"
+description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`."
+optional = false
+python-versions = ">=3.8"
+groups = ["dev"]
+files = [
+    {file = "platformdirs-4.3.6-py3-none-any.whl", hash = "sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb"},
+    {file = "platformdirs-4.3.6.tar.gz", hash = "sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907"},
+]
+
+[package.extras]
+docs = ["furo (>=2024.8.6)", "proselint (>=0.14)", "sphinx (>=8.0.2)", "sphinx-autodoc-typehints (>=2.4)"]
+test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=8.3.2)", "pytest-cov (>=5)", "pytest-mock (>=3.14)"]
+type = ["mypy (>=1.11.2)"]
+
 [[package]]
 name = "pluggy"
 version = "1.5.0"
@@ -1701,6 +1806,18 @@ doc = ["ablog (>=0.11.8)", "colorama", "graphviz", "ipykernel", "ipyleaflet", "i
 i18n = ["Babel", "jinja2"]
 test = ["pytest", "pytest-cov", "pytest-regressions", "sphinx[test]"]
 
+[[package]]
+name = "pyflakes"
+version = "3.2.0"
+description = "passive checker of Python programs"
+optional = false
+python-versions = ">=3.8"
+groups = ["dev"]
+files = [
+    {file = "pyflakes-3.2.0-py2.py3-none-any.whl", hash = "sha256:84b5be138a2dfbb40689ca07e2152deb896a65c3a3e24c251c5c62489568074a"},
+    {file = "pyflakes-3.2.0.tar.gz", hash = "sha256:1c61603ff154621fb2a9172037d84dca3500def8c8b630657d1701f026f8af3f"},
+]
+
 [[package]]
 name = "pygments"
 version = "2.19.1"
@@ -2762,11 +2879,12 @@ version = "4.12.2"
 description = "Backported and Experimental Type Hints for Python 3.8+"
 optional = false
 python-versions = ">=3.8"
-groups = ["main", "docs"]
+groups = ["main", "dev", "docs"]
 files = [
     {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"},
     {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"},
 ]
+markers = {dev = "python_version < \"3.11\""}
 
 [[package]]
 name = "typing-inspect"
@@ -3185,4 +3303,4 @@ test = ["pytest (>=8.1,<9.0)", "pytest-rerunfailures (>=14.0,<15.0)"]
 [metadata]
 lock-version = "2.1"
 python-versions = ">=3.10,<3.13"
-content-hash = "2d0a953383901fe12e97f6f56a76a9d8008788695425792eedbf739a18585188"
+content-hash = "fec634c8347a109ba12df7837d1da4c2522d388a58148ab955dafa141a2743e0"
diff --git a/pyproject.toml b/pyproject.toml
index 30693d5..5ad057b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -64,6 +64,8 @@ pytest = "^8.3.4"
 autopep8 = "^2.3.1"
 pytest-loguru = "^0.4.0"
 pytest-mock = "^3.14.0"
+flake8 = "^7.1.2"
+black = "^25.1.0"
 
 [tool.poetry.group.docs.dependencies]
 sphinx = "^8.1.3"

From cbb0414e5fbcd2f2431b312e340ae497dd97a003 Mon Sep 17 00:00:00 2001
From: erinhmclark <erinhannahmary.clark@gmail.com>
Date: Mon, 10 Mar 2025 16:05:23 +0000
Subject: [PATCH 08/55] Switch to ruff

---
 poetry.lock    | 156 +++++++++++--------------------------------------
 pyproject.toml |  24 +++++++-
 2 files changed, 54 insertions(+), 126 deletions(-)

diff --git a/poetry.lock b/poetry.lock
index bc3ffc9..1580583 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -170,53 +170,6 @@ charset-normalizer = ["charset-normalizer"]
 html5lib = ["html5lib"]
 lxml = ["lxml"]
 
-[[package]]
-name = "black"
-version = "25.1.0"
-description = "The uncompromising code formatter."
-optional = false
-python-versions = ">=3.9"
-groups = ["dev"]
-files = [
-    {file = "black-25.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:759e7ec1e050a15f89b770cefbf91ebee8917aac5c20483bc2d80a6c3a04df32"},
-    {file = "black-25.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0e519ecf93120f34243e6b0054db49c00a35f84f195d5bce7e9f5cfc578fc2da"},
-    {file = "black-25.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:055e59b198df7ac0b7efca5ad7ff2516bca343276c466be72eb04a3bcc1f82d7"},
-    {file = "black-25.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:db8ea9917d6f8fc62abd90d944920d95e73c83a5ee3383493e35d271aca872e9"},
-    {file = "black-25.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a39337598244de4bae26475f77dda852ea00a93bd4c728e09eacd827ec929df0"},
-    {file = "black-25.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:96c1c7cd856bba8e20094e36e0f948718dc688dba4a9d78c3adde52b9e6c2299"},
-    {file = "black-25.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bce2e264d59c91e52d8000d507eb20a9aca4a778731a08cfff7e5ac4a4bb7096"},
-    {file = "black-25.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:172b1dbff09f86ce6f4eb8edf9dede08b1fce58ba194c87d7a4f1a5aa2f5b3c2"},
-    {file = "black-25.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4b60580e829091e6f9238c848ea6750efed72140b91b048770b64e74fe04908b"},
-    {file = "black-25.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1e2978f6df243b155ef5fa7e558a43037c3079093ed5d10fd84c43900f2d8ecc"},
-    {file = "black-25.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3b48735872ec535027d979e8dcb20bf4f70b5ac75a8ea99f127c106a7d7aba9f"},
-    {file = "black-25.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:ea0213189960bda9cf99be5b8c8ce66bb054af5e9e861249cd23471bd7b0b3ba"},
-    {file = "black-25.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8f0b18a02996a836cc9c9c78e5babec10930862827b1b724ddfe98ccf2f2fe4f"},
-    {file = "black-25.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:afebb7098bfbc70037a053b91ae8437c3857482d3a690fefc03e9ff7aa9a5fd3"},
-    {file = "black-25.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:030b9759066a4ee5e5aca28c3c77f9c64789cdd4de8ac1df642c40b708be6171"},
-    {file = "black-25.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:a22f402b410566e2d1c950708c77ebf5ebd5d0d88a6a2e87c86d9fb48afa0d18"},
-    {file = "black-25.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a1ee0a0c330f7b5130ce0caed9936a904793576ef4d2b98c40835d6a65afa6a0"},
-    {file = "black-25.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f3df5f1bf91d36002b0a75389ca8663510cf0531cca8aa5c1ef695b46d98655f"},
-    {file = "black-25.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d9e6827d563a2c820772b32ce8a42828dc6790f095f441beef18f96aa6f8294e"},
-    {file = "black-25.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:bacabb307dca5ebaf9c118d2d2f6903da0d62c9faa82bd21a33eecc319559355"},
-    {file = "black-25.1.0-py3-none-any.whl", hash = "sha256:95e8176dae143ba9097f351d174fdaf0ccd29efb414b362ae3fd72bf0f710717"},
-    {file = "black-25.1.0.tar.gz", hash = "sha256:33496d5cd1222ad73391352b4ae8da15253c5de89b93a80b3e2c8d9a19ec2666"},
-]
-
-[package.dependencies]
-click = ">=8.0.0"
-mypy-extensions = ">=0.4.3"
-packaging = ">=22.0"
-pathspec = ">=0.9.0"
-platformdirs = ">=2"
-tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""}
-typing-extensions = {version = ">=4.0.1", markers = "python_version < \"3.11\""}
-
-[package.extras]
-colorama = ["colorama (>=0.4.3)"]
-d = ["aiohttp (>=3.10)"]
-jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"]
-uvloop = ["uvloop (>=0.15.2)"]
-
 [[package]]
 name = "boto3"
 version = "1.37.8"
@@ -636,7 +589,7 @@ version = "8.1.8"
 description = "Composable command line interface toolkit"
 optional = false
 python-versions = ">=3.7"
-groups = ["dev", "docs"]
+groups = ["docs"]
 files = [
     {file = "click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2"},
     {file = "click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a"},
@@ -656,7 +609,7 @@ files = [
     {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"},
     {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"},
 ]
-markers = {main = "sys_platform == \"win32\" or platform_system == \"Windows\"", dev = "sys_platform == \"win32\" or platform_system == \"Windows\""}
+markers = {main = "sys_platform == \"win32\" or platform_system == \"Windows\"", dev = "sys_platform == \"win32\""}
 
 [[package]]
 name = "cryptography"
@@ -789,23 +742,6 @@ future = "*"
 [package.extras]
 dev = ["Sphinx (==2.1.0)", "future (==0.17.1)", "numpy (==1.16.4)", "pytest (==4.6.1)", "pytest-mock (==1.10.4)", "tox (==3.12.1)"]
 
-[[package]]
-name = "flake8"
-version = "7.1.2"
-description = "the modular source code checker: pep8 pyflakes and co"
-optional = false
-python-versions = ">=3.8.1"
-groups = ["dev"]
-files = [
-    {file = "flake8-7.1.2-py2.py3-none-any.whl", hash = "sha256:1cbc62e65536f65e6d754dfe6f1bada7f5cf392d6f5db3c2b85892466c3e7c1a"},
-    {file = "flake8-7.1.2.tar.gz", hash = "sha256:c586ffd0b41540951ae41af572e6790dbd49fc12b3aa2541685d253d9bd504bd"},
-]
-
-[package.dependencies]
-mccabe = ">=0.7.0,<0.8.0"
-pycodestyle = ">=2.12.0,<2.13.0"
-pyflakes = ">=3.2.0,<3.3.0"
-
 [[package]]
 name = "future"
 version = "1.0.0"
@@ -1241,18 +1177,6 @@ dev = ["marshmallow[tests]", "pre-commit (>=3.5,<5.0)", "tox"]
 docs = ["autodocsumm (==0.2.14)", "furo (==2024.8.6)", "sphinx (==8.1.3)", "sphinx-copybutton (==0.5.2)", "sphinx-issues (==5.0.0)", "sphinxext-opengraph (==0.9.1)"]
 tests = ["pytest", "simplejson"]
 
-[[package]]
-name = "mccabe"
-version = "0.7.0"
-description = "McCabe checker, plugin for flake8"
-optional = false
-python-versions = ">=3.6"
-groups = ["dev"]
-files = [
-    {file = "mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"},
-    {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"},
-]
-
 [[package]]
 name = "mdit-py-plugins"
 version = "0.4.2"
@@ -1303,7 +1227,7 @@ version = "1.0.0"
 description = "Type system extensions for programs checked with the mypy type checker."
 optional = false
 python-versions = ">=3.5"
-groups = ["main", "dev"]
+groups = ["main"]
 files = [
     {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"},
     {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"},
@@ -1479,18 +1403,6 @@ files = [
     {file = "packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f"},
 ]
 
-[[package]]
-name = "pathspec"
-version = "0.12.1"
-description = "Utility library for gitignore style pattern matching of file paths."
-optional = false
-python-versions = ">=3.8"
-groups = ["dev"]
-files = [
-    {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"},
-    {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"},
-]
-
 [[package]]
 name = "pdqhash"
 version = "0.2.7"
@@ -1601,23 +1513,6 @@ tests = ["check-manifest", "coverage (>=7.4.2)", "defusedxml", "markdown2", "ole
 typing = ["typing-extensions"]
 xmp = ["defusedxml"]
 
-[[package]]
-name = "platformdirs"
-version = "4.3.6"
-description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`."
-optional = false
-python-versions = ">=3.8"
-groups = ["dev"]
-files = [
-    {file = "platformdirs-4.3.6-py3-none-any.whl", hash = "sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb"},
-    {file = "platformdirs-4.3.6.tar.gz", hash = "sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907"},
-]
-
-[package.extras]
-docs = ["furo (>=2024.8.6)", "proselint (>=0.14)", "sphinx (>=8.0.2)", "sphinx-autodoc-typehints (>=2.4)"]
-test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=8.3.2)", "pytest-cov (>=5)", "pytest-mock (>=3.14)"]
-type = ["mypy (>=1.11.2)"]
-
 [[package]]
 name = "pluggy"
 version = "1.5.0"
@@ -1806,18 +1701,6 @@ doc = ["ablog (>=0.11.8)", "colorama", "graphviz", "ipykernel", "ipyleaflet", "i
 i18n = ["Babel", "jinja2"]
 test = ["pytest", "pytest-cov", "pytest-regressions", "sphinx[test]"]
 
-[[package]]
-name = "pyflakes"
-version = "3.2.0"
-description = "passive checker of Python programs"
-optional = false
-python-versions = ">=3.8"
-groups = ["dev"]
-files = [
-    {file = "pyflakes-3.2.0-py2.py3-none-any.whl", hash = "sha256:84b5be138a2dfbb40689ca07e2152deb896a65c3a3e24c251c5c62489568074a"},
-    {file = "pyflakes-3.2.0.tar.gz", hash = "sha256:1c61603ff154621fb2a9172037d84dca3500def8c8b630657d1701f026f8af3f"},
-]
-
 [[package]]
 name = "pygments"
 version = "2.19.1"
@@ -2363,6 +2246,34 @@ files = [
     {file = "ruamel.yaml.clib-0.2.12.tar.gz", hash = "sha256:6c8fbb13ec503f99a91901ab46e0b07ae7941cd527393187039aec586fdfd36f"},
 ]
 
+[[package]]
+name = "ruff"
+version = "0.9.10"
+description = "An extremely fast Python linter and code formatter, written in Rust."
+optional = false
+python-versions = ">=3.7"
+groups = ["main"]
+files = [
+    {file = "ruff-0.9.10-py3-none-linux_armv6l.whl", hash = "sha256:eb4d25532cfd9fe461acc83498361ec2e2252795b4f40b17e80692814329e42d"},
+    {file = "ruff-0.9.10-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:188a6638dab1aa9bb6228a7302387b2c9954e455fb25d6b4470cb0641d16759d"},
+    {file = "ruff-0.9.10-py3-none-macosx_11_0_arm64.whl", hash = "sha256:5284dcac6b9dbc2fcb71fdfc26a217b2ca4ede6ccd57476f52a587451ebe450d"},
+    {file = "ruff-0.9.10-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:47678f39fa2a3da62724851107f438c8229a3470f533894b5568a39b40029c0c"},
+    {file = "ruff-0.9.10-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:99713a6e2766b7a17147b309e8c915b32b07a25c9efd12ada79f217c9c778b3e"},
+    {file = "ruff-0.9.10-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:524ee184d92f7c7304aa568e2db20f50c32d1d0caa235d8ddf10497566ea1a12"},
+    {file = "ruff-0.9.10-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:df92aeac30af821f9acf819fc01b4afc3dfb829d2782884f8739fb52a8119a16"},
+    {file = "ruff-0.9.10-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de42e4edc296f520bb84954eb992a07a0ec5a02fecb834498415908469854a52"},
+    {file = "ruff-0.9.10-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d257f95b65806104b6b1ffca0ea53f4ef98454036df65b1eda3693534813ecd1"},
+    {file = "ruff-0.9.10-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b60dec7201c0b10d6d11be00e8f2dbb6f40ef1828ee75ed739923799513db24c"},
+    {file = "ruff-0.9.10-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:d838b60007da7a39c046fcdd317293d10b845001f38bcb55ba766c3875b01e43"},
+    {file = "ruff-0.9.10-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:ccaf903108b899beb8e09a63ffae5869057ab649c1e9231c05ae354ebc62066c"},
+    {file = "ruff-0.9.10-py3-none-musllinux_1_2_i686.whl", hash = "sha256:f9567d135265d46e59d62dc60c0bfad10e9a6822e231f5b24032dba5a55be6b5"},
+    {file = "ruff-0.9.10-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:5f202f0d93738c28a89f8ed9eaba01b7be339e5d8d642c994347eaa81c6d75b8"},
+    {file = "ruff-0.9.10-py3-none-win32.whl", hash = "sha256:bfb834e87c916521ce46b1788fbb8484966e5113c02df216680102e9eb960029"},
+    {file = "ruff-0.9.10-py3-none-win_amd64.whl", hash = "sha256:f2160eeef3031bf4b17df74e307d4c5fb689a6f3a26a2de3f7ef4044e3c484f1"},
+    {file = "ruff-0.9.10-py3-none-win_arm64.whl", hash = "sha256:5fd804c0327a5e5ea26615550e706942f348b197d5475ff34c19733aee4b2e69"},
+    {file = "ruff-0.9.10.tar.gz", hash = "sha256:9bacb735d7bada9cfb0f2c227d3658fc443d90a727b47f206fb33f52f3c0eac7"},
+]
+
 [[package]]
 name = "s3transfer"
 version = "0.11.4"
@@ -2879,12 +2790,11 @@ version = "4.12.2"
 description = "Backported and Experimental Type Hints for Python 3.8+"
 optional = false
 python-versions = ">=3.8"
-groups = ["main", "dev", "docs"]
+groups = ["main", "docs"]
 files = [
     {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"},
     {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"},
 ]
-markers = {dev = "python_version < \"3.11\""}
 
 [[package]]
 name = "typing-inspect"
@@ -3303,4 +3213,4 @@ test = ["pytest (>=8.1,<9.0)", "pytest-rerunfailures (>=14.0,<15.0)"]
 [metadata]
 lock-version = "2.1"
 python-versions = ">=3.10,<3.13"
-content-hash = "fec634c8347a109ba12df7837d1da4c2522d388a58148ab955dafa141a2743e0"
+content-hash = "57907984411be7c5bb9c9f0628476156c9ab59ba32de6e0e42a51b396e8b696d"
diff --git a/pyproject.toml b/pyproject.toml
index 5ad057b..18f7397 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -57,6 +57,7 @@ dependencies = [
     "certvalidator (>=0.0.0)",
     "rich-argparse (>=1.6.0,<2.0.0)",
     "ruamel-yaml (>=0.18.10,<0.19.0)",
+    "ruff (>=0.9.10,<0.10.0)",
 ]
 
 [tool.poetry.group.dev.dependencies]
@@ -64,8 +65,6 @@ pytest = "^8.3.4"
 autopep8 = "^2.3.1"
 pytest-loguru = "^0.4.0"
 pytest-mock = "^3.14.0"
-flake8 = "^7.1.2"
-black = "^25.1.0"
 
 [tool.poetry.group.docs.dependencies]
 sphinx = "^8.1.3"
@@ -91,4 +90,23 @@ documentation = "https://github.com/bellingcat/auto-archiver"
 markers = [
     "download: marks tests that download content from the network",
     "incremental: marks a class to run tests incrementally. If a test fails in the class, the remaining tests will be skipped",
-]
\ No newline at end of file
+]
+
+[tool.ruff]
+#exclude = ["docs"]
+line-length = 120
+
+[tool.ruff.lint]
+#add bugbear?
+#extend-select = ["B"]
+
+# E701 - multiple statements on one line (I vote to keep this but I notice it's used quite a lot!)
+ignore = []
+
+[tool.ruff.lint.per-file-ignores]
+# Ignore import violations in __init__.py files
+"__init__.py" = ["F401"]
+
+[tool.ruff.format]
+docstring-code-format = false
+

From 770f4c8a3de0a3bd973dd099113bffb7b74a8cfc Mon Sep 17 00:00:00 2001
From: Patrick Robertson <robertson.patrick@gmail.com>
Date: Mon, 10 Mar 2025 16:39:31 +0000
Subject: [PATCH 09/55] Refactoring of storage code:

1. Fix some bugs in local_storage
2. Refactor unit tests to not set Media.key explicitly (unless it's well-known beforehand, which it isn't)
3. Limit length of URL for 'url' type path_generator
4. Throw an error if 'save_to' of local storage is too long
5. A few other tidyups
---
 src/auto_archiver/core/config.py              |  1 -
 src/auto_archiver/core/consts.py              |  2 +
 src/auto_archiver/core/extractor.py           |  3 +-
 src/auto_archiver/core/media.py               |  9 ++--
 src/auto_archiver/core/orchestrator.py        |  4 +-
 src/auto_archiver/core/storage.py             | 42 ++++++++----------
 .../modules/cli_feeder/cli_feeder.py          |  1 -
 .../modules/local_storage/local_storage.py    | 43 ++++++++-----------
 .../modules/s3_storage/s3_storage.py          | 13 ++----
 tests/enrichers/test_pdq_hash_enricher.py     |  4 +-
 tests/storages/test_S3_storage.py             |  8 ++--
 tests/storages/test_atlos_storage.py          |  2 +-
 tests/storages/test_gdrive_storage.py         |  2 +-
 tests/storages/test_local_storage.py          | 29 ++++++-------
 tests/storages/test_storage_base.py           |  7 +--
 15 files changed, 74 insertions(+), 96 deletions(-)

diff --git a/src/auto_archiver/core/config.py b/src/auto_archiver/core/config.py
index 66d2ffb..0282d41 100644
--- a/src/auto_archiver/core/config.py
+++ b/src/auto_archiver/core/config.py
@@ -14,7 +14,6 @@ from loguru import logger
 from copy import deepcopy
 from auto_archiver.core.consts import MODULE_TYPES
 
-from typing import Any, List, Type, Tuple
 
 _yaml: YAML = YAML()
 
diff --git a/src/auto_archiver/core/consts.py b/src/auto_archiver/core/consts.py
index a49884f..597ce4e 100644
--- a/src/auto_archiver/core/consts.py
+++ b/src/auto_archiver/core/consts.py
@@ -1,3 +1,5 @@
+class SetupError(ValueError):
+    pass
 
 MODULE_TYPES = [
     'feeder',
diff --git a/src/auto_archiver/core/extractor.py b/src/auto_archiver/core/extractor.py
index 484a09d..f84be98 100644
--- a/src/auto_archiver/core/extractor.py
+++ b/src/auto_archiver/core/extractor.py
@@ -81,7 +81,8 @@ class Extractor(BaseModule):
             if len(to_filename) > 64:
                 to_filename = to_filename[-64:]
         to_filename = os.path.join(self.tmp_dir, to_filename)
-        if verbose: logger.debug(f"downloading {url[0:50]=} {to_filename=}")
+        if verbose:
+            logger.debug(f"downloading {url[0:50]=} {to_filename=}")
         headers = {
             'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36'
         }
diff --git a/src/auto_archiver/core/media.py b/src/auto_archiver/core/media.py
index 0370c00..ecaef19 100644
--- a/src/auto_archiver/core/media.py
+++ b/src/auto_archiver/core/media.py
@@ -21,14 +21,13 @@ class Media:
     Represents a media file with associated properties and storage details.
 
     Attributes:
-    - filename: The file path of the media.
-    - key: An optional identifier for the media.
+    - filename: The file path of the media as saved locally (temporarily, before uploading to the storage).
     - urls: A list of URLs where the media is stored or accessible.
     - properties: Additional metadata or transformations for the media.
     - _mimetype: The media's mimetype (e.g., image/jpeg, video/mp4).
     """
     filename: str
-    key: str = None
+    _key: str = None
     urls: List[str] = field(default_factory=list)
     properties: dict = field(default_factory=dict)
     _mimetype: str = None  # eg: image/jpeg
@@ -67,6 +66,10 @@ class Media:
         # checks if the media is already stored in the given storage
         return len(self.urls) > 0 and len(self.urls) == len(in_storage.config["steps"]["storages"])
 
+    @property
+    def key(self) -> str:
+        return self._key
+
     def set(self, key: str, value: Any) -> Media:
         self.properties[key] = value
         return self
diff --git a/src/auto_archiver/core/orchestrator.py b/src/auto_archiver/core/orchestrator.py
index b78c7e7..6a95046 100644
--- a/src/auto_archiver/core/orchestrator.py
+++ b/src/auto_archiver/core/orchestrator.py
@@ -23,15 +23,13 @@ from .config import read_yaml, store_yaml, to_dot_notation, merge_dicts, is_vali
     DefaultValidatingParser, UniqueAppendAction, AuthenticationJsonParseAction, DEFAULT_CONFIG_FILE
 from .module import ModuleFactory, LazyBaseModule
 from . import validators, Feeder, Extractor, Database, Storage, Formatter, Enricher
-from .consts import MODULE_TYPES
+from .consts import MODULE_TYPES, SetupError
 from auto_archiver.utils.url import check_url_or_raise
 
 if TYPE_CHECKING:
     from .base_module import BaseModule
     from .module import LazyBaseModule
 
-class SetupError(ValueError):
-    pass
 class ArchivingOrchestrator:
 
     # instance variables
diff --git a/src/auto_archiver/core/storage.py b/src/auto_archiver/core/storage.py
index 2a14073..4867146 100644
--- a/src/auto_archiver/core/storage.py
+++ b/src/auto_archiver/core/storage.py
@@ -6,6 +6,7 @@ from __future__ import annotations
 from abc import abstractmethod
 from typing import IO
 import os
+import platform
 
 from loguru import logger
 from slugify import slugify
@@ -43,17 +44,30 @@ class Storage(BaseModule):
     def uploadf(self, file: IO[bytes], key: str, **kwargs: dict) -> bool:
         """
         Uploads (or saves) a file to the storage service/location.
+
+        This method should not be called directly, but instead through the 'store' method,
+        which sets up the media for storage.
         """
         pass
 
     def upload(self, media: Media, **kwargs) -> bool:
+        """
+        Uploads a media object to the storage service.
+
+        This method should not be called directly, but instead be called through the 'store' method,
+        which sets up the media for storage.
+        """
         logger.debug(f'[{self.__class__.__name__}] storing file {media.filename} with key {media.key}')
         with open(media.filename, 'rb') as f:
             return self.uploadf(f, media, **kwargs)
 
     def set_key(self, media: Media, url: str, metadata: Metadata) -> None:
         """takes the media and optionally item info and generates a key"""
-        if media.key is not None and len(media.key) > 0: return
+        
+        if media.key is not None and len(media.key) > 0:
+            # media key is already set
+            return
+
         folder = metadata.get_context('folder', '')
         filename, ext = os.path.splitext(media.filename)
 
@@ -61,10 +75,8 @@ class Storage(BaseModule):
         path_generator = self.path_generator
         if path_generator == "flat":
             path = ""
-            # TODO: this is never used
-            filename = slugify(filename)  # Ensure filename is slugified
         elif path_generator == "url":
-            path = slugify(url)
+            path = slugify(url)[:70]
         elif path_generator == "random":
             path = random_str(24)
         else:
@@ -81,25 +93,7 @@ class Storage(BaseModule):
             filename = hd[:24]
         else:
             raise ValueError(f"Invalid filename_generator: {filename_generator}")
-
+        
         key = os.path.join(folder, path, f"{filename}{ext}")
-        if len(key) > self.max_file_length():
-            # truncate the path
-            max_path_length = self.max_file_length() - len(filename) - len(ext) - len(folder) - 1
-            path = path[:max_path_length]
-            logger.warning(f'Filename too long ({len(key)} characters), truncating to {self.max_file_length()} characters')
-            key = os.path.join(folder, path, f"{filename}{ext}")
-
-
-        media.key = key
-
-
-    def max_file_length(self) -> int:
-        """
-        Returns the maximum length of a file name that can be stored in the storage service.
-
-        Files are truncated if they exceed this length.
-        Override this method in subclasses if the storage service has a different maximum file length.
-        """
-        return 255 # safe max file length for most filesystems (macOS, Windows, Linux)
 
+        media._key = key
\ No newline at end of file
diff --git a/src/auto_archiver/modules/cli_feeder/cli_feeder.py b/src/auto_archiver/modules/cli_feeder/cli_feeder.py
index 20ca6ae..fac584d 100644
--- a/src/auto_archiver/modules/cli_feeder/cli_feeder.py
+++ b/src/auto_archiver/modules/cli_feeder/cli_feeder.py
@@ -15,7 +15,6 @@ class CLIFeeder(Feeder):
         for url in urls:
             logger.debug(f"Processing {url}")
             m = Metadata().set_url(url)
-            m.set_context("folder", "cli")
             yield m
 
         logger.success(f"Processed {len(urls)} URL(s)")
\ No newline at end of file
diff --git a/src/auto_archiver/modules/local_storage/local_storage.py b/src/auto_archiver/modules/local_storage/local_storage.py
index c07f682..6f1e217 100644
--- a/src/auto_archiver/modules/local_storage/local_storage.py
+++ b/src/auto_archiver/modules/local_storage/local_storage.py
@@ -6,37 +6,34 @@ from loguru import logger
 
 from auto_archiver.core import Media
 from auto_archiver.core import Storage
-
+from auto_archiver.core.consts import SetupError
 
 class LocalStorage(Storage):
 
-    MAX_FILE_LENGTH = 255
+
+    def setup(self) -> None:
+        if len(self.save_to) > 200:
+            raise SetupError(f"Your save_to path is  long, this will cause issues saving files on your computer. Please use a shorter path")
 
     def get_cdn_url(self, media: Media) -> str:
-        # TODO: is this viable with Storage.configs on path/filename?
-        dest = os.path.join(self.save_to, media.key)
+        dest = media.key
+
         if self.save_absolute:
             dest = os.path.abspath(dest)
         return dest
 
+    def set_key(self, media, url, metadata):
+        # clarify we want to save the file to the save_to folder
+
+        old_folder = metadata.get('folder', '')
+        metadata.set_context('folder', os.path.join(self.save_to, metadata.get('folder', '')))
+        super().set_key(media, url, metadata)
+        # don't impact other storages that might want a different 'folder' set
+        metadata.set_context('folder', old_folder)
+
     def upload(self, media: Media, **kwargs) -> bool:
         # override parent so that we can use shutil.copy2 and keep metadata
-        dest = os.path.join(self.save_to, media.key)
-
-        if len(dest) > self.max_file_length():
-            old_dest_length = len(dest)
-            filename, ext = os.path.splitext(media.key)
-            dir, filename = os.path.split(filename)
-            # see whether we should truncate filename or dir
-            if len(dir) > len(filename):
-                dir = dir[:self.MAX_FILE_LENGTH - len(self.save_to) - len(ext) - len(filename) - 1]
-            else:
-                filename = filename[:self.MAX_FILE_LENGTH - len(self.save_to) - len(ext) - len(filename) - 1]
-
-            # override media.key
-            media.key = os.path.join(dir, f"{filename}{ext}")
-            dest = os.path.join(self.save_to, dir, f"{filename}{ext}")
-            logger.warning(f'Filename too long ({old_dest_length} characters), truncating to {len(dest)} characters')
+        dest = media.key
 
         os.makedirs(os.path.dirname(dest), exist_ok=True)
         logger.debug(f'[{self.__class__.__name__}] storing file {media.filename} with key {media.key} to {dest}')
@@ -46,7 +43,5 @@ class LocalStorage(Storage):
         return True
 
     # must be implemented even if unused
-    def uploadf(self, file: IO[bytes], key: str, **kwargs: dict) -> bool: pass
-
-    def max_file_length(self):
-        return self.MAX_FILE_LENGTH
\ No newline at end of file
+    def uploadf(self, file: IO[bytes], key: str, **kwargs: dict) -> bool:
+        pass
\ No newline at end of file
diff --git a/src/auto_archiver/modules/s3_storage/s3_storage.py b/src/auto_archiver/modules/s3_storage/s3_storage.py
index c8569b6..183a944 100644
--- a/src/auto_archiver/modules/s3_storage/s3_storage.py
+++ b/src/auto_archiver/modules/s3_storage/s3_storage.py
@@ -42,7 +42,7 @@ class S3Storage(Storage):
                 logger.warning(f"Unable to get mimetype for {media.key=}, error: {e}")
         self.s3.upload_fileobj(file, Bucket=self.bucket, Key=media.key, ExtraArgs=extra_args)
         return True
-    
+
     def is_upload_needed(self, media: Media) -> bool:
         if self.random_no_duplicate:
             # checks if a folder with the hash already exists, if so it skips the upload
@@ -50,13 +50,13 @@ class S3Storage(Storage):
             path = os.path.join(NO_DUPLICATES_FOLDER, hd[:24])
 
             if existing_key:=self.file_in_folder(path):
-                media.key = existing_key
+                media._key = existing_key
                 media.set("previously archived", True)
                 logger.debug(f"skipping upload of {media.filename} because it already exists in {media.key}")
                 return False
             
             _, ext = os.path.splitext(media.key)
-            media.key = os.path.join(path, f"{random_str(24)}{ext}")
+            media._key = os.path.join(path, f"{random_str(24)}{ext}")
         return True
 
     def file_in_folder(self, path:str) -> str:
@@ -66,9 +66,4 @@ class S3Storage(Storage):
         resp = self.s3.list_objects(Bucket=self.bucket, Prefix=path, Delimiter='/', MaxKeys=1)
         if 'Contents' in resp:
             return resp['Contents'][0]['Key']
-        return False
-    
-    def max_file_length(self):
-        # Amazon AWS max file length is 1024, but we will use 1000 to be safe
-        return 1000
-
+        return False
\ No newline at end of file
diff --git a/tests/enrichers/test_pdq_hash_enricher.py b/tests/enrichers/test_pdq_hash_enricher.py
index a8470fb..61fa778 100644
--- a/tests/enrichers/test_pdq_hash_enricher.py
+++ b/tests/enrichers/test_pdq_hash_enricher.py
@@ -14,8 +14,8 @@ def enricher(setup_module):
 def metadata_with_images():
     m = Metadata()
     m.set_url("https://example.com")
-    m.add_media(Media(filename="image1.jpg", key="image1"))
-    m.add_media(Media(filename="image2.jpg", key="image2"))
+    m.add_media(Media(filename="image1.jpg", _key="image1"))
+    m.add_media(Media(filename="image2.jpg", _key="image2"))
     return m
 
 
diff --git a/tests/storages/test_S3_storage.py b/tests/storages/test_S3_storage.py
index fe60329..abf9763 100644
--- a/tests/storages/test_S3_storage.py
+++ b/tests/storages/test_S3_storage.py
@@ -37,10 +37,10 @@ class TestS3Storage:
     def test_get_cdn_url_generation(self):
         """Test CDN URL formatting """
         media = Media("test.txt")
-        media.key = "path/to/file.txt"
+        media._key = "path/to/file.txt"
         url = self.storage.get_cdn_url(media)
         assert url == "https://cdn.example.com/path/to/file.txt"
-        media.key = "another/path.jpg"
+        media._key = "another/path.jpg"
         assert self.storage.get_cdn_url(media) == "https://cdn.example.com/another/path.jpg"
 
     def test_uploadf_sets_acl_public(self, mocker):
@@ -72,7 +72,7 @@ class TestS3Storage:
         self.storage.random_no_duplicate = True
         mock_file_in_folder = mocker.patch.object(S3Storage, 'file_in_folder', return_value="existing_folder/existing_file.txt")
         media = Media("test.txt")
-        media.key = "original_path.txt"
+        media._key = "original_path.txt"
         mock_calculate_hash = mocker.patch('auto_archiver.modules.s3_storage.s3_storage.calculate_file_hash', return_value="beepboop123beepboop123beepboop123")
         assert self.storage.is_upload_needed(media) is False
         assert media.key == "existing_folder/existing_file.txt"
@@ -84,7 +84,7 @@ class TestS3Storage:
 
     def test_uploads_with_correct_parameters(self, mocker):
         media = Media("test.txt")
-        media.key = "original_key.txt"
+        media._key = "original_key.txt"
         mocker.patch.object(S3Storage, 'is_upload_needed', return_value=True)
         media.mimetype = 'image/png'
         mock_file = mocker.MagicMock()
diff --git a/tests/storages/test_atlos_storage.py b/tests/storages/test_atlos_storage.py
index bcd8f18..d33ff2f 100644
--- a/tests/storages/test_atlos_storage.py
+++ b/tests/storages/test_atlos_storage.py
@@ -44,7 +44,7 @@ def media(tmp_path) -> Media:
     file_path.write_bytes(content)
     media = Media(filename=str(file_path))
     media.properties = {"something": "Title"}
-    media.key = "key"
+    media._key = "key"
     return media
 
 
diff --git a/tests/storages/test_gdrive_storage.py b/tests/storages/test_gdrive_storage.py
index f5ff87c..d87f5e8 100644
--- a/tests/storages/test_gdrive_storage.py
+++ b/tests/storages/test_gdrive_storage.py
@@ -50,7 +50,7 @@ def test_get_id_from_parent_and_name(gdrive_storage, mocker):
 
 def test_path_parts():
     media = Media(filename="test.jpg")
-    media.key = "folder1/folder2/test.jpg"
+    media._key = "folder1/folder2/test.jpg"
 
 
 
diff --git a/tests/storages/test_local_storage.py b/tests/storages/test_local_storage.py
index 7282e03..1fd6774 100644
--- a/tests/storages/test_local_storage.py
+++ b/tests/storages/test_local_storage.py
@@ -4,9 +4,9 @@ from pathlib import Path
 
 import pytest
 
-from auto_archiver.core import Media
+from auto_archiver.core import Media, Metadata
 from auto_archiver.modules.local_storage import LocalStorage
-
+from auto_archiver.core.consts import SetupError
 
 @pytest.fixture
 def local_storage(setup_module, tmp_path) -> LocalStorage:
@@ -25,37 +25,32 @@ def sample_media(tmp_path) -> Media:
     """Fixture creating a Media object with temporary source file"""
     src_file = tmp_path / "source.txt"
     src_file.write_text("test content")
-    return Media(key="subdir/test.txt", filename=str(src_file))
+    return Media(filename=str(src_file))
 
-def test_really_long_website_url_save(local_storage, tmp_path):
-    long_filename = os.path.join(local_storage.save_to, "file"*100 + ".txt")
-    src_file = tmp_path / "source.txt"
-    src_file.write_text("test content")
-    media = Media(key=long_filename, filename=str(src_file))
-    assert local_storage.upload(media) is True
-    assert src_file.read_text() == Path(local_storage.get_cdn_url(media)).read_text()
+def test_too_long_save_path(setup_module):
+    with pytest.raises(SetupError):
+        setup_module("local_storage", {"save_to": "long"*100})
 
 def test_get_cdn_url_relative(local_storage):
-    media = Media(key="test.txt", filename="dummy.txt")
+    media = Media(filename="dummy.txt")
+    local_storage.set_key(media, "https://example.com", Metadata())
     expected = os.path.join(local_storage.save_to, media.key)
     assert local_storage.get_cdn_url(media) == expected
 
-
-
 def test_get_cdn_url_absolute(local_storage):
-    media = Media(key="test.txt", filename="dummy.txt")
+    media = Media(filename="dummy.txt")
     local_storage.save_absolute = True
+    local_storage.set_key(media, "https://example.com", Metadata())
     expected = os.path.abspath(os.path.join(local_storage.save_to, media.key))
     assert local_storage.get_cdn_url(media) == expected
 
 def test_upload_file_contents_and_metadata(local_storage, sample_media):
+    local_storage.store(sample_media, "https://example.com", Metadata())
     dest = os.path.join(local_storage.save_to, sample_media.key)
-    assert local_storage.upload(sample_media) is True
     assert Path(sample_media.filename).read_text() == Path(dest).read_text()
 
-
 def test_upload_nonexistent_source(local_storage):
-    media = Media(key="missing.txt", filename="nonexistent.txt")
+    media = Media(_key="missing.txt", filename="nonexistent.txt")
     with pytest.raises(FileNotFoundError):
         local_storage.upload(media)
 
diff --git a/tests/storages/test_storage_base.py b/tests/storages/test_storage_base.py
index fd07c32..30a91b9 100644
--- a/tests/storages/test_storage_base.py
+++ b/tests/storages/test_storage_base.py
@@ -54,7 +54,7 @@ def storage_base():
 
     ],
 )
-def test_storage_setup(storage_base, path_generator, filename_generator, url, expected_key, mocker):
+def test_storage_name_generation(storage_base, path_generator, filename_generator, url, expected_key, mocker):
     mock_random = mocker.patch("auto_archiver.core.storage.random_str")
     mock_random.return_value = "pretend-random"
 
@@ -92,7 +92,4 @@ def test_really_long_name(storage_base):
     url = f"https://example.com/{'file'*100}"
     media = Media(filename="dummy.txt")
     storage.set_key(media, url, Metadata())
-    assert len(media.key) <= storage.max_file_length()
-    assert media.key == "https-example-com-filefilefilefilefilefilefilefilefilefilefilefile\
-filefilefilefilefilefilefilefilefilefilefilefilefilefilefilefilefilefilefilefilefilefilefile\
-filefilefilefilefilefilefilefilefilefilefilefilefilefilefilefilefile/6ae8a75555209fd6c44157c0.txt"
\ No newline at end of file
+    assert media.key == f"https-example-com-{'file'*13}/6ae8a75555209fd6c44157c0.txt"
\ No newline at end of file

From a9c34772896e1971e74005cfb9bb95b7ef39dd92 Mon Sep 17 00:00:00 2001
From: Patrick Robertson <robertson.patrick@gmail.com>
Date: Mon, 10 Mar 2025 16:43:14 +0000
Subject: [PATCH 10/55] Improve docs on the path_generator and
 filename_generator config options

---
 src/auto_archiver/core/storage.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/src/auto_archiver/core/storage.py b/src/auto_archiver/core/storage.py
index 4867146..63ccf8d 100644
--- a/src/auto_archiver/core/storage.py
+++ b/src/auto_archiver/core/storage.py
@@ -1,5 +1,22 @@
 """
 Base module for Storage modules – modular components that store media objects in various locations.
+
+If you are looking to implement a new storage module, you should subclass the `Storage` class and
+implement the `get_cdn_url` and `uploadf` methods.
+
+Your module **must** also have two config variables 'path_generator' and 'filename_generator' which
+determine how the key is generated for the media object. The 'path_generator' and 'filename_generator'
+variables can be set to one of the following values:
+- 'flat': A flat structure with no subfolders
+- 'url': A structure based on the URL of the media object
+- 'random': A random structure
+
+The 'filename_generator' variable can be set to one of the following values:
+- 'random': A random string
+- 'static': A replicable strategy such as a hash
+
+If you don't want to use this naming convention, you can override the `set_key` method in your subclass.
+
 """
 
 from __future__ import annotations

From 2b91dc95146a860e55f9b9003d0969527431f473 Mon Sep 17 00:00:00 2001
From: Patrick Robertson <robertson.patrick@gmail.com>
Date: Mon, 10 Mar 2025 16:51:16 +0000
Subject: [PATCH 11/55] Fix up unit tests

---
 tests/storages/test_local_storage.py |  3 +++
 tests/storages/test_storage_base.py  | 24 ++++++++++++------------
 2 files changed, 15 insertions(+), 12 deletions(-)

diff --git a/tests/storages/test_local_storage.py b/tests/storages/test_local_storage.py
index 1fd6774..c3581df 100644
--- a/tests/storages/test_local_storage.py
+++ b/tests/storages/test_local_storage.py
@@ -32,12 +32,15 @@ def test_too_long_save_path(setup_module):
         setup_module("local_storage", {"save_to": "long"*100})
 
 def test_get_cdn_url_relative(local_storage):
+    local_storage.filename_generator = "random"
     media = Media(filename="dummy.txt")
     local_storage.set_key(media, "https://example.com", Metadata())
     expected = os.path.join(local_storage.save_to, media.key)
     assert local_storage.get_cdn_url(media) == expected
 
 def test_get_cdn_url_absolute(local_storage):
+    local_storage.filename_generator = "random"
+
     media = Media(filename="dummy.txt")
     local_storage.save_absolute = True
     local_storage.set_key(media, "https://example.com", Metadata())
diff --git a/tests/storages/test_storage_base.py b/tests/storages/test_storage_base.py
index 30a91b9..53dfbd7 100644
--- a/tests/storages/test_storage_base.py
+++ b/tests/storages/test_storage_base.py
@@ -32,6 +32,13 @@ class TestBaseStorage(Storage):
     def uploadf(self, file, key, **kwargs):
         return True
 
+@pytest.fixture
+def dummy_file(tmp_path):
+    # create dummy.txt file
+    dummy_file = tmp_path / "dummy.txt"
+    dummy_file.write_text("test content")
+    return str(dummy_file)
+
 @pytest.fixture
 def storage_base():
     def _storage_base(config):
@@ -54,14 +61,11 @@ def storage_base():
 
     ],
 )
-def test_storage_name_generation(storage_base, path_generator, filename_generator, url, expected_key, mocker):
+def test_storage_name_generation(storage_base, path_generator, filename_generator, url, 
+                                 expected_key, mocker, tmp_path, dummy_file):
     mock_random = mocker.patch("auto_archiver.core.storage.random_str")
     mock_random.return_value = "pretend-random"
 
-    # create dummy.txt file
-    with open("dummy.txt", "w") as f:
-        f.write("test content")
-
     config: dict = {
         "path_generator": path_generator,
         "filename_generator": filename_generator,
@@ -72,24 +76,20 @@ def test_storage_name_generation(storage_base, path_generator, filename_generato
 
     metadata = Metadata()
     metadata.set_context("folder", "folder")
-    media = Media(filename="dummy.txt")
+    media = Media(filename=dummy_file)
     storage.set_key(media, url, metadata)
     print(media.key)
     assert media.key == expected_key
 
 
-def test_really_long_name(storage_base):
+def test_really_long_name(storage_base, dummy_file):
     config: dict = {
         "path_generator": "url",
         "filename_generator": "static",
     }
     storage: Storage = storage_base(config)
 
-    # create dummy.txt file
-    with open("dummy.txt", "w") as f:
-        f.write("test content")
-
     url = f"https://example.com/{'file'*100}"
-    media = Media(filename="dummy.txt")
+    media = Media(filename=dummy_file)
     storage.set_key(media, url, Metadata())
     assert media.key == f"https-example-com-{'file'*13}/6ae8a75555209fd6c44157c0.txt"
\ No newline at end of file

From 3fcec57492b9383535b1bc5960ba9e3d40d1e030 Mon Sep 17 00:00:00 2001
From: Miguel Sozinho Ramalho <19508417+msramalho@users.noreply.github.com>
Date: Mon, 10 Mar 2025 17:17:59 +0000
Subject: [PATCH 12/55] minor string fix

---
 src/auto_archiver/modules/local_storage/local_storage.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/auto_archiver/modules/local_storage/local_storage.py b/src/auto_archiver/modules/local_storage/local_storage.py
index 6f1e217..2b1a101 100644
--- a/src/auto_archiver/modules/local_storage/local_storage.py
+++ b/src/auto_archiver/modules/local_storage/local_storage.py
@@ -13,7 +13,7 @@ class LocalStorage(Storage):
 
     def setup(self) -> None:
         if len(self.save_to) > 200:
-            raise SetupError(f"Your save_to path is  long, this will cause issues saving files on your computer. Please use a shorter path")
+            raise SetupError(f"Your save_to path is too long, this will cause issues saving files on your computer. Please use a shorter path.")
 
     def get_cdn_url(self, media: Media) -> str:
         dest = media.key

From 85abe1837a3c876edad9762d8e8766678a20eaea Mon Sep 17 00:00:00 2001
From: erinhmclark <erinhannahmary.clark@gmail.com>
Date: Mon, 10 Mar 2025 18:44:54 +0000
Subject: [PATCH 13/55] Ruff format with defaults.

---
 .github/workflows/ruff.yaml                   |  22 ++
 docs/scripts/__init__.py                      |   2 +-
 docs/scripts/scripts.py                       |  67 ++--
 docs/source/conf.py                           |  47 +--
 pyproject.toml                                |   3 +
 scripts/create_update_gdrive_oauth_token.py   |   6 +-
 scripts/generate_settings_schema.py           |  46 +--
 scripts/telegram_setup.py                     |   2 -
 src/auto_archiver/__main__.py                 |   8 +-
 src/auto_archiver/core/__init__.py            |   5 +-
 src/auto_archiver/core/base_module.py         |  49 ++-
 src/auto_archiver/core/config.py              |  54 ++--
 src/auto_archiver/core/consts.py              |  30 +-
 src/auto_archiver/core/database.py            |   7 +-
 src/auto_archiver/core/enricher.py            |   4 +-
 src/auto_archiver/core/extractor.py           |  28 +-
 src/auto_archiver/core/feeder.py              |   8 +-
 src/auto_archiver/core/formatter.py           |   4 +-
 src/auto_archiver/core/media.py               |  16 +-
 src/auto_archiver/core/metadata.py            |  80 +++--
 src/auto_archiver/core/module.py              |  97 +++---
 src/auto_archiver/core/orchestrator.py        | 303 ++++++++++++------
 src/auto_archiver/core/storage.py             |  15 +-
 src/auto_archiver/core/validators.py          |   5 +-
 src/auto_archiver/modules/api_db/__init__.py  |   2 +-
 .../modules/api_db/__manifest__.py            |   3 +-
 src/auto_archiver/modules/api_db/api_db.py    |  26 +-
 .../atlos_feeder_db_storage/__init__.py       |   2 +-
 .../atlos_feeder_db_storage/__manifest__.py   |   6 +-
 .../atlos_feeder_db_storage.py                |  16 +-
 .../modules/cli_feeder/__manifest__.py        |  22 +-
 .../modules/cli_feeder/cli_feeder.py          |  12 +-
 .../modules/console_db/__init__.py            |   2 +-
 .../modules/console_db/console_db.py          |   8 +-
 src/auto_archiver/modules/csv_db/__init__.py  |   2 +-
 .../modules/csv_db/__manifest__.py            |   9 +-
 src/auto_archiver/modules/csv_db/csv_db.py    |   7 +-
 .../modules/csv_feeder/__init__.py            |   2 +-
 .../modules/csv_feeder/__manifest__.py        |  33 +-
 .../modules/csv_feeder/csv_feeder.py          |  11 +-
 .../modules/gdrive_storage/__init__.py        |   2 +-
 .../modules/gdrive_storage/__manifest__.py    |  19 +-
 .../modules/gdrive_storage/gdrive_storage.py  | 103 +++---
 .../modules/generic_extractor/__init__.py     |   2 +-
 .../modules/generic_extractor/bluesky.py      |  16 +-
 .../modules/generic_extractor/dropin.py       |  13 +-
 .../modules/generic_extractor/facebook.py     |  13 +-
 .../generic_extractor/generic_extractor.py    | 228 ++++++++-----
 .../modules/generic_extractor/truth.py        |  41 ++-
 .../modules/generic_extractor/twitter.py      |  36 +--
 .../modules/gsheet_feeder_db/__init__.py      |   2 +-
 .../modules/gsheet_feeder_db/__manifest__.py  |   6 +-
 .../gsheet_feeder_db/gsheet_feeder_db.py      |  40 +--
 .../modules/gsheet_feeder_db/gworksheet.py    |  47 ++-
 .../modules/hash_enricher/__init__.py         |   2 +-
 .../modules/hash_enricher/__manifest__.py     |  15 +-
 .../modules/hash_enricher/hash_enricher.py    |   7 +-
 .../modules/html_formatter/__init__.py        |   2 +-
 .../modules/html_formatter/__manifest__.py    |  13 +-
 .../modules/html_formatter/html_formatter.py  |  15 +-
 .../instagram_api_extractor/__manifest__.py   |  20 +-
 .../instagram_api_extractor.py                |  85 ++---
 .../modules/instagram_extractor/__init__.py   |   2 +-
 .../instagram_extractor/__manifest__.py       |   3 +-
 .../instagram_extractor.py                    |  67 ++--
 .../instagram_tbot_extractor/__manifest__.py  |  21 +-
 .../instagram_tbot_extractor.py               |  15 +-
 .../modules/local_storage/__init__.py         |   2 +-
 .../modules/local_storage/__manifest__.py     |  10 +-
 .../modules/local_storage/local_storage.py    |   7 +-
 .../modules/meta_enricher/__manifest__.py     |   2 +-
 .../modules/meta_enricher/meta_enricher.py    |   7 +-
 .../modules/metadata_enricher/__init__.py     |   2 +-
 .../modules/metadata_enricher/__manifest__.py |   7 +-
 .../metadata_enricher/metadata_enricher.py    |   5 +-
 .../modules/mute_formatter/__manifest__.py    |   3 +-
 .../modules/mute_formatter/mute_formatter.py  |   4 +-
 .../modules/pdq_hash_enricher/__init__.py     |   2 +-
 .../modules/pdq_hash_enricher/__manifest__.py |   2 +-
 .../pdq_hash_enricher/pdq_hash_enricher.py    |  12 +-
 .../modules/s3_storage/__init__.py            |   2 +-
 .../modules/s3_storage/__manifest__.py        |  22 +-
 .../modules/s3_storage/s3_storage.py          |  41 +--
 .../screenshot_enricher/__manifest__.py       |  43 +--
 .../screenshot_enricher.py                    |  13 +-
 .../modules/ssl_enricher/__init__.py          |   2 +-
 .../modules/ssl_enricher/__manifest__.py      |  12 +-
 .../modules/ssl_enricher/ssl_enricher.py      |  10 +-
 .../modules/telegram_extractor/__init__.py    |   2 +-
 .../telegram_extractor/telegram_extractor.py  |  25 +-
 .../modules/telethon_extractor/__init__.py    |   2 +-
 .../telethon_extractor/__manifest__.py        |  47 +--
 .../telethon_extractor/telethon_extractor.py  |  47 ++-
 .../thumbnail_enricher/__manifest__.py        |  23 +-
 .../thumbnail_enricher/thumbnail_enricher.py  |  15 +-
 .../timestamping_enricher/__manifest__.py     |  49 ++-
 .../timestamping_enricher.py                  |  23 +-
 .../modules/twitter_api_extractor/__init__.py |   2 +-
 .../twitter_api_extractor/__manifest__.py     |  34 +-
 .../twitter_api_extractor.py                  |  71 ++--
 .../modules/vk_extractor/__manifest__.py      |   6 +-
 .../modules/vk_extractor/vk_extractor.py      |   8 +-
 .../wacz_extractor_enricher/__manifest__.py   |  54 ++--
 .../wacz_extractor_enricher.py                | 110 ++++---
 .../wayback_extractor_enricher/__init__.py    |   2 +-
 .../wayback_extractor_enricher.py             |  33 +-
 .../modules/whisper_enricher/__init__.py      |   2 +-
 .../modules/whisper_enricher/__manifest__.py  |  35 +-
 .../whisper_enricher/whisper_enricher.py      |  85 +++--
 src/auto_archiver/utils/__init__.py           |   5 +-
 src/auto_archiver/utils/misc.py               |  35 +-
 src/auto_archiver/utils/url.py                |  71 ++--
 src/auto_archiver/utils/webdriver.py          |  87 +++--
 src/auto_archiver/version.py                  |   7 +-
 tests/conftest.py                             |  30 +-
 tests/data/dropin.py                          |   3 +-
 .../test_modules/example_module/__init__.py   |   2 +-
 .../example_module/__manifest__.py            |  12 +-
 .../example_module/example_module.py          |   7 +-
 tests/databases/test_api_db.py                |  30 +-
 tests/databases/test_atlos_db.py              |   4 +-
 tests/databases/test_csv_db.py                |  16 +-
 tests/databases/test_gsheet_db.py             |  65 ++--
 tests/enrichers/test_hash_enricher.py         |  36 ++-
 tests/enrichers/test_meta_enricher.py         |   5 +-
 tests/enrichers/test_metadata_enricher.py     |   6 +-
 tests/enrichers/test_pdq_hash_enricher.py     |   3 +-
 tests/enrichers/test_screenshot_enricher.py   |  22 +-
 tests/enrichers/test_ssl_enricher.py          |   1 -
 tests/enrichers/test_thumbnail_enricher.py    |  74 +++--
 tests/enrichers/test_wayback_enricher.py      |  72 ++---
 tests/enrichers/test_whisper_enricher.py      |  27 +-
 tests/extractors/test_extractor_base.py       |   3 +-
 tests/extractors/test_generic_extractor.py    | 153 +++++----
 .../test_instagram_api_extractor.py           |  94 +++---
 tests/extractors/test_instagram_extractor.py  |  29 +-
 .../test_instagram_tbot_extractor.py          |  50 +--
 .../extractors/test_twitter_api_extractor.py  | 126 +++++---
 tests/extractors/test_vk_extractor.py         |  15 +-
 tests/feeders/test_atlos_feeder.py            |  94 ++++--
 tests/feeders/test_csv_feeder.py              |   8 +-
 tests/feeders/test_gsheet_feeder.py           |  69 ++--
 tests/feeders/test_gworksheet.py              |  23 +-
 tests/formatters/test_html_formatter.py       |   6 +-
 tests/storages/test_S3_storage.py             |  56 ++--
 tests/storages/test_atlos_storage.py          |   5 +-
 tests/storages/test_gdrive_storage.py         |  50 +--
 tests/storages/test_local_storage.py          |   5 +-
 tests/storages/test_storage_base.py           |   9 +-
 tests/test_config.py                          |  32 +-
 tests/test_implementation.py                  |  13 +-
 tests/test_metadata.py                        |  16 +-
 tests/test_modules.py                         |  12 +-
 tests/test_orchestrator.py                    |  92 ++++--
 tests/utils/test_misc.py                      |  76 +++--
 155 files changed, 2539 insertions(+), 1908 deletions(-)
 create mode 100644 .github/workflows/ruff.yaml

diff --git a/.github/workflows/ruff.yaml b/.github/workflows/ruff.yaml
new file mode 100644
index 0000000..50407e5
--- /dev/null
+++ b/.github/workflows/ruff.yaml
@@ -0,0 +1,22 @@
+name: Ruff Formatting & Linting
+
+on: [push, pull_request]
+
+jobs:
+  ruff:
+    name: Run Ruff Checks
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Run Ruff (Lint & Format Check)
+        uses: astral-sh/ruff-action@v1
+        with:
+          args: "check . --output-format=concise"
+
+      - name: Run Ruff Format Check
+        uses: astral-sh/ruff-action@v1
+        with:
+          args: "format --check ."
diff --git a/docs/scripts/__init__.py b/docs/scripts/__init__.py
index ba9737c..b76d0fe 100644
--- a/docs/scripts/__init__.py
+++ b/docs/scripts/__init__.py
@@ -1 +1 @@
-from scripts import generate_module_docs
\ No newline at end of file
+from scripts import generate_module_docs
diff --git a/docs/scripts/scripts.py b/docs/scripts/scripts.py
index 66ba14d..ca1d348 100644
--- a/docs/scripts/scripts.py
+++ b/docs/scripts/scripts.py
@@ -10,12 +10,12 @@ MODULES_FOLDER = Path(__file__).parent.parent.parent.parent / "src" / "auto_arch
 SAVE_FOLDER = Path(__file__).parent.parent / "source" / "modules" / "autogen"
 
 type_color = {
-    'feeder': "<span style='color: #FFA500'>[feeder](/core_modules.md#feeder-modules)</a></span>",
-    'extractor': "<span style='color: #00FF00'>[extractor](/core_modules.md#extractor-modules)</a></span>",
-    'enricher': "<span style='color: #0000FF'>[enricher](/core_modules.md#enricher-modules)</a></span>",
-    'database': "<span style='color: #FF00FF'>[database](/core_modules.md#database-modules)</a></span>",
-    'storage': "<span style='color: #FFFF00'>[storage](/core_modules.md#storage-modules)</a></span>",
-    'formatter': "<span style='color: #00FFFF'>[formatter](/core_modules.md#formatter-modules)</a></span>",
+    "feeder": "<span style='color: #FFA500'>[feeder](/core_modules.md#feeder-modules)</a></span>",
+    "extractor": "<span style='color: #00FF00'>[extractor](/core_modules.md#extractor-modules)</a></span>",
+    "enricher": "<span style='color: #0000FF'>[enricher](/core_modules.md#enricher-modules)</a></span>",
+    "database": "<span style='color: #FF00FF'>[database](/core_modules.md#database-modules)</a></span>",
+    "storage": "<span style='color: #FFFF00'>[storage](/core_modules.md#storage-modules)</a></span>",
+    "formatter": "<span style='color: #00FFFF'>[formatter](/core_modules.md#formatter-modules)</a></span>",
 }
 
 TABLE_HEADER = ("Option", "Description", "Default", "Type")
@@ -34,6 +34,7 @@ steps:
 
 """
 
+
 def generate_module_docs():
     yaml = YAML()
     SAVE_FOLDER.mkdir(exist_ok=True)
@@ -48,49 +49,49 @@ def generate_module_docs():
         # generate the markdown file from the __manifest__.py file.
 
         manifest = module.manifest
-        for type in manifest['type']:
+        for type in manifest["type"]:
             modules_by_type.setdefault(type, []).append(module)
 
-        description = "\n".join(l.lstrip() for l in manifest['description'].split("\n"))
-        types = ", ".join(type_color[t] for t in manifest['type'])
+        description = "\n".join(l.lstrip() for l in manifest["description"].split("\n"))
+        types = ", ".join(type_color[t] for t in manifest["type"])
         readme_str = f"""
-# {manifest['name']}
+# {manifest["name"]}
 ```{{admonition}} Module type
 
 {types}
 ```
 {description}
-"""     
-        steps_str = "\n".join(f"  {t}s:\n  - {module.name}" for t in manifest['type'])
+"""
+        steps_str = "\n".join(f"  {t}s:\n  - {module.name}" for t in manifest["type"])
 
-        if not manifest['configs']:
+        if not manifest["configs"]:
             config_string = f"# No configuration options for {module.name}.*\n"
         else:
-
             config_table = header_row
             config_yaml = {}
 
             global_yaml[module.name] = CommentedMap()
-            global_yaml.yaml_set_comment_before_after_key(module.name, f"\n\n{module.display_name} configuration options")
+            global_yaml.yaml_set_comment_before_after_key(
+                module.name, f"\n\n{module.display_name} configuration options"
+            )
 
-
-            for key, value in manifest['configs'].items():
-                type = value.get('type', 'string')
-                if type == 'json_loader':
-                    value['type'] = 'json'
-                elif type == 'str':
+            for key, value in manifest["configs"].items():
+                type = value.get("type", "string")
+                if type == "json_loader":
+                    value["type"] = "json"
+                elif type == "str":
                     type = "string"
-                
-                default = value.get('default', '')
+
+                default = value.get("default", "")
                 config_yaml[key] = default
 
                 global_yaml[module.name][key] = default
 
-                if value.get('help', ''):
-                    global_yaml[module.name].yaml_add_eol_comment(value.get('help', ''), key)
+                if value.get("help", ""):
+                    global_yaml[module.name].yaml_add_eol_comment(value.get("help", ""), key)
 
-                help = "**Required**. " if value.get('required', False) else "Optional. "
-                help += value.get('help', '')
+                help = "**Required**. " if value.get("required", False) else "Optional. "
+                help += value.get("help", "")
                 config_table += f"| `{module.name}.{key}` | {help} | {value.get('default', '')} | {type} |\n"
                 global_table += f"| `{module.name}.{key}` | {help} | {default} | {type} |\n"
             readme_str += "\n## Configuration Options\n"
@@ -98,18 +99,18 @@ def generate_module_docs():
 
             config_string = io.BytesIO()
             yaml.dump({module.name: config_yaml}, config_string)
-            config_string = config_string.getvalue().decode('utf-8')
+            config_string = config_string.getvalue().decode("utf-8")
         yaml_string = EXAMPLE_YAML.format(steps_str=steps_str, config_string=config_string)
         readme_str += f"```{{code}} yaml\n{yaml_string}\n```\n"
 
-        if manifest['configs']:
+        if manifest["configs"]:
             readme_str += "\n### Command Line:\n"
             readme_str += config_table
 
         # add a link to the autodoc refs
         readme_str += f"\n[API Reference](../../../autoapi/{module.name}/index)\n"
         # create the module.type folder, use the first type just for where to store the file
-        for type in manifest['type']:
+        for type in manifest["type"]:
             type_folder = SAVE_FOLDER / type
             type_folder.mkdir(exist_ok=True)
             with open(type_folder / f"{module.name}.md", "w") as f:
@@ -117,10 +118,10 @@ def generate_module_docs():
                 f.write(readme_str)
         generate_index(modules_by_type)
 
-    del global_yaml['placeholder']
+    del global_yaml["placeholder"]
     global_string = io.BytesIO()
     global_yaml = yaml.dump(global_yaml, global_string)
-    global_string = global_string.getvalue().decode('utf-8')
+    global_string = global_string.getvalue().decode("utf-8")
     global_yaml = f"```yaml\n{global_string}\n```"
     with open(SAVE_FOLDER / "configs_cheatsheet.md", "w") as f:
         f.write("### Configuration File\n" + global_yaml + "\n### Command Line\n" + global_table)
@@ -144,4 +145,4 @@ def generate_index(modules_by_type):
 
 
 if __name__ == "__main__":
-    generate_module_docs()
\ No newline at end of file
+    generate_module_docs()
diff --git a/docs/source/conf.py b/docs/source/conf.py
index ee6416e..8cfbd30 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -5,7 +5,7 @@ import os
 from importlib.metadata import metadata
 from datetime import datetime
 
-sys.path.append(os.path.abspath('../scripts'))
+sys.path.append(os.path.abspath("../scripts"))
 from scripts import generate_module_docs
 from auto_archiver.version import __version__
 
@@ -20,33 +20,35 @@ project = package_metadata["name"]
 copyright = str(datetime.now().year)
 author = "Bellingcat"
 release = package_metadata["version"]
-language = 'en'
+language = "en"
 
 # -- General configuration ---------------------------------------------------
 extensions = [
-    "myst_parser",                  # Markdown support
-    "autoapi.extension",            # Generate API documentation from docstrings
-    "sphinxcontrib.mermaid",        # Mermaid diagrams
-    "sphinx.ext.viewcode",          # Source code links
+    "myst_parser",  # Markdown support
+    "autoapi.extension",  # Generate API documentation from docstrings
+    "sphinxcontrib.mermaid",  # Mermaid diagrams
+    "sphinx.ext.viewcode",  # Source code links
     "sphinx_copybutton",
-    "sphinx.ext.napoleon",          # Google-style and NumPy-style docstrings
+    "sphinx.ext.napoleon",  # Google-style and NumPy-style docstrings
     "sphinx.ext.autosectionlabel",
     # 'sphinx.ext.autosummary',       # Summarize module/class/function docs
 ]
 
-templates_path = ['_templates']
+templates_path = ["_templates"]
 exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", ""]
 
 
 # -- AutoAPI Configuration ---------------------------------------------------
-autoapi_type = 'python'
+autoapi_type = "python"
 autoapi_dirs = ["../../src/auto_archiver/core/", "../../src/auto_archiver/utils/"]
 # get all the modules and add them to the autoapi_dirs
 autoapi_dirs.extend([f"../../src/auto_archiver/modules/{m}" for m in os.listdir("../../src/auto_archiver/modules")])
-autodoc_typehints = "signature"     # Include type hints in the signature
-autoapi_ignore = ["*/version.py", ]                 # Ignore specific modules
-autoapi_keep_files = True          # Option to retain intermediate JSON files for debugging
-autoapi_add_toctree_entry = True    # Include API docs in the TOC
+autodoc_typehints = "signature"  # Include type hints in the signature
+autoapi_ignore = [
+    "*/version.py",
+]  # Ignore specific modules
+autoapi_keep_files = True  # Option to retain intermediate JSON files for debugging
+autoapi_add_toctree_entry = True  # Include API docs in the TOC
 autoapi_python_use_implicit_namespaces = True
 autoapi_template_dir = "../_templates/autoapi"
 autoapi_options = [
@@ -59,13 +61,13 @@ autoapi_options = [
 
 # -- Markdown Support --------------------------------------------------------
 myst_enable_extensions = [
-    "deflist",              # Definition lists
-    "html_admonition",      # HTML-style admonitions
-    "html_image",           # Inline HTML images
-    "replacements",         # Substitutions like (C)
-    "smartquotes",          # Smart quotes
-    "linkify",              # Auto-detect links
-    "substitution",         # Text substitutions
+    "deflist",  # Definition lists
+    "html_admonition",  # HTML-style admonitions
+    "html_image",  # Inline HTML images
+    "replacements",  # Substitutions like (C)
+    "smartquotes",  # Smart quotes
+    "linkify",  # Auto-detect links
+    "substitution",  # Text substitutions
 ]
 myst_heading_anchors = 2
 myst_fence_as_directive = ["mermaid"]
@@ -76,7 +78,7 @@ source_suffix = {
 }
 
 # -- Options for HTML output -------------------------------------------------
-html_theme = 'sphinx_book_theme'
+html_theme = "sphinx_book_theme"
 html_static_path = ["../_static"]
 html_css_files = ["custom.css"]
 html_title = f"Auto Archiver v{__version__}"
@@ -87,7 +89,6 @@ html_theme_options = {
 }
 
 
-
 copybutton_prompt_text = r">>> |\.\.\."
 copybutton_prompt_is_regexp = True
-copybutton_only_copy_prompt_lines = False
\ No newline at end of file
+copybutton_only_copy_prompt_lines = False
diff --git a/pyproject.toml b/pyproject.toml
index 18f7397..44cf89b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -98,6 +98,9 @@ line-length = 120
 
 [tool.ruff.lint]
 #add bugbear?
+# I : isort
+# UP : upgrade, e.g. use fstrings
+# ANN : annotations
 #extend-select = ["B"]
 
 # E701 - multiple statements on one line (I vote to keep this but I notice it's used quite a lot!)
diff --git a/scripts/create_update_gdrive_oauth_token.py b/scripts/create_update_gdrive_oauth_token.py
index eb6fdbe..a57043e 100644
--- a/scripts/create_update_gdrive_oauth_token.py
+++ b/scripts/create_update_gdrive_oauth_token.py
@@ -70,11 +70,7 @@ def main(credentials, token):
         print(emailAddress)
 
         # Call the Drive v3 API and return some files
-        results = (
-            service.files()
-            .list(pageSize=10, fields="nextPageToken, files(id, name)")
-            .execute()
-        )
+        results = service.files().list(pageSize=10, fields="nextPageToken, files(id, name)").execute()
         items = results.get("files", [])
 
         if not items:
diff --git a/scripts/generate_settings_schema.py b/scripts/generate_settings_schema.py
index 16cb22f..fa7aaf6 100644
--- a/scripts/generate_settings_schema.py
+++ b/scripts/generate_settings_schema.py
@@ -8,12 +8,14 @@ from auto_archiver.core.module import ModuleFactory
 from auto_archiver.core.consts import MODULE_TYPES
 from auto_archiver.core.config import EMPTY_CONFIG
 
+
 class SchemaEncoder(json.JSONEncoder):
     def default(self, obj):
         if isinstance(obj, set):
             return list(obj)
         return json.JSONEncoder.default(self, obj)
 
+
 # Get available modules
 module_factory = ModuleFactory()
 available_modules = module_factory.available_modules()
@@ -21,32 +23,40 @@ available_modules = module_factory.available_modules()
 modules_by_type = {}
 # Categorize modules by type
 for module in available_modules:
-    for type in module.manifest.get('type', []):
+    for type in module.manifest.get("type", []):
         modules_by_type.setdefault(type, []).append(module)
 
-all_modules_ordered_by_type = sorted(available_modules, key=lambda x: (MODULE_TYPES.index(x.type[0]), not x.requires_setup))
+all_modules_ordered_by_type = sorted(
+    available_modules, key=lambda x: (MODULE_TYPES.index(x.type[0]), not x.requires_setup)
+)
 
 yaml: YAML = YAML()
 
 config_string = io.BytesIO()
 yaml.dump(EMPTY_CONFIG, config_string)
-config_string = config_string.getvalue().decode('utf-8')
+config_string = config_string.getvalue().decode("utf-8")
 output_schema = {
-    'modules': dict((module.name, 
-                     {
-                         'name': module.name,
-                         'display_name': module.display_name,
-                         'manifest': module.manifest,
-                         'configs': module.configs or None
-                         }
-                         ) for module in all_modules_ordered_by_type),
-    'steps': dict((f"{module_type}s", [module.name for module in modules_by_type[module_type]]) for module_type in MODULE_TYPES),
-    'configs': [m.name for m in all_modules_ordered_by_type if m.configs],
-    'module_types': MODULE_TYPES,
-    'empty_config': config_string
+    "modules": dict(
+        (
+            module.name,
+            {
+                "name": module.name,
+                "display_name": module.display_name,
+                "manifest": module.manifest,
+                "configs": module.configs or None,
+            },
+        )
+        for module in all_modules_ordered_by_type
+    ),
+    "steps": dict(
+        (f"{module_type}s", [module.name for module in modules_by_type[module_type]]) for module_type in MODULE_TYPES
+    ),
+    "configs": [m.name for m in all_modules_ordered_by_type if m.configs],
+    "module_types": MODULE_TYPES,
+    "empty_config": config_string,
 }
 
 current_file_dir = os.path.dirname(os.path.abspath(__file__))
-output_file = os.path.join(current_file_dir, 'settings/src/schema.json')
-with open(output_file, 'w') as file:
-    json.dump(output_schema, file, indent=4, cls=SchemaEncoder)
\ No newline at end of file
+output_file = os.path.join(current_file_dir, "settings/src/schema.json")
+with open(output_file, "w") as file:
+    json.dump(output_schema, file, indent=4, cls=SchemaEncoder)
diff --git a/scripts/telegram_setup.py b/scripts/telegram_setup.py
index e6fa43c..9480cd8 100644
--- a/scripts/telegram_setup.py
+++ b/scripts/telegram_setup.py
@@ -12,7 +12,6 @@ Then run this script to create a new session file.
 You will need to provide your phone number and a 2FA code the first time you run this script.
 """
 
-
 import os
 from telethon.sync import TelegramClient
 from loguru import logger
@@ -26,4 +25,3 @@ SESSION_FILE = "secrets/anon-insta"
 os.makedirs("secrets", exist_ok=True)
 with TelegramClient(SESSION_FILE, API_ID, API_HASH) as client:
     logger.success(f"New session file created: {SESSION_FILE}.session")
-
diff --git a/src/auto_archiver/__main__.py b/src/auto_archiver/__main__.py
index f901d21..615486e 100644
--- a/src/auto_archiver/__main__.py
+++ b/src/auto_archiver/__main__.py
@@ -1,9 +1,13 @@
-""" Entry point for the auto_archiver package. """
+"""Entry point for the auto_archiver package."""
+
 from auto_archiver.core.orchestrator import ArchivingOrchestrator
 import sys
 
+
 def main():
-    for _ in ArchivingOrchestrator()._command_line_run(sys.argv[1:]): pass
+    for _ in ArchivingOrchestrator()._command_line_run(sys.argv[1:]):
+        pass
+
 
 if __name__ == "__main__":
     main()
diff --git a/src/auto_archiver/core/__init__.py b/src/auto_archiver/core/__init__.py
index 78d9a3d..ef1ec57 100644
--- a/src/auto_archiver/core/__init__.py
+++ b/src/auto_archiver/core/__init__.py
@@ -1,6 +1,5 @@
-""" Core modules to handle things such as orchestration, metadata and configs..
+"""Core modules to handle things such as orchestration, metadata and configs.."""
 
-"""
 from .metadata import Metadata
 from .media import Media
 from .base_module import BaseModule
@@ -14,4 +13,4 @@ from .enricher import Enricher
 from .feeder import Feeder
 from .storage import Storage
 from .extractor import Extractor
-from .formatter import Formatter
\ No newline at end of file
+from .formatter import Formatter
diff --git a/src/auto_archiver/core/base_module.py b/src/auto_archiver/core/base_module.py
index d6e4455..d809e59 100644
--- a/src/auto_archiver/core/base_module.py
+++ b/src/auto_archiver/core/base_module.py
@@ -1,7 +1,6 @@
-
 from __future__ import annotations
 
-from typing import  Mapping, Any, Type, TYPE_CHECKING
+from typing import Mapping, Any, Type, TYPE_CHECKING
 from abc import ABC
 from copy import deepcopy, copy
 from tempfile import TemporaryDirectory
@@ -13,8 +12,8 @@ from loguru import logger
 if TYPE_CHECKING:
     from .module import ModuleFactory
 
-class BaseModule(ABC):
 
+class BaseModule(ABC):
     """
     Base module class. All modules should inherit from this class.
 
@@ -46,14 +45,13 @@ class BaseModule(ABC):
 
     @property
     def storages(self) -> list:
-        return self.config.get('storages', [])
+        return self.config.get("storages", [])
 
     def config_setup(self, config: dict):
-
         # this is important. Each instance is given its own deepcopied config, so modules cannot
         # change values to affect other modules
         config = deepcopy(config)
-        authentication = deepcopy(config.pop('authentication', {}))
+        authentication = deepcopy(config.pop("authentication", {}))
 
         self.authentication = authentication
         self.config = config
@@ -68,7 +66,7 @@ class BaseModule(ABC):
         """
         Returns the authentication information for a given site. This is used to authenticate
         with a site before extracting data. The site should be the domain of the site, e.g. 'twitter.com'
-        
+
         :param site: the domain of the site to get authentication information for
         :param extract_cookies: whether or not to extract cookies from the given browser/file and return the cookie jar (disabling can speed up processing if you don't actually need the cookies jar).
 
@@ -94,7 +92,6 @@ class BaseModule(ABC):
         # add the 'www' version of the site to the list of sites to check
         authdict = {}
 
-
         for to_try in [site, f"www.{site}"]:
             if to_try in self.authentication:
                 authdict.update(self.authentication[to_try])
@@ -104,17 +101,20 @@ class BaseModule(ABC):
         if not authdict:
             for key in self.authentication.keys():
                 if key in site or site in key:
-                    logger.debug(f"Could not find exact authentication information for site '{site}'. \
+                    logger.debug(
+                        f"Could not find exact authentication information for site '{site}'. \
 did find information for '{key}' which is close, is this what you meant? \
-If so, edit your authentication settings to make sure it exactly matches.")
+If so, edit your authentication settings to make sure it exactly matches."
+                    )
 
         def get_ytdlp_cookiejar(args):
             import yt_dlp
             from yt_dlp import parse_options
+
             logger.debug(f"Extracting cookies from settings: {args[1]}")
             # parse_options returns a named tuple as follows, we only need the ydl_options part
             # collections.namedtuple('ParsedOptions', ('parser', 'options', 'urls', 'ydl_opts'))
-            ytdlp_opts = getattr(parse_options(args), 'ydl_opts')
+            ytdlp_opts = getattr(parse_options(args), "ydl_opts")
             return yt_dlp.YoutubeDL(ytdlp_opts).cookiejar
 
         get_cookiejar_options = None
@@ -125,22 +125,21 @@ If so, edit your authentication settings to make sure it exactly matches.")
         # 3. cookies_from_browser setting in global config
         # 4. cookies_file setting in global config
 
-        if 'cookies_from_browser' in authdict:
-            get_cookiejar_options = ['--cookies-from-browser', authdict['cookies_from_browser']]
-        elif 'cookies_file' in authdict:
-            get_cookiejar_options = ['--cookies', authdict['cookies_file']]
-        elif 'cookies_from_browser' in self.authentication:
-            authdict['cookies_from_browser'] = self.authentication['cookies_from_browser']
-            get_cookiejar_options = ['--cookies-from-browser', self.authentication['cookies_from_browser']]
-        elif 'cookies_file' in self.authentication:
-            authdict['cookies_file'] = self.authentication['cookies_file']
-            get_cookiejar_options = ['--cookies', self.authentication['cookies_file']]
+        if "cookies_from_browser" in authdict:
+            get_cookiejar_options = ["--cookies-from-browser", authdict["cookies_from_browser"]]
+        elif "cookies_file" in authdict:
+            get_cookiejar_options = ["--cookies", authdict["cookies_file"]]
+        elif "cookies_from_browser" in self.authentication:
+            authdict["cookies_from_browser"] = self.authentication["cookies_from_browser"]
+            get_cookiejar_options = ["--cookies-from-browser", self.authentication["cookies_from_browser"]]
+        elif "cookies_file" in self.authentication:
+            authdict["cookies_file"] = self.authentication["cookies_file"]
+            get_cookiejar_options = ["--cookies", self.authentication["cookies_file"]]
 
-        
         if get_cookiejar_options:
-            authdict['cookies_jar'] = get_ytdlp_cookiejar(get_cookiejar_options)
+            authdict["cookies_jar"] = get_ytdlp_cookiejar(get_cookiejar_options)
 
         return authdict
-    
+
     def repr(self):
-        return f"Module<'{self.display_name}' (config: {self.config[self.name]})>"
\ No newline at end of file
+        return f"Module<'{self.display_name}' (config: {self.config[self.name]})>"
diff --git a/src/auto_archiver/core/config.py b/src/auto_archiver/core/config.py
index 66d2ffb..8122809 100644
--- a/src/auto_archiver/core/config.py
+++ b/src/auto_archiver/core/config.py
@@ -20,12 +20,14 @@ _yaml: YAML = YAML()
 
 DEFAULT_CONFIG_FILE = "secrets/orchestration.yaml"
 
-EMPTY_CONFIG = _yaml.load("""
+EMPTY_CONFIG = _yaml.load(
+    """
 # Auto Archiver Configuration
 
 # Steps are the modules that will be run in the order they are defined
-steps:""" + "".join([f"\n   {module}s: []" for module in MODULE_TYPES]) + \
-"""
+steps:"""
+    + "".join([f"\n   {module}s: []" for module in MODULE_TYPES])
+    + """
 
 # Global configuration
 
@@ -52,14 +54,14 @@ authentication: {}
 logging:
   level: INFO
 
-""")
+"""
+)
 # note: 'logging' is explicitly added above in order to better format the config file
 
 
 # Arg Parse Actions/Classes
 class AuthenticationJsonParseAction(argparse.Action):
     def __call__(self, parser, namespace, values, option_string=None):
-
         try:
             auth_dict = json.loads(values)
             setattr(namespace, self.dest, auth_dict)
@@ -68,34 +70,38 @@ class AuthenticationJsonParseAction(argparse.Action):
 
         def load_from_file(path):
             try:
-                with open(path, 'r') as f:
+                with open(path, "r") as f:
                     try:
                         auth_dict = json.load(f)
                     except json.JSONDecodeError:
                         f.seek(0)
                         # maybe it's yaml, try that
                         auth_dict = _yaml.load(f)
-                    if auth_dict.get('authentication'):
-                        auth_dict = auth_dict['authentication']
-                    auth_dict['load_from_file']  = path
+                    if auth_dict.get("authentication"):
+                        auth_dict = auth_dict["authentication"]
+                    auth_dict["load_from_file"] = path
                     return auth_dict
             except:
                 return None
 
-        if isinstance(auth_dict, dict) and auth_dict.get('from_file'):
-            auth_dict = load_from_file(auth_dict['from_file'])
+        if isinstance(auth_dict, dict) and auth_dict.get("from_file"):
+            auth_dict = load_from_file(auth_dict["from_file"])
         elif isinstance(auth_dict, str):
             # if it's a string
             auth_dict = load_from_file(auth_dict)
-        
+
         if not isinstance(auth_dict, dict):
-            raise argparse.ArgumentTypeError("Authentication must be a dictionary of site names and their authentication methods")
-        global_options = ['cookies_from_browser', 'cookies_file', 'load_from_file']
+            raise argparse.ArgumentTypeError(
+                "Authentication must be a dictionary of site names and their authentication methods"
+            )
+        global_options = ["cookies_from_browser", "cookies_file", "load_from_file"]
         for key, auth in auth_dict.items():
             if key in global_options:
                 continue
             if not isinstance(key, str) or not isinstance(auth, dict):
-                raise argparse.ArgumentTypeError(f"Authentication must be a dictionary of site names and their authentication methods. Valid global configs are {global_options}")
+                raise argparse.ArgumentTypeError(
+                    f"Authentication must be a dictionary of site names and their authentication methods. Valid global configs are {global_options}"
+                )
 
         setattr(namespace, self.dest, auth_dict)
 
@@ -106,8 +112,8 @@ class UniqueAppendAction(argparse.Action):
             if value not in getattr(namespace, self.dest):
                 getattr(namespace, self.dest).append(value)
 
-class DefaultValidatingParser(argparse.ArgumentParser):
 
+class DefaultValidatingParser(argparse.ArgumentParser):
     def error(self, message):
         """
         Override of error to format a nicer looking error message using logger
@@ -136,8 +142,10 @@ class DefaultValidatingParser(argparse.ArgumentParser):
 
         return super().parse_known_args(args, namespace)
 
+
 # Config Utils
 
+
 def to_dot_notation(yaml_conf: CommentedMap | dict) -> dict:
     dotdict = {}
 
@@ -151,6 +159,7 @@ def to_dot_notation(yaml_conf: CommentedMap | dict) -> dict:
     process_subdict(yaml_conf)
     return dotdict
 
+
 def from_dot_notation(dotdict: dict) -> dict:
     normal_dict = {}
 
@@ -171,9 +180,11 @@ def from_dot_notation(dotdict: dict) -> dict:
 def is_list_type(value):
     return isinstance(value, list) or isinstance(value, tuple) or isinstance(value, set)
 
+
 def is_dict_type(value):
     return isinstance(value, dict) or isinstance(value, CommentedMap)
 
+
 def merge_dicts(dotdict: dict, yaml_dict: CommentedMap) -> CommentedMap:
     yaml_dict: CommentedMap = deepcopy(yaml_dict)
 
@@ -184,7 +195,7 @@ def merge_dicts(dotdict: dict, yaml_dict: CommentedMap) -> CommentedMap:
                 yaml_subdict[key] = value
                 continue
 
-            if key == 'steps':
+            if key == "steps":
                 for module_type, modules in value.items():
                     # overwrite the 'steps' from the config file with the ones from the CLI
                     yaml_subdict[key][module_type] = modules
@@ -199,6 +210,7 @@ def merge_dicts(dotdict: dict, yaml_dict: CommentedMap) -> CommentedMap:
     update_dict(from_dot_notation(dotdict), yaml_dict)
     return yaml_dict
 
+
 def read_yaml(yaml_filename: str) -> CommentedMap:
     config = None
     try:
@@ -212,6 +224,7 @@ def read_yaml(yaml_filename: str) -> CommentedMap:
 
     return config
 
+
 # TODO: make this tidier/find a way to notify of which keys should not be stored
 
 
@@ -219,13 +232,14 @@ def store_yaml(config: CommentedMap, yaml_filename: str) -> None:
     config_to_save = deepcopy(config)
 
     auth_dict = config_to_save.get("authentication", {})
-    if auth_dict and auth_dict.get('load_from_file'):
+    if auth_dict and auth_dict.get("load_from_file"):
         # remove all other values from the config, don't want to store it in the config file
         auth_dict = {"load_from_file": auth_dict["load_from_file"]}
 
-    config_to_save.pop('urls', None)
+    config_to_save.pop("urls", None)
     with open(yaml_filename, "w", encoding="utf-8") as outf:
         _yaml.dump(config_to_save, outf)
 
+
 def is_valid_config(config: CommentedMap) -> bool:
-    return config and config != EMPTY_CONFIG
\ No newline at end of file
+    return config and config != EMPTY_CONFIG
diff --git a/src/auto_archiver/core/consts.py b/src/auto_archiver/core/consts.py
index a49884f..054be84 100644
--- a/src/auto_archiver/core/consts.py
+++ b/src/auto_archiver/core/consts.py
@@ -1,23 +1,15 @@
-
-MODULE_TYPES = [
-    'feeder',
-    'extractor',
-    'enricher',
-    'database',
-    'storage',
-    'formatter'
-]
+MODULE_TYPES = ["feeder", "extractor", "enricher", "database", "storage", "formatter"]
 
 MANIFEST_FILE = "__manifest__.py"
 
 DEFAULT_MANIFEST = {
-    'name': '', # the display name of the module
-    'author': 'Bellingcat', # creator of the module, leave this as Bellingcat or set your own name!
-    'type': [], # the type of the module, can be one or more of MODULE_TYPES
-    'requires_setup': True, # whether or not this module requires additional setup such as setting API Keys or installing additional software
-    'description': '', # a description of the module
-    'dependencies': {}, # external dependencies, e.g. python packages or binaries, in dictionary format
-    'entry_point': '', # the entry point for the module, in the format 'module_name::ClassName'. This can be left blank to use the default entry point of module_name::ModuleName
-    'version': '1.0', # the version of the module
-    'configs': {} # any configuration options this module has, these will be exposed to the user in the config file or via the command line
-}
\ No newline at end of file
+    "name": "",  # the display name of the module
+    "author": "Bellingcat",  # creator of the module, leave this as Bellingcat or set your own name!
+    "type": [],  # the type of the module, can be one or more of MODULE_TYPES
+    "requires_setup": True,  # whether or not this module requires additional setup such as setting API Keys or installing additional software
+    "description": "",  # a description of the module
+    "dependencies": {},  # external dependencies, e.g. python packages or binaries, in dictionary format
+    "entry_point": "",  # the entry point for the module, in the format 'module_name::ClassName'. This can be left blank to use the default entry point of module_name::ModuleName
+    "version": "1.0",  # the version of the module
+    "configs": {},  # any configuration options this module has, these will be exposed to the user in the config file or via the command line
+}
diff --git a/src/auto_archiver/core/database.py b/src/auto_archiver/core/database.py
index a6e76e5..85575c1 100644
--- a/src/auto_archiver/core/database.py
+++ b/src/auto_archiver/core/database.py
@@ -1,6 +1,6 @@
 """
 Database module for the auto-archiver that defines the interface for implementing database modules
-in the media archiving framework. 
+in the media archiving framework.
 """
 
 from __future__ import annotations
@@ -9,6 +9,7 @@ from typing import Union
 
 from auto_archiver.core import Metadata, BaseModule
 
+
 class Database(BaseModule):
     """
     Base class for implementing database modules in the media archiving framework.
@@ -20,7 +21,7 @@ class Database(BaseModule):
         """signals the DB that the given item archival has started"""
         pass
 
-    def failed(self, item: Metadata, reason:str) -> None:
+    def failed(self, item: Metadata, reason: str) -> None:
         """update DB accordingly for failure"""
         pass
 
@@ -34,6 +35,6 @@ class Database(BaseModule):
         return False
 
     @abstractmethod
-    def done(self, item: Metadata, cached: bool=False) -> None:
+    def done(self, item: Metadata, cached: bool = False) -> None:
         """archival result ready - should be saved to DB"""
         pass
diff --git a/src/auto_archiver/core/enricher.py b/src/auto_archiver/core/enricher.py
index a862223..9b8e19a 100644
--- a/src/auto_archiver/core/enricher.py
+++ b/src/auto_archiver/core/enricher.py
@@ -8,13 +8,15 @@ the archiving step and before storage or formatting.
 
 Enrichers are optional but highly useful for making the archived data more powerful.
 """
+
 from __future__ import annotations
 from abc import abstractmethod
 from auto_archiver.core import Metadata, BaseModule
 
+
 class Enricher(BaseModule):
     """Base classes and utilities for enrichers in the Auto Archiver system.
-    
+
     Enricher modules must implement the `enrich` method to define their behavior.
     """
 
diff --git a/src/auto_archiver/core/extractor.py b/src/auto_archiver/core/extractor.py
index 484a09d..8ad13f5 100644
--- a/src/auto_archiver/core/extractor.py
+++ b/src/auto_archiver/core/extractor.py
@@ -1,10 +1,11 @@
-""" The `extractor` module defines the base functionality for implementing extractors in the media archiving framework.
-    This class provides common utility methods and a standard interface for extractors.
+"""The `extractor` module defines the base functionality for implementing extractors in the media archiving framework.
+This class provides common utility methods and a standard interface for extractors.
 
-    Factory method to initialize an extractor instance based on its name.
+Factory method to initialize an extractor instance based on its name.
 
 
 """
+
 from __future__ import annotations
 from pathlib import Path
 from abc import abstractmethod
@@ -39,7 +40,7 @@ class Extractor(BaseModule):
         Used to clean unnecessary URL parameters OR unfurl redirect links
         """
         return url
-    
+
     def match_link(self, url: str) -> re.Match:
         """
         Returns a match object if the given URL matches the valid_url pattern or False/None if not.
@@ -58,7 +59,7 @@ class Extractor(BaseModule):
         """
         if self.valid_url:
             return self.match_link(url) is not None
-        
+
         return True
 
     def _guess_file_type(self, path: str) -> str:
@@ -74,16 +75,17 @@ class Extractor(BaseModule):
     @retry(wait_random_min=500, wait_random_max=3500, stop_max_attempt_number=5)
     def download_from_url(self, url: str, to_filename: str = None, verbose=True) -> str:
         """
-            downloads a URL to provided filename, or inferred from URL, returns local filename
+        downloads a URL to provided filename, or inferred from URL, returns local filename
         """
         if not to_filename:
-            to_filename = url.split('/')[-1].split('?')[0]
+            to_filename = url.split("/")[-1].split("?")[0]
             if len(to_filename) > 64:
                 to_filename = to_filename[-64:]
         to_filename = os.path.join(self.tmp_dir, to_filename)
-        if verbose: logger.debug(f"downloading {url[0:50]=} {to_filename=}")
+        if verbose:
+            logger.debug(f"downloading {url[0:50]=} {to_filename=}")
         headers = {
-            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36'
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36"
         }
         try:
             d = requests.get(url, stream=True, headers=headers, timeout=30)
@@ -91,12 +93,12 @@ class Extractor(BaseModule):
 
             # get mimetype from the response headers
             if not mimetypes.guess_type(to_filename)[0]:
-                content_type = d.headers.get('Content-Type') or self._guess_file_type(url)
+                content_type = d.headers.get("Content-Type") or self._guess_file_type(url)
                 extension = mimetypes.guess_extension(content_type)
                 if extension:
                     to_filename += extension
 
-            with open(to_filename, 'wb') as f:
+            with open(to_filename, "wb") as f:
                 for chunk in d.iter_content(chunk_size=8192):
                     f.write(chunk)
             return to_filename
@@ -108,8 +110,8 @@ class Extractor(BaseModule):
     def download(self, item: Metadata) -> Metadata | False:
         """
         Downloads the media from the given URL and returns a Metadata object with the downloaded media.
-        
+
         If the URL is not supported or the download fails, this method should return False.
 
         """
-        pass
\ No newline at end of file
+        pass
diff --git a/src/auto_archiver/core/feeder.py b/src/auto_archiver/core/feeder.py
index e8302e6..dfcddb9 100644
--- a/src/auto_archiver/core/feeder.py
+++ b/src/auto_archiver/core/feeder.py
@@ -1,5 +1,5 @@
 """
-The feeder base module defines the interface for implementing feeders in the media archiving framework. 
+The feeder base module defines the interface for implementing feeders in the media archiving framework.
 """
 
 from __future__ import annotations
@@ -7,8 +7,8 @@ from abc import abstractmethod
 from auto_archiver.core import Metadata
 from auto_archiver.core import BaseModule
 
-class Feeder(BaseModule):
 
+class Feeder(BaseModule):
     """
     Base class for implementing feeders in the media archiving framework.
 
@@ -19,7 +19,7 @@ class Feeder(BaseModule):
     def __iter__(self) -> Metadata:
         """
         Returns an iterator (use `yield`) over the items to be archived.
-        
+
         These should be instances of Metadata, typically created with Metadata().set_url(url).
         """
-        return None
\ No newline at end of file
+        return None
diff --git a/src/auto_archiver/core/formatter.py b/src/auto_archiver/core/formatter.py
index 3bfc250..0c63c7f 100644
--- a/src/auto_archiver/core/formatter.py
+++ b/src/auto_archiver/core/formatter.py
@@ -12,7 +12,7 @@ from auto_archiver.core import Metadata, Media, BaseModule
 class Formatter(BaseModule):
     """
     Base class for implementing formatters in the media archiving framework.
-    
+
     Subclasses must implement the `format` method to define their behavior.
     """
 
@@ -21,4 +21,4 @@ class Formatter(BaseModule):
         """
         Formats a Metadata object into a user-viewable format (e.g. HTML) and stores it if needed.
         """
-        return None
\ No newline at end of file
+        return None
diff --git a/src/auto_archiver/core/media.py b/src/auto_archiver/core/media.py
index b6820ab..439e056 100644
--- a/src/auto_archiver/core/media.py
+++ b/src/auto_archiver/core/media.py
@@ -27,6 +27,7 @@ class Media:
     - properties: Additional metadata or transformations for the media.
     - _mimetype: The media's mimetype (e.g., image/jpeg, video/mp4).
     """
+
     filename: str
     key: str = None
     urls: List[str] = field(default_factory=list)
@@ -52,14 +53,15 @@ class Media:
         This function returns a generator for all the inner media.
 
         """
-        if include_self: yield self
+        if include_self:
+            yield self
         for prop in self.properties.values():
-            if isinstance(prop, Media): 
+            if isinstance(prop, Media):
                 for inner_media in prop.all_inner_media(include_self=True):
                     yield inner_media
             if isinstance(prop, list):
                 for prop_media in prop:
-                    if isinstance(prop_media, Media): 
+                    if isinstance(prop_media, Media):
                         for inner_media in prop_media.all_inner_media(include_self=True):
                             yield inner_media
 
@@ -110,15 +112,17 @@ class Media:
         # checks for video streams with ffmpeg, or min file size for a video
         # self.is_video() should be used together with this method
         try:
-            streams = ffmpeg.probe(self.filename, select_streams='v')['streams']
+            streams = ffmpeg.probe(self.filename, select_streams="v")["streams"]
             logger.warning(f"STREAMS FOR {self.filename} {streams}")
             return any(s.get("duration_ts", 0) > 0 for s in streams)
-        except Error: return False # ffmpeg errors when reading bad files
+        except Error:
+            return False  # ffmpeg errors when reading bad files
         except Exception as e:
             logger.error(e)
             logger.error(traceback.format_exc())
             try:
                 fsize = os.path.getsize(self.filename)
                 return fsize > 20_000
-            except: pass
+            except:
+                pass
         return True
diff --git a/src/auto_archiver/core/metadata.py b/src/auto_archiver/core/metadata.py
index a8d2ad4..9c696a2 100644
--- a/src/auto_archiver/core/metadata.py
+++ b/src/auto_archiver/core/metadata.py
@@ -21,6 +21,7 @@ from loguru import logger
 
 from .media import Media
 
+
 @dataclass_json  # annotation order matters
 @dataclass
 class Metadata:
@@ -40,7 +41,8 @@ class Metadata:
         - If `True`, this instance's values are overwritten by `right`.
         - If `False`, the inverse applies.
         """
-        if not right: return self
+        if not right:
+            return self
         if overwrite_left:
             if right.status and len(right.status):
                 self.status = right.status
@@ -50,8 +52,10 @@ class Metadata:
                 if type(v) not in [dict, list, set] or k not in self.metadata:
                     self.set(k, v)
                 else:  # key conflict
-                    if type(v) in [dict, set]: self.set(k, self.get(k) | v)
-                    elif type(v) == list: self.set(k, self.get(k) + v)
+                    if type(v) in [dict, set]:
+                        self.set(k, self.get(k) | v)
+                    elif type(v) == list:
+                        self.set(k, self.get(k) + v)
             self.media.extend(right.media)
         else:  # invert and do same logic
             return right.merge(self)
@@ -69,7 +73,7 @@ class Metadata:
 
     def append(self, key: str, val: Any) -> Metadata:
         if key not in self.metadata:
-            self.metadata[key] = []    
+            self.metadata[key] = []
         self.metadata[key] = val
         return self
 
@@ -80,24 +84,26 @@ class Metadata:
         return self.metadata.get(key, default)
 
     def success(self, context: str = None) -> Metadata:
-        if context: self.status = f"{context}: success"
-        else: self.status = "success"
+        if context:
+            self.status = f"{context}: success"
+        else:
+            self.status = "success"
         return self
 
     def is_success(self) -> bool:
         return "success" in self.status
 
     def is_empty(self) -> bool:
-        meaningfull_ids = set(self.metadata.keys()) - set(["_processed_at", "url", "total_bytes", "total_size", "archive_duration_seconds"])
+        meaningfull_ids = set(self.metadata.keys()) - set(
+            ["_processed_at", "url", "total_bytes", "total_size", "archive_duration_seconds"]
+        )
         return not self.is_success() and len(self.media) == 0 and len(meaningfull_ids) == 0
 
     @property  # getter .netloc
     def netloc(self) -> str:
         return urlparse(self.get_url()).netloc
 
-
-# custom getter/setters
-
+    # custom getter/setters
 
     def set_url(self, url: str) -> Metadata:
         assert type(url) is str and len(url) > 0, "invalid URL"
@@ -127,12 +133,17 @@ class Metadata:
 
     def get_timestamp(self, utc=True, iso=True) -> datetime.datetime:
         ts = self.get("timestamp")
-        if not ts: return 
+        if not ts:
+            return
         try:
-            if type(ts) == str: ts = datetime.datetime.fromisoformat(ts)
-            if type(ts) == float: ts = datetime.datetime.fromtimestamp(ts)
-            if utc: ts = ts.replace(tzinfo=datetime.timezone.utc)
-            if iso: return ts.isoformat()
+            if type(ts) == str:
+                ts = datetime.datetime.fromisoformat(ts)
+            if type(ts) == float:
+                ts = datetime.datetime.fromtimestamp(ts)
+            if utc:
+                ts = ts.replace(tzinfo=datetime.timezone.utc)
+            if iso:
+                return ts.isoformat()
             return ts
         except Exception as e:
             logger.error(f"Unable to parse timestamp {ts}: {e}")
@@ -140,16 +151,20 @@ class Metadata:
 
     def add_media(self, media: Media, id: str = None) -> Metadata:
         # adds a new media, optionally including an id
-        if media is None: return
+        if media is None:
+            return
         if id is not None:
-            assert not len([1 for m in self.media if m.get("id") == id]), f"cannot add 2 pieces of media with the same id {id}"
+            assert not len([1 for m in self.media if m.get("id") == id]), (
+                f"cannot add 2 pieces of media with the same id {id}"
+            )
             media.set("id", id)
         self.media.append(media)
         return media
 
     def get_media_by_id(self, id: str, default=None) -> Media:
         for m in self.media:
-            if m.get("id") == id: return m
+            if m.get("id") == id:
+                return m
         return default
 
     def remove_duplicate_media_by_hash(self) -> None:
@@ -159,7 +174,8 @@ class Metadata:
             with open(filename, "rb") as f:
                 while True:
                     buf = f.read(chunksize)
-                    if not buf: break
+                    if not buf:
+                        break
                     hash_algo.update(buf)
             return hash_algo.hexdigest()
 
@@ -167,15 +183,18 @@ class Metadata:
         new_media = []
         for m in self.media:
             h = m.get("hash")
-            if not h: h = calculate_hash_in_chunks(hashlib.sha256(), int(1.6e7), m.filename)
-            if len(h) and h in media_hashes: continue
+            if not h:
+                h = calculate_hash_in_chunks(hashlib.sha256(), int(1.6e7), m.filename)
+            if len(h) and h in media_hashes:
+                continue
             media_hashes.add(h)
             new_media.append(m)
         self.media = new_media
 
     def get_first_image(self, default=None) -> Media:
         for m in self.media:
-            if "image" in m.mimetype: return m
+            if "image" in m.mimetype:
+                return m
         return default
 
     def set_final_media(self, final: Media) -> Metadata:
@@ -193,22 +212,25 @@ class Metadata:
     def __str__(self) -> str:
         return self.__repr__()
 
-
     @staticmethod
     def choose_most_complete(results: List[Metadata]) -> Metadata:
         # returns the most complete result from a list of results
         # prioritizes results with more media, then more metadata
-        if len(results) == 0: return None
-        if len(results) == 1: return results[0]
+        if len(results) == 0:
+            return None
+        if len(results) == 1:
+            return results[0]
         most_complete = results[0]
         for r in results[1:]:
-            if len(r.media) > len(most_complete.media): most_complete = r
-            elif len(r.media) == len(most_complete.media) and len(r.metadata) > len(most_complete.metadata): most_complete = r
+            if len(r.media) > len(most_complete.media):
+                most_complete = r
+            elif len(r.media) == len(most_complete.media) and len(r.metadata) > len(most_complete.metadata):
+                most_complete = r
         return most_complete
 
     def set_context(self, key: str, val: Any) -> Metadata:
         self._context[key] = val
         return self
-    
+
     def get_context(self, key: str, default: Any = None) -> Any:
-        return self._context.get(key, default)
\ No newline at end of file
+        return self._context.get(key, default)
diff --git a/src/auto_archiver/core/module.py b/src/auto_archiver/core/module.py
index 2c6617d..3e2110a 100644
--- a/src/auto_archiver/core/module.py
+++ b/src/auto_archiver/core/module.py
@@ -3,6 +3,7 @@ Defines the Step abstract base class, which acts as a blueprint for steps in the
 by handling user configuration, validating the steps properties, and implementing dynamic instantiation.
 
 """
+
 from __future__ import annotations
 
 from dataclasses import dataclass
@@ -24,17 +25,17 @@ if TYPE_CHECKING:
 
 HAS_SETUP_PATHS = False
 
-class ModuleFactory:
 
+class ModuleFactory:
     def __init__(self):
         self._lazy_modules = {}
 
     def setup_paths(self, paths: list[str]) -> None:
         """
         Sets up the paths for the modules to be loaded from
-        
+
         This is necessary for the modules to be imported correctly
-        
+
         """
         global HAS_SETUP_PATHS
 
@@ -46,11 +47,13 @@ class ModuleFactory:
 
             # see odoo/module/module.py -> initialize_sys_path
             if path not in auto_archiver.modules.__path__:
-                    if HAS_SETUP_PATHS == True:
-                        logger.warning(f"You are attempting to re-initialise the module paths with: '{path}' for a 2nd time. \
+                if HAS_SETUP_PATHS == True:
+                    logger.warning(
+                        f"You are attempting to re-initialise the module paths with: '{path}' for a 2nd time. \
                                        This could lead to unexpected behaviour. It is recommended to only use a single modules path. \
-                                       If you wish to load modules from different paths then load a 2nd python interpreter (e.g. using multiprocessing).")
-                    auto_archiver.modules.__path__.append(path)
+                                       If you wish to load modules from different paths then load a 2nd python interpreter (e.g. using multiprocessing)."
+                    )
+                auto_archiver.modules.__path__.append(path)
 
         # sort based on the length of the path, so that the longest path is last in the list
         auto_archiver.modules.__path__ = sorted(auto_archiver.modules.__path__, key=len, reverse=True)
@@ -60,20 +63,20 @@ class ModuleFactory:
     def get_module(self, module_name: str, config: dict) -> BaseModule:
         """
         Gets and sets up a module using the provided config
-        
+
         This will actually load and instantiate the module, and load all its dependencies (i.e. not lazy)
-        
+
         """
         return self.get_module_lazy(module_name).load(config)
 
     def get_module_lazy(self, module_name: str, suppress_warnings: bool = False) -> LazyBaseModule:
         """
         Lazily loads a module, returning a LazyBaseModule
-        
+
         This has all the information about the module, but does not load the module itself or its dependencies
-        
+
         To load an actual module, call .setup() on a lazy module
-        
+
         """
         if module_name in self._lazy_modules:
             return self._lazy_modules[module_name]
@@ -81,13 +84,14 @@ class ModuleFactory:
         available = self.available_modules(limit_to_modules=[module_name], suppress_warnings=suppress_warnings)
         if not available:
             message = f"Module '{module_name}' not found. Are you sure it's installed/exists?"
-            if 'archiver' in module_name:
+            if "archiver" in module_name:
                 message += f" Did you mean {module_name.replace('archiver', 'extractor')}?"
             raise IndexError(message)
         return available[0]
 
-    def available_modules(self, limit_to_modules: List[str]= [], suppress_warnings: bool = False) -> List[LazyBaseModule]:
-        
+    def available_modules(
+        self, limit_to_modules: List[str] = [], suppress_warnings: bool = False
+    ) -> List[LazyBaseModule]:
         # search through all valid 'modules' paths. Default is 'modules' in the current directory
 
         # see odoo/modules/module.py -> get_modules
@@ -119,7 +123,7 @@ class ModuleFactory:
                 self._lazy_modules[possible_module] = lazy_module
 
                 all_modules.append(lazy_module)
-        
+
         if not suppress_warnings:
             for module in limit_to_modules:
                 if not any(module == m.name for m in all_modules):
@@ -127,15 +131,16 @@ class ModuleFactory:
 
         return all_modules
 
+
 @dataclass
 class LazyBaseModule:
-
     """
     A lazy module class, which only loads the manifest and does not load the module itself.
 
     This is useful for getting information about a module without actually loading it.
 
     """
+
     name: str
     description: str
     path: str
@@ -152,30 +157,30 @@ class LazyBaseModule:
 
     @property
     def type(self):
-        return self.manifest['type']
+        return self.manifest["type"]
 
     @property
     def entry_point(self):
-        if not self._entry_point and not self.manifest['entry_point']:
+        if not self._entry_point and not self.manifest["entry_point"]:
             # try to create the entry point from the module name
             self._entry_point = f"{self.name}::{self.name.replace('_', ' ').title().replace(' ', '')}"
         return self._entry_point
 
     @property
     def dependencies(self) -> dict:
-        return self.manifest['dependencies']
-    
+        return self.manifest["dependencies"]
+
     @property
     def configs(self) -> dict:
-        return self.manifest['configs']
-    
+        return self.manifest["configs"]
+
     @property
     def requires_setup(self) -> bool:
-        return self.manifest['requires_setup']
-    
+        return self.manifest["requires_setup"]
+
     @property
     def display_name(self) -> str:
-        return self.manifest['name']
+        return self.manifest["name"]
 
     @property
     def manifest(self) -> dict:
@@ -190,16 +195,15 @@ class LazyBaseModule:
                 manifest.update(ast.literal_eval(f.read()))
             except (ValueError, TypeError, SyntaxError, MemoryError, RecursionError) as e:
                 raise ValueError(f"Error loading manifest from file {self.path}/{MANIFEST_FILE}: {e}")
-            
+
         self._manifest = manifest
-        self._entry_point = manifest['entry_point']
-        self.description = manifest['description']
-        self.version = manifest['version']
+        self._entry_point = manifest["entry_point"]
+        self.description = manifest["description"]
+        self.version = manifest["version"]
 
         return manifest
 
     def load(self, config) -> BaseModule:
-
         if self._instance:
             return self._instance
 
@@ -210,8 +214,10 @@ class LazyBaseModule:
                     # clear out any empty strings that a user may have erroneously added
                     continue
                 if not check(dep):
-                    logger.error(f"Module '{self.name}' requires external dependency '{dep}' which is not available/setup. \
-                                 Have you installed the required dependencies for the '{self.name}' module? See the README for more information.")
+                    logger.error(
+                        f"Module '{self.name}' requires external dependency '{dep}' which is not available/setup. \
+                                 Have you installed the required dependencies for the '{self.name}' module? See the README for more information."
+                    )
                     exit(1)
 
         def check_python_dep(dep):
@@ -219,7 +225,7 @@ class LazyBaseModule:
             try:
                 m = self.module_factory.get_module_lazy(dep, suppress_warnings=True)
                 try:
-                # we must now load this module and set it up with the config
+                    # we must now load this module and set it up with the config
                     m.load(config)
                     return True
                 except:
@@ -231,13 +237,12 @@ class LazyBaseModule:
 
             return find_spec(dep)
 
-        check_deps(self.dependencies.get('python', []), check_python_dep)
-        check_deps(self.dependencies.get('bin', []), lambda dep: shutil.which(dep))
-
+        check_deps(self.dependencies.get("python", []), check_python_dep)
+        check_deps(self.dependencies.get("bin", []), lambda dep: shutil.which(dep))
 
         logger.debug(f"Loading module '{self.display_name}'...")
 
-        for qualname in [self.name, f'auto_archiver.modules.{self.name}']:
+        for qualname in [self.name, f"auto_archiver.modules.{self.name}"]:
             try:
                 # first import the whole module, to make sure it's working properly
                 __import__(qualname)
@@ -246,10 +251,10 @@ class LazyBaseModule:
                 pass
 
         # then import the file for the entry point
-        file_name, class_name = self.entry_point.split('::')
-        sub_qualname = f'{qualname}.{file_name}'
+        file_name, class_name = self.entry_point.split("::")
+        sub_qualname = f"{qualname}.{file_name}"
 
-        __import__(f'{qualname}.{file_name}', fromlist=[self.entry_point])
+        __import__(f"{qualname}.{file_name}", fromlist=[self.entry_point])
         # finally, get the class instance
         instance: BaseModule = getattr(sys.modules[sub_qualname], class_name)()
 
@@ -257,11 +262,11 @@ class LazyBaseModule:
         instance.name = self.name
         instance.display_name = self.display_name
         instance.module_factory = self.module_factory
-        
-        # merge the default config with the user config
-        default_config = dict((k, v['default']) for k, v in self.configs.items() if 'default' in v)
 
-        config[self.name] = default_config  | config.get(self.name, {})
+        # merge the default config with the user config
+        default_config = dict((k, v["default"]) for k, v in self.configs.items() if "default" in v)
+
+        config[self.name] = default_config | config.get(self.name, {})
         instance.config_setup(config)
         instance.setup()
 
@@ -270,4 +275,4 @@ class LazyBaseModule:
         return instance
 
     def __repr__(self):
-        return f"Module<'{self.display_name}' ({self.name})>"
\ No newline at end of file
+        return f"Module<'{self.display_name}' ({self.name})>"
diff --git a/src/auto_archiver/core/orchestrator.py b/src/auto_archiver/core/orchestrator.py
index b78c7e7..ba00995 100644
--- a/src/auto_archiver/core/orchestrator.py
+++ b/src/auto_archiver/core/orchestrator.py
@@ -1,6 +1,6 @@
-""" Orchestrates all archiving steps, including feeding items,
-    archiving them with specific archivers, enrichment, storage,
-    formatting, database operations and clean up.
+"""Orchestrates all archiving steps, including feeding items,
+archiving them with specific archivers, enrichment, storage,
+formatting, database operations and clean up.
 
 """
 
@@ -19,8 +19,17 @@ import requests
 
 from .metadata import Metadata, Media
 from auto_archiver.version import __version__
-from .config import read_yaml, store_yaml, to_dot_notation, merge_dicts, is_valid_config, \
-    DefaultValidatingParser, UniqueAppendAction, AuthenticationJsonParseAction, DEFAULT_CONFIG_FILE
+from .config import (
+    read_yaml,
+    store_yaml,
+    to_dot_notation,
+    merge_dicts,
+    is_valid_config,
+    DefaultValidatingParser,
+    UniqueAppendAction,
+    AuthenticationJsonParseAction,
+    DEFAULT_CONFIG_FILE,
+)
 from .module import ModuleFactory, LazyBaseModule
 from . import validators, Feeder, Extractor, Database, Storage, Formatter, Enricher
 from .consts import MODULE_TYPES
@@ -30,10 +39,12 @@ if TYPE_CHECKING:
     from .base_module import BaseModule
     from .module import LazyBaseModule
 
+
 class SetupError(ValueError):
     pass
-class ArchivingOrchestrator:
 
+
+class ArchivingOrchestrator:
     # instance variables
     module_factory: ModuleFactory
     setup_finished: bool
@@ -63,30 +74,63 @@ class ArchivingOrchestrator:
             epilog="Check the code at https://github.com/bellingcat/auto-archiver",
             formatter_class=RichHelpFormatter,
         )
-        parser.add_argument('--help', '-h', action='store_true', dest='help', help='show a full help message and exit')
-        parser.add_argument('--version', action='version', version=__version__)
-        parser.add_argument('--config', action='store', dest="config_file", help='the filename of the YAML configuration file (defaults to \'config.yaml\')', default=DEFAULT_CONFIG_FILE)
-        parser.add_argument('--mode', action='store', dest='mode', type=str, choices=['simple', 'full'], help='the mode to run the archiver in', default='simple')
+        parser.add_argument("--help", "-h", action="store_true", dest="help", help="show a full help message and exit")
+        parser.add_argument("--version", action="version", version=__version__)
+        parser.add_argument(
+            "--config",
+            action="store",
+            dest="config_file",
+            help="the filename of the YAML configuration file (defaults to 'config.yaml')",
+            default=DEFAULT_CONFIG_FILE,
+        )
+        parser.add_argument(
+            "--mode",
+            action="store",
+            dest="mode",
+            type=str,
+            choices=["simple", "full"],
+            help="the mode to run the archiver in",
+            default="simple",
+        )
         # override the default 'help' so we can inject all the configs and show those
-        parser.add_argument('-s', '--store', dest='store', default=False, help='Store the created config in the config file', action=argparse.BooleanOptionalAction)
-        parser.add_argument('--module_paths', dest='module_paths', nargs='+', default=[], help='additional paths to search for modules', action=UniqueAppendAction)
+        parser.add_argument(
+            "-s",
+            "--store",
+            dest="store",
+            default=False,
+            help="Store the created config in the config file",
+            action=argparse.BooleanOptionalAction,
+        )
+        parser.add_argument(
+            "--module_paths",
+            dest="module_paths",
+            nargs="+",
+            default=[],
+            help="additional paths to search for modules",
+            action=UniqueAppendAction,
+        )
 
         self.basic_parser = parser
         return parser
-    
+
     def check_steps(self, config):
         for module_type in MODULE_TYPES:
-            if not config['steps'].get(f"{module_type}s", []):
-                if module_type == 'feeder' or module_type == 'formatter' and config['steps'].get(f"{module_type}"):
-                    raise SetupError(f"It appears you have '{module_type}' set under 'steps' in your configuration file, but as of version 0.13.0 of Auto Archiver, you must use '{module_type}s'. Change this in your configuration file and try again. \
-Here's how that would look: \n\nsteps:\n  {module_type}s:\n  - [your_{module_type}_name_here]\n  {'extractors:...' if module_type == 'feeder' else '...'}\n")
-                if module_type == 'extractor' and config['steps'].get('archivers'):
-                    raise SetupError(f"As of version 0.13.0 of Auto Archiver, the 'archivers' step name has been changed to 'extractors'. Change this in your configuration file and try again. \
-Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_here]\n  enrichers:...\n")
-                raise SetupError(f"No {module_type}s were configured. Make sure to set at least one {module_type} in your configuration file or on the command line (using --{module_type}s)")
+            if not config["steps"].get(f"{module_type}s", []):
+                if module_type == "feeder" or module_type == "formatter" and config["steps"].get(f"{module_type}"):
+                    raise SetupError(
+                        f"It appears you have '{module_type}' set under 'steps' in your configuration file, but as of version 0.13.0 of Auto Archiver, you must use '{module_type}s'. Change this in your configuration file and try again. \
+Here's how that would look: \n\nsteps:\n  {module_type}s:\n  - [your_{module_type}_name_here]\n  {'extractors:...' if module_type == 'feeder' else '...'}\n"
+                    )
+                if module_type == "extractor" and config["steps"].get("archivers"):
+                    raise SetupError(
+                        f"As of version 0.13.0 of Auto Archiver, the 'archivers' step name has been changed to 'extractors'. Change this in your configuration file and try again. \
+Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_here]\n  enrichers:...\n"
+                    )
+                raise SetupError(
+                    f"No {module_type}s were configured. Make sure to set at least one {module_type} in your configuration file or on the command line (using --{module_type}s)"
+                )
 
     def setup_complete_parser(self, basic_config: dict, yaml_config: dict, unused_args: list[str]) -> None:
-
         # modules parser to get the overridden 'steps' values
         modules_parser = argparse.ArgumentParser(
             add_help=False,
@@ -94,7 +138,9 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
         self.add_modules_args(modules_parser)
         cli_modules, unused_args = modules_parser.parse_known_args(unused_args)
         for module_type in MODULE_TYPES:
-            yaml_config['steps'][f"{module_type}s"] = getattr(cli_modules, f"{module_type}s", []) or yaml_config['steps'].get(f"{module_type}s", [])
+            yaml_config["steps"][f"{module_type}s"] = getattr(cli_modules, f"{module_type}s", []) or yaml_config[
+                "steps"
+            ].get(f"{module_type}s", [])
 
         parser = DefaultValidatingParser(
             add_help=False,
@@ -117,30 +163,32 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
             enabled_modules = []
             # first loads the modules from the config file, then from the command line
             for module_type in MODULE_TYPES:
-                enabled_modules.extend(yaml_config['steps'].get(f"{module_type}s", []))
+                enabled_modules.extend(yaml_config["steps"].get(f"{module_type}s", []))
 
             # clear out duplicates, but keep the order
             enabled_modules = list(dict.fromkeys(enabled_modules))
-            avail_modules = self.module_factory.available_modules(limit_to_modules=enabled_modules, suppress_warnings=True)
+            avail_modules = self.module_factory.available_modules(
+                limit_to_modules=enabled_modules, suppress_warnings=True
+            )
             self.add_individual_module_args(avail_modules, parser)
-        elif basic_config.mode == 'simple':
+        elif basic_config.mode == "simple":
             simple_modules = [module for module in self.module_factory.available_modules() if not module.requires_setup]
             self.add_individual_module_args(simple_modules, parser)
 
             # add them to the config
             for module in simple_modules:
                 for module_type in module.type:
-                    yaml_config['steps'].setdefault(f"{module_type}s", []).append(module.name)
+                    yaml_config["steps"].setdefault(f"{module_type}s", []).append(module.name)
         else:
             # load all modules, they're not using the 'simple' mode
             all_modules = self.module_factory.available_modules()
             # add all the modules to the steps
             for module in all_modules:
                 for module_type in module.type:
-                    yaml_config['steps'].setdefault(f"{module_type}s", []).append(module.name)
+                    yaml_config["steps"].setdefault(f"{module_type}s", []).append(module.name)
 
             self.add_individual_module_args(all_modules, parser)
-        
+
         parser.set_defaults(**to_dot_notation(yaml_config))
 
         # reload the parser with the new arguments, now that we have them
@@ -166,43 +214,76 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
             store_yaml(config, basic_config.config_file)
 
         return config
-    
+
     def add_modules_args(self, parser: argparse.ArgumentParser = None):
         if not parser:
             parser = self.parser
 
         # Module loading from the command line
         for module_type in MODULE_TYPES:
-            parser.add_argument(f'--{module_type}s', dest=f'{module_type}s', nargs='+', help=f'the {module_type}s to use', default=[], action=UniqueAppendAction)
+            parser.add_argument(
+                f"--{module_type}s",
+                dest=f"{module_type}s",
+                nargs="+",
+                help=f"the {module_type}s to use",
+                default=[],
+                action=UniqueAppendAction,
+            )
 
     def add_additional_args(self, parser: argparse.ArgumentParser = None):
         if not parser:
             parser = self.parser
 
-        parser.add_argument('--authentication', dest='authentication', help='A dictionary of sites and their authentication methods \
+        parser.add_argument(
+            "--authentication",
+            dest="authentication",
+            help="A dictionary of sites and their authentication methods \
                                                                             (token, username etc.) that extractors can use to log into \
                                                                             a website. If passing this on the command line, use a JSON string. \
-                                                                            You may also pass a path to a valid JSON/YAML file which will be parsed.',
-                                                                            default={},
-                                                                            nargs="?",
-                                                                            action=AuthenticationJsonParseAction)
+                                                                            You may also pass a path to a valid JSON/YAML file which will be parsed.",
+            default={},
+            nargs="?",
+            action=AuthenticationJsonParseAction,
+        )
 
         # logging arguments
-        parser.add_argument('--logging.level', action='store', dest='logging.level', choices=['INFO', 'DEBUG', 'ERROR', 'WARNING'], help='the logging level to use', default='INFO', type=str.upper)
-        parser.add_argument('--logging.file', action='store', dest='logging.file', help='the logging file to write to', default=None)
-        parser.add_argument('--logging.rotation', action='store', dest='logging.rotation', help='the logging rotation to use', default=None)
-
-    def add_individual_module_args(self, modules: list[LazyBaseModule] = None, parser: argparse.ArgumentParser = None) -> None:
+        parser.add_argument(
+            "--logging.level",
+            action="store",
+            dest="logging.level",
+            choices=["INFO", "DEBUG", "ERROR", "WARNING"],
+            help="the logging level to use",
+            default="INFO",
+            type=str.upper,
+        )
+        parser.add_argument(
+            "--logging.file", action="store", dest="logging.file", help="the logging file to write to", default=None
+        )
+        parser.add_argument(
+            "--logging.rotation",
+            action="store",
+            dest="logging.rotation",
+            help="the logging rotation to use",
+            default=None,
+        )
 
+    def add_individual_module_args(
+        self, modules: list[LazyBaseModule] = None, parser: argparse.ArgumentParser = None
+    ) -> None:
         if not modules:
             modules = self.module_factory.available_modules()
-        
+
         for module in modules:
-            if module.name == 'cli_feeder':
+            if module.name == "cli_feeder":
                 # special case. For the CLI feeder, allow passing URLs directly on the command line without setting --cli_feeder.urls=
-                parser.add_argument('urls', nargs='*', default=[], help='URL(s) to archive, either a single URL or a list of urls, should not come from config.yaml')
+                parser.add_argument(
+                    "urls",
+                    nargs="*",
+                    default=[],
+                    help="URL(s) to archive, either a single URL or a list of urls, should not come from config.yaml",
+                )
                 continue
-                
+
             if not module.configs:
                 # this module has no configs, don't show anything in the help
                 # (TODO: do we want to show something about this module though, like a description?)
@@ -211,21 +292,21 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
             group = parser.add_argument_group(module.display_name or module.name, f"{module.description[:100]}...")
 
             for name, kwargs in module.configs.items():
-                if not kwargs.get('metavar', None):
+                if not kwargs.get("metavar", None):
                     # make a nicer metavar, metavar is what's used in the help, e.g. --cli_feeder.urls [METAVAR]
-                    kwargs['metavar'] = name.upper()
+                    kwargs["metavar"] = name.upper()
 
-                if kwargs.get('required', False):
+                if kwargs.get("required", False):
                     # required args shouldn't have a 'default' value, remove it
-                    kwargs.pop('default', None)
+                    kwargs.pop("default", None)
 
-                kwargs.pop('cli_set', None)
-                should_store = kwargs.pop('should_store', False)
-                kwargs['dest'] = f"{module.name}.{kwargs.pop('dest', name)}"
+                kwargs.pop("cli_set", None)
+                should_store = kwargs.pop("should_store", False)
+                kwargs["dest"] = f"{module.name}.{kwargs.pop('dest', name)}"
                 try:
-                    kwargs['type'] = getattr(validators, kwargs.get('type', '__invalid__'))
+                    kwargs["type"] = getattr(validators, kwargs.get("type", "__invalid__"))
                 except AttributeError:
-                    kwargs['type'] = __builtins__.get(kwargs.get('type'), str)
+                    kwargs["type"] = __builtins__.get(kwargs.get("type"), str)
                 arg = group.add_argument(f"--{module.name}.{name}", **kwargs)
                 arg.should_store = should_store
 
@@ -240,12 +321,11 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
         self.basic_parser.exit()
 
     def setup_logging(self, config):
+        logging_config = config["logging"]
 
-        logging_config = config['logging']
-
-        if logging_config.get('enabled', True) is False:
+        if logging_config.get("enabled", True) is False:
             # disabled logging settings, they're set on a higher level
-            logger.disable('auto_archiver')
+            logger.disable("auto_archiver")
             return
 
         # setup loguru logging
@@ -255,38 +335,45 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
             pass
 
         # add other logging info
-        if self.logger_id is None: # note - need direct comparison to None since need to consider falsy value 0
-            self.logger_id = logger.add(sys.stderr, level=logging_config['level'])
-            if log_file := logging_config['file']:
-                logger.add(log_file) if not logging_config['rotation'] else logger.add(log_file, rotation=logging_config['rotation'])
+        if self.logger_id is None:  # note - need direct comparison to None since need to consider falsy value 0
+            self.logger_id = logger.add(sys.stderr, level=logging_config["level"])
+            if log_file := logging_config["file"]:
+                logger.add(log_file) if not logging_config["rotation"] else logger.add(
+                    log_file, rotation=logging_config["rotation"]
+                )
 
     def install_modules(self, modules_by_type):
         """
-        Traverses all modules in 'steps' and loads them into the orchestrator, storing them in the 
+        Traverses all modules in 'steps' and loads them into the orchestrator, storing them in the
         orchestrator's attributes (self.feeders, self.extractors etc.). If no modules of a certain type
         are loaded, the program will exit with an error message.
         """
 
         invalid_modules = []
         for module_type in MODULE_TYPES:
-
             step_items = []
             modules_to_load = modules_by_type[f"{module_type}s"]
             if not modules_to_load:
-                raise SetupError(f"No {module_type}s were configured. Make sure to set at least one {module_type} in your configuration file or on the command line (using --{module_type}s)")
+                raise SetupError(
+                    f"No {module_type}s were configured. Make sure to set at least one {module_type} in your configuration file or on the command line (using --{module_type}s)"
+                )
 
             def check_steps_ok():
                 if not len(step_items):
                     if len(modules_to_load):
-                        logger.error(f"Unable to load any {module_type}s. Tried the following, but none were available: {modules_to_load}")
-                    raise SetupError(f"NO {module_type.upper()}S LOADED. Please check your configuration and try again.")
-                
+                        logger.error(
+                            f"Unable to load any {module_type}s. Tried the following, but none were available: {modules_to_load}"
+                        )
+                    raise SetupError(
+                        f"NO {module_type.upper()}S LOADED. Please check your configuration and try again."
+                    )
 
-                if (module_type == 'feeder' or module_type == 'formatter') and len(step_items) > 1:
-                    raise SetupError(f"Only one {module_type} is allowed, found {len(step_items)} {module_type}s. Please remove one of the following from your configuration file: {modules_to_load}")
+                if (module_type == "feeder" or module_type == "formatter") and len(step_items) > 1:
+                    raise SetupError(
+                        f"Only one {module_type} is allowed, found {len(step_items)} {module_type}s. Please remove one of the following from your configuration file: {modules_to_load}"
+                    )
 
             for module in modules_to_load:
-
                 if module in invalid_modules:
                     continue
 
@@ -295,7 +382,7 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
                     loaded_module: BaseModule = self.module_factory.get_module(module, self.config)
                 except (KeyboardInterrupt, Exception) as e:
                     logger.error(f"Error during setup of modules: {e}\n{traceback.format_exc()}")
-                    if loaded_module and module_type == 'extractor':
+                    if loaded_module and module_type == "extractor":
                         loaded_module.cleanup()
                     raise e
 
@@ -310,11 +397,13 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
 
     def load_config(self, config_file: str) -> dict:
         if not os.path.exists(config_file) and config_file != DEFAULT_CONFIG_FILE:
-            logger.error(f"The configuration file {config_file} was  not found. Make sure the file exists and try again, or run without the --config file to use the default settings.")
+            logger.error(
+                f"The configuration file {config_file} was  not found. Make sure the file exists and try again, or run without the --config file to use the default settings."
+            )
             raise FileNotFoundError(f"Configuration file {config_file} not found")
 
         return read_yaml(config_file)
-    
+
     def setup_config(self, args: list) -> dict:
         """
         Sets up the configuration file, merging the default config with the user's config
@@ -337,13 +426,13 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
         yaml_config = self.load_config(basic_config.config_file)
 
         return self.setup_complete_parser(basic_config, yaml_config, unused_args)
-    
+
     def check_for_updates(self):
         response = requests.get("https://pypi.org/pypi/auto-archiver/json").json()
-        latest_version = response['info']['version']
+        latest_version = response["info"]["version"]
         # check version compared to current version
         if latest_version != __version__:
-            if os.environ.get('RUNNING_IN_DOCKER'):
+            if os.environ.get("RUNNING_IN_DOCKER"):
                 update_cmd = "`docker pull bellingcat/auto-archiver:latest`"
             else:
                 update_cmd = "`pip install --upgrade auto-archiver`"
@@ -353,33 +442,36 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
             logger.warning(f"Make sure to update to the latest version using: {update_cmd}")
             logger.warning("")
 
-        
     def setup(self, args: list):
         """
         Function to configure all setup of the orchestrator: setup configs and load modules.
-        
+
         This method should only ever be called once
         """
 
         self.check_for_updates()
 
         if self.setup_finished:
-            logger.warning("The `setup_config()` function should only ever be run once. \
+            logger.warning(
+                "The `setup_config()` function should only ever be run once. \
                            If you need to re-run the setup, please re-instantiate a new instance of the orchestrator. \
                            For code implementatations, you should call .setup_config() once then you may call .feed() \
-                           multiple times to archive multiple URLs.")
+                           multiple times to archive multiple URLs."
+            )
             return
 
         self.setup_basic_parser()
         self.config = self.setup_config(args)
 
         logger.info(f"======== Welcome to the AUTO ARCHIVER ({__version__}) ==========")
-        self.install_modules(self.config['steps'])
+        self.install_modules(self.config["steps"])
 
         # log out the modules that were loaded
         for module_type in MODULE_TYPES:
-            logger.info(f"{module_type.upper()}S: " + ", ".join(m.display_name for m in getattr(self, f"{module_type}s")))
-        
+            logger.info(
+                f"{module_type.upper()}S: " + ", ".join(m.display_name for m in getattr(self, f"{module_type}s"))
+            )
+
         self.setup_finished = True
 
     def _command_line_run(self, args: list) -> Generator[Metadata]:
@@ -387,9 +479,9 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
         This is the main entry point for the orchestrator, when run from the command line.
 
         :param args: list of arguments to pass to the orchestrator - these are the command line args
-        
+
         You should not call this method from code implementations.
-          
+
         This method sets up the configuration, loads the modules, and runs the feed.
         If you wish to make code invocations yourself, you should use the 'setup' and 'feed' methods separately.
         To test configurations, without loading any modules you can also first call 'setup_configs'
@@ -407,7 +499,6 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
             e.cleanup()
 
     def feed(self) -> Generator[Metadata]:
-        
         url_count = 0
         for feeder in self.feeders:
             for item in feeder:
@@ -438,7 +529,7 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
             self.cleanup()
             exit()
         except Exception as e:
-            logger.error(f'Got unexpected error on item {item}: {e}\n{traceback.format_exc()}')
+            logger.error(f"Got unexpected error on item {item}: {e}\n{traceback.format_exc()}")
             for d in self.databases:
                 if type(e) == AssertionError:
                     d.failed(item, str(e))
@@ -453,13 +544,13 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
 
     def archive(self, result: Metadata) -> Union[Metadata, None]:
         """
-            Runs the archiving process for a single URL
-            1. Each archiver can sanitize its own URLs
-            2. Check for cached results in Databases, and signal start to the databases
-            3. Call Archivers until one succeeds
-            4. Call Enrichers
-            5. Store all downloaded/generated media
-            6. Call selected Formatter and store formatted if needed
+        Runs the archiving process for a single URL
+        1. Each archiver can sanitize its own URLs
+        2. Check for cached results in Databases, and signal start to the databases
+        3. Call Archivers until one succeeds
+        4. Call Enrichers
+        5. Store all downloaded/generated media
+        6. Call selected Formatter and store formatted if needed
         """
 
         original_url = result.get_url().strip()
@@ -475,7 +566,8 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
             url = a.sanitize_url(url)
 
         result.set_url(url)
-        if original_url != url: result.set("original_url", original_url)
+        if original_url != url:
+            result.set("original_url", original_url)
 
         # 2 - notify start to DBs, propagate already archived if feature enabled in DBs
         cached_result = None
@@ -486,7 +578,8 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
         if cached_result:
             logger.debug("Found previously archived entry")
             for d in self.databases:
-                try: d.done(cached_result, cached=True)
+                try:
+                    d.done(cached_result, cached=True)
                 except Exception as e:
                     logger.error(f"ERROR database {d.name}: {e}: {traceback.format_exc()}")
             return cached_result
@@ -496,13 +589,15 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
             logger.info(f"Trying extractor {a.name} for {url}")
             try:
                 result.merge(a.download(result))
-                if result.is_success(): break
+                if result.is_success():
+                    break
             except Exception as e:
                 logger.error(f"ERROR archiver {a.name}: {e}: {traceback.format_exc()}")
 
         # 4 - call enrichers to work with archived content
         for e in self.enrichers:
-            try: e.enrich(result)
+            try:
+                e.enrich(result)
             except Exception as exc:
                 logger.error(f"ERROR enricher {e.name}: {exc}: {traceback.format_exc()}")
 
@@ -520,12 +615,12 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
 
         # signal completion to databases and archivers
         for d in self.databases:
-            try: d.done(result)
+            try:
+                d.done(result)
             except Exception as e:
                 logger.error(f"ERROR database {d.name}: {e}: {traceback.format_exc()}")
 
         return result
-    
 
     def setup_authentication(self, config: dict) -> dict:
         """
@@ -534,7 +629,7 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
         Split up strings into multiple sites if they are comma separated
         """
 
-        authentication = config.get('authentication', {})
+        authentication = config.get("authentication", {})
 
         # extract out concatenated sites
         for key, val in copy(authentication).items():
@@ -543,8 +638,8 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
                     site = site.strip()
                     authentication[site] = val
                 del authentication[key]
-        
-        config['authentication'] = authentication
+
+        config["authentication"] = authentication
         return config
 
     # Helper Properties
diff --git a/src/auto_archiver/core/storage.py b/src/auto_archiver/core/storage.py
index 1535eab..2c007a4 100644
--- a/src/auto_archiver/core/storage.py
+++ b/src/auto_archiver/core/storage.py
@@ -15,16 +15,16 @@ from auto_archiver.utils.misc import random_str
 from auto_archiver.core import Media, BaseModule, Metadata
 from auto_archiver.modules.hash_enricher.hash_enricher import HashEnricher
 
+
 class Storage(BaseModule):
-    
     """
     Base class for implementing storage modules in the media archiving framework.
 
     Subclasses must implement the `get_cdn_url` and `uploadf` methods to define their behavior.
     """
 
-    def store(self, media: Media, url: str, metadata: Metadata=None) -> None:
-        if media.is_stored(in_storage=self): 
+    def store(self, media: Media, url: str, metadata: Metadata = None) -> None:
+        if media.is_stored(in_storage=self):
             logger.debug(f"{media.key} already stored, skipping")
             return
         self.set_key(media, url, metadata)
@@ -46,14 +46,15 @@ class Storage(BaseModule):
         pass
 
     def upload(self, media: Media, **kwargs) -> bool:
-        logger.debug(f'[{self.__class__.__name__}] storing file {media.filename} with key {media.key}')
-        with open(media.filename, 'rb') as f:
+        logger.debug(f"[{self.__class__.__name__}] storing file {media.filename} with key {media.key}")
+        with open(media.filename, "rb") as f:
             return self.uploadf(f, media, **kwargs)
 
     def set_key(self, media: Media, url, metadata: Metadata) -> None:
         """takes the media and optionally item info and generates a key"""
-        if media.key is not None and len(media.key) > 0: return
-        folder = metadata.get_context('folder', '')
+        if media.key is not None and len(media.key) > 0:
+            return
+        folder = metadata.get_context("folder", "")
         filename, ext = os.path.splitext(media.filename)
 
         # Handle path_generator logic
diff --git a/src/auto_archiver/core/validators.py b/src/auto_archiver/core/validators.py
index 0d3f01f..765a02b 100644
--- a/src/auto_archiver/core/validators.py
+++ b/src/auto_archiver/core/validators.py
@@ -3,11 +3,13 @@ from pathlib import Path
 import argparse
 import json
 
+
 def example_validator(value):
     if "example" not in value:
         raise argparse.ArgumentTypeError(f"{value} is not a valid value for this argument")
     return value
 
+
 def positive_number(value):
     if value < 0:
         raise argparse.ArgumentTypeError(f"{value} is not a positive number")
@@ -19,5 +21,6 @@ def valid_file(value):
         raise argparse.ArgumentTypeError(f"File '{value}' does not exist.")
     return value
 
+
 def json_loader(cli_val):
-    return json.loads(cli_val)
\ No newline at end of file
+    return json.loads(cli_val)
diff --git a/src/auto_archiver/modules/api_db/__init__.py b/src/auto_archiver/modules/api_db/__init__.py
index a4f39a1..e73511d 100644
--- a/src/auto_archiver/modules/api_db/__init__.py
+++ b/src/auto_archiver/modules/api_db/__init__.py
@@ -1 +1 @@
-from .api_db import AAApiDb
\ No newline at end of file
+from .api_db import AAApiDb
diff --git a/src/auto_archiver/modules/api_db/__manifest__.py b/src/auto_archiver/modules/api_db/__manifest__.py
index e67b31a..bf33b5a 100644
--- a/src/auto_archiver/modules/api_db/__manifest__.py
+++ b/src/auto_archiver/modules/api_db/__manifest__.py
@@ -11,8 +11,7 @@
             "required": True,
             "help": "API endpoint where calls are made to",
         },
-        "api_token": {"default": None,
-                      "help": "API Bearer token."},
+        "api_token": {"default": None, "help": "API Bearer token."},
         "public": {
             "default": False,
             "type": "bool",
diff --git a/src/auto_archiver/modules/api_db/api_db.py b/src/auto_archiver/modules/api_db/api_db.py
index 753ff3f..c422248 100644
--- a/src/auto_archiver/modules/api_db/api_db.py
+++ b/src/auto_archiver/modules/api_db/api_db.py
@@ -12,10 +12,11 @@ class AAApiDb(Database):
     """Connects to auto-archiver-api instance"""
 
     def fetch(self, item: Metadata) -> Union[Metadata, bool]:
-        """ query the database for the existence of this item.
-            Helps avoid re-archiving the same URL multiple times.
+        """query the database for the existence of this item.
+        Helps avoid re-archiving the same URL multiple times.
         """
-        if not self.use_api_cache: return
+        if not self.use_api_cache:
+            return
 
         params = {"url": item.get_url(), "limit": 15}
         headers = {"Authorization": f"Bearer {self.api_token}", "accept": "application/json"}
@@ -32,22 +33,25 @@ class AAApiDb(Database):
 
     def done(self, item: Metadata, cached: bool = False) -> None:
         """archival result ready - should be saved to DB"""
-        if not self.store_results: return
+        if not self.store_results:
+            return
         if cached:
             logger.debug(f"skipping saving archive of {item.get_url()} to the AA API because it was cached")
             return
         logger.debug(f"saving archive of {item.get_url()} to the AA API.")
 
         payload = {
-            'author_id': self.author_id,
-            'url': item.get_url(),
-            'public': self.public,
-            'group_id': self.group_id,
-            'tags': list(self.tags),
-            'result': item.to_json(),
+            "author_id": self.author_id,
+            "url": item.get_url(),
+            "public": self.public,
+            "group_id": self.group_id,
+            "tags": list(self.tags),
+            "result": item.to_json(),
         }
         headers = {"Authorization": f"Bearer {self.api_token}"}
-        response = requests.post(os.path.join(self.api_endpoint, "interop/submit-archive"), json=payload, headers=headers)
+        response = requests.post(
+            os.path.join(self.api_endpoint, "interop/submit-archive"), json=payload, headers=headers
+        )
 
         if response.status_code == 201:
             logger.success(f"AA API: {response.json()}")
diff --git a/src/auto_archiver/modules/atlos_feeder_db_storage/__init__.py b/src/auto_archiver/modules/atlos_feeder_db_storage/__init__.py
index 8d62823..8ffadf6 100644
--- a/src/auto_archiver/modules/atlos_feeder_db_storage/__init__.py
+++ b/src/auto_archiver/modules/atlos_feeder_db_storage/__init__.py
@@ -1 +1 @@
-from .atlos_feeder_db_storage import AtlosFeederDbStorage
\ No newline at end of file
+from .atlos_feeder_db_storage import AtlosFeederDbStorage
diff --git a/src/auto_archiver/modules/atlos_feeder_db_storage/__manifest__.py b/src/auto_archiver/modules/atlos_feeder_db_storage/__manifest__.py
index 2ea8f8f..eda3784 100644
--- a/src/auto_archiver/modules/atlos_feeder_db_storage/__manifest__.py
+++ b/src/auto_archiver/modules/atlos_feeder_db_storage/__manifest__.py
@@ -1,7 +1,7 @@
 {
     "name": "Atlos Feeder Database Storage",
     "type": ["feeder", "database", "storage"],
-"entry_point": "atlos_feeder_db_storage::AtlosFeederDbStorage",
+    "entry_point": "atlos_feeder_db_storage::AtlosFeederDbStorage",
     "requires_setup": True,
     "dependencies": {
         "python": ["loguru", "requests"],
@@ -15,7 +15,7 @@
         "atlos_url": {
             "default": "https://platform.atlos.org",
             "help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.",
-            "type": "str"
+            "type": "str",
         },
     },
     "description": """
@@ -42,5 +42,5 @@
     - Requires an Atlos account with a project and a valid API token for authentication.
     - Ensures only unprocessed, visible, and ready-to-archive URLs are returned.
     - Feches any media items within an Atlos project, regardless of separation into incidents.
-    """
+    """,
 }
diff --git a/src/auto_archiver/modules/atlos_feeder_db_storage/atlos_feeder_db_storage.py b/src/auto_archiver/modules/atlos_feeder_db_storage/atlos_feeder_db_storage.py
index 87b4f82..c84abd6 100644
--- a/src/auto_archiver/modules/atlos_feeder_db_storage/atlos_feeder_db_storage.py
+++ b/src/auto_archiver/modules/atlos_feeder_db_storage/atlos_feeder_db_storage.py
@@ -10,7 +10,6 @@ from auto_archiver.utils import calculate_file_hash
 
 
 class AtlosFeederDbStorage(Feeder, Database, Storage):
-
     def setup(self) -> requests.Session:
         """create and return a persistent session."""
         self.session = requests.Session()
@@ -18,9 +17,7 @@ class AtlosFeederDbStorage(Feeder, Database, Storage):
     def _get(self, endpoint: str, params: Optional[dict] = None) -> dict:
         """Wrapper for GET requests to the Atlos API."""
         url = f"{self.atlos_url}{endpoint}"
-        response = self.session.get(
-            url, headers={"Authorization": f"Bearer {self.api_token}"}, params=params
-        )
+        response = self.session.get(url, headers={"Authorization": f"Bearer {self.api_token}"}, params=params)
         response.raise_for_status()
         return response.json()
 
@@ -85,10 +82,7 @@ class AtlosFeederDbStorage(Feeder, Database, Storage):
     def _process_metadata(self, item: Metadata) -> dict:
         """Process metadata for storage on Atlos. Will convert any datetime
         objects to ISO format."""
-        return {
-            k: v.isoformat() if hasattr(v, "isoformat") else v
-            for k, v in item.metadata.items()
-        }
+        return {k: v.isoformat() if hasattr(v, "isoformat") else v for k, v in item.metadata.items()}
 
     def done(self, item: Metadata, cached: bool = False) -> None:
         """Mark an item as successfully archived in Atlos."""
@@ -129,10 +123,7 @@ class AtlosFeederDbStorage(Feeder, Database, Storage):
 
         # Check whether the media has already been uploaded
         source_material = self._get(f"/api/v2/source_material/{atlos_id}")["result"]
-        existing_media = [
-            artifact.get("file_hash_sha256")
-            for artifact in source_material.get("artifacts", [])
-        ]
+        existing_media = [artifact.get("file_hash_sha256") for artifact in source_material.get("artifacts", [])]
         if media_hash in existing_media:
             logger.info(f"{media.filename} with SHA256 {media_hash} already uploaded to Atlos")
             return True
@@ -150,4 +141,3 @@ class AtlosFeederDbStorage(Feeder, Database, Storage):
     def uploadf(self, file: IO[bytes], key: str, **kwargs: dict) -> bool:
         """Upload a file-like object; not implemented."""
         pass
-
diff --git a/src/auto_archiver/modules/cli_feeder/__manifest__.py b/src/auto_archiver/modules/cli_feeder/__manifest__.py
index 609aa3e..218f7d0 100644
--- a/src/auto_archiver/modules/cli_feeder/__manifest__.py
+++ b/src/auto_archiver/modules/cli_feeder/__manifest__.py
@@ -1,16 +1,16 @@
 {
-    'name': 'Command Line Feeder',
-    'type': ['feeder'],
-    'entry_point': 'cli_feeder::CLIFeeder',
-    'requires_setup': False,
-    'description': 'Feeds URLs to orchestrator from the command line',
-    'configs': {
-        'urls': {
-            'default': None,
-            'help': 'URL(s) to archive, either a single URL or a list of urls, should not come from config.yaml',
+    "name": "Command Line Feeder",
+    "type": ["feeder"],
+    "entry_point": "cli_feeder::CLIFeeder",
+    "requires_setup": False,
+    "description": "Feeds URLs to orchestrator from the command line",
+    "configs": {
+        "urls": {
+            "default": None,
+            "help": "URL(s) to archive, either a single URL or a list of urls, should not come from config.yaml",
         },
     },
-    'description': """
+    "description": """
 The Command Line Feeder is the default enabled feeder for the Auto Archiver. It allows you to pass URLs directly to the orchestrator from the command line 
 without the need to specify any additional configuration or command line arguments:
 
@@ -20,4 +20,4 @@ You can pass multiple URLs by separating them with a space. The URLs will be pro
 
 `auto-archiver --feeder cli_feeder -- https://example.com/1/ https://example.com/2/`
 """,
-}
\ No newline at end of file
+}
diff --git a/src/auto_archiver/modules/cli_feeder/cli_feeder.py b/src/auto_archiver/modules/cli_feeder/cli_feeder.py
index 20ca6ae..d730bd1 100644
--- a/src/auto_archiver/modules/cli_feeder/cli_feeder.py
+++ b/src/auto_archiver/modules/cli_feeder/cli_feeder.py
@@ -3,19 +3,21 @@ from loguru import logger
 from auto_archiver.core.feeder import Feeder
 from auto_archiver.core.metadata import Metadata
 
-class CLIFeeder(Feeder):
 
+class CLIFeeder(Feeder):
     def setup(self) -> None:
-        self.urls = self.config['urls']
+        self.urls = self.config["urls"]
         if not self.urls:
-            raise ValueError("No URLs provided. Please provide at least one URL via the command line, or set up an alternative feeder. Use --help for more information.")
+            raise ValueError(
+                "No URLs provided. Please provide at least one URL via the command line, or set up an alternative feeder. Use --help for more information."
+            )
 
     def __iter__(self) -> Metadata:
-        urls = self.config['urls']
+        urls = self.config["urls"]
         for url in urls:
             logger.debug(f"Processing {url}")
             m = Metadata().set_url(url)
             m.set_context("folder", "cli")
             yield m
 
-        logger.success(f"Processed {len(urls)} URL(s)")
\ No newline at end of file
+        logger.success(f"Processed {len(urls)} URL(s)")
diff --git a/src/auto_archiver/modules/console_db/__init__.py b/src/auto_archiver/modules/console_db/__init__.py
index 343f09c..831beb1 100644
--- a/src/auto_archiver/modules/console_db/__init__.py
+++ b/src/auto_archiver/modules/console_db/__init__.py
@@ -1 +1 @@
-from .console_db import ConsoleDb
\ No newline at end of file
+from .console_db import ConsoleDb
diff --git a/src/auto_archiver/modules/console_db/console_db.py b/src/auto_archiver/modules/console_db/console_db.py
index b26a605..c6711c5 100644
--- a/src/auto_archiver/modules/console_db/console_db.py
+++ b/src/auto_archiver/modules/console_db/console_db.py
@@ -6,18 +6,18 @@ from auto_archiver.core import Metadata
 
 class ConsoleDb(Database):
     """
-        Outputs results to the console
+    Outputs results to the console
     """
 
     def started(self, item: Metadata) -> None:
         logger.info(f"STARTED {item}")
 
-    def failed(self, item: Metadata, reason:str) -> None:
+    def failed(self, item: Metadata, reason: str) -> None:
         logger.error(f"FAILED {item}: {reason}")
 
     def aborted(self, item: Metadata) -> None:
         logger.warning(f"ABORTED {item}")
 
-    def done(self, item: Metadata, cached: bool=False) -> None:
+    def done(self, item: Metadata, cached: bool = False) -> None:
         """archival result ready - should be saved to DB"""
-        logger.success(f"DONE {item}")
\ No newline at end of file
+        logger.success(f"DONE {item}")
diff --git a/src/auto_archiver/modules/csv_db/__init__.py b/src/auto_archiver/modules/csv_db/__init__.py
index 1092cb2..bc9eb85 100644
--- a/src/auto_archiver/modules/csv_db/__init__.py
+++ b/src/auto_archiver/modules/csv_db/__init__.py
@@ -1 +1 @@
-from .csv_db import CSVDb
\ No newline at end of file
+from .csv_db import CSVDb
diff --git a/src/auto_archiver/modules/csv_db/__manifest__.py b/src/auto_archiver/modules/csv_db/__manifest__.py
index d9733b2..0db9cc8 100644
--- a/src/auto_archiver/modules/csv_db/__manifest__.py
+++ b/src/auto_archiver/modules/csv_db/__manifest__.py
@@ -2,12 +2,11 @@
     "name": "CSV Database",
     "type": ["database"],
     "requires_setup": False,
-    "dependencies": {"python": ["loguru"]
-                              },
-    'entry_point': 'csv_db::CSVDb',
+    "dependencies": {"python": ["loguru"]},
+    "entry_point": "csv_db::CSVDb",
     "configs": {
-            "csv_file": {"default": "db.csv", "help": "CSV file name to save metadata to"},
-        },
+        "csv_file": {"default": "db.csv", "help": "CSV file name to save metadata to"},
+    },
     "description": """
 Handles exporting archival results to a CSV file.
 
diff --git a/src/auto_archiver/modules/csv_db/csv_db.py b/src/auto_archiver/modules/csv_db/csv_db.py
index b5985e2..ac31027 100644
--- a/src/auto_archiver/modules/csv_db/csv_db.py
+++ b/src/auto_archiver/modules/csv_db/csv_db.py
@@ -9,14 +9,15 @@ from auto_archiver.core import Metadata
 
 class CSVDb(Database):
     """
-        Outputs results to a CSV file
+    Outputs results to a CSV file
     """
 
-    def done(self, item: Metadata, cached: bool=False) -> None:
+    def done(self, item: Metadata, cached: bool = False) -> None:
         """archival result ready - should be saved to DB"""
         logger.success(f"DONE {item}")
         is_empty = not os.path.isfile(self.csv_file) or os.path.getsize(self.csv_file) == 0
         with open(self.csv_file, "a", encoding="utf-8") as outf:
             writer = DictWriter(outf, fieldnames=asdict(Metadata()))
-            if is_empty: writer.writeheader()
+            if is_empty:
+                writer.writeheader()
             writer.writerow(asdict(item))
diff --git a/src/auto_archiver/modules/csv_feeder/__init__.py b/src/auto_archiver/modules/csv_feeder/__init__.py
index 161b78d..14dbd75 100644
--- a/src/auto_archiver/modules/csv_feeder/__init__.py
+++ b/src/auto_archiver/modules/csv_feeder/__init__.py
@@ -1 +1 @@
-from .csv_feeder import CSVFeeder
\ No newline at end of file
+from .csv_feeder import CSVFeeder
diff --git a/src/auto_archiver/modules/csv_feeder/__manifest__.py b/src/auto_archiver/modules/csv_feeder/__manifest__.py
index 6d4c7bf..ffb4e24 100644
--- a/src/auto_archiver/modules/csv_feeder/__manifest__.py
+++ b/src/auto_archiver/modules/csv_feeder/__manifest__.py
@@ -2,26 +2,23 @@
     "name": "CSV Feeder",
     "type": ["feeder"],
     "requires_setup": False,
-    "dependencies": {
-        "python": ["loguru"],
-        "bin": [""]
-    },
-    'requires_setup': True,
-    'entry_point': "csv_feeder::CSVFeeder",
+    "dependencies": {"python": ["loguru"], "bin": [""]},
+    "requires_setup": True,
+    "entry_point": "csv_feeder::CSVFeeder",
     "configs": {
-            "files": {
-                "default": None,
-                "help": "Path to the input file(s) to read the URLs from, comma separated. \
+        "files": {
+            "default": None,
+            "help": "Path to the input file(s) to read the URLs from, comma separated. \
                         Input files should be formatted with one URL per line",
-                "required": True,
-                "type": "valid_file",
-                "nargs": "+",
-            },
-            "column": {
-                "default": None,
-                "help": "Column number or name to read the URLs from, 0-indexed",
-            }
+            "required": True,
+            "type": "valid_file",
+            "nargs": "+",
         },
+        "column": {
+            "default": None,
+            "help": "Column number or name to read the URLs from, 0-indexed",
+        },
+    },
     "description": """
     Reads URLs from CSV files and feeds them into the archiving process.
 
@@ -33,5 +30,5 @@
     ### Setup
     - Input files should be formatted with one URL per line, with or without a header row.
     - If you have a header row, you can specify the column number or name to read URLs from using the 'column' config option.
-    """
+    """,
 }
diff --git a/src/auto_archiver/modules/csv_feeder/csv_feeder.py b/src/auto_archiver/modules/csv_feeder/csv_feeder.py
index c3f6eea..9c72162 100644
--- a/src/auto_archiver/modules/csv_feeder/csv_feeder.py
+++ b/src/auto_archiver/modules/csv_feeder/csv_feeder.py
@@ -5,11 +5,10 @@ from auto_archiver.core import Feeder
 from auto_archiver.core import Metadata
 from auto_archiver.utils import url_or_none
 
+
 class CSVFeeder(Feeder):
-
     column = None
 
-
     def __iter__(self) -> Metadata:
         for file in self.files:
             with open(file, "r") as f:
@@ -20,9 +19,11 @@ class CSVFeeder(Feeder):
                     try:
                         url_column = first_row.index(url_column)
                     except ValueError:
-                        logger.error(f"Column {url_column} not found in header row: {first_row}. Did you set the 'column' config correctly?")
+                        logger.error(
+                            f"Column {url_column} not found in header row: {first_row}. Did you set the 'column' config correctly?"
+                        )
                         return
-                elif not(url_or_none(first_row[url_column])):
+                elif not (url_or_none(first_row[url_column])):
                     # it's a header row, but we've been given a column number already
                     logger.debug(f"Skipping header row: {first_row}")
                 else:
@@ -35,4 +36,4 @@ class CSVFeeder(Feeder):
                         continue
                     url = row[url_column]
                     logger.debug(f"Processing {url}")
-                    yield Metadata().set_url(url)
\ No newline at end of file
+                    yield Metadata().set_url(url)
diff --git a/src/auto_archiver/modules/gdrive_storage/__init__.py b/src/auto_archiver/modules/gdrive_storage/__init__.py
index 2765e4b..bd326bb 100644
--- a/src/auto_archiver/modules/gdrive_storage/__init__.py
+++ b/src/auto_archiver/modules/gdrive_storage/__init__.py
@@ -1 +1 @@
-from .gdrive_storage import GDriveStorage
\ No newline at end of file
+from .gdrive_storage import GDriveStorage
diff --git a/src/auto_archiver/modules/gdrive_storage/__manifest__.py b/src/auto_archiver/modules/gdrive_storage/__manifest__.py
index 73784b8..bff6636 100644
--- a/src/auto_archiver/modules/gdrive_storage/__manifest__.py
+++ b/src/auto_archiver/modules/gdrive_storage/__manifest__.py
@@ -22,11 +22,18 @@
             "help": "how to name stored files: 'random' creates a random string; 'static' uses a replicable strategy such as a hash.",
             "choices": ["random", "static"],
         },
-        "root_folder_id": {"required": True,
-                           "help": "root google drive folder ID to use as storage, found in URL: 'https://drive.google.com/drive/folders/FOLDER_ID'"},
-        "oauth_token": {"default": None,
-                        "help": "JSON filename with Google Drive OAuth token: check auto-archiver repository scripts folder for create_update_gdrive_oauth_token.py. NOTE: storage used will count towards owner of GDrive folder, therefore it is best to use oauth_token_filename over service_account."},
-        "service_account": {"default": "secrets/service_account.json", "help": "service account JSON file path, same as used for Google Sheets. NOTE: storage used will count towards the developer account."},
+        "root_folder_id": {
+            "required": True,
+            "help": "root google drive folder ID to use as storage, found in URL: 'https://drive.google.com/drive/folders/FOLDER_ID'",
+        },
+        "oauth_token": {
+            "default": None,
+            "help": "JSON filename with Google Drive OAuth token: check auto-archiver repository scripts folder for create_update_gdrive_oauth_token.py. NOTE: storage used will count towards owner of GDrive folder, therefore it is best to use oauth_token_filename over service_account.",
+        },
+        "service_account": {
+            "default": "secrets/service_account.json",
+            "help": "service account JSON file path, same as used for Google Sheets. NOTE: storage used will count towards the developer account.",
+        },
     },
     "description": """
     
@@ -94,5 +101,5 @@ This module integrates Google Drive as a storage backend, enabling automatic fol
     https://davemateer.com/2022/04/28/google-drive-with-python#tokens
     
     
-"""
+""",
 }
diff --git a/src/auto_archiver/modules/gdrive_storage/gdrive_storage.py b/src/auto_archiver/modules/gdrive_storage/gdrive_storage.py
index 4971030..f01ea4e 100644
--- a/src/auto_archiver/modules/gdrive_storage/gdrive_storage.py
+++ b/src/auto_archiver/modules/gdrive_storage/gdrive_storage.py
@@ -1,4 +1,3 @@
-
 import json
 import os
 import time
@@ -15,12 +14,9 @@ from auto_archiver.core import Media
 from auto_archiver.core import Storage
 
 
-
-
 class GDriveStorage(Storage):
-
     def setup(self) -> None:
-        self.scopes = ['https://www.googleapis.com/auth/drive']
+        self.scopes = ["https://www.googleapis.com/auth/drive"]
         # Initialize Google Drive service
         self._setup_google_drive_service()
 
@@ -37,25 +33,25 @@ class GDriveStorage(Storage):
 
     def _initialize_with_oauth_token(self):
         """Initialize Google Drive service with OAuth token."""
-        with open(self.oauth_token, 'r') as stream:
+        with open(self.oauth_token, "r") as stream:
             creds_json = json.load(stream)
-            creds_json['refresh_token'] = creds_json.get("refresh_token", "")
+            creds_json["refresh_token"] = creds_json.get("refresh_token", "")
 
         creds = Credentials.from_authorized_user_info(creds_json, self.scopes)
         if not creds.valid and creds.expired and creds.refresh_token:
             creds.refresh(Request())
-            with open(self.oauth_token, 'w') as token_file:
+            with open(self.oauth_token, "w") as token_file:
                 logger.debug("Saving refreshed OAuth token.")
                 token_file.write(creds.to_json())
         elif not creds.valid:
             raise ValueError("Invalid OAuth token. Please regenerate the token.")
 
-        return build('drive', 'v3', credentials=creds)
+        return build("drive", "v3", credentials=creds)
 
     def _initialize_with_service_account(self):
         """Initialize Google Drive service with service account."""
         creds = service_account.Credentials.from_service_account_file(self.service_account, scopes=self.scopes)
-        return build('drive', 'v3', credentials=creds)
+        return build("drive", "v3", credentials=creds)
 
     def get_cdn_url(self, media: Media) -> str:
         """
@@ -79,7 +75,7 @@ class GDriveStorage(Storage):
         return f"https://drive.google.com/file/d/{file_id}/view?usp=sharing"
 
     def upload(self, media: Media, **kwargs) -> bool:
-        logger.debug(f'[{self.__class__.__name__}] storing file {media.filename} with key {media.key}')
+        logger.debug(f"[{self.__class__.__name__}] storing file {media.filename} with key {media.key}")
         """
         1. for each sub-folder in the path check if exists or create
         2. upload file to root_id/other_paths.../filename
@@ -95,25 +91,30 @@ class GDriveStorage(Storage):
             parent_id = upload_to
 
         # upload file to gd
-        logger.debug(f'uploading {filename=} to folder id {upload_to}')
-        file_metadata = {
-            'name': [filename],
-            'parents': [upload_to]
-        }
+        logger.debug(f"uploading {filename=} to folder id {upload_to}")
+        file_metadata = {"name": [filename], "parents": [upload_to]}
         media = MediaFileUpload(media.filename, resumable=True)
-        gd_file = self.service.files().create(supportsAllDrives=True, body=file_metadata, media_body=media, fields='id').execute()
-        logger.debug(f'uploadf: uploaded file {gd_file["id"]} successfully in folder={upload_to}')
+        gd_file = (
+            self.service.files()
+            .create(supportsAllDrives=True, body=file_metadata, media_body=media, fields="id")
+            .execute()
+        )
+        logger.debug(f"uploadf: uploaded file {gd_file['id']} successfully in folder={upload_to}")
 
     # must be implemented even if unused
-    def uploadf(self, file: IO[bytes], key: str, **kwargs: dict) -> bool: pass
+    def uploadf(self, file: IO[bytes], key: str, **kwargs: dict) -> bool:
+        pass
 
-    def _get_id_from_parent_and_name(self, parent_id: str,
-                                     name: str,
-                                     retries: int = 1,
-                                     sleep_seconds: int = 10,
-                                     use_mime_type: bool = False,
-                                     raise_on_missing: bool = True,
-                                     use_cache=False):
+    def _get_id_from_parent_and_name(
+        self,
+        parent_id: str,
+        name: str,
+        retries: int = 1,
+        sleep_seconds: int = 10,
+        use_mime_type: bool = False,
+        raise_on_missing: bool = True,
+        use_cache=False,
+    ):
         """
         Retrieves the id of a folder or file from its @name and the @parent_id folder
         Optionally does multiple @retries and sleeps @sleep_seconds between them
@@ -137,29 +138,36 @@ class GDriveStorage(Storage):
             query_string += f" and mimeType='application/vnd.google-apps.folder' "
 
         for attempt in range(retries):
-            results = self.service.files().list(
-                # both below for Google Shared Drives
-                supportsAllDrives=True,
-                includeItemsFromAllDrives=True,
-                q=query_string,
-                spaces='drive',  # ie not appDataFolder or photos
-                fields='files(id, name)'
-            ).execute()
-            items = results.get('files', [])
+            results = (
+                self.service.files()
+                .list(
+                    # both below for Google Shared Drives
+                    supportsAllDrives=True,
+                    includeItemsFromAllDrives=True,
+                    q=query_string,
+                    spaces="drive",  # ie not appDataFolder or photos
+                    fields="files(id, name)",
+                )
+                .execute()
+            )
+            items = results.get("files", [])
 
             if len(items) > 0:
-                logger.debug(f"{debug_header} found {len(items)} matches, returning last of {','.join([i['id'] for i in items])}")
-                _id = items[-1]['id']
-                if use_cache: self.api_cache[cache_key] = _id
+                logger.debug(
+                    f"{debug_header} found {len(items)} matches, returning last of {','.join([i['id'] for i in items])}"
+                )
+                _id = items[-1]["id"]
+                if use_cache:
+                    self.api_cache[cache_key] = _id
                 return _id
             else:
-                logger.debug(f'{debug_header} not found, attempt {attempt+1}/{retries}.')
+                logger.debug(f"{debug_header} not found, attempt {attempt + 1}/{retries}.")
                 if attempt < retries - 1:
-                    logger.debug(f'sleeping for {sleep_seconds} second(s)')
+                    logger.debug(f"sleeping for {sleep_seconds} second(s)")
                     time.sleep(sleep_seconds)
 
         if raise_on_missing:
-            raise ValueError(f'{debug_header} not found after {retries} attempt(s)')
+            raise ValueError(f"{debug_header} not found after {retries} attempt(s)")
         return None
 
     def _mkdir(self, name: str, parent_id: str):
@@ -167,12 +175,7 @@ class GDriveStorage(Storage):
         Creates a new GDrive folder @name inside folder @parent_id
         Returns id of the created folder
         """
-        logger.debug(f'Creating new folder with {name=} inside {parent_id=}')
-        file_metadata = {
-            'name': [name],
-            'mimeType': 'application/vnd.google-apps.folder',
-            'parents': [parent_id]
-        }
-        gd_folder = self.service.files().create(supportsAllDrives=True, body=file_metadata, fields='id').execute()
-        return gd_folder.get('id')
-
+        logger.debug(f"Creating new folder with {name=} inside {parent_id=}")
+        file_metadata = {"name": [name], "mimeType": "application/vnd.google-apps.folder", "parents": [parent_id]}
+        gd_folder = self.service.files().create(supportsAllDrives=True, body=file_metadata, fields="id").execute()
+        return gd_folder.get("id")
diff --git a/src/auto_archiver/modules/generic_extractor/__init__.py b/src/auto_archiver/modules/generic_extractor/__init__.py
index 5bfcd01..d573b3d 100644
--- a/src/auto_archiver/modules/generic_extractor/__init__.py
+++ b/src/auto_archiver/modules/generic_extractor/__init__.py
@@ -1 +1 @@
-from .generic_extractor import GenericExtractor
\ No newline at end of file
+from .generic_extractor import GenericExtractor
diff --git a/src/auto_archiver/modules/generic_extractor/bluesky.py b/src/auto_archiver/modules/generic_extractor/bluesky.py
index 5eef520..5baad6c 100644
--- a/src/auto_archiver/modules/generic_extractor/bluesky.py
+++ b/src/auto_archiver/modules/generic_extractor/bluesky.py
@@ -4,15 +4,16 @@ from auto_archiver.core.extractor import Extractor
 from auto_archiver.core.metadata import Metadata, Media
 from .dropin import GenericDropin, InfoExtractor
 
-class Bluesky(GenericDropin):
 
+class Bluesky(GenericDropin):
     def create_metadata(self, post: dict, ie_instance: InfoExtractor, archiver: Extractor, url: str) -> Metadata:
         result = Metadata()
         result.set_url(url)
         result.set_title(post["record"]["text"])
         result.set_timestamp(post["record"]["createdAt"])
         for k, v in self._get_post_data(post).items():
-            if v: result.set(k, v)
+            if v:
+                result.set(k, v)
 
         # download if embeds present (1 video XOR >=1 images)
         for media in self._download_bsky_embeds(post, archiver):
@@ -23,12 +24,12 @@ class Bluesky(GenericDropin):
 
     def extract_post(self, url: str, ie_instance: InfoExtractor) -> dict:
         # TODO: If/when this PR (https://github.com/yt-dlp/yt-dlp/pull/12098) is merged on ytdlp, remove the comments and delete the code below
-        handle, video_id = ie_instance._match_valid_url(url).group('handle', 'id')
+        handle, video_id = ie_instance._match_valid_url(url).group("handle", "id")
         return ie_instance._extract_post(handle=handle, post_id=video_id)
 
     def _download_bsky_embeds(self, post: dict, archiver: Extractor) -> list[Media]:
         """
-        Iterates over image(s) or video in a Bluesky post and downloads them        
+        Iterates over image(s) or video in a Bluesky post and downloads them
         """
         media = []
         embed = post.get("record", {}).get("embed", {})
@@ -37,16 +38,15 @@ class Bluesky(GenericDropin):
 
         media_url = "https://bsky.social/xrpc/com.atproto.sync.getBlob?cid={}&did={}"
         for image_media in image_medias:
-            url = media_url.format(image_media['image']['ref']['$link'], post['author']['did'])
+            url = media_url.format(image_media["image"]["ref"]["$link"], post["author"]["did"])
             image_media = archiver.download_from_url(url)
             media.append(Media(image_media))
         for video_media in video_medias:
-            url = media_url.format(video_media['ref']['$link'], post['author']['did'])
+            url = media_url.format(video_media["ref"]["$link"], post["author"]["did"])
             video_media = archiver.download_from_url(url)
             media.append(Media(video_media))
         return media
 
-
     def _get_post_data(self, post: dict) -> dict:
         """
         Extracts relevant information returned by the .getPostThread api call (excluding text/created_at): author, mentions, tags, links.
@@ -74,4 +74,4 @@ class Bluesky(GenericDropin):
             res["tags"] = tags
         if links:
             res["links"] = links
-        return res
\ No newline at end of file
+        return res
diff --git a/src/auto_archiver/modules/generic_extractor/dropin.py b/src/auto_archiver/modules/generic_extractor/dropin.py
index c5749ff..723c8fc 100644
--- a/src/auto_archiver/modules/generic_extractor/dropin.py
+++ b/src/auto_archiver/modules/generic_extractor/dropin.py
@@ -2,11 +2,12 @@ from yt_dlp.extractor.common import InfoExtractor
 from auto_archiver.core.metadata import Metadata
 from auto_archiver.core.extractor import Extractor
 
+
 class GenericDropin:
     """Base class for dropins for the generic extractor.
-    
+
     In many instances, an extractor will exist in ytdlp, but it will only process videos.
-    Dropins can be created and used to make use of the already-written private code of a 
+    Dropins can be created and used to make use of the already-written private code of a
     specific extractor from ytdlp.
 
     The dropin should be able to handle the following methods:
@@ -28,21 +29,19 @@ class GenericDropin:
         This method should return the post data from the url.
         """
         raise NotImplementedError("This method should be implemented in the subclass")
-    
 
     def create_metadata(self, post: dict, ie_instance: InfoExtractor, archiver: Extractor, url: str) -> Metadata:
         """
         This method should create a Metadata object from the post data.
         """
         raise NotImplementedError("This method should be implemented in the subclass")
-    
 
     def skip_ytdlp_download(self, url: str, ie_instance: InfoExtractor):
         """
         This method should return True if you want to skip the ytdlp download method.
         """
         return False
-    
+
     def keys_to_clean(self, video_data: dict, info_extractor: InfoExtractor):
         """
         This method should return a list of strings (keys) to clean from the video_data dict.
@@ -50,9 +49,9 @@ class GenericDropin:
         E.g. ["uploader", "uploader_id", "tiktok_specific_field"]
         """
         return []
-    
+
     def download_additional_media(self, video_data: dict, info_extractor: InfoExtractor, metadata: Metadata):
         """
         This method should download any additional media from the post.
         """
-        return metadata
\ No newline at end of file
+        return metadata
diff --git a/src/auto_archiver/modules/generic_extractor/facebook.py b/src/auto_archiver/modules/generic_extractor/facebook.py
index fed8e09..36c5e60 100644
--- a/src/auto_archiver/modules/generic_extractor/facebook.py
+++ b/src/auto_archiver/modules/generic_extractor/facebook.py
@@ -3,16 +3,15 @@ from .dropin import GenericDropin
 
 class Facebook(GenericDropin):
     def extract_post(self, url: str, ie_instance):
-        video_id = ie_instance._match_valid_url(url).group('id')
-        ie_instance._download_webpage(
-            url.replace('://m.facebook.com/', '://www.facebook.com/'), video_id)
-        webpage = ie_instance._download_webpage(url, ie_instance._match_valid_url(url).group('id'))
+        video_id = ie_instance._match_valid_url(url).group("id")
+        ie_instance._download_webpage(url.replace("://m.facebook.com/", "://www.facebook.com/"), video_id)
+        webpage = ie_instance._download_webpage(url, ie_instance._match_valid_url(url).group("id"))
 
         # TODO: fix once https://github.com/yt-dlp/yt-dlp/pull/12275 is merged
         post_data = ie_instance._extract_metadata(webpage)
         return post_data
-    
+
     def create_metadata(self, post: dict, ie_instance, archiver, url):
         metadata = archiver.create_metadata(url)
-        metadata.set_title(post.get('title')).set_content(post.get('description')).set_post_data(post)
-        return metadata
\ No newline at end of file
+        metadata.set_title(post.get("title")).set_content(post.get("description")).set_post_data(post)
+        return metadata
diff --git a/src/auto_archiver/modules/generic_extractor/generic_extractor.py b/src/auto_archiver/modules/generic_extractor/generic_extractor.py
index 5acce46..107ce93 100644
--- a/src/auto_archiver/modules/generic_extractor/generic_extractor.py
+++ b/src/auto_archiver/modules/generic_extractor/generic_extractor.py
@@ -12,6 +12,7 @@ from loguru import logger
 from auto_archiver.core.extractor import Extractor
 from auto_archiver.core import Metadata, Media
 
+
 class GenericExtractor(Extractor):
     _dropins = {}
 
@@ -19,14 +20,14 @@ class GenericExtractor(Extractor):
         # check for file .ytdlp-update in the secrets folder
         if self.ytdlp_update_interval < 0:
             return
-        
-        use_secrets = os.path.exists('secrets')
-        path = os.path.join('secrets' if use_secrets else '', '.ytdlp-update')
+
+        use_secrets = os.path.exists("secrets")
+        path = os.path.join("secrets" if use_secrets else "", ".ytdlp-update")
         next_update_check = None
         if os.path.exists(path):
             with open(path, "r") as f:
                 next_update_check = datetime.datetime.fromisoformat(f.read())
-        
+
         if not next_update_check or next_update_check < datetime.datetime.now():
             self.update_ytdlp()
 
@@ -36,8 +37,11 @@ class GenericExtractor(Extractor):
 
     def update_ytdlp(self):
         logger.info("Checking and updating yt-dlp...")
-        logger.info(f"Tip: change the 'ytdlp_update_interval' setting to control how often yt-dlp is updated. Set to -1 to disable or 0 to enable on every run. Current setting: {self.ytdlp_update_interval}")
+        logger.info(
+            f"Tip: change the 'ytdlp_update_interval' setting to control how often yt-dlp is updated. Set to -1 to disable or 0 to enable on every run. Current setting: {self.ytdlp_update_interval}"
+        )
         from importlib.metadata import version as get_version
+
         old_version = get_version("yt-dlp")
         try:
             # try and update with pip (this works inside poetry environment and in a normal virtualenv)
@@ -59,15 +63,17 @@ class GenericExtractor(Extractor):
         for info_extractor in yt_dlp.YoutubeDL()._ies.values():
             if info_extractor.suitable(url) and info_extractor.working():
                 yield info_extractor
-        
+
     def suitable(self, url: str) -> bool:
         """
         Checks for valid URLs out of all ytdlp extractors.
         Returns False for the GenericIE, which as labelled by yt-dlp: 'Generic downloader that works on some sites'
         """
         return any(self.suitable_extractors(url))
-    
-    def download_additional_media(self, video_data: dict, info_extractor: InfoExtractor, metadata: Metadata) -> Metadata:
+
+    def download_additional_media(
+        self, video_data: dict, info_extractor: InfoExtractor, metadata: Metadata
+    ) -> Metadata:
         """
         Downloads additional media like images, comments, subtitles, etc.
 
@@ -76,7 +82,7 @@ class GenericExtractor(Extractor):
 
         # Just get the main thumbnail. More thumbnails are available in
         # video_data['thumbnails'] should they be required
-        thumbnail_url = video_data.get('thumbnail')
+        thumbnail_url = video_data.get("thumbnail")
         if thumbnail_url:
             try:
                 cover_image_path = self.download_from_url(thumbnail_url)
@@ -99,15 +105,65 @@ class GenericExtractor(Extractor):
         Clean up the ytdlp generic video data to make it more readable and remove unnecessary keys that ytdlp adds
         """
 
-        base_keys = ['formats', 'thumbnail', 'display_id', 'epoch', 'requested_downloads',
-                     'duration_string', 'thumbnails', 'http_headers', 'webpage_url_basename', 'webpage_url_domain',
-                     'extractor', 'extractor_key', 'playlist', 'playlist_index', 'duration_string', 'protocol', 'requested_subtitles',
-                     'format_id', 'acodec', 'vcodec', 'ext', 'epoch', '_has_drm', 'filesize', 'audio_ext', 'video_ext', 'vbr', 'abr',
-                     'resolution', 'dynamic_range', 'aspect_ratio', 'cookies', 'format', 'quality', 'preference', 'artists',
-                     'channel_id', 'subtitles', 'tbr', 'url', 'original_url', 'automatic_captions', 'playable_in_embed', 'live_status',
-                     '_format_sort_fields', 'chapters', 'requested_formats', 'format_note',
-                     'audio_channels', 'asr', 'fps', 'was_live', 'is_live', 'heatmap', 'age_limit', 'stretched_ratio']
-        
+        base_keys = [
+            "formats",
+            "thumbnail",
+            "display_id",
+            "epoch",
+            "requested_downloads",
+            "duration_string",
+            "thumbnails",
+            "http_headers",
+            "webpage_url_basename",
+            "webpage_url_domain",
+            "extractor",
+            "extractor_key",
+            "playlist",
+            "playlist_index",
+            "duration_string",
+            "protocol",
+            "requested_subtitles",
+            "format_id",
+            "acodec",
+            "vcodec",
+            "ext",
+            "epoch",
+            "_has_drm",
+            "filesize",
+            "audio_ext",
+            "video_ext",
+            "vbr",
+            "abr",
+            "resolution",
+            "dynamic_range",
+            "aspect_ratio",
+            "cookies",
+            "format",
+            "quality",
+            "preference",
+            "artists",
+            "channel_id",
+            "subtitles",
+            "tbr",
+            "url",
+            "original_url",
+            "automatic_captions",
+            "playable_in_embed",
+            "live_status",
+            "_format_sort_fields",
+            "chapters",
+            "requested_formats",
+            "format_note",
+            "audio_channels",
+            "asr",
+            "fps",
+            "was_live",
+            "is_live",
+            "heatmap",
+            "age_limit",
+            "stretched_ratio",
+        ]
+
         dropin = self.dropin_for_name(info_extractor.ie_key())
         if dropin:
             try:
@@ -116,8 +172,8 @@ class GenericExtractor(Extractor):
                 pass
 
         return base_keys
-    
-    def add_metadata(self, video_data: dict, info_extractor: InfoExtractor, url:str, result: Metadata) -> Metadata:
+
+    def add_metadata(self, video_data: dict, info_extractor: InfoExtractor, url: str, result: Metadata) -> Metadata:
         """
         Creates a Metadata object from the given video_data
         """
@@ -126,29 +182,36 @@ class GenericExtractor(Extractor):
         result = self.download_additional_media(video_data, info_extractor, result)
 
         # keep both 'title' and 'fulltitle', but prefer 'title', falling back to 'fulltitle' if it doesn't exist
-        result.set_title(video_data.pop('title', video_data.pop('fulltitle', "")))
+        result.set_title(video_data.pop("title", video_data.pop("fulltitle", "")))
         result.set_url(url)
-        if "description" in video_data: result.set_content(video_data["description"])
+        if "description" in video_data:
+            result.set_content(video_data["description"])
         # extract comments if enabled
         if self.comments:
-            result.set("comments", [{
-                "text": c["text"],
-                "author": c["author"], 
-                "timestamp": datetime.datetime.fromtimestamp(c.get("timestamp"), tz = datetime.timezone.utc)
-            } for c in video_data.get("comments", [])])
+            result.set(
+                "comments",
+                [
+                    {
+                        "text": c["text"],
+                        "author": c["author"],
+                        "timestamp": datetime.datetime.fromtimestamp(c.get("timestamp"), tz=datetime.timezone.utc),
+                    }
+                    for c in video_data.get("comments", [])
+                ],
+            )
 
         # then add the common metadata
         if timestamp := video_data.pop("timestamp", None):
-            timestamp = datetime.datetime.fromtimestamp(timestamp, tz = datetime.timezone.utc).isoformat()
+            timestamp = datetime.datetime.fromtimestamp(timestamp, tz=datetime.timezone.utc).isoformat()
             result.set_timestamp(timestamp)
         if upload_date := video_data.pop("upload_date", None):
-            upload_date = datetime.datetime.strptime(upload_date, '%Y%m%d').replace(tzinfo=datetime.timezone.utc)
+            upload_date = datetime.datetime.strptime(upload_date, "%Y%m%d").replace(tzinfo=datetime.timezone.utc)
             result.set("upload_date", upload_date)
-        
+
         # then clean away any keys we don't want
         for clean_key in self.keys_to_clean(info_extractor, video_data):
             video_data.pop(clean_key, None)
-        
+
         # then add the rest of the video data
         for k, v in video_data.items():
             if v:
@@ -169,22 +232,24 @@ class GenericExtractor(Extractor):
             logger.debug(f"""Could not find valid dropin for {info_extractor.IE_NAME}.
                      Why not try creating your own, and make sure it has a valid function called 'create_metadata'. Learn more: https://auto-archiver.readthedocs.io/en/latest/user_guidelines.html#""")
             return False
-        
+
         post_data = dropin.extract_post(url, ie_instance)
         return dropin.create_metadata(post_data, ie_instance, self, url)
 
-    def get_metadata_for_video(self, data: dict, info_extractor: Type[InfoExtractor], url: str, ydl: yt_dlp.YoutubeDL) -> Metadata:
-
+    def get_metadata_for_video(
+        self, data: dict, info_extractor: Type[InfoExtractor], url: str, ydl: yt_dlp.YoutubeDL
+    ) -> Metadata:
         # this time download
-        ydl.params['getcomments'] = self.comments
-        #TODO: for playlist or long lists of videos, how to download one at a time so they can be stored before the next one is downloaded?
+        ydl.params["getcomments"] = self.comments
+        # TODO: for playlist or long lists of videos, how to download one at a time so they can be stored before the next one is downloaded?
         data = ydl.extract_info(url, ie_key=info_extractor.ie_key(), download=True)
         if "entries" in data:
             entries = data.get("entries", [])
             if not len(entries):
-                logger.warning('YoutubeDLArchiver could not find any video')
+                logger.warning("YoutubeDLArchiver could not find any video")
                 return False
-        else: entries = [data]
+        else:
+            entries = [data]
 
         result = Metadata()
 
@@ -192,17 +257,18 @@ class GenericExtractor(Extractor):
             try:
                 filename = ydl.prepare_filename(entry)
                 if not os.path.exists(filename):
-                    filename = filename.split('.')[0] + '.mkv'
+                    filename = filename.split(".")[0] + ".mkv"
 
                 new_media = Media(filename)
                 for x in ["duration", "original_url", "fulltitle", "description", "upload_date"]:
-                    if x in entry: new_media.set(x, entry[x])
+                    if x in entry:
+                        new_media.set(x, entry[x])
 
                 # read text from subtitles if enabled
                 if self.subtitles:
-                    for lang, val in (data.get('requested_subtitles') or {}).items():
-                        try:    
-                            subs = pysubs2.load(val.get('filepath'), encoding="utf-8")
+                    for lang, val in (data.get("requested_subtitles") or {}).items():
+                        try:
+                            subs = pysubs2.load(val.get("filepath"), encoding="utf-8")
                             text = " ".join([line.text for line in subs])
                             new_media.set(f"subtitles_{lang}", text)
                         except Exception as e:
@@ -212,8 +278,8 @@ class GenericExtractor(Extractor):
                 logger.error(f"Error processing entry {entry}: {e}")
 
         return self.add_metadata(data, info_extractor, url, result)
-    
-    def dropin_for_name(self, dropin_name: str, additional_paths = [], package=__package__) -> Type[InfoExtractor]:
+
+    def dropin_for_name(self, dropin_name: str, additional_paths=[], package=__package__) -> Type[InfoExtractor]:
         dropin_name = dropin_name.lower()
 
         if dropin_name == "generic":
@@ -221,6 +287,7 @@ class GenericExtractor(Extractor):
             return None
 
         dropin_class_name = dropin_name.title()
+
         def _load_dropin(dropin):
             dropin_class = getattr(dropin, dropin_class_name)()
             return self._dropins.setdefault(dropin_name, dropin_class)
@@ -244,7 +311,7 @@ class GenericExtractor(Extractor):
                 return _load_dropin(dropin)
             except (FileNotFoundError, ModuleNotFoundError):
                 pass
-        
+
         # fallback to loading the dropins within auto-archiver
         try:
             return _load_dropin(importlib.import_module(f".{dropin_name}", package=package))
@@ -256,12 +323,12 @@ class GenericExtractor(Extractor):
     def download_for_extractor(self, info_extractor: InfoExtractor, url: str, ydl: yt_dlp.YoutubeDL) -> Metadata:
         """
         Tries to download the given url using the specified extractor
-        
+
         It first tries to use ytdlp directly to download the video. If the post is not a video, it will then try to
         use the extractor's _extract_post method to get the post metadata if possible.
         """
         # when getting info without download, we also don't need the comments
-        ydl.params['getcomments'] = False
+        ydl.params["getcomments"] = False
         result = False
 
         dropin_submodule = self.dropin_for_name(info_extractor.ie_key())
@@ -272,7 +339,7 @@ class GenericExtractor(Extractor):
 
             # don't download since it can be a live stream
             data = ydl.extract_info(url, ie_key=info_extractor.ie_key(), download=False)
-            if data.get('is_live', False) and not self.livestreams:
+            if data.get("is_live", False) and not self.livestreams:
                 logger.warning("Livestream detected, skipping due to 'livestreams' configuration setting")
                 return False
             # it's a valid video, that the youtubdedl can download out of the box
@@ -283,16 +350,21 @@ class GenericExtractor(Extractor):
                 # don't clutter the logs with issues about the 'generic' extractor not having a dropin
                 return False
 
-            logger.debug(f'Issue using "{info_extractor.IE_NAME}" extractor to download video (error: {repr(e)}), attempting to use extractor to get post data instead')
+            logger.debug(
+                f'Issue using "{info_extractor.IE_NAME}" extractor to download video (error: {repr(e)}), attempting to use extractor to get post data instead'
+            )
             try:
                 result = self.get_metadata_for_post(info_extractor, url, ydl)
             except (yt_dlp.utils.DownloadError, yt_dlp.utils.ExtractorError) as post_e:
-                logger.error(f'Error downloading metadata for post: {post_e}')
+                logger.error(f"Error downloading metadata for post: {post_e}")
                 return False
             except Exception as generic_e:
-                logger.debug(f'Attempt to extract using ytdlp extractor "{info_extractor.IE_NAME}" failed:  \n  {repr(generic_e)}', exc_info=True)
+                logger.debug(
+                    f'Attempt to extract using ytdlp extractor "{info_extractor.IE_NAME}" failed:  \n  {repr(generic_e)}',
+                    exc_info=True,
+                )
                 return False
-        
+
         if result:
             extractor_name = "yt-dlp"
             if info_extractor:
@@ -308,43 +380,49 @@ class GenericExtractor(Extractor):
     def download(self, item: Metadata) -> Metadata:
         url = item.get_url()
 
-        #TODO: this is a temporary hack until this issue is closed: https://github.com/yt-dlp/yt-dlp/issues/11025
+        # TODO: this is a temporary hack until this issue is closed: https://github.com/yt-dlp/yt-dlp/issues/11025
         if url.startswith("https://ya.ru"):
             url = url.replace("https://ya.ru", "https://yandex.ru")
             item.set("replaced_url", url)
 
+        ydl_options = {
+            "outtmpl": os.path.join(self.tmp_dir, f"%(id)s.%(ext)s"),
+            "quiet": False,
+            "noplaylist": not self.allow_playlist,
+            "writesubtitles": self.subtitles,
+            "writeautomaticsub": self.subtitles,
+            "live_from_start": self.live_from_start,
+            "proxy": self.proxy,
+            "max_downloads": self.max_downloads,
+            "playlistend": self.max_downloads,
+        }
 
-        ydl_options = {'outtmpl': os.path.join(self.tmp_dir, f'%(id)s.%(ext)s'), 
-                       'quiet': False, 'noplaylist': not self.allow_playlist ,
-                       'writesubtitles': self.subtitles,'writeautomaticsub': self.subtitles,
-                       "live_from_start": self.live_from_start, "proxy": self.proxy,
-                       "max_downloads": self.max_downloads, "playlistend": self.max_downloads}
-        
         # set up auth
         auth = self.auth_for_site(url, extract_cookies=False)
 
         # order of importance: username/pasword -> api_key -> cookie -> cookies_from_browser -> cookies_file
         if auth:
-            if 'username' in auth and 'password' in auth:
-                logger.debug(f'Using provided auth username and password for {url}')
-                ydl_options['username'] = auth['username']
-                ydl_options['password'] = auth['password']
-            elif 'cookie' in auth:
-                logger.debug(f'Using provided auth cookie for {url}')
-                yt_dlp.utils.std_headers['cookie'] = auth['cookie']
-            elif 'cookies_from_browser' in auth:
-                logger.debug(f'Using extracted cookies from browser {auth["cookies_from_browser"]} for {url}')
-                ydl_options['cookiesfrombrowser'] = auth['cookies_from_browser']
-            elif 'cookies_file' in auth:
-                logger.debug(f'Using cookies from file {auth["cookies_file"]} for {url}')
-                ydl_options['cookiefile'] = auth['cookies_file']
+            if "username" in auth and "password" in auth:
+                logger.debug(f"Using provided auth username and password for {url}")
+                ydl_options["username"] = auth["username"]
+                ydl_options["password"] = auth["password"]
+            elif "cookie" in auth:
+                logger.debug(f"Using provided auth cookie for {url}")
+                yt_dlp.utils.std_headers["cookie"] = auth["cookie"]
+            elif "cookies_from_browser" in auth:
+                logger.debug(f"Using extracted cookies from browser {auth['cookies_from_browser']} for {url}")
+                ydl_options["cookiesfrombrowser"] = auth["cookies_from_browser"]
+            elif "cookies_file" in auth:
+                logger.debug(f"Using cookies from file {auth['cookies_file']} for {url}")
+                ydl_options["cookiefile"] = auth["cookies_file"]
 
-        ydl = yt_dlp.YoutubeDL(ydl_options) # allsubtitles and subtitleslangs not working as expected, so default lang is always "en"
+        ydl = yt_dlp.YoutubeDL(
+            ydl_options
+        )  # allsubtitles and subtitleslangs not working as expected, so default lang is always "en"
 
         for info_extractor in self.suitable_extractors(url):
             result = self.download_for_extractor(info_extractor, url, ydl)
             if result:
                 return result
 
-
         return False
diff --git a/src/auto_archiver/modules/generic_extractor/truth.py b/src/auto_archiver/modules/generic_extractor/truth.py
index e65b4b1..345f1cd 100644
--- a/src/auto_archiver/modules/generic_extractor/truth.py
+++ b/src/auto_archiver/modules/generic_extractor/truth.py
@@ -9,11 +9,11 @@ from dateutil.parser import parse as parse_dt
 
 from .dropin import GenericDropin
 
-class Truth(GenericDropin):
 
+class Truth(GenericDropin):
     def extract_post(self, url, ie_instance: InfoExtractor) -> dict:
         video_id = ie_instance._match_id(url)
-        truthsocial_url = f'https://truthsocial.com/api/v1/statuses/{video_id}'
+        truthsocial_url = f"https://truthsocial.com/api/v1/statuses/{video_id}"
         return ie_instance._download_json(truthsocial_url, video_id)
 
     def skip_ytdlp_download(self, url, ie_instance: Type[InfoExtractor]) -> bool:
@@ -22,31 +22,42 @@ class Truth(GenericDropin):
     def create_metadata(self, post: dict, ie_instance: InfoExtractor, archiver: Extractor, url: str) -> Metadata:
         """
         Creates metadata from a truth social post
-        
+
         Only used for posts that contain no media. ytdlp.TruthIE extractor can handle posts with media
-        
+
         Format is:
-        
+
         {'id': '109598702184774628', 'created_at': '2022-12-29T19:51:18.161Z', 'in_reply_to_id': None, 'quote_id': None, 'in_reply_to_account_id': None, 'sensitive': False, 'spoiler_text': '', 'visibility': 'public', 'language': 'en', 'uri': 'https://truthsocial.com/@bbcnewa/109598702184774628', 'url': 'https://truthsocial.com/@bbcnewa/109598702184774628', 'content': '<p>Pele, regarded by many as football\'s greatest ever player, has died in Brazil at the age of 82. <a href="https://www.bbc.com/sport/football/42751517" rel="nofollow noopener noreferrer" target="_blank"><span class="invisible">https://www.</span><span class="ellipsis">bbc.com/sport/football/4275151</span><span class="invisible">7</span></a></p>', 'account': {'id': '107905163010312793', 'username': 'bbcnewa', 'acct': 'bbcnewa', 'display_name': 'BBC News', 'locked': False, 'bot': False, 'discoverable': True, 'group': False, 'created_at': '2022-03-05T17:42:01.159Z', 'note': '<p>News, features and analysis by the BBC</p>', 'url': 'https://truthsocial.com/@bbcnewa', 'avatar': 'https://static-assets-1.truthsocial.com/tmtg:prime-ts-assets/accounts/avatars/107/905/163/010/312/793/original/e7c07550dc22c23a.jpeg', 'avatar_static': 'https://static-assets-1.truthsocial.com/tmtg:prime-ts-assets/accounts/avatars/107/905/163/010/312/793/original/e7c07550dc22c23a.jpeg', 'header': 'https://static-assets-1.truthsocial.com/tmtg:prime-ts-assets/accounts/headers/107/905/163/010/312/793/original/a00eeec2b57206c7.jpeg', 'header_static': 'https://static-assets-1.truthsocial.com/tmtg:prime-ts-assets/accounts/headers/107/905/163/010/312/793/original/a00eeec2b57206c7.jpeg', 'followers_count': 1131, 'following_count': 3, 'statuses_count': 9, 'last_status_at': '2024-11-12', 'verified': False, 'location': '', 'website': 'https://www.bbc.com/news', 'unauth_visibility': True, 'chats_onboarded': True, 'feeds_onboarded': True, 'accepting_messages': False, 'show_nonmember_group_statuses': None, 'emojis': [], 'fields': [], 'tv_onboarded': True, 'tv_account': False}, 'media_attachments': [], 'mentions': [], 'tags': [], 'card': None, 'group': None, 'quote': None, 'in_reply_to': None, 'reblog': None, 'sponsored': False, 'replies_count': 1, 'reblogs_count': 0, 'favourites_count': 2, 'favourited': False, 'reblogged': False, 'muted': False, 'pinned': False, 'bookmarked': False, 'poll': None, 'emojis': []}
         """
 
         result = Metadata()
         result.set_url(url)
-        timestamp = post['created_at'] # format is 2022-12-29T19:51:18.161Z
+        timestamp = post["created_at"]  # format is 2022-12-29T19:51:18.161Z
         result.set_timestamp(parse_dt(timestamp))
-        result.set('description', post['content'])
-        result.set('author', post['account']['username'])
+        result.set("description", post["content"])
+        result.set("author", post["account"]["username"])
 
-        for key in ['replies_count', 'reblogs_count', 'favourites_count', ('account', 'followers_count'), ('account', 'following_count'), ('account', 'statuses_count'), ('account', 'display_name'), 'language', 'in_reply_to_account', 'replies_count']:
+        for key in [
+            "replies_count",
+            "reblogs_count",
+            "favourites_count",
+            ("account", "followers_count"),
+            ("account", "following_count"),
+            ("account", "statuses_count"),
+            ("account", "display_name"),
+            "language",
+            "in_reply_to_account",
+            "replies_count",
+        ]:
             if isinstance(key, tuple):
                 store_key = " ".join(key)
             else:
                 store_key = key
             result.set(store_key, traverse_obj(post, key))
-        
-        # add the media
-        for media in post.get('media_attachments', []):
-            filename = archiver.download_from_url(media['url'])
-            result.add_media(Media(filename), id=media.get('id'))
 
-        return result
\ No newline at end of file
+        # add the media
+        for media in post.get("media_attachments", []):
+            filename = archiver.download_from_url(media["url"])
+            result.add_media(Media(filename), id=media.get("id"))
+
+        return result
diff --git a/src/auto_archiver/modules/generic_extractor/twitter.py b/src/auto_archiver/modules/generic_extractor/twitter.py
index 3faed6b..5b8468c 100644
--- a/src/auto_archiver/modules/generic_extractor/twitter.py
+++ b/src/auto_archiver/modules/generic_extractor/twitter.py
@@ -10,9 +10,8 @@ from auto_archiver.core.extractor import Extractor
 
 from .dropin import GenericDropin, InfoExtractor
 
+
 class Twitter(GenericDropin):
-
-
     def choose_variant(self, variants):
         # choosing the highest quality possible
         variant, width, height = None, 0, 0
@@ -27,9 +26,9 @@ class Twitter(GenericDropin):
             else:
                 variant = var if not variant else variant
         return variant
-    
+
     def extract_post(self, url: str, ie_instance: InfoExtractor):
-        twid = ie_instance._match_valid_url(url).group('id')
+        twid = ie_instance._match_valid_url(url).group("id")
         return ie_instance._extract_status(twid=twid)
 
     def create_metadata(self, tweet: dict, ie_instance: InfoExtractor, archiver: Extractor, url: str) -> Metadata:
@@ -41,30 +40,29 @@ class Twitter(GenericDropin):
         except (ValueError, KeyError) as ex:
             logger.warning(f"Unable to parse tweet: {str(ex)}\nRetreived tweet data: {tweet}")
             return False
-                
-        result\
-            .set_title(tweet.get('full_text', ''))\
-            .set_content(json.dumps(tweet, ensure_ascii=False))\
-            .set_timestamp(timestamp)
+
+        result.set_title(tweet.get("full_text", "")).set_content(json.dumps(tweet, ensure_ascii=False)).set_timestamp(
+            timestamp
+        )
         if not tweet.get("entities", {}).get("media"):
-            logger.debug('No media found, archiving tweet text only')
+            logger.debug("No media found, archiving tweet text only")
             result.status = "twitter-ytdl"
             return result
         for i, tw_media in enumerate(tweet["entities"]["media"]):
             media = Media(filename="")
             mimetype = ""
             if tw_media["type"] == "photo":
-                media.set("src", UrlUtil.twitter_best_quality_url(tw_media['media_url_https']))
+                media.set("src", UrlUtil.twitter_best_quality_url(tw_media["media_url_https"]))
                 mimetype = "image/jpeg"
             elif tw_media["type"] == "video":
-                variant = self.choose_variant(tw_media['video_info']['variants'])
-                media.set("src", variant['url'])
-                mimetype = variant['content_type']
+                variant = self.choose_variant(tw_media["video_info"]["variants"])
+                media.set("src", variant["url"])
+                mimetype = variant["content_type"]
             elif tw_media["type"] == "animated_gif":
-                variant = tw_media['video_info']['variants'][0]
-                media.set("src", variant['url'])
-                mimetype = variant['content_type']
+                variant = tw_media["video_info"]["variants"][0]
+                media.set("src", variant["url"])
+                mimetype = variant["content_type"]
             ext = mimetypes.guess_extension(mimetype)
-            media.filename = archiver.download_from_url(media.get("src"), f'{slugify(url)}_{i}{ext}')
+            media.filename = archiver.download_from_url(media.get("src"), f"{slugify(url)}_{i}{ext}")
             result.add_media(media)
-        return result
\ No newline at end of file
+        return result
diff --git a/src/auto_archiver/modules/gsheet_feeder_db/__init__.py b/src/auto_archiver/modules/gsheet_feeder_db/__init__.py
index 2e9ac02..fbd37b9 100644
--- a/src/auto_archiver/modules/gsheet_feeder_db/__init__.py
+++ b/src/auto_archiver/modules/gsheet_feeder_db/__init__.py
@@ -1,2 +1,2 @@
 from .gworksheet import GWorksheet
-from .gsheet_feeder_db import GsheetsFeederDB
\ No newline at end of file
+from .gsheet_feeder_db import GsheetsFeederDB
diff --git a/src/auto_archiver/modules/gsheet_feeder_db/__manifest__.py b/src/auto_archiver/modules/gsheet_feeder_db/__manifest__.py
index 9dd6b87..6c9d071 100644
--- a/src/auto_archiver/modules/gsheet_feeder_db/__manifest__.py
+++ b/src/auto_archiver/modules/gsheet_feeder_db/__manifest__.py
@@ -12,9 +12,7 @@
             "default": None,
             "help": "the id of the sheet to archive (alternative to 'sheet' config)",
         },
-        "header": {"default": 1,
-                   "type": "int",
-                   "help": "index of the header row (starts at 1)", "type": "int"},
+        "header": {"default": 1, "type": "int", "help": "index of the header row (starts at 1)", "type": "int"},
         "service_account": {
             "default": "secrets/service_account.json",
             "help": "service account JSON file path. Learn how to create one: https://gspread.readthedocs.io/en/latest/oauth2.html",
@@ -65,7 +63,7 @@
             "default": True,
             "type": "bool",
             "help": "if True the stored files path will include 'workbook_name/worksheet_name/...'",
-        }
+        },
     },
     "description": """
     GsheetsFeederDatabase
diff --git a/src/auto_archiver/modules/gsheet_feeder_db/gsheet_feeder_db.py b/src/auto_archiver/modules/gsheet_feeder_db/gsheet_feeder_db.py
index 406eeb4..4a9c9b3 100644
--- a/src/auto_archiver/modules/gsheet_feeder_db/gsheet_feeder_db.py
+++ b/src/auto_archiver/modules/gsheet_feeder_db/gsheet_feeder_db.py
@@ -8,6 +8,7 @@ The filtered rows are processed into `Metadata` objects.
 - validates the sheet's structure and filters rows based on input configurations.
 - Ensures only rows with valid URLs and unprocessed statuses are included.
 """
+
 import os
 from typing import Tuple, Union
 from urllib.parse import quote
@@ -23,7 +24,6 @@ from auto_archiver.utils.misc import calculate_file_hash, get_current_timestamp
 
 
 class GsheetsFeederDB(Feeder, Database):
-
     def setup(self) -> None:
         self.gsheets_client = gspread.service_account(filename=self.service_account)
         # TODO mv to validators
@@ -42,24 +42,28 @@ class GsheetsFeederDB(Feeder, Database):
             if not self.should_process_sheet(worksheet.title):
                 logger.debug(f"SKIPPED worksheet '{worksheet.title}' due to allow/block rules")
                 continue
-            logger.info(f'Opening worksheet {ii=}: {worksheet.title=} header={self.header}')
+            logger.info(f"Opening worksheet {ii=}: {worksheet.title=} header={self.header}")
             gw = GWorksheet(worksheet, header_row=self.header, columns=self.columns)
             if len(missing_cols := self.missing_required_columns(gw)):
-                logger.warning(f"SKIPPED worksheet '{worksheet.title}' due to missing required column(s) for {missing_cols}")
+                logger.warning(
+                    f"SKIPPED worksheet '{worksheet.title}' due to missing required column(s) for {missing_cols}"
+                )
                 continue
 
             # process and yield metadata here:
             yield from self._process_rows(gw)
-            logger.success(f'Finished worksheet {worksheet.title}')
+            logger.success(f"Finished worksheet {worksheet.title}")
 
     def _process_rows(self, gw: GWorksheet):
         for row in range(1 + self.header, gw.count_rows() + 1):
-            url = gw.get_cell(row, 'url').strip()
-            if not len(url): continue
-            original_status = gw.get_cell(row, 'status')
-            status = gw.get_cell(row, 'status', fresh=original_status in ['', None])
+            url = gw.get_cell(row, "url").strip()
+            if not len(url):
+                continue
+            original_status = gw.get_cell(row, "status")
+            status = gw.get_cell(row, "status", fresh=original_status in ["", None])
             # TODO: custom status parser(?) aka should_retry_from_status
-            if status not in ['', None]: continue
+            if status not in ["", None]:
+                continue
 
             # All checks done - archival process starts here
             m = Metadata().set_url(url)
@@ -70,10 +74,10 @@ class GsheetsFeederDB(Feeder, Database):
         # TODO: Check folder value not being recognised
         m.set_context("gsheet", {"row": row, "worksheet": gw})
 
-        if gw.get_cell_or_default(row, 'folder', "") is None:
-            folder = ''
+        if gw.get_cell_or_default(row, "folder", "") is None:
+            folder = ""
         else:
-            folder = slugify(gw.get_cell_or_default(row, 'folder', "").strip())
+            folder = slugify(gw.get_cell_or_default(row, "folder", "").strip())
         if len(folder):
             if self.use_sheet_names_in_stored_paths:
                 m.set_context("folder", os.path.join(folder, slugify(self.sheet), slugify(gw.wks.title)))
@@ -91,12 +95,11 @@ class GsheetsFeederDB(Feeder, Database):
 
     def missing_required_columns(self, gw: GWorksheet) -> list:
         missing = []
-        for required_col in ['url', 'status']:
+        for required_col in ["url", "status"]:
             if not gw.col_exists(required_col):
                 missing.append(required_col)
         return missing
 
-
     def started(self, item: Metadata) -> None:
         logger.warning(f"STARTED {item}")
         gw, row = self._retrieve_gsheet(item)
@@ -155,9 +158,7 @@ class GsheetsFeederDB(Feeder, Database):
         if len(pdq_hashes):
             batch_if_valid("pdq_hash", ",".join(pdq_hashes))
 
-        if (screenshot := item.get_media_by_id("screenshot")) and hasattr(
-            screenshot, "urls"
-        ):
+        if (screenshot := item.get_media_by_id("screenshot")) and hasattr(screenshot, "urls"):
             batch_if_valid("screenshot", "\n".join(screenshot.urls))
 
         if thumbnail := item.get_first_image("thumbnail"):
@@ -186,11 +187,12 @@ class GsheetsFeederDB(Feeder, Database):
             logger.debug(f"Unable to update sheet: {e}")
 
     def _retrieve_gsheet(self, item: Metadata) -> Tuple[GWorksheet, int]:
-
         if gsheet := item.get_context("gsheet"):
             gw: GWorksheet = gsheet.get("worksheet")
             row: int = gsheet.get("row")
         elif self.sheet_id:
-            logger.error(f"Unable to retrieve Gsheet for {item.get_url()}, GsheetDB must be used alongside GsheetFeeder.")
+            logger.error(
+                f"Unable to retrieve Gsheet for {item.get_url()}, GsheetDB must be used alongside GsheetFeeder."
+            )
 
         return gw, row
diff --git a/src/auto_archiver/modules/gsheet_feeder_db/gworksheet.py b/src/auto_archiver/modules/gsheet_feeder_db/gworksheet.py
index ba2d691..84cd45e 100644
--- a/src/auto_archiver/modules/gsheet_feeder_db/gworksheet.py
+++ b/src/auto_archiver/modules/gsheet_feeder_db/gworksheet.py
@@ -5,24 +5,25 @@ class GWorksheet:
     """
     This class makes read/write operations to the a worksheet easier.
     It can read the headers from a custom row number, but the row references
-    should always include the offset of the header. 
-    eg: if header=4, row 5 will be the first with data. 
+    should always include the offset of the header.
+    eg: if header=4, row 5 will be the first with data.
     """
+
     COLUMN_NAMES = {
-        'url': 'link',
-        'status': 'archive status',
-        'folder': 'destination folder',
-        'archive': 'archive location',
-        'date': 'archive date',
-        'thumbnail': 'thumbnail',
-        'timestamp': 'upload timestamp',
-        'title': 'upload title',
-        'text': 'text content',
-        'screenshot': 'screenshot',
-        'hash': 'hash',
-        'pdq_hash': 'perceptual hashes',
-        'wacz': 'wacz',
-        'replaywebpage': 'replaywebpage',
+        "url": "link",
+        "status": "archive status",
+        "folder": "destination folder",
+        "archive": "archive location",
+        "date": "archive date",
+        "thumbnail": "thumbnail",
+        "timestamp": "upload timestamp",
+        "title": "upload title",
+        "text": "text content",
+        "screenshot": "screenshot",
+        "hash": "hash",
+        "pdq_hash": "perceptual hashes",
+        "wacz": "wacz",
+        "replaywebpage": "replaywebpage",
     }
 
     def __init__(self, worksheet, columns=COLUMN_NAMES, header_row=1):
@@ -36,7 +37,7 @@ class GWorksheet:
 
     def _check_col_exists(self, col: str):
         if col not in self.columns:
-            raise Exception(f'Column {col} is not in the configured column names: {self.columns.keys()}')
+            raise Exception(f"Column {col} is not in the configured column names: {self.columns.keys()}")
 
     def _col_index(self, col: str):
         self._check_col_exists(col)
@@ -58,7 +59,7 @@ class GWorksheet:
 
     def get_cell(self, row, col: str, fresh=False):
         """
-        returns the cell value from (row, col), 
+        returns the cell value from (row, col),
         where row can be an index (1-based) OR list of values
         as received from self.get_row(row)
         if fresh=True, the sheet is queried again for this cell
@@ -71,7 +72,7 @@ class GWorksheet:
             row = self.get_row(row)
 
         if col_index >= len(row):
-            return ''
+            return ""
         return row[col_index]
 
     def get_cell_or_default(self, row, col: str, default: str = None, fresh=False, when_empty_use_default=True):
@@ -96,13 +97,9 @@ class GWorksheet:
         receives a list of [(row:int, col:str, val)] and batch updates it, the parameters are the same as in the self.set_cell() method
         """
         cell_updates = [
-            {
-                'range': self.to_a1(row, col),
-                'values': [[str(val)[0:49999]]]
-            }
-            for row, col, val in cell_updates
+            {"range": self.to_a1(row, col), "values": [[str(val)[0:49999]]]} for row, col, val in cell_updates
         ]
-        self.wks.batch_update(cell_updates, value_input_option='USER_ENTERED')
+        self.wks.batch_update(cell_updates, value_input_option="USER_ENTERED")
 
     def to_a1(self, row: int, col: str):
         # row is 1-based
diff --git a/src/auto_archiver/modules/hash_enricher/__init__.py b/src/auto_archiver/modules/hash_enricher/__init__.py
index 18ec885..3532e93 100644
--- a/src/auto_archiver/modules/hash_enricher/__init__.py
+++ b/src/auto_archiver/modules/hash_enricher/__init__.py
@@ -1 +1 @@
-from .hash_enricher import HashEnricher
\ No newline at end of file
+from .hash_enricher import HashEnricher
diff --git a/src/auto_archiver/modules/hash_enricher/__manifest__.py b/src/auto_archiver/modules/hash_enricher/__manifest__.py
index c7a023e..4f638de 100644
--- a/src/auto_archiver/modules/hash_enricher/__manifest__.py
+++ b/src/auto_archiver/modules/hash_enricher/__manifest__.py
@@ -3,16 +3,17 @@
     "type": ["enricher"],
     "requires_setup": False,
     "dependencies": {
-                          "python": ["loguru"],
+        "python": ["loguru"],
     },
     "configs": {
-            "algorithm": {"default": "SHA-256", "help": "hash algorithm to use", "choices": ["SHA-256", "SHA3-512"]},
-            # TODO add non-negative requirement to match previous implementation?
-            "chunksize": {"default": 16000000,
-                          "help": "number of bytes to use when reading files in chunks (if this value is too large you will run out of RAM), default is 16MB",
-                          'type': 'int',
-                          },
+        "algorithm": {"default": "SHA-256", "help": "hash algorithm to use", "choices": ["SHA-256", "SHA3-512"]},
+        # TODO add non-negative requirement to match previous implementation?
+        "chunksize": {
+            "default": 16000000,
+            "help": "number of bytes to use when reading files in chunks (if this value is too large you will run out of RAM), default is 16MB",
+            "type": "int",
         },
+    },
     "description": """
 Generates cryptographic hashes for media files to ensure data integrity and authenticity.
 
diff --git a/src/auto_archiver/modules/hash_enricher/hash_enricher.py b/src/auto_archiver/modules/hash_enricher/hash_enricher.py
index 7a0587c..71425f2 100644
--- a/src/auto_archiver/modules/hash_enricher/hash_enricher.py
+++ b/src/auto_archiver/modules/hash_enricher/hash_enricher.py
@@ -1,4 +1,4 @@
-""" Hash Enricher for generating cryptographic hashes of media files.
+"""Hash Enricher for generating cryptographic hashes of media files.
 
 The `HashEnricher` calculates cryptographic hashes (e.g., SHA-256, SHA3-512)
 for media files stored in `Metadata` objects. These hashes are used for
@@ -7,6 +7,7 @@ exact duplicates. The hash is computed by reading the file's bytes in chunks,
 making it suitable for handling large files efficiently.
 
 """
+
 import hashlib
 from loguru import logger
 
@@ -20,7 +21,6 @@ class HashEnricher(Enricher):
     Calculates hashes for Media instances
     """
 
-
     def enrich(self, to_enrich: Metadata) -> None:
         url = to_enrich.get_url()
         logger.debug(f"calculating media hashes for {url=} (using {self.algorithm})")
@@ -35,5 +35,6 @@ class HashEnricher(Enricher):
             hash_algo = hashlib.sha256
         elif self.algorithm == "SHA3-512":
             hash_algo = hashlib.sha3_512
-        else: return ""
+        else:
+            return ""
         return calculate_file_hash(filename, hash_algo, self.chunksize)
diff --git a/src/auto_archiver/modules/html_formatter/__init__.py b/src/auto_archiver/modules/html_formatter/__init__.py
index 432ef33..fd1bb70 100644
--- a/src/auto_archiver/modules/html_formatter/__init__.py
+++ b/src/auto_archiver/modules/html_formatter/__init__.py
@@ -1 +1 @@
-from .html_formatter import HtmlFormatter
\ No newline at end of file
+from .html_formatter import HtmlFormatter
diff --git a/src/auto_archiver/modules/html_formatter/__manifest__.py b/src/auto_archiver/modules/html_formatter/__manifest__.py
index 6e51c7a..6501e4f 100644
--- a/src/auto_archiver/modules/html_formatter/__manifest__.py
+++ b/src/auto_archiver/modules/html_formatter/__manifest__.py
@@ -2,14 +2,13 @@
     "name": "HTML Formatter",
     "type": ["formatter"],
     "requires_setup": False,
-    "dependencies": {
-                          "python": ["hash_enricher", "loguru", "jinja2"],
-                          "bin": [""]
-    },
+    "dependencies": {"python": ["hash_enricher", "loguru", "jinja2"], "bin": [""]},
     "configs": {
-            "detect_thumbnails": {"default": True,
-                                  "help": "if true will group by thumbnails generated by thumbnail enricher by id 'thumbnail_00'",
-                                  "type": "bool"},
+        "detect_thumbnails": {
+            "default": True,
+            "help": "if true will group by thumbnails generated by thumbnail enricher by id 'thumbnail_00'",
+            "type": "bool",
         },
+    },
     "description": """ """,
 }
diff --git a/src/auto_archiver/modules/html_formatter/html_formatter.py b/src/auto_archiver/modules/html_formatter/html_formatter.py
index deb4b44..88a9eca 100644
--- a/src/auto_archiver/modules/html_formatter/html_formatter.py
+++ b/src/auto_archiver/modules/html_formatter/html_formatter.py
@@ -11,6 +11,7 @@ from auto_archiver.core import Metadata, Media
 from auto_archiver.core import Formatter
 from auto_archiver.utils.misc import random_str
 
+
 class HtmlFormatter(Formatter):
     environment: Environment = None
     template: any = None
@@ -21,9 +22,9 @@ class HtmlFormatter(Formatter):
         self.environment = Environment(loader=FileSystemLoader(template_dir), autoescape=True)
 
         # JinjaHelper class static methods are added as filters
-        self.environment.filters.update({
-            k: v.__func__ for k, v in JinjaHelpers.__dict__.items() if isinstance(v, staticmethod)
-        })
+        self.environment.filters.update(
+            {k: v.__func__ for k, v in JinjaHelpers.__dict__.items() if isinstance(v, staticmethod)}
+        )
 
         # Load a specific template or default to "html_template.html"
         template_name = self.config.get("template_name", "html_template.html")
@@ -36,11 +37,7 @@ class HtmlFormatter(Formatter):
             return
 
         content = self.template.render(
-            url=url,
-            title=item.get_title(),
-            media=item.media,
-            metadata=item.metadata,
-            version=__version__
+            url=url, title=item.get_title(), media=item.media, metadata=item.metadata, version=__version__
         )
 
         html_path = os.path.join(self.tmp_dir, f"formatted{random_str(24)}.html")
@@ -49,7 +46,7 @@ class HtmlFormatter(Formatter):
         final_media = Media(filename=html_path, _mimetype="text/html")
 
         # get the already instantiated hash_enricher module
-        he = self.module_factory.get_module('hash_enricher', self.config)
+        he = self.module_factory.get_module("hash_enricher", self.config)
         if len(hd := he.calculate_hash(final_media.filename)):
             final_media.set("hash", f"{he.algorithm}:{hd}")
 
diff --git a/src/auto_archiver/modules/instagram_api_extractor/__manifest__.py b/src/auto_archiver/modules/instagram_api_extractor/__manifest__.py
index 2d8f1d9..e10bd1e 100644
--- a/src/auto_archiver/modules/instagram_api_extractor/__manifest__.py
+++ b/src/auto_archiver/modules/instagram_api_extractor/__manifest__.py
@@ -2,18 +2,18 @@
     "name": "Instagram API Extractor",
     "type": ["extractor"],
     "entry_point": "instagram_api_extractor::InstagramAPIExtractor",
-    "dependencies":
-        {"python": ["requests",
-                    "loguru",
-                    "retrying",
-                    "tqdm",],
-         },
+    "dependencies": {
+        "python": [
+            "requests",
+            "loguru",
+            "retrying",
+            "tqdm",
+        ],
+    },
     "requires_setup": True,
     "configs": {
-        "access_token": {"default": None,
-                         "help": "a valid instagrapi-api token"},
-        "api_endpoint": {"required": True,
-                         "help": "API endpoint to use"},
+        "access_token": {"default": None, "help": "a valid instagrapi-api token"},
+        "api_endpoint": {"required": True, "help": "API endpoint to use"},
         "full_profile": {
             "default": False,
             "type": "bool",
diff --git a/src/auto_archiver/modules/instagram_api_extractor/instagram_api_extractor.py b/src/auto_archiver/modules/instagram_api_extractor/instagram_api_extractor.py
index a75e065..bb37df2 100644
--- a/src/auto_archiver/modules/instagram_api_extractor/instagram_api_extractor.py
+++ b/src/auto_archiver/modules/instagram_api_extractor/instagram_api_extractor.py
@@ -36,21 +36,16 @@ class InstagramAPIExtractor(Extractor):
         if self.api_endpoint[-1] == "/":
             self.api_endpoint = self.api_endpoint[:-1]
 
-
     def download(self, item: Metadata) -> Metadata:
         url = item.get_url()
 
-        url.replace("instagr.com", "instagram.com").replace(
-            "instagr.am", "instagram.com"
-        )
+        url.replace("instagr.com", "instagram.com").replace("instagr.am", "instagram.com")
         insta_matches = self.valid_url.findall(url)
         logger.info(f"{insta_matches=}")
         if not len(insta_matches) or len(insta_matches[0]) != 3:
             return
         if len(insta_matches) > 1:
-            logger.warning(
-                f"Multiple instagram matches found in {url=}, using the first one"
-            )
+            logger.warning(f"Multiple instagram matches found in {url=}, using the first one")
             return
         g1, g2, g3 = insta_matches[0][0], insta_matches[0][1], insta_matches[0][2]
         if g1 == "":
@@ -73,9 +68,7 @@ class InstagramAPIExtractor(Extractor):
     def call_api(self, path: str, params: dict) -> dict:
         headers = {"accept": "application/json", "x-access-key": self.access_token}
         logger.debug(f"calling {self.api_endpoint}/{path} with {params=}")
-        return requests.get(
-            f"{self.api_endpoint}/{path}", headers=headers, params=params
-        ).json()
+        return requests.get(f"{self.api_endpoint}/{path}", headers=headers, params=params).json()
 
     def cleanup_dict(self, d: dict | list) -> dict:
         # repeats 3 times to remove nested empty values
@@ -88,8 +81,7 @@ class InstagramAPIExtractor(Extractor):
         return {
             k: clean_v
             for k, v in d.items()
-            if (clean_v := self.cleanup_dict(v))
-            not in [0.0, 0, [], {}, "", None, "null"]
+            if (clean_v := self.cleanup_dict(v)) not in [0.0, 0, [], {}, "", None, "null"]
             and k not in ["x", "y", "width", "height"]
         }
 
@@ -126,9 +118,7 @@ class InstagramAPIExtractor(Extractor):
             try:
                 self.download_all_tagged(result, user_id)
             except Exception as e:
-                result.append(
-                    "errors", f"Error downloading tagged posts for {username}"
-                )
+                result.append("errors", f"Error downloading tagged posts for {username}")
                 logger.error(f"Error downloading tagged posts for {username}: {e}")
 
             # download all highlights
@@ -153,22 +143,13 @@ class InstagramAPIExtractor(Extractor):
                     "errors",
                     f"Error downloading highlight id{h.get('pk')} for {username}",
                 )
-                logger.error(
-                    f"Error downloading highlight id{h.get('pk')} for {username}: {e}"
-                )
-            if (
-                self.full_profile_max_posts
-                and count_highlights >= self.full_profile_max_posts
-            ):
-                logger.info(
-                    f"HIGHLIGHTS reached full_profile_max_posts={self.full_profile_max_posts}"
-                )
+                logger.error(f"Error downloading highlight id{h.get('pk')} for {username}: {e}")
+            if self.full_profile_max_posts and count_highlights >= self.full_profile_max_posts:
+                logger.info(f"HIGHLIGHTS reached full_profile_max_posts={self.full_profile_max_posts}")
                 break
         result.set("#highlights", count_highlights)
 
-    def download_post(
-        self, result: Metadata, code: str = None, id: str = None, context: str = None
-    ) -> Metadata:
+    def download_post(self, result: Metadata, code: str = None, id: str = None, context: str = None) -> Metadata:
         if id:
             post = self.call_api(f"v1/media/by/id", {"id": id})
         else:
@@ -196,11 +177,7 @@ class InstagramAPIExtractor(Extractor):
         h_info = full_h.get("response", {}).get("reels", {}).get(f"highlight:{id}")
         assert h_info, f"Highlight {id} not found: {full_h=}"
 
-        if (
-            cover_media := h_info.get("cover_media", {})
-            .get("cropped_image_version", {})
-            .get("url")
-        ):
+        if cover_media := h_info.get("cover_media", {}).get("cropped_image_version", {}).get("url"):
             filename = self.download_from_url(cover_media)
             result.add_media(Media(filename=filename), id=f"cover_media highlight {id}")
 
@@ -210,9 +187,7 @@ class InstagramAPIExtractor(Extractor):
                 self.scrape_item(result, h, "highlight")
             except Exception as e:
                 result.append("errors", f"Error downloading highlight {h.get('id')}")
-                logger.error(
-                    f"Error downloading highlight, skipping {h.get('id')}: {e}"
-                )
+                logger.error(f"Error downloading highlight, skipping {h.get('id')}: {e}")
 
         return h_info
 
@@ -244,9 +219,7 @@ class InstagramAPIExtractor(Extractor):
 
         post_count = 0
         while end_cursor != "":
-            posts = self.call_api(
-                f"v1/user/medias/chunk", {"user_id": user_id, "end_cursor": end_cursor}
-            )
+            posts = self.call_api(f"v1/user/medias/chunk", {"user_id": user_id, "end_cursor": end_cursor})
             if not len(posts) or not type(posts) == list or len(posts) != 2:
                 break
             posts, end_cursor = posts[0], posts[1]
@@ -260,13 +233,8 @@ class InstagramAPIExtractor(Extractor):
                     logger.error(f"Error downloading post, skipping {p.get('id')}: {e}")
                 pbar.update(1)
                 post_count += 1
-            if (
-                self.full_profile_max_posts
-                and post_count >= self.full_profile_max_posts
-            ):
-                logger.info(
-                    f"POSTS reached full_profile_max_posts={self.full_profile_max_posts}"
-                )
+            if self.full_profile_max_posts and post_count >= self.full_profile_max_posts:
+                logger.info(f"POSTS reached full_profile_max_posts={self.full_profile_max_posts}")
                 break
         result.set("#posts", post_count)
 
@@ -276,9 +244,7 @@ class InstagramAPIExtractor(Extractor):
 
         tagged_count = 0
         while next_page_id != None:
-            resp = self.call_api(
-                f"v2/user/tag/medias", {"user_id": user_id, "page_id": next_page_id}
-            )
+            resp = self.call_api(f"v2/user/tag/medias", {"user_id": user_id, "page_id": next_page_id})
             posts = resp.get("response", {}).get("items", [])
             if not len(posts):
                 break
@@ -290,21 +256,12 @@ class InstagramAPIExtractor(Extractor):
                 try:
                     self.scrape_item(result, p, "tagged")
                 except Exception as e:
-                    result.append(
-                        "errors", f"Error downloading tagged post {p.get('id')}"
-                    )
-                    logger.error(
-                        f"Error downloading tagged post, skipping {p.get('id')}: {e}"
-                    )
+                    result.append("errors", f"Error downloading tagged post {p.get('id')}")
+                    logger.error(f"Error downloading tagged post, skipping {p.get('id')}: {e}")
                 pbar.update(1)
                 tagged_count += 1
-            if (
-                self.full_profile_max_posts
-                and tagged_count >= self.full_profile_max_posts
-            ):
-                logger.info(
-                    f"TAGS reached full_profile_max_posts={self.full_profile_max_posts}"
-                )
+            if self.full_profile_max_posts and tagged_count >= self.full_profile_max_posts:
+                logger.info(f"TAGS reached full_profile_max_posts={self.full_profile_max_posts}")
                 break
         result.set("#tagged", tagged_count)
 
@@ -318,9 +275,7 @@ class InstagramAPIExtractor(Extractor):
         context can be used to give specific id prefixes to media
         """
         if "clips_metadata" in item:
-            if reusable_text := item.get("clips_metadata", {}).get(
-                "reusable_text_attribute_string"
-            ):
+            if reusable_text := item.get("clips_metadata", {}).get("reusable_text_attribute_string"):
                 item["clips_metadata_text"] = reusable_text
             if self.minimize_json_output:
                 del item["clips_metadata"]
diff --git a/src/auto_archiver/modules/instagram_extractor/__init__.py b/src/auto_archiver/modules/instagram_extractor/__init__.py
index 6f39171..cefbcc5 100644
--- a/src/auto_archiver/modules/instagram_extractor/__init__.py
+++ b/src/auto_archiver/modules/instagram_extractor/__init__.py
@@ -1 +1 @@
-from .instagram_extractor import InstagramExtractor
\ No newline at end of file
+from .instagram_extractor import InstagramExtractor
diff --git a/src/auto_archiver/modules/instagram_extractor/__manifest__.py b/src/auto_archiver/modules/instagram_extractor/__manifest__.py
index a66389f..6067c92 100644
--- a/src/auto_archiver/modules/instagram_extractor/__manifest__.py
+++ b/src/auto_archiver/modules/instagram_extractor/__manifest__.py
@@ -9,8 +9,7 @@
     },
     "requires_setup": True,
     "configs": {
-        "username": {"required": True,
-                     "help": "A valid Instagram username."},
+        "username": {"required": True, "help": "A valid Instagram username."},
         "password": {
             "required": True,
             "help": "The corresponding Instagram account password.",
diff --git a/src/auto_archiver/modules/instagram_extractor/instagram_extractor.py b/src/auto_archiver/modules/instagram_extractor/instagram_extractor.py
index 7e195ad..f310771 100644
--- a/src/auto_archiver/modules/instagram_extractor/instagram_extractor.py
+++ b/src/auto_archiver/modules/instagram_extractor/instagram_extractor.py
@@ -1,8 +1,9 @@
-""" Uses the Instaloader library to download content from Instagram. This class handles both individual posts
-    and user profiles, downloading as much information as possible, including images, videos, text, stories,
-    highlights, and tagged posts. Authentication is required via username/password or a session file.
+"""Uses the Instaloader library to download content from Instagram. This class handles both individual posts
+and user profiles, downloading as much information as possible, including images, videos, text, stories,
+highlights, and tagged posts. Authentication is required via username/password or a session file.
 
 """
+
 import re, os, shutil
 import instaloader
 from loguru import logger
@@ -11,6 +12,7 @@ from auto_archiver.core import Extractor
 from auto_archiver.core import Metadata
 from auto_archiver.core import Media
 
+
 class InstagramExtractor(Extractor):
     """
     Uses Instaloader to download either a post (inc images, videos, text) or as much as possible from a profile (posts, stories, highlights, ...)
@@ -25,13 +27,12 @@ class InstagramExtractor(Extractor):
     # TODO: links to stories
 
     def setup(self) -> None:
-
         self.insta = instaloader.Instaloader(
             download_geotags=True,
             download_comments=True,
             compress_json=False,
             dirname_pattern=self.download_folder,
-            filename_pattern="{date_utc}_UTC_{target}__{typename}"
+            filename_pattern="{date_utc}_UTC_{target}__{typename}",
         )
         try:
             self.insta.load_session_from_file(self.username, self.session_file)
@@ -44,7 +45,6 @@ class InstagramExtractor(Extractor):
             except Exception as e:
                 logger.error(f"Failed to setup Instagram Extractor with Instagrapi. {e}")
 
-
     def download(self, item: Metadata) -> Metadata:
         url = item.get_url()
 
@@ -53,7 +53,8 @@ class InstagramExtractor(Extractor):
         profile_matches = self.profile_pattern.findall(url)
 
         # return if not a valid instagram link
-        if not len(post_matches) and not len(profile_matches): return
+        if not len(post_matches) and not len(profile_matches):
+            return
 
         result = None
         try:
@@ -65,7 +66,9 @@ class InstagramExtractor(Extractor):
             elif len(profile_matches):
                 result = self.download_profile(url, profile_matches[0])
         except Exception as e:
-            logger.error(f"Failed to download with instagram extractor due to: {e}, make sure your account credentials are valid.")
+            logger.error(
+                f"Failed to download with instagram extractor due to: {e}, make sure your account credentials are valid."
+            )
         finally:
             shutil.rmtree(self.download_folder, ignore_errors=True)
         return result
@@ -84,35 +87,50 @@ class InstagramExtractor(Extractor):
         profile = instaloader.Profile.from_username(self.insta.context, username)
         try:
             for post in profile.get_posts():
-                try: self.insta.download_post(post, target=f"profile_post_{post.owner_username}")
-                except Exception as e: logger.error(f"Failed to download post: {post.shortcode}: {e}")
-        except Exception as e: logger.error(f"Failed profile.get_posts: {e}")
+                try:
+                    self.insta.download_post(post, target=f"profile_post_{post.owner_username}")
+                except Exception as e:
+                    logger.error(f"Failed to download post: {post.shortcode}: {e}")
+        except Exception as e:
+            logger.error(f"Failed profile.get_posts: {e}")
 
         try:
             for post in profile.get_tagged_posts():
-                try: self.insta.download_post(post, target=f"tagged_post_{post.owner_username}")
-                except Exception as e: logger.error(f"Failed to download tagged post: {post.shortcode}: {e}")
-        except Exception as e: logger.error(f"Failed profile.get_tagged_posts: {e}")
+                try:
+                    self.insta.download_post(post, target=f"tagged_post_{post.owner_username}")
+                except Exception as e:
+                    logger.error(f"Failed to download tagged post: {post.shortcode}: {e}")
+        except Exception as e:
+            logger.error(f"Failed profile.get_tagged_posts: {e}")
 
         try:
             for post in profile.get_igtv_posts():
-                try: self.insta.download_post(post, target=f"igtv_post_{post.owner_username}")
-                except Exception as e: logger.error(f"Failed to download igtv post: {post.shortcode}: {e}")
-        except Exception as e: logger.error(f"Failed profile.get_igtv_posts: {e}")
+                try:
+                    self.insta.download_post(post, target=f"igtv_post_{post.owner_username}")
+                except Exception as e:
+                    logger.error(f"Failed to download igtv post: {post.shortcode}: {e}")
+        except Exception as e:
+            logger.error(f"Failed profile.get_igtv_posts: {e}")
 
         try:
             for story in self.insta.get_stories([profile.userid]):
                 for item in story.get_items():
-                    try: self.insta.download_storyitem(item, target=f"story_item_{story.owner_username}")
-                    except Exception as e: logger.error(f"Failed to download story item: {item}: {e}")
-        except Exception as e: logger.error(f"Failed get_stories: {e}")
+                    try:
+                        self.insta.download_storyitem(item, target=f"story_item_{story.owner_username}")
+                    except Exception as e:
+                        logger.error(f"Failed to download story item: {item}: {e}")
+        except Exception as e:
+            logger.error(f"Failed get_stories: {e}")
 
         try:
             for highlight in self.insta.get_highlights(profile.userid):
                 for item in highlight.get_items():
-                    try: self.insta.download_storyitem(item, target=f"highlight_item_{highlight.owner_username}")
-                    except Exception as e: logger.error(f"Failed to download highlight item: {item}: {e}")
-        except Exception as e: logger.error(f"Failed get_highlights: {e}")
+                    try:
+                        self.insta.download_storyitem(item, target=f"highlight_item_{highlight.owner_username}")
+                    except Exception as e:
+                        logger.error(f"Failed to download highlight item: {item}: {e}")
+        except Exception as e:
+            logger.error(f"Failed get_highlights: {e}")
 
         return self.process_downloads(url, f"@{username}", profile._asdict(), None)
 
@@ -124,7 +142,8 @@ class InstagramExtractor(Extractor):
             all_media = []
             for f in os.listdir(self.download_folder):
                 if os.path.isfile((filename := os.path.join(self.download_folder, f))):
-                    if filename[-4:] == ".txt": continue
+                    if filename[-4:] == ".txt":
+                        continue
                     all_media.append(Media(filename))
 
             assert len(all_media) > 1, "No uploaded media found"
diff --git a/src/auto_archiver/modules/instagram_tbot_extractor/__manifest__.py b/src/auto_archiver/modules/instagram_tbot_extractor/__manifest__.py
index a24a864..e3f94d0 100644
--- a/src/auto_archiver/modules/instagram_tbot_extractor/__manifest__.py
+++ b/src/auto_archiver/modules/instagram_tbot_extractor/__manifest__.py
@@ -1,16 +1,21 @@
 {
     "name": "Instagram Telegram Bot Extractor",
     "type": ["extractor"],
-    "dependencies": {"python": ["loguru", "telethon",],
-                              },
+    "dependencies": {
+        "python": [
+            "loguru",
+            "telethon",
+        ],
+    },
     "requires_setup": True,
     "configs": {
-            "api_id": {"default": None, "help": "telegram API_ID value, go to https://my.telegram.org/apps"},
-            "api_hash": {"default": None, "help": "telegram API_HASH value, go to https://my.telegram.org/apps"},
-            "session_file": {"default": "secrets/anon-insta", "help": "optional, records the telegram login session for future usage, '.session' will be appended to the provided value."},
-            "timeout": {"default": 45,
-                        "type": "int",
-                        "help": "timeout to fetch the instagram content in seconds."},
+        "api_id": {"default": None, "help": "telegram API_ID value, go to https://my.telegram.org/apps"},
+        "api_hash": {"default": None, "help": "telegram API_HASH value, go to https://my.telegram.org/apps"},
+        "session_file": {
+            "default": "secrets/anon-insta",
+            "help": "optional, records the telegram login session for future usage, '.session' will be appended to the provided value.",
+        },
+        "timeout": {"default": 45, "type": "int", "help": "timeout to fetch the instagram content in seconds."},
     },
     "description": """
 The `InstagramTbotExtractor` module uses a Telegram bot (`instagram_load_bot`) to fetch and archive Instagram content,
diff --git a/src/auto_archiver/modules/instagram_tbot_extractor/instagram_tbot_extractor.py b/src/auto_archiver/modules/instagram_tbot_extractor/instagram_tbot_extractor.py
index 1416da9..39ed893 100644
--- a/src/auto_archiver/modules/instagram_tbot_extractor/instagram_tbot_extractor.py
+++ b/src/auto_archiver/modules/instagram_tbot_extractor/instagram_tbot_extractor.py
@@ -65,15 +65,15 @@ class InstagramTbotExtractor(Extractor):
         session_file_name = self.session_file + ".session"
         if os.path.exists(session_file_name):
             os.remove(session_file_name)
-        
+
     def download(self, item: Metadata) -> Metadata:
         url = item.get_url()
-        if not "instagram.com" in url: return False
+        if not "instagram.com" in url:
+            return False
 
         result = Metadata()
         tmp_dir = self.tmp_dir
         with self.client.start():
-
             chat, since_id = self._send_url_to_bot(url)
             message = self._process_messages(chat, since_id, tmp_dir, result)
 
@@ -110,13 +110,14 @@ class InstagramTbotExtractor(Extractor):
             for post in self.client.iter_messages(chat, min_id=since_id):
                 since_id = max(since_id, post.id)
                 # Skip known filler message:
-                if post.message == 'The bot receives information through https://hikerapi.com/p/hJqpppqi':
+                if post.message == "The bot receives information through https://hikerapi.com/p/hJqpppqi":
                     continue
                 if post.media and post.id not in seen_media:
-                    filename_dest = os.path.join(tmp_dir, f'{chat.id}_{post.id}')
+                    filename_dest = os.path.join(tmp_dir, f"{chat.id}_{post.id}")
                     media = self.client.download_media(post.media, filename_dest)
                     if media:
                         result.add_media(Media(media))
                         seen_media.append(post.id)
-                if post.message: message += post.message
-        return message.strip()
\ No newline at end of file
+                if post.message:
+                    message += post.message
+        return message.strip()
diff --git a/src/auto_archiver/modules/local_storage/__init__.py b/src/auto_archiver/modules/local_storage/__init__.py
index d23147d..e9c81f8 100644
--- a/src/auto_archiver/modules/local_storage/__init__.py
+++ b/src/auto_archiver/modules/local_storage/__init__.py
@@ -1 +1 @@
-from .local_storage import LocalStorage
\ No newline at end of file
+from .local_storage import LocalStorage
diff --git a/src/auto_archiver/modules/local_storage/__manifest__.py b/src/auto_archiver/modules/local_storage/__manifest__.py
index 8ad6381..d1cdf7f 100644
--- a/src/auto_archiver/modules/local_storage/__manifest__.py
+++ b/src/auto_archiver/modules/local_storage/__manifest__.py
@@ -17,9 +17,11 @@
             "choices": ["random", "static"],
         },
         "save_to": {"default": "./local_archive", "help": "folder where to save archived content"},
-        "save_absolute": {"default": False, 
-                          "type": "bool",
-                          "help": "whether the path to the stored file is absolute or relative in the output result inc. formatters (WARN: leaks the file structure)"},
+        "save_absolute": {
+            "default": False,
+            "type": "bool",
+            "help": "whether the path to the stored file is absolute or relative in the output result inc. formatters (WARN: leaks the file structure)",
+        },
     },
     "description": """
     LocalStorage: A storage module for saving archived content locally on the filesystem.
@@ -33,5 +35,5 @@
     ### Notes
     - Default storage folder is `./archived`, but this can be changed via the `save_to` configuration.
     - The `save_absolute` option can reveal the file structure in output formats; use with caution.
-    """
+    """,
 }
diff --git a/src/auto_archiver/modules/local_storage/local_storage.py b/src/auto_archiver/modules/local_storage/local_storage.py
index b995577..fce0c64 100644
--- a/src/auto_archiver/modules/local_storage/local_storage.py
+++ b/src/auto_archiver/modules/local_storage/local_storage.py
@@ -1,4 +1,3 @@
-
 import shutil
 from typing import IO
 import os
@@ -9,7 +8,6 @@ from auto_archiver.core import Storage
 
 
 class LocalStorage(Storage):
-
     def get_cdn_url(self, media: Media) -> str:
         # TODO: is this viable with Storage.configs on path/filename?
         dest = os.path.join(self.save_to, media.key)
@@ -21,10 +19,11 @@ class LocalStorage(Storage):
         # override parent so that we can use shutil.copy2 and keep metadata
         dest = os.path.join(self.save_to, media.key)
         os.makedirs(os.path.dirname(dest), exist_ok=True)
-        logger.debug(f'[{self.__class__.__name__}] storing file {media.filename} with key {media.key} to {dest}')
+        logger.debug(f"[{self.__class__.__name__}] storing file {media.filename} with key {media.key} to {dest}")
         res = shutil.copy2(media.filename, dest)
         logger.info(res)
         return True
 
     # must be implemented even if unused
-    def uploadf(self, file: IO[bytes], key: str, **kwargs: dict) -> bool: pass
+    def uploadf(self, file: IO[bytes], key: str, **kwargs: dict) -> bool:
+        pass
diff --git a/src/auto_archiver/modules/meta_enricher/__manifest__.py b/src/auto_archiver/modules/meta_enricher/__manifest__.py
index 37c9201..d3732a3 100644
--- a/src/auto_archiver/modules/meta_enricher/__manifest__.py
+++ b/src/auto_archiver/modules/meta_enricher/__manifest__.py
@@ -3,7 +3,7 @@
     "type": ["enricher"],
     "requires_setup": False,
     "dependencies": {
-                          "python": ["loguru"],
+        "python": ["loguru"],
     },
     "description": """ 
     Adds metadata information about the archive operations, Adds metadata about archive operations, including file sizes and archive duration./
diff --git a/src/auto_archiver/modules/meta_enricher/meta_enricher.py b/src/auto_archiver/modules/meta_enricher/meta_enricher.py
index 03fb01e..9356b16 100644
--- a/src/auto_archiver/modules/meta_enricher/meta_enricher.py
+++ b/src/auto_archiver/modules/meta_enricher/meta_enricher.py
@@ -23,7 +23,9 @@ class MetaEnricher(Enricher):
         self.enrich_archive_duration(to_enrich)
 
     def enrich_file_sizes(self, to_enrich: Metadata):
-        logger.debug(f"calculating archive file sizes for url={to_enrich.get_url()} ({len(to_enrich.media)} media files)")
+        logger.debug(
+            f"calculating archive file sizes for url={to_enrich.get_url()} ({len(to_enrich.media)} media files)"
+        )
         total_size = 0
         for media in to_enrich.get_all_media():
             file_stats = os.stat(media.filename)
@@ -34,7 +36,6 @@ class MetaEnricher(Enricher):
         to_enrich.set("total_bytes", total_size)
         to_enrich.set("total_size", self.human_readable_bytes(total_size))
 
-
     def human_readable_bytes(self, size: int) -> str:
         # receives number of bytes and returns human readble size
         for unit in ["bytes", "KB", "MB", "GB", "TB"]:
@@ -46,4 +47,4 @@ class MetaEnricher(Enricher):
         logger.debug(f"calculating archive duration for url={to_enrich.get_url()} ")
 
         archive_duration = datetime.datetime.now(datetime.timezone.utc) - to_enrich.get("_processed_at")
-        to_enrich.set("archive_duration_seconds", archive_duration.seconds)
\ No newline at end of file
+        to_enrich.set("archive_duration_seconds", archive_duration.seconds)
diff --git a/src/auto_archiver/modules/metadata_enricher/__init__.py b/src/auto_archiver/modules/metadata_enricher/__init__.py
index 020bd4a..2fe894a 100644
--- a/src/auto_archiver/modules/metadata_enricher/__init__.py
+++ b/src/auto_archiver/modules/metadata_enricher/__init__.py
@@ -1 +1 @@
-from .metadata_enricher import MetadataEnricher
\ No newline at end of file
+from .metadata_enricher import MetadataEnricher
diff --git a/src/auto_archiver/modules/metadata_enricher/__manifest__.py b/src/auto_archiver/modules/metadata_enricher/__manifest__.py
index f8ccdc6..3727551 100644
--- a/src/auto_archiver/modules/metadata_enricher/__manifest__.py
+++ b/src/auto_archiver/modules/metadata_enricher/__manifest__.py
@@ -2,10 +2,7 @@
     "name": "Media Metadata Enricher",
     "type": ["enricher"],
     "requires_setup": True,
-    "dependencies": {
-        "python": ["loguru"], 
-        "bin": ["exiftool"]
-    },
+    "dependencies": {"python": ["loguru"], "bin": ["exiftool"]},
     "description": """
     Extracts metadata information from files using ExifTool.
 
@@ -17,5 +14,5 @@
     ### Notes
     - Requires ExifTool to be installed and accessible via the system's PATH.
     - Skips enrichment for files where metadata extraction fails.
-    """
+    """,
 }
diff --git a/src/auto_archiver/modules/metadata_enricher/metadata_enricher.py b/src/auto_archiver/modules/metadata_enricher/metadata_enricher.py
index c052d0a..e4fac44 100644
--- a/src/auto_archiver/modules/metadata_enricher/metadata_enricher.py
+++ b/src/auto_archiver/modules/metadata_enricher/metadata_enricher.py
@@ -11,7 +11,6 @@ class MetadataEnricher(Enricher):
     Extracts metadata information from files using exiftool.
     """
 
-
     def enrich(self, to_enrich: Metadata) -> None:
         url = to_enrich.get_url()
         logger.debug(f"extracting EXIF metadata for {url=}")
@@ -23,13 +22,13 @@ class MetadataEnricher(Enricher):
     def get_metadata(self, filename: str) -> dict:
         try:
             # Run ExifTool command to extract metadata from the file
-            cmd = ['exiftool', filename]
+            cmd = ["exiftool", filename]
             result = subprocess.run(cmd, capture_output=True, text=True)
 
             # Process the output to extract individual metadata fields
             metadata = {}
             for line in result.stdout.splitlines():
-                field, value = line.strip().split(':', 1)
+                field, value = line.strip().split(":", 1)
                 metadata[field.strip()] = value.strip()
             return metadata
         except FileNotFoundError:
diff --git a/src/auto_archiver/modules/mute_formatter/__manifest__.py b/src/auto_archiver/modules/mute_formatter/__manifest__.py
index e81dc4c..185645e 100644
--- a/src/auto_archiver/modules/mute_formatter/__manifest__.py
+++ b/src/auto_archiver/modules/mute_formatter/__manifest__.py
@@ -2,8 +2,7 @@
     "name": "Mute Formatter",
     "type": ["formatter"],
     "requires_setup": True,
-    "dependencies": {
-    },
+    "dependencies": {},
     "description": """ Default formatter.
     """,
 }
diff --git a/src/auto_archiver/modules/mute_formatter/mute_formatter.py b/src/auto_archiver/modules/mute_formatter/mute_formatter.py
index 129ddcb..b7c0ba5 100644
--- a/src/auto_archiver/modules/mute_formatter/mute_formatter.py
+++ b/src/auto_archiver/modules/mute_formatter/mute_formatter.py
@@ -5,5 +5,5 @@ from auto_archiver.core import Formatter
 
 
 class MuteFormatter(Formatter):
-
-    def format(self, item: Metadata) -> Media: return None
+    def format(self, item: Metadata) -> Media:
+        return None
diff --git a/src/auto_archiver/modules/pdq_hash_enricher/__init__.py b/src/auto_archiver/modules/pdq_hash_enricher/__init__.py
index b444197..88a964b 100644
--- a/src/auto_archiver/modules/pdq_hash_enricher/__init__.py
+++ b/src/auto_archiver/modules/pdq_hash_enricher/__init__.py
@@ -1 +1 @@
-from .pdq_hash_enricher import PdqHashEnricher
\ No newline at end of file
+from .pdq_hash_enricher import PdqHashEnricher
diff --git a/src/auto_archiver/modules/pdq_hash_enricher/__manifest__.py b/src/auto_archiver/modules/pdq_hash_enricher/__manifest__.py
index 133fef7..9c7a5c8 100644
--- a/src/auto_archiver/modules/pdq_hash_enricher/__manifest__.py
+++ b/src/auto_archiver/modules/pdq_hash_enricher/__manifest__.py
@@ -17,5 +17,5 @@
     ### Notes
     - Best used after enrichers like `thumbnail_enricher` or `screenshot_enricher` to ensure images are available.
     - Uses the `pdqhash` library to compute 256-bit perceptual hashes, which are stored as hexadecimal strings.
-    """
+    """,
 }
diff --git a/src/auto_archiver/modules/pdq_hash_enricher/pdq_hash_enricher.py b/src/auto_archiver/modules/pdq_hash_enricher/pdq_hash_enricher.py
index e812e8b..c7d4a47 100644
--- a/src/auto_archiver/modules/pdq_hash_enricher/pdq_hash_enricher.py
+++ b/src/auto_archiver/modules/pdq_hash_enricher/pdq_hash_enricher.py
@@ -10,6 +10,7 @@ This enricher is typically used after thumbnail or screenshot enrichers
 to ensure images are available for hashing.
 
 """
+
 import traceback
 import pdqhash
 import numpy as np
@@ -34,7 +35,12 @@ class PdqHashEnricher(Enricher):
         for m in to_enrich.media:
             for media in m.all_inner_media(True):
                 media_id = media.get("id", "")
-                if media.is_image() and "screenshot" not in media_id and "warc-file-" not in media_id and len(hd := self.calculate_pdq_hash(media.filename)):
+                if (
+                    media.is_image()
+                    and "screenshot" not in media_id
+                    and "warc-file-" not in media_id
+                    and len(hd := self.calculate_pdq_hash(media.filename))
+                ):
                     media.set("pdq_hash", hd)
                     media_with_hashes.append(media.filename)
 
@@ -51,5 +57,7 @@ class PdqHashEnricher(Enricher):
                 hash = "".join(str(b) for b in hash_array)
                 return hex(int(hash, 2))[2:]
         except UnidentifiedImageError as e:
-            logger.error(f"Image {filename=} is likely corrupted or in unsupported format {e}: {traceback.format_exc()}")
+            logger.error(
+                f"Image {filename=} is likely corrupted or in unsupported format {e}: {traceback.format_exc()}"
+            )
         return ""
diff --git a/src/auto_archiver/modules/s3_storage/__init__.py b/src/auto_archiver/modules/s3_storage/__init__.py
index cbf3237..5e388d1 100644
--- a/src/auto_archiver/modules/s3_storage/__init__.py
+++ b/src/auto_archiver/modules/s3_storage/__init__.py
@@ -1 +1 @@
-from .s3_storage import S3Storage
\ No newline at end of file
+from .s3_storage import S3Storage
diff --git a/src/auto_archiver/modules/s3_storage/__manifest__.py b/src/auto_archiver/modules/s3_storage/__manifest__.py
index bf032e7..b07cea3 100644
--- a/src/auto_archiver/modules/s3_storage/__manifest__.py
+++ b/src/auto_archiver/modules/s3_storage/__manifest__.py
@@ -20,20 +20,20 @@
         "region": {"default": None, "help": "S3 region name"},
         "key": {"default": None, "help": "S3 API key"},
         "secret": {"default": None, "help": "S3 API secret"},
-        "random_no_duplicate": {"default": False,
-                                "type": "bool",
-                                "help": "if set, it will override `path_generator`, `filename_generator` and `folder`. It will check if the file already exists and if so it will not upload it again. Creates a new root folder path `no-dups/`"},
+        "random_no_duplicate": {
+            "default": False,
+            "type": "bool",
+            "help": "if set, it will override `path_generator`, `filename_generator` and `folder`. It will check if the file already exists and if so it will not upload it again. Creates a new root folder path `no-dups/`",
+        },
         "endpoint_url": {
-            "default": 'https://{region}.digitaloceanspaces.com',
-            "help": "S3 bucket endpoint, {region} are inserted at runtime"
+            "default": "https://{region}.digitaloceanspaces.com",
+            "help": "S3 bucket endpoint, {region} are inserted at runtime",
         },
         "cdn_url": {
-            "default": 'https://{bucket}.{region}.cdn.digitaloceanspaces.com/{key}',
-            "help": "S3 CDN url, {bucket}, {region} and {key} are inserted at runtime"
+            "default": "https://{bucket}.{region}.cdn.digitaloceanspaces.com/{key}",
+            "help": "S3 CDN url, {bucket}, {region} and {key} are inserted at runtime",
         },
-        "private": {"default": False,
-                    "type": "bool",
-                    "help": "if true S3 files will not be readable online"},
+        "private": {"default": False, "type": "bool", "help": "if true S3 files will not be readable online"},
     },
     "description": """
     S3Storage: A storage module for saving media files to an S3-compatible object storage.
@@ -50,5 +50,5 @@
     - The `random_no_duplicate` option ensures no duplicate uploads by leveraging hash-based folder structures.
     - Uses `boto3` for interaction with the S3 API.
     - Depends on the `HashEnricher` module for hash calculation.
-    """
+    """,
 }
diff --git a/src/auto_archiver/modules/s3_storage/s3_storage.py b/src/auto_archiver/modules/s3_storage/s3_storage.py
index 6590ac9..540d50d 100644
--- a/src/auto_archiver/modules/s3_storage/s3_storage.py
+++ b/src/auto_archiver/modules/s3_storage/s3_storage.py
@@ -1,4 +1,3 @@
-
 from typing import IO
 
 import boto3
@@ -11,60 +10,62 @@ from auto_archiver.utils.misc import calculate_file_hash, random_str
 
 NO_DUPLICATES_FOLDER = "no-dups/"
 
-class S3Storage(Storage):
 
+class S3Storage(Storage):
     def setup(self) -> None:
         self.s3 = boto3.client(
-            's3',
+            "s3",
             region_name=self.region,
             endpoint_url=self.endpoint_url.format(region=self.region),
             aws_access_key_id=self.key,
-            aws_secret_access_key=self.secret
+            aws_secret_access_key=self.secret,
         )
         if self.random_no_duplicate:
-            logger.warning("random_no_duplicate is set to True, this will override `path_generator`, `filename_generator` and `folder`.")
+            logger.warning(
+                "random_no_duplicate is set to True, this will override `path_generator`, `filename_generator` and `folder`."
+            )
 
     def get_cdn_url(self, media: Media) -> str:
         return self.cdn_url.format(bucket=self.bucket, region=self.region, key=media.key)
 
     def uploadf(self, file: IO[bytes], media: Media, **kwargs: dict) -> None:
-        if not self.is_upload_needed(media): return True
+        if not self.is_upload_needed(media):
+            return True
 
         extra_args = kwargs.get("extra_args", {})
-        if not self.private and 'ACL' not in extra_args:
-            extra_args['ACL'] = 'public-read'
+        if not self.private and "ACL" not in extra_args:
+            extra_args["ACL"] = "public-read"
 
-        if 'ContentType' not in extra_args:
+        if "ContentType" not in extra_args:
             try:
                 if media.mimetype:
-                    extra_args['ContentType'] = media.mimetype
+                    extra_args["ContentType"] = media.mimetype
             except Exception as e:
                 logger.warning(f"Unable to get mimetype for {media.key=}, error: {e}")
         self.s3.upload_fileobj(file, Bucket=self.bucket, Key=media.key, ExtraArgs=extra_args)
         return True
-    
+
     def is_upload_needed(self, media: Media) -> bool:
         if self.random_no_duplicate:
             # checks if a folder with the hash already exists, if so it skips the upload
             hd = calculate_file_hash(media.filename)
             path = os.path.join(NO_DUPLICATES_FOLDER, hd[:24])
 
-            if existing_key:=self.file_in_folder(path):
+            if existing_key := self.file_in_folder(path):
                 media.key = existing_key
                 media.set("previously archived", True)
                 logger.debug(f"skipping upload of {media.filename} because it already exists in {media.key}")
                 return False
-            
+
             _, ext = os.path.splitext(media.key)
             media.key = os.path.join(path, f"{random_str(24)}{ext}")
         return True
 
-    def file_in_folder(self, path:str) -> str:
+    def file_in_folder(self, path: str) -> str:
         # checks if path exists and is not an empty folder
-        if not path.endswith('/'):
-            path = path + '/' 
-        resp = self.s3.list_objects(Bucket=self.bucket, Prefix=path, Delimiter='/', MaxKeys=1)
-        if 'Contents' in resp:
-            return resp['Contents'][0]['Key']
+        if not path.endswith("/"):
+            path = path + "/"
+        resp = self.s3.list_objects(Bucket=self.bucket, Prefix=path, Delimiter="/", MaxKeys=1)
+        if "Contents" in resp:
+            return resp["Contents"][0]["Key"]
         return False
-
diff --git a/src/auto_archiver/modules/screenshot_enricher/__manifest__.py b/src/auto_archiver/modules/screenshot_enricher/__manifest__.py
index 92c0883..db04e6c 100644
--- a/src/auto_archiver/modules/screenshot_enricher/__manifest__.py
+++ b/src/auto_archiver/modules/screenshot_enricher/__manifest__.py
@@ -6,26 +6,29 @@
         "python": ["loguru", "selenium"],
     },
     "configs": {
-            "width": {"default": 1280,
-                      "type": "int",
-                      "help": "width of the screenshots"},
-            "height": {"default": 1024,
-                        "type": "int",
-                       "help": "height of the screenshots"},
-            "timeout": {"default": 60,
-                        "type": "int",
-                        "help": "timeout for taking the screenshot"},
-            "sleep_before_screenshot": {"default": 4,
-                                        "type": "int",
-                                        "help": "seconds to wait for the pages to load before taking screenshot"},
-            "http_proxy": {"default": "", "help": "http proxy to use for the webdriver, eg http://proxy-user:password@proxy-ip:port"},
-            "save_to_pdf": {"default": False,
-                            "type": "bool",
-                            "help": "save the page as pdf along with the screenshot. PDF saving options can be adjusted with the 'print_options' parameter"},
-            "print_options": {"default": {},
-                              "help": "options to pass to the pdf printer, in JSON format. See https://www.selenium.dev/documentation/webdriver/interactions/print_page/ for more information",
-                              "type": "json_loader"},
+        "width": {"default": 1280, "type": "int", "help": "width of the screenshots"},
+        "height": {"default": 1024, "type": "int", "help": "height of the screenshots"},
+        "timeout": {"default": 60, "type": "int", "help": "timeout for taking the screenshot"},
+        "sleep_before_screenshot": {
+            "default": 4,
+            "type": "int",
+            "help": "seconds to wait for the pages to load before taking screenshot",
         },
+        "http_proxy": {
+            "default": "",
+            "help": "http proxy to use for the webdriver, eg http://proxy-user:password@proxy-ip:port",
+        },
+        "save_to_pdf": {
+            "default": False,
+            "type": "bool",
+            "help": "save the page as pdf along with the screenshot. PDF saving options can be adjusted with the 'print_options' parameter",
+        },
+        "print_options": {
+            "default": {},
+            "help": "options to pass to the pdf printer, in JSON format. See https://www.selenium.dev/documentation/webdriver/interactions/print_page/ for more information",
+            "type": "json_loader",
+        },
+    },
     "description": """
     Captures screenshots and optionally saves web pages as PDFs using a WebDriver.
 
@@ -37,5 +40,5 @@
 
     ### Notes
     - Requires a WebDriver (e.g., ChromeDriver) installed and accessible via the system's PATH.
-    """
+    """,
 }
diff --git a/src/auto_archiver/modules/screenshot_enricher/screenshot_enricher.py b/src/auto_archiver/modules/screenshot_enricher/screenshot_enricher.py
index 832d0f8..9fa2d62 100644
--- a/src/auto_archiver/modules/screenshot_enricher/screenshot_enricher.py
+++ b/src/auto_archiver/modules/screenshot_enricher/screenshot_enricher.py
@@ -9,8 +9,8 @@ from auto_archiver.core import Enricher
 from auto_archiver.utils import Webdriver, url as UrlUtil, random_str
 from auto_archiver.core import Media, Metadata
 
-class ScreenshotEnricher(Enricher):
 
+class ScreenshotEnricher(Enricher):
     def __init__(self, webdriver_factory=None):
         super().__init__()
         self.webdriver_factory = webdriver_factory or Webdriver
@@ -25,8 +25,14 @@ class ScreenshotEnricher(Enricher):
         logger.debug(f"Enriching screenshot for {url=}")
         auth = self.auth_for_site(url)
         with self.webdriver_factory(
-                self.width, self.height, self.timeout, facebook_accept_cookies='facebook.com' in url,
-                       http_proxy=self.http_proxy, print_options=self.print_options, auth=auth) as driver:
+            self.width,
+            self.height,
+            self.timeout,
+            facebook_accept_cookies="facebook.com" in url,
+            http_proxy=self.http_proxy,
+            print_options=self.print_options,
+            auth=auth,
+        ) as driver:
             try:
                 driver.get(url)
                 time.sleep(int(self.sleep_before_screenshot))
@@ -43,4 +49,3 @@ class ScreenshotEnricher(Enricher):
                 logger.info("TimeoutException loading page for screenshot")
             except Exception as e:
                 logger.error(f"Got error while loading webdriver for screenshot enricher: {e}")
-
diff --git a/src/auto_archiver/modules/ssl_enricher/__init__.py b/src/auto_archiver/modules/ssl_enricher/__init__.py
index 23d2bee..86b9638 100644
--- a/src/auto_archiver/modules/ssl_enricher/__init__.py
+++ b/src/auto_archiver/modules/ssl_enricher/__init__.py
@@ -1 +1 @@
-from .ssl_enricher import SSLEnricher
\ No newline at end of file
+from .ssl_enricher import SSLEnricher
diff --git a/src/auto_archiver/modules/ssl_enricher/__manifest__.py b/src/auto_archiver/modules/ssl_enricher/__manifest__.py
index 097cd21..959fe2f 100644
--- a/src/auto_archiver/modules/ssl_enricher/__manifest__.py
+++ b/src/auto_archiver/modules/ssl_enricher/__manifest__.py
@@ -5,11 +5,13 @@
     "dependencies": {
         "python": ["loguru", "slugify"],
     },
-    'entry_point': 'ssl_enricher::SSLEnricher',
+    "entry_point": "ssl_enricher::SSLEnricher",
     "configs": {
-        "skip_when_nothing_archived": {"default": True,
-                                       "type": 'bool',
-                                       "help": "if true, will skip enriching when no media is archived"},
+        "skip_when_nothing_archived": {
+            "default": True,
+            "type": "bool",
+            "help": "if true, will skip enriching when no media is archived",
+        },
     },
     "description": """
     Retrieves SSL certificate information for a domain and stores it as a file.
@@ -21,5 +23,5 @@
 
     ### Notes
     - Requires the target URL to use the HTTPS scheme; other schemes are not supported.
-    """
+    """,
 }
diff --git a/src/auto_archiver/modules/ssl_enricher/ssl_enricher.py b/src/auto_archiver/modules/ssl_enricher/ssl_enricher.py
index b429163..74d80ce 100644
--- a/src/auto_archiver/modules/ssl_enricher/ssl_enricher.py
+++ b/src/auto_archiver/modules/ssl_enricher/ssl_enricher.py
@@ -13,16 +13,18 @@ class SSLEnricher(Enricher):
     """
 
     def enrich(self, to_enrich: Metadata) -> None:
-        if not to_enrich.media and self.skip_when_nothing_archived: return
-        
+        if not to_enrich.media and self.skip_when_nothing_archived:
+            return
+
         url = to_enrich.get_url()
         parsed = urlparse(url)
         assert parsed.scheme in ["https"], f"Invalid URL scheme {url=}"
-        
+
         domain = parsed.netloc
         logger.debug(f"fetching SSL certificate for {domain=} in {url=}")
 
         cert = ssl.get_server_certificate((domain, 443))
         cert_fn = os.path.join(self.tmp_dir, f"{slugify(domain)}.pem")
-        with open(cert_fn, "w") as f: f.write(cert)
+        with open(cert_fn, "w") as f:
+            f.write(cert)
         to_enrich.add_media(Media(filename=cert_fn), id="ssl_certificate")
diff --git a/src/auto_archiver/modules/telegram_extractor/__init__.py b/src/auto_archiver/modules/telegram_extractor/__init__.py
index 1fd80c2..18b73c2 100644
--- a/src/auto_archiver/modules/telegram_extractor/__init__.py
+++ b/src/auto_archiver/modules/telegram_extractor/__init__.py
@@ -1 +1 @@
-from .telegram_extractor import TelegramExtractor
\ No newline at end of file
+from .telegram_extractor import TelegramExtractor
diff --git a/src/auto_archiver/modules/telegram_extractor/telegram_extractor.py b/src/auto_archiver/modules/telegram_extractor/telegram_extractor.py
index d612e24..5184024 100644
--- a/src/auto_archiver/modules/telegram_extractor/telegram_extractor.py
+++ b/src/auto_archiver/modules/telegram_extractor/telegram_extractor.py
@@ -15,11 +15,11 @@ class TelegramExtractor(Extractor):
     def download(self, item: Metadata) -> Metadata:
         url = item.get_url()
         # detect URLs that we definitely cannot handle
-        if 't.me' != item.netloc:
+        if "t.me" != item.netloc:
             return False
 
         headers = {
-            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36'
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36"
         }
 
         # TODO: check if we can do this more resilient to variable URLs
@@ -27,11 +27,11 @@ class TelegramExtractor(Extractor):
             url += "?embed=1"
 
         t = requests.get(url, headers=headers)
-        s = BeautifulSoup(t.content, 'html.parser')
+        s = BeautifulSoup(t.content, "html.parser")
 
         result = Metadata()
         result.set_content(html.escape(str(t.content)))
-        if (timestamp := (s.find_all('time') or [{}])[0].get('datetime')):
+        if timestamp := (s.find_all("time") or [{}])[0].get("datetime"):
             result.set_timestamp(timestamp)
 
         video = s.find("video")
@@ -41,25 +41,26 @@ class TelegramExtractor(Extractor):
 
             image_urls = []
             for im in image_tags:
-                urls = [u.replace("'", "") for u in re.findall(r'url\((.*?)\)', im['style'])]
+                urls = [u.replace("'", "") for u in re.findall(r"url\((.*?)\)", im["style"])]
                 image_urls += urls
 
-            if not len(image_urls): return False
+            if not len(image_urls):
+                return False
             for img_url in image_urls:
                 result.add_media(Media(self.download_from_url(img_url)))
         else:
-            video_url = video.get('src')
+            video_url = video.get("src")
             m_video = Media(self.download_from_url(video_url))
             # extract duration from HTML
             try:
-                duration = s.find_all('time')[0].contents[0]
-                if ':' in duration:
-                    duration = float(duration.split(
-                        ':')[0]) * 60 + float(duration.split(':')[1])
+                duration = s.find_all("time")[0].contents[0]
+                if ":" in duration:
+                    duration = float(duration.split(":")[0]) * 60 + float(duration.split(":")[1])
                 else:
                     duration = float(duration)
                 m_video.set("duration", duration)
-            except: pass
+            except:
+                pass
             result.add_media(m_video)
 
         return result.success("telegram")
diff --git a/src/auto_archiver/modules/telethon_extractor/__init__.py b/src/auto_archiver/modules/telethon_extractor/__init__.py
index 2eaa57c..9d5e963 100644
--- a/src/auto_archiver/modules/telethon_extractor/__init__.py
+++ b/src/auto_archiver/modules/telethon_extractor/__init__.py
@@ -1 +1 @@
-from .telethon_extractor import TelethonExtractor
\ No newline at end of file
+from .telethon_extractor import TelethonExtractor
diff --git a/src/auto_archiver/modules/telethon_extractor/__manifest__.py b/src/auto_archiver/modules/telethon_extractor/__manifest__.py
index 5e58203..150b62c 100644
--- a/src/auto_archiver/modules/telethon_extractor/__manifest__.py
+++ b/src/auto_archiver/modules/telethon_extractor/__manifest__.py
@@ -3,26 +3,35 @@
     "type": ["extractor"],
     "requires_setup": True,
     "dependencies": {
-        "python": ["telethon",
-                   "loguru",
-                   "tqdm",
-                   ],
-        "bin": [""]
+        "python": [
+            "telethon",
+            "loguru",
+            "tqdm",
+        ],
+        "bin": [""],
     },
     "configs": {
-            "api_id": {"default": None, "help": "telegram API_ID value, go to https://my.telegram.org/apps"},
-            "api_hash": {"default": None, "help": "telegram API_HASH value, go to https://my.telegram.org/apps"},
-            "bot_token": {"default": None, "help": "optional, but allows access to more content such as large videos, talk to @botfather"},
-            "session_file": {"default": "secrets/anon", "help": "optional, records the telegram login session for future usage, '.session' will be appended to the provided value."},
-            "join_channels": {"default": True,
-                              "type": "bool",
-                              "help": "disables the initial setup with channel_invites config, useful if you have a lot and get stuck"},
-            "channel_invites": {
-                "default": {},
-                "help": "(JSON string) private channel invite links (format: t.me/joinchat/HASH OR t.me/+HASH) and (optional but important to avoid hanging for minutes on startup) channel id (format: CHANNEL_ID taken from a post url like https://t.me/c/CHANNEL_ID/1), the telegram account will join any new channels on setup",
-                "type": "json_loader",
-            }
+        "api_id": {"default": None, "help": "telegram API_ID value, go to https://my.telegram.org/apps"},
+        "api_hash": {"default": None, "help": "telegram API_HASH value, go to https://my.telegram.org/apps"},
+        "bot_token": {
+            "default": None,
+            "help": "optional, but allows access to more content such as large videos, talk to @botfather",
         },
+        "session_file": {
+            "default": "secrets/anon",
+            "help": "optional, records the telegram login session for future usage, '.session' will be appended to the provided value.",
+        },
+        "join_channels": {
+            "default": True,
+            "type": "bool",
+            "help": "disables the initial setup with channel_invites config, useful if you have a lot and get stuck",
+        },
+        "channel_invites": {
+            "default": {},
+            "help": "(JSON string) private channel invite links (format: t.me/joinchat/HASH OR t.me/+HASH) and (optional but important to avoid hanging for minutes on startup) channel id (format: CHANNEL_ID taken from a post url like https://t.me/c/CHANNEL_ID/1), the telegram account will join any new channels on setup",
+            "type": "json_loader",
+        },
+    },
     "description": """
 The `TelethonExtractor` uses the Telethon library to archive posts and media from Telegram channels and groups. 
 It supports private and public channels, downloading grouped posts with media, and can join channels using invite links 
@@ -46,5 +55,5 @@ To use the `TelethonExtractor`, you must configure the following:
 The first time you run, you will be prompted to do a authentication with the phone number associated, alternatively you can put your `anon.session` in the root.
 
 
-"""
-}
\ No newline at end of file
+""",
+}
diff --git a/src/auto_archiver/modules/telethon_extractor/telethon_extractor.py b/src/auto_archiver/modules/telethon_extractor/telethon_extractor.py
index 65ea8cd..be878a2 100644
--- a/src/auto_archiver/modules/telethon_extractor/telethon_extractor.py
+++ b/src/auto_archiver/modules/telethon_extractor/telethon_extractor.py
@@ -1,9 +1,13 @@
-
 import shutil
 from telethon.sync import TelegramClient
 from telethon.errors import ChannelInvalidError
 from telethon.tl.functions.messages import ImportChatInviteRequest
-from telethon.errors.rpcerrorlist import UserAlreadyParticipantError, FloodWaitError, InviteRequestSentError, InviteHashExpiredError
+from telethon.errors.rpcerrorlist import (
+    UserAlreadyParticipantError,
+    FloodWaitError,
+    InviteRequestSentError,
+    InviteHashExpiredError,
+)
 from loguru import logger
 from tqdm import tqdm
 import re, time, os
@@ -17,9 +21,7 @@ class TelethonExtractor(Extractor):
     valid_url = re.compile(r"https:\/\/t\.me(\/c){0,1}\/(.+)\/(\d+)")
     invite_pattern = re.compile(r"t.me(\/joinchat){0,1}\/\+?(.+)")
 
-
     def setup(self) -> None:
-
         """
         1. makes a copy of session_file that is removed in cleanup
         2. trigger login process for telegram or proceed if already saved in a session file
@@ -34,7 +36,7 @@ class TelethonExtractor(Extractor):
 
         # initiate the client
         self.client = TelegramClient(self.session_file, self.api_id, self.api_hash)
-        
+
         with self.client.start():
             logger.success(f"SETUP {self.name} login works.")
 
@@ -52,13 +54,15 @@ class TelethonExtractor(Extractor):
                     channel_invite = self.channel_invites[i]
                     channel_id = channel_invite.get("id", False)
                     invite = channel_invite["invite"]
-                    if (match := self.invite_pattern.search(invite)):
+                    if match := self.invite_pattern.search(invite):
                         try:
                             if channel_id:
                                 ent = self.client.get_entity(int(channel_id))  # fails if not a member
                             else:
                                 ent = self.client.get_entity(invite)  # fails if not a member
-                                logger.warning(f"please add the property id='{ent.id}' to the 'channel_invites' configuration where {invite=}, not doing so can lead to a minutes-long setup time due to telegram's rate limiting.")
+                                logger.warning(
+                                    f"please add the property id='{ent.id}' to the 'channel_invites' configuration where {invite=}, not doing so can lead to a minutes-long setup time due to telegram's rate limiting."
+                                )
                         except ValueError as e:
                             logger.info(f"joining new channel {invite=}")
                             try:
@@ -95,7 +99,8 @@ class TelethonExtractor(Extractor):
         # detect URLs that we definitely cannot handle
         match = self.valid_url.search(url)
         logger.debug(f"TELETHON: {match=}")
-        if not match: return False
+        if not match:
+            return False
 
         is_private = match.group(1) == "/c"
         chat = int(match.group(2)) if is_private else match.group(2)
@@ -105,45 +110,53 @@ class TelethonExtractor(Extractor):
 
         # NB: not using bot_token since then private channels cannot be archived: self.client.start(bot_token=self.bot_token)
         with self.client.start():
-        # with self.client.start(bot_token=self.bot_token):
+            # with self.client.start(bot_token=self.bot_token):
             try:
                 post = self.client.get_messages(chat, ids=post_id)
             except ValueError as e:
                 logger.error(f"Could not fetch telegram {url} possibly it's private: {e}")
                 return False
             except ChannelInvalidError as e:
-                logger.error(f"Could not fetch telegram {url}. This error may be fixed if you setup a bot_token in addition to api_id and api_hash (but then private channels will not be archived, we need to update this logic to handle both): {e}")
+                logger.error(
+                    f"Could not fetch telegram {url}. This error may be fixed if you setup a bot_token in addition to api_id and api_hash (but then private channels will not be archived, we need to update this logic to handle both): {e}"
+                )
                 return False
 
             logger.debug(f"TELETHON GOT POST {post=}")
-            if post is None: return False
+            if post is None:
+                return False
 
             media_posts = self._get_media_posts_in_group(chat, post)
-            logger.debug(f'got {len(media_posts)=} for {url=}')
+            logger.debug(f"got {len(media_posts)=} for {url=}")
 
             tmp_dir = self.tmp_dir
 
             group_id = post.grouped_id if post.grouped_id is not None else post.id
             title = post.message
             for mp in media_posts:
-                if len(mp.message) > len(title): title = mp.message  # save the longest text found (usually only 1)
+                if len(mp.message) > len(title):
+                    title = mp.message  # save the longest text found (usually only 1)
 
                 # media can also be in entities
                 if mp.entities:
-                    other_media_urls = [e.url for e in mp.entities if hasattr(e, "url") and e.url and self._guess_file_type(e.url) in ["video", "image", "audio"]]
+                    other_media_urls = [
+                        e.url
+                        for e in mp.entities
+                        if hasattr(e, "url") and e.url and self._guess_file_type(e.url) in ["video", "image", "audio"]
+                    ]
                     if len(other_media_urls):
                         logger.debug(f"Got {len(other_media_urls)} other media urls from {mp.id=}: {other_media_urls}")
                     for i, om_url in enumerate(other_media_urls):
-                        filename = self.download_from_url(om_url, f'{chat}_{group_id}_{i}')
+                        filename = self.download_from_url(om_url, f"{chat}_{group_id}_{i}")
                         result.add_media(Media(filename=filename), id=f"{group_id}_{i}")
 
-                filename_dest = os.path.join(tmp_dir, f'{chat}_{group_id}', str(mp.id))
+                filename_dest = os.path.join(tmp_dir, f"{chat}_{group_id}", str(mp.id))
                 filename = self.client.download_media(mp.media, filename_dest)
                 if not filename:
                     logger.debug(f"Empty media found, skipping {str(mp)=}")
                     continue
                 result.add_media(Media(filename))
-            
+
             result.set_title(title).set_timestamp(post.date).set("api_data", post.to_dict())
             if post.message != title:
                 result.set_content(post.message)
diff --git a/src/auto_archiver/modules/thumbnail_enricher/__manifest__.py b/src/auto_archiver/modules/thumbnail_enricher/__manifest__.py
index 1bd23b5..b11d17b 100644
--- a/src/auto_archiver/modules/thumbnail_enricher/__manifest__.py
+++ b/src/auto_archiver/modules/thumbnail_enricher/__manifest__.py
@@ -2,18 +2,19 @@
     "name": "Thumbnail Enricher",
     "type": ["enricher"],
     "requires_setup": False,
-    "dependencies": {
-        "python": ["loguru", "ffmpeg"],
-        "bin": ["ffmpeg"]
-    },
+    "dependencies": {"python": ["loguru", "ffmpeg"], "bin": ["ffmpeg"]},
     "configs": {
-            "thumbnails_per_minute": {"default": 60,
-                                      "type": "int",
-                                      "help": "how many thumbnails to generate per minute of video, can be limited by max_thumbnails"},
-            "max_thumbnails": {"default": 16,
-                               "type": "int",
-                               "help": "limit the number of thumbnails to generate per video, 0 means no limit"},
+        "thumbnails_per_minute": {
+            "default": 60,
+            "type": "int",
+            "help": "how many thumbnails to generate per minute of video, can be limited by max_thumbnails",
         },
+        "max_thumbnails": {
+            "default": 16,
+            "type": "int",
+            "help": "limit the number of thumbnails to generate per video, 0 means no limit",
+        },
+    },
     "description": """
     Generates thumbnails for video files to provide visual previews.
 
@@ -27,5 +28,5 @@
     - Requires `ffmpeg` to be installed and accessible via the system's PATH.
     - Handles videos without pre-existing duration metadata by probing with `ffmpeg`.
     - Skips enrichment for non-video media files.
-    """
+    """,
 }
diff --git a/src/auto_archiver/modules/thumbnail_enricher/thumbnail_enricher.py b/src/auto_archiver/modules/thumbnail_enricher/thumbnail_enricher.py
index 8178cd8..2f50c6b 100644
--- a/src/auto_archiver/modules/thumbnail_enricher/thumbnail_enricher.py
+++ b/src/auto_archiver/modules/thumbnail_enricher/thumbnail_enricher.py
@@ -6,6 +6,7 @@ visual snapshots of the video's keyframes, helping users preview content
 and identify important moments without watching the entire video.
 
 """
+
 import ffmpeg, os
 from loguru import logger
 
@@ -18,7 +19,7 @@ class ThumbnailEnricher(Enricher):
     """
     Generates thumbnails for all the media
     """
-    
+
     def enrich(self, to_enrich: Metadata) -> None:
         """
         Uses or reads the video duration to generate thumbnails
@@ -36,7 +37,9 @@ class ThumbnailEnricher(Enricher):
                 if duration is None:
                     try:
                         probe = ffmpeg.probe(m.filename)
-                        duration = float(next(stream for stream in probe['streams'] if stream['codec_type'] == 'video')['duration'])
+                        duration = float(
+                            next(stream for stream in probe["streams"] if stream["codec_type"] == "video")["duration"]
+                        )
                         to_enrich.media[m_id].set("duration", duration)
                     except Exception as e:
                         logger.error(f"error getting duration of video {m.filename}: {e}")
@@ -48,11 +51,13 @@ class ThumbnailEnricher(Enricher):
                 thumbnails_media = []
                 for index, timestamp in enumerate(timestamps):
                     output_path = os.path.join(folder, f"out{index}.jpg")
-                    ffmpeg.input(m.filename, ss=timestamp).filter('scale', 512, -1).output(output_path, vframes=1, loglevel="quiet").run()
+                    ffmpeg.input(m.filename, ss=timestamp).filter("scale", 512, -1).output(
+                        output_path, vframes=1, loglevel="quiet"
+                    ).run()
 
                     try:
-                        thumbnails_media.append(Media(
-                            filename=output_path)
+                        thumbnails_media.append(
+                            Media(filename=output_path)
                             .set("id", f"thumbnail_{index}")
                             .set("timestamp", "%.3fs" % timestamp)
                         )
diff --git a/src/auto_archiver/modules/timestamping_enricher/__manifest__.py b/src/auto_archiver/modules/timestamping_enricher/__manifest__.py
index 6ad9c57..c451437 100644
--- a/src/auto_archiver/modules/timestamping_enricher/__manifest__.py
+++ b/src/auto_archiver/modules/timestamping_enricher/__manifest__.py
@@ -3,38 +3,29 @@
     "type": ["enricher"],
     "requires_setup": True,
     "dependencies": {
-        "python": [
-            "loguru",
-            "slugify",
-            "tsp_client",
-            "asn1crypto",
-            "certvalidator",
-            "certifi"
-        ],
+        "python": ["loguru", "slugify", "tsp_client", "asn1crypto", "certvalidator", "certifi"],
     },
     "configs": {
         "tsa_urls": {
             "default": [
-                    # [Adobe Approved Trust List] and [Windows Cert Store]
-                    "http://timestamp.digicert.com",
-                    "http://timestamp.identrust.com",
-                    # "https://timestamp.entrust.net/TSS/RFC3161sha2TS", # not valid for timestamping
-                    # "https://timestamp.sectigo.com", # wait 15 seconds between each request.
-
-                    # [Adobe: European Union Trusted Lists].
-                    # "https://timestamp.sectigo.com/qualified", # wait 15 seconds between each request.
-
-                    # [Windows Cert Store]
-                    "http://timestamp.globalsign.com/tsa/r6advanced1",
-                    # [Adobe: European Union Trusted Lists] and [Windows Cert Store]
-                    # "http://ts.quovadisglobal.com/eu", # not valid for timestamping
-                    # "http://tsa.belgium.be/connect", # self-signed certificate in certificate chain
-                    # "https://timestamp.aped.gov.gr/qtss", # self-signed certificate in certificate chain
-                    # "http://tsa.sep.bg", # self-signed certificate in certificate chain
-                    # "http://tsa.izenpe.com", #unable to get local issuer certificate
-                    # "http://kstamp.keynectis.com/KSign", # unable to get local issuer certificate
-                    "http://tss.accv.es:8318/tsa",
-                ],
+                # [Adobe Approved Trust List] and [Windows Cert Store]
+                "http://timestamp.digicert.com",
+                "http://timestamp.identrust.com",
+                # "https://timestamp.entrust.net/TSS/RFC3161sha2TS", # not valid for timestamping
+                # "https://timestamp.sectigo.com", # wait 15 seconds between each request.
+                # [Adobe: European Union Trusted Lists].
+                # "https://timestamp.sectigo.com/qualified", # wait 15 seconds between each request.
+                # [Windows Cert Store]
+                "http://timestamp.globalsign.com/tsa/r6advanced1",
+                # [Adobe: European Union Trusted Lists] and [Windows Cert Store]
+                # "http://ts.quovadisglobal.com/eu", # not valid for timestamping
+                # "http://tsa.belgium.be/connect", # self-signed certificate in certificate chain
+                # "https://timestamp.aped.gov.gr/qtss", # self-signed certificate in certificate chain
+                # "http://tsa.sep.bg", # self-signed certificate in certificate chain
+                # "http://tsa.izenpe.com", #unable to get local issuer certificate
+                # "http://kstamp.keynectis.com/KSign", # unable to get local issuer certificate
+                "http://tss.accv.es:8318/tsa",
+            ],
             "help": "List of RFC3161 Time Stamp Authorities to use, separate with commas if passed via the command line.",
         }
     },
@@ -50,5 +41,5 @@
     ### Notes
     - Should be run after the `hash_enricher` to ensure file hashes are available.
     - Requires internet access to interact with the configured TSAs.
-    """
+    """,
 }
diff --git a/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py b/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py
index 078c1ba..586b7f8 100644
--- a/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py
+++ b/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py
@@ -11,6 +11,7 @@ import certifi
 from auto_archiver.core import Enricher
 from auto_archiver.core import Metadata, Media
 
+
 class TimestampingEnricher(Enricher):
     """
     Uses several RFC3161 Time Stamp Authorities to generate a timestamp token that will be preserved. This can be used to prove that a certain file existed at a certain time, useful for legal purposes, for example, to prove that a certain file was not tampered with after a certain date.
@@ -25,27 +26,30 @@ class TimestampingEnricher(Enricher):
         logger.debug(f"RFC3161 timestamping existing files for {url=}")
 
         # create a new text file with the existing media hashes
-        hashes = [m.get("hash").replace("SHA-256:", "").replace("SHA3-512:", "") for m in to_enrich.media if m.get("hash")]
+        hashes = [
+            m.get("hash").replace("SHA-256:", "").replace("SHA3-512:", "") for m in to_enrich.media if m.get("hash")
+        ]
 
         if not len(hashes):
             logger.warning(f"No hashes found in {url=}")
             return
-        
+
         tmp_dir = self.tmp_dir
         hashes_fn = os.path.join(tmp_dir, "hashes.txt")
 
         data_to_sign = "\n".join(hashes)
-        with open(hashes_fn, "w") as f: 
+        with open(hashes_fn, "w") as f:
             f.write(data_to_sign)
         hashes_media = Media(filename=hashes_fn)
 
         timestamp_tokens = []
         from slugify import slugify
+
         for tsa_url in self.tsa_urls:
             try:
                 signing_settings = SigningSettings(tsp_server=tsa_url, digest_algorithm=DigestAlgorithm.SHA256)
                 signer = TSPSigner()
-                message = bytes(data_to_sign, encoding='utf8')
+                message = bytes(data_to_sign, encoding="utf8")
                 # send TSQ and get TSR from the TSA server
                 signed = signer.sign(message=message, signing_settings=signing_settings)
                 # fail if there's any issue with the certificates, uses certifi list of trusted CAs
@@ -54,7 +58,8 @@ class TimestampingEnricher(Enricher):
                 cert_chain = self.download_and_verify_certificate(signed)
                 # continue with saving the timestamp token
                 tst_fn = os.path.join(tmp_dir, f"timestamp_token_{slugify(tsa_url)}")
-                with open(tst_fn, "wb") as f: f.write(signed)
+                with open(tst_fn, "wb") as f:
+                    f.write(signed)
                 timestamp_tokens.append(Media(filename=tst_fn).set("tsa", tsa_url).set("cert_chain", cert_chain))
             except Exception as e:
                 logger.warning(f"Error while timestamping {url=} with {tsa_url=}: {e}")
@@ -75,7 +80,7 @@ class TimestampingEnricher(Enricher):
         tst = ContentInfo.load(signed)
 
         trust_roots = []
-        with open(certifi.where(), 'rb') as f:
+        with open(certifi.where(), "rb") as f:
             for _, _, der_bytes in pem.unarmor(f.read(), multiple=True):
                 trust_roots.append(der_bytes)
         context = ValidationContext(trust_roots=trust_roots)
@@ -83,11 +88,11 @@ class TimestampingEnricher(Enricher):
         certificates = tst["content"]["certificates"]
         first_cert = certificates[0].dump()
         intermediate_certs = []
-        for i in range(1, len(certificates)): # cannot use list comprehension [1:]
+        for i in range(1, len(certificates)):  # cannot use list comprehension [1:]
             intermediate_certs.append(certificates[i].dump())
 
         validator = CertificateValidator(first_cert, intermediate_certs=intermediate_certs, validation_context=context)
-        path = validator.validate_usage({'digital_signature'}, extended_key_usage={'time_stamping'})
+        path = validator.validate_usage({"digital_signature"}, extended_key_usage={"time_stamping"})
 
         cert_chain = []
         for cert in path:
@@ -96,4 +101,4 @@ class TimestampingEnricher(Enricher):
                 f.write(cert.dump())
             cert_chain.append(Media(filename=cert_fn).set("subject", cert.subject.native["common_name"]))
 
-        return cert_chain
\ No newline at end of file
+        return cert_chain
diff --git a/src/auto_archiver/modules/twitter_api_extractor/__init__.py b/src/auto_archiver/modules/twitter_api_extractor/__init__.py
index 7005965..54e7b6c 100644
--- a/src/auto_archiver/modules/twitter_api_extractor/__init__.py
+++ b/src/auto_archiver/modules/twitter_api_extractor/__init__.py
@@ -1 +1 @@
-from .twitter_api_extractor import TwitterApiExtractor
\ No newline at end of file
+from .twitter_api_extractor import TwitterApiExtractor
diff --git a/src/auto_archiver/modules/twitter_api_extractor/__manifest__.py b/src/auto_archiver/modules/twitter_api_extractor/__manifest__.py
index 05d1ac0..203155f 100644
--- a/src/auto_archiver/modules/twitter_api_extractor/__manifest__.py
+++ b/src/auto_archiver/modules/twitter_api_extractor/__manifest__.py
@@ -3,21 +3,28 @@
     "type": ["extractor"],
     "requires_setup": True,
     "dependencies": {
-        "python": ["requests",
-                   "loguru",
-                   "pytwitter",
-                   "slugify",],
-        "bin": [""]
+        "python": [
+            "requests",
+            "loguru",
+            "pytwitter",
+            "slugify",
+        ],
+        "bin": [""],
     },
     "configs": {
-            "bearer_token": {"default": None, "help": "[deprecated: see bearer_tokens] twitter API bearer_token which is enough for archiving, if not provided you will need consumer_key, consumer_secret, access_token, access_secret"},
-            "bearer_tokens": {"default": [], "help": " a list of twitter API bearer_token which is enough for archiving, if not provided you will need consumer_key, consumer_secret, access_token, access_secret, if provided you can still add those for better rate limits. CSV of bearer tokens if provided via the command line",
-                              },
-            "consumer_key": {"default": None, "help": "twitter API consumer_key"},
-            "consumer_secret": {"default": None, "help": "twitter API consumer_secret"},
-            "access_token": {"default": None, "help": "twitter API access_token"},
-            "access_secret": {"default": None, "help": "twitter API access_secret"},
+        "bearer_token": {
+            "default": None,
+            "help": "[deprecated: see bearer_tokens] twitter API bearer_token which is enough for archiving, if not provided you will need consumer_key, consumer_secret, access_token, access_secret",
         },
+        "bearer_tokens": {
+            "default": [],
+            "help": " a list of twitter API bearer_token which is enough for archiving, if not provided you will need consumer_key, consumer_secret, access_token, access_secret, if provided you can still add those for better rate limits. CSV of bearer tokens if provided via the command line",
+        },
+        "consumer_key": {"default": None, "help": "twitter API consumer_key"},
+        "consumer_secret": {"default": None, "help": "twitter API consumer_secret"},
+        "access_token": {"default": None, "help": "twitter API access_token"},
+        "access_secret": {"default": None, "help": "twitter API access_secret"},
+    },
     "description": """
         The `TwitterApiExtractor` fetches tweets and associated media using the Twitter API. 
         It supports multiple API configurations for extended rate limits and reliable access. 
@@ -39,6 +46,5 @@
         - **Access Token and Secret**: Complements the consumer key for enhanced API capabilities.
         
         Credentials can be obtained by creating a Twitter developer account at [Twitter Developer Platform](https://developer.twitter.com/en).
-        """
-,
+        """,
 }
diff --git a/src/auto_archiver/modules/twitter_api_extractor/twitter_api_extractor.py b/src/auto_archiver/modules/twitter_api_extractor/twitter_api_extractor.py
index 72fd2f2..a7af607 100644
--- a/src/auto_archiver/modules/twitter_api_extractor/twitter_api_extractor.py
+++ b/src/auto_archiver/modules/twitter_api_extractor/twitter_api_extractor.py
@@ -11,8 +11,8 @@ from slugify import slugify
 from auto_archiver.core import Extractor
 from auto_archiver.core import Metadata, Media
 
-class TwitterApiExtractor(Extractor):
 
+class TwitterApiExtractor(Extractor):
     valid_url: re.Pattern = re.compile(r"(?:twitter|x).com\/(?:\#!\/)?(\w+)\/status(?:es)?\/(\d+)")
 
     def setup(self) -> None:
@@ -23,30 +23,38 @@ class TwitterApiExtractor(Extractor):
         if self.bearer_token:
             self.apis.append(Api(bearer_token=self.bearer_token))
         if self.consumer_key and self.consumer_secret and self.access_token and self.access_secret:
-            self.apis.append(Api(consumer_key=self.consumer_key, consumer_secret=self.consumer_secret,
-                             access_token=self.access_token, access_secret=self.access_secret))
-        assert self.api_client is not None, "Missing Twitter API configurations, please provide either AND/OR (consumer_key, consumer_secret, access_token, access_secret) to use this archiver, you can provide both for better rate-limit results."
+            self.apis.append(
+                Api(
+                    consumer_key=self.consumer_key,
+                    consumer_secret=self.consumer_secret,
+                    access_token=self.access_token,
+                    access_secret=self.access_secret,
+                )
+            )
+        assert self.api_client is not None, (
+            "Missing Twitter API configurations, please provide either AND/OR (consumer_key, consumer_secret, access_token, access_secret) to use this archiver, you can provide both for better rate-limit results."
+        )
 
     @property  # getter .mimetype
     def api_client(self) -> str:
         return self.apis[self.api_index]
-    
+
     def sanitize_url(self, url: str) -> str:
         # expand URL if t.co and clean tracker GET params
-        if 'https://t.co/' in url:
+        if "https://t.co/" in url:
             try:
                 r = requests.get(url, timeout=30)
-                logger.debug(f'Expanded url {url} to {r.url}')
+                logger.debug(f"Expanded url {url} to {r.url}")
                 url = r.url
             except:
-                logger.error(f'Failed to expand url {url}')
+                logger.error(f"Failed to expand url {url}")
         return url
 
-
     def download(self, item: Metadata) -> Metadata:
         # call download retry until success or no more apis
         while self.api_index < len(self.apis):
-            if res := self.download_retry(item): return res
+            if res := self.download_retry(item):
+                return res
             self.api_index += 1
         self.api_index = 0
         return False
@@ -54,7 +62,8 @@ class TwitterApiExtractor(Extractor):
     def get_username_tweet_id(self, url):
         # detect URLs that we definitely cannot handle
         matches = self.valid_url.findall(url)
-        if not len(matches): return False, False
+        if not len(matches):
+            return False, False
 
         username, tweet_id = matches[0]  # only one URL supported
         logger.debug(f"Found {username=} and {tweet_id=} in {url=}")
@@ -65,10 +74,16 @@ class TwitterApiExtractor(Extractor):
         url = item.get_url()
         # detect URLs that we definitely cannot handle
         username, tweet_id = self.get_username_tweet_id(url)
-        if not username: return False
+        if not username:
+            return False
 
         try:
-            tweet = self.api_client.get_tweet(tweet_id, expansions=["attachments.media_keys"], media_fields=["type", "duration_ms", "url", "variants"], tweet_fields=["attachments", "author_id", "created_at", "entities", "id", "text", "possibly_sensitive"])
+            tweet = self.api_client.get_tweet(
+                tweet_id,
+                expansions=["attachments.media_keys"],
+                media_fields=["type", "duration_ms", "url", "variants"],
+                tweet_fields=["attachments", "author_id", "created_at", "entities", "id", "text", "possibly_sensitive"],
+            )
             logger.debug(tweet)
         except Exception as e:
             logger.error(f"Could not get tweet: {e}")
@@ -88,29 +103,35 @@ class TwitterApiExtractor(Extractor):
                     mimetype = "image/jpeg"
                 elif hasattr(m, "variants"):
                     variant = self.choose_variant(m.variants)
-                    if not variant: continue
+                    if not variant:
+                        continue
                     media.set("src", variant.url)
                     mimetype = variant.content_type
                 else:
                     continue
                 logger.info(f"Found media {media}")
                 ext = mimetypes.guess_extension(mimetype)
-                media.filename = self.download_from_url(media.get("src"), f'{slugify(url)}_{i}{ext}')
+                media.filename = self.download_from_url(media.get("src"), f"{slugify(url)}_{i}{ext}")
                 result.add_media(media)
 
-        result.set_content(json.dumps({
-            "id": tweet.data.id,
-            "text": tweet.data.text,
-            "created_at": tweet.data.created_at,
-            "author_id": tweet.data.author_id,
-            "geo": tweet.data.geo,
-            "lang": tweet.data.lang,
-            "media": urls
-        }, ensure_ascii=False, indent=4))
+        result.set_content(
+            json.dumps(
+                {
+                    "id": tweet.data.id,
+                    "text": tweet.data.text,
+                    "created_at": tweet.data.created_at,
+                    "author_id": tweet.data.author_id,
+                    "geo": tweet.data.geo,
+                    "lang": tweet.data.lang,
+                    "media": urls,
+                },
+                ensure_ascii=False,
+                indent=4,
+            )
+        )
         return result.success("twitter-api")
 
     def choose_variant(self, variants):
-
         """
         Chooses the highest quality variable possible out of a list of variants
         """
diff --git a/src/auto_archiver/modules/vk_extractor/__manifest__.py b/src/auto_archiver/modules/vk_extractor/__manifest__.py
index 61e454e..ed16331 100644
--- a/src/auto_archiver/modules/vk_extractor/__manifest__.py
+++ b/src/auto_archiver/modules/vk_extractor/__manifest__.py
@@ -7,10 +7,8 @@
         "python": ["loguru", "vk_url_scraper"],
     },
     "configs": {
-        "username": {"required": True,
-                     "help": "valid VKontakte username"},
-        "password": {"required": True,
-                     "help": "valid VKontakte password"},
+        "username": {"required": True, "help": "valid VKontakte username"},
+        "password": {"required": True, "help": "valid VKontakte password"},
         "session_file": {
             "default": "secrets/vk_config.v2.json",
             "help": "valid VKontakte password",
diff --git a/src/auto_archiver/modules/vk_extractor/vk_extractor.py b/src/auto_archiver/modules/vk_extractor/vk_extractor.py
index 99527c4..997b0a8 100644
--- a/src/auto_archiver/modules/vk_extractor/vk_extractor.py
+++ b/src/auto_archiver/modules/vk_extractor/vk_extractor.py
@@ -7,7 +7,7 @@ from auto_archiver.core import Metadata, Media
 
 
 class VkExtractor(Extractor):
-    """"
+    """ "
     VK videos are handled by YTDownloader, this archiver gets posts text and images.
     Currently only works for /wall posts
     """
@@ -18,11 +18,13 @@ class VkExtractor(Extractor):
     def download(self, item: Metadata) -> Metadata:
         url = item.get_url()
 
-        if "vk.com" not in item.netloc: return False
+        if "vk.com" not in item.netloc:
+            return False
 
         # some urls can contain multiple wall/photo/... parts and all will be fetched
         vk_scrapes = self.vks.scrape(url)
-        if not len(vk_scrapes): return False
+        if not len(vk_scrapes):
+            return False
         logger.debug(f"VK: got {len(vk_scrapes)} scraped instances")
 
         result = Metadata()
diff --git a/src/auto_archiver/modules/wacz_extractor_enricher/__manifest__.py b/src/auto_archiver/modules/wacz_extractor_enricher/__manifest__.py
index 9b373b9..d2477b4 100644
--- a/src/auto_archiver/modules/wacz_extractor_enricher/__manifest__.py
+++ b/src/auto_archiver/modules/wacz_extractor_enricher/__manifest__.py
@@ -4,34 +4,38 @@
     "entry_point": "wacz_extractor_enricher::WaczExtractorEnricher",
     "requires_setup": True,
     "dependencies": {
-        "python": [
-            "loguru",
-            "jsonlines",
-            "warcio"
-        ],
+        "python": ["loguru", "jsonlines", "warcio"],
         # TODO?
-        "bin": [
-            "docker"
-        ]
+        "bin": ["docker"],
     },
     "configs": {
-            "profile": {"default": None, "help": "browsertrix-profile (for profile generation see https://github.com/webrecorder/browsertrix-crawler#creating-and-using-browser-profiles)."},
-            "docker_commands": {"default": None, "help":"if a custom docker invocation is needed"},
-            "timeout": {"default": 120,
-                        "type": "int",
-                        "help": "timeout for WACZ generation in seconds", "type": "int"},
-            "extract_media": {"default": False, 
-                              "type": 'bool',
-                              "help": "If enabled all the images/videos/audio present in the WACZ archive will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched."
-                              },
-            "extract_screenshot": {"default": True,
-                                    "type": 'bool',
-                                   "help": "If enabled the screenshot captured by browsertrix will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched."
-                                   },
-            "socks_proxy_host": {"default": None, "help": "SOCKS proxy host for browsertrix-crawler, use in combination with socks_proxy_port. eg: user:password@host"},
-            "socks_proxy_port": {"default": None, "type":"int", "help": "SOCKS proxy port for browsertrix-crawler, use in combination with socks_proxy_host. eg 1234"},
-            "proxy_server": {"default": None, "help": "SOCKS server proxy URL, in development"},
+        "profile": {
+            "default": None,
+            "help": "browsertrix-profile (for profile generation see https://github.com/webrecorder/browsertrix-crawler#creating-and-using-browser-profiles).",
         },
+        "docker_commands": {"default": None, "help": "if a custom docker invocation is needed"},
+        "timeout": {"default": 120, "type": "int", "help": "timeout for WACZ generation in seconds", "type": "int"},
+        "extract_media": {
+            "default": False,
+            "type": "bool",
+            "help": "If enabled all the images/videos/audio present in the WACZ archive will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched.",
+        },
+        "extract_screenshot": {
+            "default": True,
+            "type": "bool",
+            "help": "If enabled the screenshot captured by browsertrix will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched.",
+        },
+        "socks_proxy_host": {
+            "default": None,
+            "help": "SOCKS proxy host for browsertrix-crawler, use in combination with socks_proxy_port. eg: user:password@host",
+        },
+        "socks_proxy_port": {
+            "default": None,
+            "type": "int",
+            "help": "SOCKS proxy port for browsertrix-crawler, use in combination with socks_proxy_host. eg 1234",
+        },
+        "proxy_server": {"default": None, "help": "SOCKS server proxy URL, in development"},
+    },
     "description": """
     Creates .WACZ archives of web pages using the `browsertrix-crawler` tool, with options for media extraction and screenshot saving.
     [Browsertrix-crawler](https://crawler.docs.browsertrix.com/user-guide/) is a headless browser-based crawler that archives web pages in WACZ format.
@@ -45,5 +49,5 @@
     ### Notes
     - Requires Docker for running `browsertrix-crawler` .
     - Configurable via parameters for timeout, media extraction, screenshots, and proxy settings.
-    """
+    """,
 }
diff --git a/src/auto_archiver/modules/wacz_extractor_enricher/wacz_extractor_enricher.py b/src/auto_archiver/modules/wacz_extractor_enricher/wacz_extractor_enricher.py
index ff7314a..ec61572 100644
--- a/src/auto_archiver/modules/wacz_extractor_enricher/wacz_extractor_enricher.py
+++ b/src/auto_archiver/modules/wacz_extractor_enricher/wacz_extractor_enricher.py
@@ -19,13 +19,12 @@ class WaczExtractorEnricher(Enricher, Extractor):
     """
 
     def setup(self) -> None:
-
-        self.use_docker = os.environ.get('WACZ_ENABLE_DOCKER') or not os.environ.get('RUNNING_IN_DOCKER')
-        self.docker_in_docker = os.environ.get('WACZ_ENABLE_DOCKER') and os.environ.get('RUNNING_IN_DOCKER')
+        self.use_docker = os.environ.get("WACZ_ENABLE_DOCKER") or not os.environ.get("RUNNING_IN_DOCKER")
+        self.docker_in_docker = os.environ.get("WACZ_ENABLE_DOCKER") and os.environ.get("RUNNING_IN_DOCKER")
 
         self.cwd_dind = f"/crawls/crawls{random_str(8)}"
-        self.browsertrix_home_host = os.environ.get('BROWSERTRIX_HOME_HOST')
-        self.browsertrix_home_container = os.environ.get('BROWSERTRIX_HOME_CONTAINER') or self.browsertrix_home_host
+        self.browsertrix_home_host = os.environ.get("BROWSERTRIX_HOME_HOST")
+        self.browsertrix_home_container = os.environ.get("BROWSERTRIX_HOME_CONTAINER") or self.browsertrix_home_host
         # create crawls folder if not exists, so it can be safely removed in cleanup
         if self.docker_in_docker:
             os.makedirs(self.cwd_dind, exist_ok=True)
@@ -55,21 +54,32 @@ class WaczExtractorEnricher(Enricher, Extractor):
 
         cmd = [
             "crawl",
-            "--url", url,
-            "--scopeType", "page",
+            "--url",
+            url,
+            "--scopeType",
+            "page",
             "--generateWACZ",
-            "--text", "to-pages",
-            "--screenshot", "fullPage",
-            "--collection", collection,
-            "--id", collection,
-            "--saveState", "never",
-            "--behaviors", "autoscroll,autoplay,autofetch,siteSpecific",
-            "--behaviorTimeout", str(self.timeout),
-            "--timeout", str(self.timeout),
-            "--diskUtilization", "99",
+            "--text",
+            "to-pages",
+            "--screenshot",
+            "fullPage",
+            "--collection",
+            collection,
+            "--id",
+            collection,
+            "--saveState",
+            "never",
+            "--behaviors",
+            "autoscroll,autoplay,autofetch,siteSpecific",
+            "--behaviorTimeout",
+            str(self.timeout),
+            "--timeout",
+            str(self.timeout),
+            "--diskUtilization",
+            "99",
             # "--blockAds" # note: this has been known to cause issues on cloudflare protected sites
         ]
-        
+
         if self.docker_in_docker:
             cmd.extend(["--cwd", self.cwd_dind])
 
@@ -80,7 +90,14 @@ class WaczExtractorEnricher(Enricher, Extractor):
             if self.docker_commands:
                 cmd = self.docker_commands + cmd
             else:
-                cmd = ["docker", "run", "--rm", "-v", f"{browsertrix_home_host}:/crawls/", "webrecorder/browsertrix-crawler"] + cmd
+                cmd = [
+                    "docker",
+                    "run",
+                    "--rm",
+                    "-v",
+                    f"{browsertrix_home_host}:/crawls/",
+                    "webrecorder/browsertrix-crawler",
+                ] + cmd
 
             if self.profile:
                 profile_fn = os.path.join(browsertrix_home_container, "profile.tar.gz")
@@ -109,7 +126,6 @@ class WaczExtractorEnricher(Enricher, Extractor):
             logger.error(f"WACZ generation failed: {e}")
             return False
 
-        
         if self.docker_in_docker:
             wacz_fn = os.path.join(self.cwd_dind, "collections", collection, f"{collection}.wacz")
         elif self.use_docker:
@@ -138,11 +154,10 @@ class WaczExtractorEnricher(Enricher, Extractor):
             logger.info(f"Parsing pages.jsonl  {jsonl_fn=}")
             with jsonlines.open(jsonl_fn) as reader:
                 for obj in reader:
-                    if 'title' in obj:
-                        to_enrich.set_title(obj['title'])
-                    if 'text' in obj:
-                        to_enrich.set_content(obj['text'])
-
+                    if "title" in obj:
+                        to_enrich.set_title(obj["title"])
+                    if "text" in obj:
+                        to_enrich.set_content(obj["text"])
 
         return True
 
@@ -155,36 +170,42 @@ class WaczExtractorEnricher(Enricher, Extractor):
         # unzipping the .wacz
         tmp_dir = self.tmp_dir
         unzipped_dir = os.path.join(tmp_dir, "unzipped")
-        with ZipFile(wacz_filename, 'r') as z_obj:
+        with ZipFile(wacz_filename, "r") as z_obj:
             z_obj.extractall(path=unzipped_dir)
 
         # if warc is split into multiple gzip chunks, merge those
         warc_dir = os.path.join(unzipped_dir, "archive")
         warc_filename = os.path.join(tmp_dir, "merged.warc")
-        with open(warc_filename, 'wb') as outfile:
+        with open(warc_filename, "wb") as outfile:
             for filename in sorted(os.listdir(warc_dir)):
-                if filename.endswith('.gz'):
+                if filename.endswith(".gz"):
                     chunk_file = os.path.join(warc_dir, filename)
-                    with open(chunk_file, 'rb') as infile:
+                    with open(chunk_file, "rb") as infile:
                         shutil.copyfileobj(infile, outfile)
 
         # get media out of .warc
         counter = 0
         seen_urls = set()
         import json
-        with open(warc_filename, 'rb') as warc_stream:
+
+        with open(warc_filename, "rb") as warc_stream:
             for record in ArchiveIterator(warc_stream):
                 # only include fetched resources
-                if record.rec_type == "resource" and record.content_type == "image/png" and self.extract_screenshot:  # screenshots
+                if (
+                    record.rec_type == "resource" and record.content_type == "image/png" and self.extract_screenshot
+                ):  # screenshots
                     fn = os.path.join(tmp_dir, f"warc-file-{counter}.png")
-                    with open(fn, "wb") as outf: outf.write(record.raw_stream.read())
+                    with open(fn, "wb") as outf:
+                        outf.write(record.raw_stream.read())
                     m = Media(filename=fn)
                     to_enrich.add_media(m, "browsertrix-screenshot")
                     counter += 1
-                if not self.extract_media: continue
+                if not self.extract_media:
+                    continue
 
-                if record.rec_type != 'response': continue
-                record_url = record.rec_headers.get_header('WARC-Target-URI')
+                if record.rec_type != "response":
+                    continue
+                record_url = record.rec_headers.get_header("WARC-Target-URI")
                 if not UrlUtil.is_relevant_url(record_url):
                     logger.debug(f"Skipping irrelevant URL {record_url} but it's still present in the WACZ.")
                     continue
@@ -194,8 +215,10 @@ class WaczExtractorEnricher(Enricher, Extractor):
 
                 # filter by media mimetypes
                 content_type = record.http_headers.get("Content-Type")
-                if not content_type: continue
-                if not any(x in content_type for x in ["video", "image", "audio"]): continue
+                if not content_type:
+                    continue
+                if not any(x in content_type for x in ["video", "image", "audio"]):
+                    continue
 
                 # create local file and add media
                 ext = mimetypes.guess_extension(content_type)
@@ -203,7 +226,8 @@ class WaczExtractorEnricher(Enricher, Extractor):
                 fn = os.path.join(tmp_dir, warc_fn)
 
                 record_url_best_qual = UrlUtil.twitter_best_quality_url(record_url)
-                with open(fn, "wb") as outf: outf.write(record.raw_stream.read())
+                with open(fn, "wb") as outf:
+                    outf.write(record.raw_stream.read())
 
                 m = Media(filename=fn)
                 m.set("src", record_url)
@@ -213,12 +237,16 @@ class WaczExtractorEnricher(Enricher, Extractor):
                         m.filename = self.download_from_url(record_url_best_qual, warc_fn)
                         m.set("src", record_url_best_qual)
                         m.set("src_alternative", record_url)
-                    except Exception as e: logger.warning(f"Unable to download best quality URL for {record_url=} got error {e}, using original in WARC.")
+                    except Exception as e:
+                        logger.warning(
+                            f"Unable to download best quality URL for {record_url=} got error {e}, using original in WARC."
+                        )
 
                 # remove bad videos
-                if m.is_video() and not m.is_valid_video(): continue
-                
+                if m.is_video() and not m.is_valid_video():
+                    continue
+
                 to_enrich.add_media(m, warc_fn)
                 counter += 1
                 seen_urls.add(record_url)
-        logger.info(f"WACZ extract_media/extract_screenshot finished, found {counter} relevant media file(s)")
\ No newline at end of file
+        logger.info(f"WACZ extract_media/extract_screenshot finished, found {counter} relevant media file(s)")
diff --git a/src/auto_archiver/modules/wayback_extractor_enricher/__init__.py b/src/auto_archiver/modules/wayback_extractor_enricher/__init__.py
index b69332d..b6fb182 100644
--- a/src/auto_archiver/modules/wayback_extractor_enricher/__init__.py
+++ b/src/auto_archiver/modules/wayback_extractor_enricher/__init__.py
@@ -1 +1 @@
-from .wayback_extractor_enricher import WaybackExtractorEnricher
\ No newline at end of file
+from .wayback_extractor_enricher import WaybackExtractorEnricher
diff --git a/src/auto_archiver/modules/wayback_extractor_enricher/wayback_extractor_enricher.py b/src/auto_archiver/modules/wayback_extractor_enricher/wayback_extractor_enricher.py
index 1763b12..2dc3545 100644
--- a/src/auto_archiver/modules/wayback_extractor_enricher/wayback_extractor_enricher.py
+++ b/src/auto_archiver/modules/wayback_extractor_enricher/wayback_extractor_enricher.py
@@ -6,11 +6,12 @@ from auto_archiver.core import Extractor, Enricher
 from auto_archiver.utils import url as UrlUtil
 from auto_archiver.core import Metadata
 
+
 class WaybackExtractorEnricher(Enricher, Extractor):
     """
     Submits the current URL to the webarchive and returns a job_id or completed archive.
 
-    The Wayback machine will rate-limit IP heavy usage. 
+    The Wayback machine will rate-limit IP heavy usage.
     """
 
     def download(self, item: Metadata) -> Metadata:
@@ -22,8 +23,10 @@ class WaybackExtractorEnricher(Enricher, Extractor):
 
     def enrich(self, to_enrich: Metadata) -> bool:
         proxies = {}
-        if self.proxy_http: proxies["http"] = self.proxy_http
-        if self.proxy_https: proxies["https"] = self.proxy_https
+        if self.proxy_http:
+            proxies["http"] = self.proxy_http
+        if self.proxy_https:
+            proxies["https"] = self.proxy_https
 
         url = to_enrich.get_url()
         if UrlUtil.is_auth_wall(url):
@@ -36,15 +39,12 @@ class WaybackExtractorEnricher(Enricher, Extractor):
             logger.info(f"Wayback enricher had already been executed: {to_enrich.get('wayback')}")
             return True
 
-        ia_headers = {
-            "Accept": "application/json",
-            "Authorization": f"LOW {self.key}:{self.secret}"
-        }
-        post_data = {'url': url}
+        ia_headers = {"Accept": "application/json", "Authorization": f"LOW {self.key}:{self.secret}"}
+        post_data = {"url": url}
         if self.if_not_archived_within:
             post_data["if_not_archived_within"] = self.if_not_archived_within
         # see https://docs.google.com/document/d/1Nsv52MvSjbLb2PCpHlat0gkzw0EvtSgpKHu4mk0MnrA for more options
-        r = requests.post('https://web.archive.org/save/', headers=ia_headers, data=post_data, proxies=proxies)
+        r = requests.post("https://web.archive.org/save/", headers=ia_headers, data=post_data, proxies=proxies)
 
         if r.status_code != 200:
             logger.error(em := f"Internet archive failed with status of {r.status_code}: {r.json()}")
@@ -53,7 +53,7 @@ class WaybackExtractorEnricher(Enricher, Extractor):
 
         # check job status
         try:
-            job_id = r.json().get('job_id')
+            job_id = r.json().get("job_id")
             if not job_id:
                 logger.error(f"Wayback failed with {r.json()}")
                 return False
@@ -61,7 +61,6 @@ class WaybackExtractorEnricher(Enricher, Extractor):
             logger.error(f"Expected a JSON with job_id from Wayback and got {r.text}")
             return False
 
-
         # waits at most timeout seconds until job is completed, otherwise only enriches the job_id information
         start_time = time.time()
         wayback_url = False
@@ -69,11 +68,13 @@ class WaybackExtractorEnricher(Enricher, Extractor):
         while not wayback_url and time.time() - start_time <= self.timeout:
             try:
                 logger.debug(f"GETting status for {job_id=} on {url=} ({attempt=})")
-                r_status = requests.get(f'https://web.archive.org/save/status/{job_id}', headers=ia_headers, proxies=proxies)
+                r_status = requests.get(
+                    f"https://web.archive.org/save/status/{job_id}", headers=ia_headers, proxies=proxies
+                )
                 r_json = r_status.json()
-                if r_status.status_code == 200 and r_json['status'] == 'success':
+                if r_status.status_code == 200 and r_json["status"] == "success":
                     wayback_url = f"https://web.archive.org/web/{r_json['timestamp']}/{r_json['original_url']}"
-                elif r_status.status_code != 200 or r_json['status'] != 'pending':
+                elif r_status.status_code != 200 or r_json["status"] != "pending":
                     logger.error(f"Wayback failed with {r_json}")
                     return False
             except requests.exceptions.RequestException as e:
@@ -91,6 +92,8 @@ class WaybackExtractorEnricher(Enricher, Extractor):
         if wayback_url:
             to_enrich.set("wayback", wayback_url)
         else:
-            to_enrich.set("wayback", {"job_id": job_id, "check_status": f'https://web.archive.org/save/status/{job_id}'})
+            to_enrich.set(
+                "wayback", {"job_id": job_id, "check_status": f"https://web.archive.org/save/status/{job_id}"}
+            )
         to_enrich.set("check wayback", f"https://web.archive.org/web/*/{url}")
         return True
diff --git a/src/auto_archiver/modules/whisper_enricher/__init__.py b/src/auto_archiver/modules/whisper_enricher/__init__.py
index d3d3526..d69bdd1 100644
--- a/src/auto_archiver/modules/whisper_enricher/__init__.py
+++ b/src/auto_archiver/modules/whisper_enricher/__init__.py
@@ -1 +1 @@
-from .whisper_enricher import WhisperEnricher
\ No newline at end of file
+from .whisper_enricher import WhisperEnricher
diff --git a/src/auto_archiver/modules/whisper_enricher/__manifest__.py b/src/auto_archiver/modules/whisper_enricher/__manifest__.py
index 0e09d03..e7af7d1 100644
--- a/src/auto_archiver/modules/whisper_enricher/__manifest__.py
+++ b/src/auto_archiver/modules/whisper_enricher/__manifest__.py
@@ -6,19 +6,26 @@
         "python": ["s3_storage", "loguru", "requests"],
     },
     "configs": {
-        "api_endpoint": {"required": True,
-                         "help": "WhisperApi api endpoint, eg: https://whisperbox-api.com/api/v1, a deployment of https://github.com/bellingcat/whisperbox-transcribe."},
-        "api_key": {"required": True,
-                    "help": "WhisperApi api key for authentication"},
-        "include_srt": {"default": False,
-                        "type": "bool",
-                        "help": "Whether to include a subtitle SRT (SubRip Subtitle file) for the video (can be used in video players)."},
-        "timeout": {"default": 90,
-                    "type": "int",
-                    "help": "How many seconds to wait at most for a successful job completion."},
-        "action": {"default": "translate",
-                   "help": "which Whisper operation to execute",
-                   "choices": ["transcribe", "translate", "language_detection"]},
+        "api_endpoint": {
+            "required": True,
+            "help": "WhisperApi api endpoint, eg: https://whisperbox-api.com/api/v1, a deployment of https://github.com/bellingcat/whisperbox-transcribe.",
+        },
+        "api_key": {"required": True, "help": "WhisperApi api key for authentication"},
+        "include_srt": {
+            "default": False,
+            "type": "bool",
+            "help": "Whether to include a subtitle SRT (SubRip Subtitle file) for the video (can be used in video players).",
+        },
+        "timeout": {
+            "default": 90,
+            "type": "int",
+            "help": "How many seconds to wait at most for a successful job completion.",
+        },
+        "action": {
+            "default": "translate",
+            "help": "which Whisper operation to execute",
+            "choices": ["transcribe", "translate", "language_detection"],
+        },
     },
     "description": """
     Integrates with a Whisper API service to transcribe, translate, or detect the language of audio and video files.
@@ -35,5 +42,5 @@
     - Only compatible with S3-compatible storage systems for media file accessibility.
     - ** This stores the media files in S3 prior to enriching them as Whisper requires public URLs to access the media files.
     - Handles multiple jobs and retries for failed or incomplete processing.
-    """
+    """,
 }
diff --git a/src/auto_archiver/modules/whisper_enricher/whisper_enricher.py b/src/auto_archiver/modules/whisper_enricher/whisper_enricher.py
index d63d2ed..0c884bb 100644
--- a/src/auto_archiver/modules/whisper_enricher/whisper_enricher.py
+++ b/src/auto_archiver/modules/whisper_enricher/whisper_enricher.py
@@ -5,6 +5,7 @@ from loguru import logger
 from auto_archiver.core import Enricher
 from auto_archiver.core import Metadata, Media
 
+
 class WhisperEnricher(Enricher):
     """
     Connects with a Whisper API service to get texts out of audio
@@ -13,15 +14,15 @@ class WhisperEnricher(Enricher):
     """
 
     def setup(self) -> None:
-        self.stores = self.config['steps']['storages']
+        self.stores = self.config["steps"]["storages"]
         self.s3 = self.module_factory.get_module("s3_storage", self.config)
         if not "s3_storage" in self.stores:
-            logger.error("WhisperEnricher: To use the WhisperEnricher you need to use S3Storage so files are accessible publicly to the whisper service being called.")
+            logger.error(
+                "WhisperEnricher: To use the WhisperEnricher you need to use S3Storage so files are accessible publicly to the whisper service being called."
+            )
             return
 
-
     def enrich(self, to_enrich: Metadata) -> None:
-
         url = to_enrich.get_url()
         logger.debug(f"WHISPER[{self.action}]: iterating media items for {url=}.")
 
@@ -36,28 +37,33 @@ class WhisperEnricher(Enricher):
                     logger.debug(f"JOB SUBMITTED: {job_id=} for {m.key=}")
                     to_enrich.media[i].set("whisper_model", {"job_id": job_id})
                 except Exception as e:
-                    logger.error(f"Failed to submit whisper job for {m.filename=} with error {e}\n{traceback.format_exc()}")
+                    logger.error(
+                        f"Failed to submit whisper job for {m.filename=} with error {e}\n{traceback.format_exc()}"
+                    )
 
         job_results = self.check_jobs(job_results)
 
         for i, m in enumerate(to_enrich.media):
             if m.is_video() or m.is_audio():
                 job_id = to_enrich.media[i].get("whisper_model", {}).get("job_id")
-                if not job_id: continue
-                to_enrich.media[i].set("whisper_model", {
-                    "job_id": job_id,
-                    "job_status_check": f"{self.api_endpoint}/jobs/{job_id}",
-                    "job_artifacts_check": f"{self.api_endpoint}/jobs/{job_id}/artifacts",
-                    **(job_results[job_id] if job_results[job_id] else {"result": "incomplete or failed job"})
-                })
+                if not job_id:
+                    continue
+                to_enrich.media[i].set(
+                    "whisper_model",
+                    {
+                        "job_id": job_id,
+                        "job_status_check": f"{self.api_endpoint}/jobs/{job_id}",
+                        "job_artifacts_check": f"{self.api_endpoint}/jobs/{job_id}/artifacts",
+                        **(job_results[job_id] if job_results[job_id] else {"result": "incomplete or failed job"}),
+                    },
+                )
                 # append the extracted text to the content of the post so it gets written to the DBs like gsheets text column
                 if job_results[job_id]:
-                    for k,v in job_results[job_id].items():
+                    for k, v in job_results[job_id].items():
                         if "_text" in k and len(v):
                             to_enrich.set_content(f"\n[automatic video transcript]: {v}")
 
     def submit_job(self, media: Media):
-
         s3_url = self.s3.get_cdn_url(media)
         assert s3_url in media.urls, f"Could not find S3 url ({s3_url}) in list of stored media urls "
         payload = {
@@ -66,10 +72,14 @@ class WhisperEnricher(Enricher):
             # "language": "string" # may be a config
         }
         logger.debug(f"calling API with {payload=}")
-        response = requests.post(f'{self.api_endpoint}/jobs', json=payload, headers={'Authorization': f'Bearer {self.api_key}'})
-        assert response.status_code == 201, f"calling the whisper api {self.api_endpoint} returned a non-success code: {response.status_code}"
+        response = requests.post(
+            f"{self.api_endpoint}/jobs", json=payload, headers={"Authorization": f"Bearer {self.api_key}"}
+        )
+        assert response.status_code == 201, (
+            f"calling the whisper api {self.api_endpoint} returned a non-success code: {response.status_code}"
+        )
         logger.debug(response.json())
-        return response.json()['id']
+        return response.json()["id"]
 
     def check_jobs(self, job_results: dict):
         start_time = time.time()
@@ -77,37 +87,50 @@ class WhisperEnricher(Enricher):
         while not all_completed and (time.time() - start_time) <= self.timeout:
             all_completed = True
             for job_id in job_results:
-                if job_results[job_id] != False: continue
+                if job_results[job_id] != False:
+                    continue
                 all_completed = False  # at least one not ready
-                try: job_results[job_id] = self.check_job(job_id)
+                try:
+                    job_results[job_id] = self.check_job(job_id)
                 except Exception as e:
                     logger.error(f"Failed to check {job_id=} with error {e}\n{traceback.format_exc()}")
-            if not all_completed: time.sleep(3)
+            if not all_completed:
+                time.sleep(3)
         return job_results
 
     def check_job(self, job_id):
-        r = requests.get(f'{self.api_endpoint}/jobs/{job_id}', headers={'Authorization': f'Bearer {self.api_key}'})
+        r = requests.get(f"{self.api_endpoint}/jobs/{job_id}", headers={"Authorization": f"Bearer {self.api_key}"})
         assert r.status_code == 200, f"Job status did not respond with 200, instead with: {r.status_code}"
         j = r.json()
         logger.debug(f"Checked job {job_id=} with status='{j['status']}'")
-        if j['status'] == "processing": return False
-        elif j['status'] == "error": return f"Error: {j['meta']['error']}"
-        elif j['status'] == "success":
-            r_res = requests.get(f'{self.api_endpoint}/jobs/{job_id}/artifacts', headers={'Authorization': f'Bearer {self.api_key}'})
-            assert r_res.status_code == 200, f"Job artifacts did not respond with 200, instead with: {r_res.status_code}"
+        if j["status"] == "processing":
+            return False
+        elif j["status"] == "error":
+            return f"Error: {j['meta']['error']}"
+        elif j["status"] == "success":
+            r_res = requests.get(
+                f"{self.api_endpoint}/jobs/{job_id}/artifacts", headers={"Authorization": f"Bearer {self.api_key}"}
+            )
+            assert r_res.status_code == 200, (
+                f"Job artifacts did not respond with 200, instead with: {r_res.status_code}"
+            )
             logger.success(r_res.json())
             result = {}
             for art_id, artifact in enumerate(r_res.json()):
                 subtitle = []
                 full_text = []
                 for i, d in enumerate(artifact.get("data")):
-                    subtitle.append(f"{i+1}\n{d.get('start')} --> {d.get('end')}\n{d.get('text').strip()}")
-                    full_text.append(d.get('text').strip())
-                if not len(subtitle): continue
-                if self.include_srt: result[f"artifact_{art_id}_subtitle"] = "\n".join(subtitle)
+                    subtitle.append(f"{i + 1}\n{d.get('start')} --> {d.get('end')}\n{d.get('text').strip()}")
+                    full_text.append(d.get("text").strip())
+                if not len(subtitle):
+                    continue
+                if self.include_srt:
+                    result[f"artifact_{art_id}_subtitle"] = "\n".join(subtitle)
                 result[f"artifact_{art_id}_text"] = "\n".join(full_text)
             # call /delete endpoint on timely success
-            r_del = requests.delete(f'{self.api_endpoint}/jobs/{job_id}', headers={'Authorization': f'Bearer {self.api_key}'})
+            r_del = requests.delete(
+                f"{self.api_endpoint}/jobs/{job_id}", headers={"Authorization": f"Bearer {self.api_key}"}
+            )
             logger.debug(f"DELETE whisper {job_id=} result: {r_del.status_code}")
             return result
         return False
diff --git a/src/auto_archiver/utils/__init__.py b/src/auto_archiver/utils/__init__.py
index 46ca191..a8fa77f 100644
--- a/src/auto_archiver/utils/__init__.py
+++ b/src/auto_archiver/utils/__init__.py
@@ -1,7 +1,8 @@
-""" Auto Archiver Utilities. """
+"""Auto Archiver Utilities."""
+
 # we need to explicitly expose the available imports here
 from .misc import *
 from .webdriver import Webdriver
 
 # handy utils from ytdlp
-from yt_dlp.utils import (clean_html, traverse_obj, strip_or_none, url_or_none)
\ No newline at end of file
+from yt_dlp.utils import clean_html, traverse_obj, strip_or_none, url_or_none
diff --git a/src/auto_archiver/utils/misc.py b/src/auto_archiver/utils/misc.py
index 49ef0b5..379bff5 100644
--- a/src/auto_archiver/utils/misc.py
+++ b/src/auto_archiver/utils/misc.py
@@ -16,20 +16,21 @@ def mkdir_if_not_exists(folder):
 
 def expand_url(url):
     # expand short URL links
-    if 'https://t.co/' in url:
+    if "https://t.co/" in url:
         try:
             r = requests.get(url)
-            logger.debug(f'Expanded url {url} to {r.url}')
+            logger.debug(f"Expanded url {url} to {r.url}")
             return r.url
         except:
-            logger.error(f'Failed to expand url {url}')
+            logger.error(f"Failed to expand url {url}")
     return url
 
 
 def getattr_or(o: object, prop: str, default=None):
     try:
         res = getattr(o, prop)
-        if res is None: raise
+        if res is None:
+            raise
         return res
     except:
         return default
@@ -61,18 +62,19 @@ def random_str(length: int = 32) -> str:
     return str(uuid.uuid4()).replace("-", "")[:length]
 
 
-def calculate_file_hash(filename: str, hash_algo = hashlib.sha256, chunksize: int = 16000000) -> str:
+def calculate_file_hash(filename: str, hash_algo=hashlib.sha256, chunksize: int = 16000000) -> str:
     hash = hash_algo()
     with open(filename, "rb") as f:
         while True:
             buf = f.read(chunksize)
-            if not buf: break
+            if not buf:
+                break
             hash.update(buf)
     return hash.hexdigest()
 
 
 def get_datetime_from_str(dt_str: str, fmt: str | None = None, dayfirst=True) -> datetime | None:
-    """ parse a datetime string with option of passing a specific format
+    """parse a datetime string with option of passing a specific format
 
     Args:
         dt_str: the datetime string to parse
@@ -88,19 +90,24 @@ def get_datetime_from_str(dt_str: str, fmt: str | None = None, dayfirst=True) ->
 
 
 def get_timestamp(ts, utc=True, iso=True, dayfirst=True) -> str | datetime | None:
-    """  Consistent parsing of timestamps.
+    """Consistent parsing of timestamps.
     Args:
          If utc=True, the timezone is set to UTC,
          if iso=True, the output is an iso string
          Use dayfirst to signify between date formats which put the date vs month first:
          e.g. DD/MM/YYYY vs MM/DD/YYYY
-     """
-    if not ts: return
+    """
+    if not ts:
+        return
     try:
-        if isinstance(ts, str): ts = parse_dt(ts, dayfirst=dayfirst)
-        if isinstance(ts, (int, float)): ts = datetime.fromtimestamp(ts)
-        if utc: ts = ts.replace(tzinfo=timezone.utc)
-        if iso: return ts.isoformat()
+        if isinstance(ts, str):
+            ts = parse_dt(ts, dayfirst=dayfirst)
+        if isinstance(ts, (int, float)):
+            ts = datetime.fromtimestamp(ts)
+        if utc:
+            ts = ts.replace(tzinfo=timezone.utc)
+        if iso:
+            return ts.isoformat()
         return ts
     except Exception as e:
         logger.error(f"Unable to parse timestamp {ts}: {e}")
diff --git a/src/auto_archiver/utils/url.py b/src/auto_archiver/utils/url.py
index 061f4aa..169ed87 100644
--- a/src/auto_archiver/utils/url.py
+++ b/src/auto_archiver/utils/url.py
@@ -4,8 +4,8 @@ from ipaddress import ip_address
 
 
 AUTHWALL_URLS = [
-    re.compile(r"https:\/\/t\.me(\/c)\/(.+)\/(\d+)"), # telegram private channels
-    re.compile(r"https:\/\/www\.instagram\.com"), # instagram
+    re.compile(r"https:\/\/t\.me(\/c)\/(.+)\/(\d+)"),  # telegram private channels
+    re.compile(r"https:\/\/www\.instagram\.com"),  # instagram
 ]
 
 
@@ -14,17 +14,16 @@ def check_url_or_raise(url: str) -> bool | ValueError:
     Blocks localhost, private, reserved, and link-local IPs and all non-http/https schemes.
     """
 
-    
     if not (url.startswith("http://") or url.startswith("https://")):
         raise ValueError(f"Invalid URL scheme for url {url}")
-    
+
     parsed = urlparse(url)
     if not parsed.hostname:
         raise ValueError(f"Invalid URL hostname for url {url}")
-    
+
     if parsed.hostname == "localhost":
         raise ValueError(f"Localhost URLs cannot be parsed for security reasons (for url {url})")
-    
+
     if parsed.scheme not in ["http", "https"]:
         raise ValueError(f"Invalid URL scheme, only http and https supported (for url {url})")
 
@@ -32,7 +31,7 @@ def check_url_or_raise(url: str) -> bool | ValueError:
         ip = ip_address(parsed.hostname)
     except ValueError:
         pass
-    
+
     else:
         if not ip.is_global:
             raise ValueError(f"IP address {ip} is not globally reachable")
@@ -42,18 +41,21 @@ def check_url_or_raise(url: str) -> bool | ValueError:
             raise ValueError(f"Link-local IP address {ip} used")
         if ip.is_private:
             raise ValueError(f"Private IP address {ip} used")
-    
+
     return True
 
+
 def domain_for_url(url: str) -> str:
     """
     SECURITY: parse the domain using urllib to avoid any potential security issues
     """
     return urlparse(url).netloc
 
+
 def clean(url: str) -> str:
     return url
 
+
 def is_auth_wall(url: str) -> bool:
     """
     checks if URL is behind an authentication wall meaning steps like wayback, wacz, ... may not work
@@ -64,13 +66,15 @@ def is_auth_wall(url: str) -> bool:
 
     return False
 
+
 def remove_get_parameters(url: str) -> str:
     # http://example.com/file.mp4?t=1 -> http://example.com/file.mp4
     # useful for mimetypes to work
     parsed_url = urlparse(url)
-    new_url = urlunparse(parsed_url._replace(query=''))
+    new_url = urlunparse(parsed_url._replace(query=""))
     return new_url
 
+
 def is_relevant_url(url: str) -> bool:
     """
     Detect if a detected media URL is recurring and therefore irrelevant to a specific archive. Useful, for example, for the enumeration of the media files in WARC files which include profile pictures, favicons, etc.
@@ -78,42 +82,59 @@ def is_relevant_url(url: str) -> bool:
     clean_url = remove_get_parameters(url)
 
     # favicons
-    if "favicon" in url: return False
+    if "favicon" in url:
+        return False
     # ifnore icons
-    if clean_url.endswith(".ico"): return False
+    if clean_url.endswith(".ico"):
+        return False
     # ignore SVGs
-    if remove_get_parameters(url).endswith(".svg"): return False
+    if remove_get_parameters(url).endswith(".svg"):
+        return False
 
     # twitter profile pictures
-    if "twimg.com/profile_images" in url: return False
-    if "twimg.com" in url and "/default_profile_images" in url: return False
+    if "twimg.com/profile_images" in url:
+        return False
+    if "twimg.com" in url and "/default_profile_images" in url:
+        return False
 
     # instagram profile pictures
-    if "https://scontent.cdninstagram.com/" in url and "150x150" in url: return False
+    if "https://scontent.cdninstagram.com/" in url and "150x150" in url:
+        return False
     # instagram recurring images
-    if "https://static.cdninstagram.com/rsrc.php/" in url: return False
+    if "https://static.cdninstagram.com/rsrc.php/" in url:
+        return False
 
     # telegram
-    if "https://telegram.org/img/emoji/" in url: return False
+    if "https://telegram.org/img/emoji/" in url:
+        return False
 
     # youtube
-    if "https://www.youtube.com/s/gaming/emoji/" in url: return False
-    if "https://yt3.ggpht.com" in url and "default-user=" in url: return False
-    if "https://www.youtube.com/s/search/audio/" in url: return False
+    if "https://www.youtube.com/s/gaming/emoji/" in url:
+        return False
+    if "https://yt3.ggpht.com" in url and "default-user=" in url:
+        return False
+    if "https://www.youtube.com/s/search/audio/" in url:
+        return False
 
     # ok
-    if " https://ok.ru/res/i/" in url: return False
+    if " https://ok.ru/res/i/" in url:
+        return False
 
     # vk
-    if "https://vk.com/emoji/" in url: return False
-    if "vk.com/images/" in url: return False
-    if "vk.com/images/reaction/" in url: return False
+    if "https://vk.com/emoji/" in url:
+        return False
+    if "vk.com/images/" in url:
+        return False
+    if "vk.com/images/reaction/" in url:
+        return False
 
     # wikipedia
-    if "wikipedia.org/static" in url: return False
+    if "wikipedia.org/static" in url:
+        return False
 
     return True
 
+
 def twitter_best_quality_url(url: str) -> str:
     """
     some twitter image URLs point to a less-than best quality
diff --git a/src/auto_archiver/utils/webdriver.py b/src/auto_archiver/utils/webdriver.py
index cb4e2a9..0690ab5 100644
--- a/src/auto_archiver/utils/webdriver.py
+++ b/src/auto_archiver/utils/webdriver.py
@@ -1,10 +1,11 @@
-""" This Webdriver class acts as a context manager for the selenium webdriver. """
+"""This Webdriver class acts as a context manager for the selenium webdriver."""
+
 from __future__ import annotations
 
 import os
 import time
 
-#import domain_for_url
+# import domain_for_url
 from urllib.parse import urlparse, urlunparse
 from http.cookiejar import MozillaCookieJar
 
@@ -19,16 +20,15 @@ from loguru import logger
 
 
 class CookieSettingDriver(webdriver.Firefox):
-
     facebook_accept_cookies: bool
     cookies: str
     cookiejar: MozillaCookieJar
 
     def __init__(self, cookies, cookiejar, facebook_accept_cookies, *args, **kwargs):
-        if os.environ.get('RUNNING_IN_DOCKER'):
+        if os.environ.get("RUNNING_IN_DOCKER"):
             # Selenium doesn't support linux-aarch64 driver, we need to set this manually
-            kwargs['service'] = webdriver.FirefoxService(executable_path='/usr/local/bin/geckodriver')
-        
+            kwargs["service"] = webdriver.FirefoxService(executable_path="/usr/local/bin/geckodriver")
+
         super(CookieSettingDriver, self).__init__(*args, **kwargs)
         self.cookies = cookies
         self.cookiejar = cookiejar
@@ -38,42 +38,43 @@ class CookieSettingDriver(webdriver.Firefox):
         if self.cookies or self.cookiejar:
             # set up the driver to make it not 'cookie averse' (needs a context/URL)
             # get the 'robots.txt' file which should be quick and easy
-            robots_url = urlunparse(urlparse(url)._replace(path='/robots.txt', query='', fragment=''))
+            robots_url = urlunparse(urlparse(url)._replace(path="/robots.txt", query="", fragment=""))
             super(CookieSettingDriver, self).get(robots_url)
 
             if self.cookies:
                 # an explicit cookie is set for this site, use that first
                 for cookie in self.cookies.split(";"):
                     for name, value in cookie.split("="):
-                        self.driver.add_cookie({'name': name, 'value': value})
+                        self.driver.add_cookie({"name": name, "value": value})
             elif self.cookiejar:
                 domain = urlparse(url).netloc.lstrip("www.")
                 for cookie in self.cookiejar:
                     if domain in cookie.domain:
                         try:
-                            self.add_cookie({
-                                'name': cookie.name,
-                                'value': cookie.value,
-                                'path': cookie.path,
-                                'domain': cookie.domain,
-                                'secure': bool(cookie.secure),
-                                'expiry': cookie.expires
-                            })
+                            self.add_cookie(
+                                {
+                                    "name": cookie.name,
+                                    "value": cookie.value,
+                                    "path": cookie.path,
+                                    "domain": cookie.domain,
+                                    "secure": bool(cookie.secure),
+                                    "expiry": cookie.expires,
+                                }
+                            )
                         except Exception as e:
                             logger.warning(f"Failed to add cookie to webdriver: {e}")
-        
+
         if self.facebook_accept_cookies:
             try:
-                logger.debug(f'Trying fb click accept cookie popup.')
+                logger.debug(f"Trying fb click accept cookie popup.")
                 super(CookieSettingDriver, self).get("http://www.facebook.com")
                 essential_only = self.find_element(By.XPATH, "//span[contains(text(), 'Decline optional cookies')]")
                 essential_only.click()
-                logger.debug(f'fb click worked')
+                logger.debug(f"fb click worked")
                 # linux server needs a sleep otherwise facebook cookie won't have worked and we'll get a popup on next page
                 time.sleep(2)
             except Exception as e:
-                logger.warning(f'Failed on fb accept cookies.', e)
-        
+                logger.warning(f"Failed on fb accept cookies.", e)
 
         # now get the actual URL
         super(CookieSettingDriver, self).get(url)
@@ -87,9 +88,14 @@ class CookieSettingDriver(webdriver.Firefox):
                 pass
 
         else:
-
             # for all other sites, try and use some common button text to reject/accept cookies
-            for text in ["Refuse non-essential cookies", "Decline optional cookies", "Reject additional cookies", "Reject all", "Accept all cookies"]:
+            for text in [
+                "Refuse non-essential cookies",
+                "Decline optional cookies",
+                "Reject additional cookies",
+                "Reject all",
+                "Accept all cookies",
+            ]:
                 try:
                     xpath = f"//*[contains(translate(text(), 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz'), '{text.lower()}')]"
                     WebDriverWait(self, 5).until(EC.element_to_be_clickable((By.XPATH, xpath))).click()
@@ -97,11 +103,18 @@ class CookieSettingDriver(webdriver.Firefox):
                 except selenium_exceptions.WebDriverException:
                     pass
 
-    
+
 class Webdriver:
-    def __init__(self, width: int, height: int, timeout_seconds: int,
-                 facebook_accept_cookies: bool = False, http_proxy: str = "",
-                 print_options: dict = {}, auth: dict = {}) -> webdriver:
+    def __init__(
+        self,
+        width: int,
+        height: int,
+        timeout_seconds: int,
+        facebook_accept_cookies: bool = False,
+        http_proxy: str = "",
+        print_options: dict = {},
+        auth: dict = {},
+    ) -> webdriver:
         self.width = width
         self.height = height
         self.timeout_seconds = timeout_seconds
@@ -116,23 +129,29 @@ class Webdriver:
     def __enter__(self) -> webdriver:
         options = webdriver.FirefoxOptions()
         options.add_argument("--headless")
-        options.add_argument(f'--proxy-server={self.http_proxy}')
-        options.set_preference('network.protocol-handler.external.tg', False)
+        options.add_argument(f"--proxy-server={self.http_proxy}")
+        options.set_preference("network.protocol-handler.external.tg", False)
         # if facebook cookie popup is present, force the browser to English since then it's easier to click the 'Decline optional cookies' option
         if self.facebook_accept_cookies:
-            options.add_argument('--lang=en')
+            options.add_argument("--lang=en")
 
         try:
-            self.driver = CookieSettingDriver(cookies=self.auth.get('cookies'), cookiejar=self.auth.get('cookies_jar'),
-                                              facebook_accept_cookies=self.facebook_accept_cookies, options=options)
+            self.driver = CookieSettingDriver(
+                cookies=self.auth.get("cookies"),
+                cookiejar=self.auth.get("cookies_jar"),
+                facebook_accept_cookies=self.facebook_accept_cookies,
+                options=options,
+            )
             self.driver.set_window_size(self.width, self.height)
             self.driver.set_page_load_timeout(self.timeout_seconds)
             self.driver.print_options = self.print_options
         except selenium_exceptions.TimeoutException as e:
-            logger.error(f"failed to get new webdriver, possibly due to insufficient system resources or timeout settings: {e}")
+            logger.error(
+                f"failed to get new webdriver, possibly due to insufficient system resources or timeout settings: {e}"
+            )
 
         return self.driver
-    
+
     def __exit__(self, exc_type, exc_val, exc_tb):
         self.driver.close()
         self.driver.quit()
diff --git a/src/auto_archiver/version.py b/src/auto_archiver/version.py
index dd700b6..d0e6a5f 100644
--- a/src/auto_archiver/version.py
+++ b/src/auto_archiver/version.py
@@ -1,7 +1,8 @@
-""" Version information for the auto_archiver package.
-    TODO: This is a placeholder to replicate previous versioning.
+"""Version information for the auto_archiver package.
+TODO: This is a placeholder to replicate previous versioning.
 
 """
+
 from importlib.metadata import version as get_version
 
 VERSION_SHORT = get_version("auto_archiver")
@@ -9,4 +10,4 @@ VERSION_SHORT = get_version("auto_archiver")
 # This is mainly for nightly builds which have the suffix ".dev$DATE". See
 # https://semver.org/#is-v123-a-semantic-version for the semantics.
 _SUFFIX = ""
-__version__ = f"{VERSION_SHORT}{_SUFFIX}"
\ No newline at end of file
+__version__ = f"{VERSION_SHORT}{_SUFFIX}"
diff --git a/tests/conftest.py b/tests/conftest.py
index a94abcd..ba1f652 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1,6 +1,7 @@
 """
 pytest conftest file, for shared fixtures and configuration
 """
+
 import os
 import pickle
 from datetime import datetime, timezone
@@ -16,32 +17,34 @@ from auto_archiver.core.module import ModuleFactory
 # that you only want to run if everything else succeeds (e.g. API calls). The order here is important
 # what comes first will be run first (at the end of all other tests not mentioned)
 # format is the name of the module (python file) without the .py extension
-TESTS_TO_RUN_LAST = ['test_twitter_api_archiver']
+TESTS_TO_RUN_LAST = ["test_twitter_api_archiver"]
+
 
 @pytest.fixture
 def setup_module(request):
     def _setup_module(module_name, config={}):
-
         module_factory = ModuleFactory()
 
         if isinstance(module_name, type):
             # get the module name:
             # if the class does not have a .name, use the name of the parent folder
-            module_name = module_name.__module__.rsplit(".",2)[-2]
+            module_name = module_name.__module__.rsplit(".", 2)[-2]
 
         m = module_factory.get_module(module_name, {module_name: config})
         # add the tmp_dir to the module
         tmp_dir = TemporaryDirectory()
         m.tmp_dir = tmp_dir.name
-        
+
         def cleanup():
             tmp_dir.cleanup()
+
         request.addfinalizer(cleanup)
 
         return m
 
     return _setup_module
 
+
 @pytest.fixture
 def check_hash():
     def _check_hash(filename: str, hash: str):
@@ -51,6 +54,7 @@ def check_hash():
 
     return _check_hash
 
+
 @pytest.fixture
 def make_item():
     def _make_item(url: str, **kwargs) -> Metadata:
@@ -62,7 +66,6 @@ def make_item():
     return _make_item
 
 
-
 def pytest_collection_modifyitems(items):
     module_mapping = {item: item.module.__name__.split(".")[-1] for item in items}
 
@@ -78,13 +81,13 @@ def pytest_collection_modifyitems(items):
     items[:] = sorted_items
 
 
-
 # Incremental testing - fail tests in a class if any previous test fails
 # taken from https://docs.pytest.org/en/latest/example/simple.html#incremental-testing-test-steps
 
 # store history of failures per test class name and per index in parametrize (if parametrize used)
 _test_failed_incremental: Dict[str, Dict[Tuple[int, ...], str]] = {}
 
+
 def pytest_runtest_makereport(item, call):
     if "incremental" in item.keywords:
         # incremental marker is used
@@ -93,17 +96,11 @@ def pytest_runtest_makereport(item, call):
             # retrieve the class name of the test
             cls_name = str(item.cls)
             # retrieve the index of the test (if parametrize is used in combination with incremental)
-            parametrize_index = (
-                tuple(item.callspec.indices.values())
-                if hasattr(item, "callspec")
-                else ()
-            )
+            parametrize_index = tuple(item.callspec.indices.values()) if hasattr(item, "callspec") else ()
             # retrieve the name of the test function
             test_name = item.originalname or item.name
             # store in _test_failed_incremental the original name of the failed test
-            _test_failed_incremental.setdefault(cls_name, {}).setdefault(
-                parametrize_index, test_name
-            )
+            _test_failed_incremental.setdefault(cls_name, {}).setdefault(parametrize_index, test_name)
 
 
 def pytest_runtest_setup(item):
@@ -119,16 +116,17 @@ def pytest_runtest_setup(item):
                 pytest.xfail(f"previous test failed ({test_name})")
 
 
-
 @pytest.fixture()
 def unpickle():
     """
     Returns a helper function that unpickles a file
     ** gets the file from the test_files directory: tests/data/ **
     """
+
     def _unpickle(path):
         with open(os.path.join("tests/data", path), "rb") as f:
             return pickle.load(f)
+
     return _unpickle
 
 
@@ -156,4 +154,4 @@ def metadata():
     metadata = Metadata()
     metadata.set("_processed_at", "2021-01-01T00:00:00")
     metadata.set_url("https://example.com")
-    return metadata
\ No newline at end of file
+    return metadata
diff --git a/tests/data/dropin.py b/tests/data/dropin.py
index 0049c48..93c7500 100644
--- a/tests/data/dropin.py
+++ b/tests/data/dropin.py
@@ -1,5 +1,6 @@
 # this is a dummy class used to test importing a dropin in the
 #  generic extractor by filename/path
 
+
 class Dropin:
-    pass
\ No newline at end of file
+    pass
diff --git a/tests/data/test_modules/example_module/__init__.py b/tests/data/test_modules/example_module/__init__.py
index 560a9b9..02986fc 100644
--- a/tests/data/test_modules/example_module/__init__.py
+++ b/tests/data/test_modules/example_module/__init__.py
@@ -1 +1 @@
-from .example_module import ExampleModule
\ No newline at end of file
+from .example_module import ExampleModule
diff --git a/tests/data/test_modules/example_module/__manifest__.py b/tests/data/test_modules/example_module/__manifest__.py
index e3a26bb..064b3c9 100644
--- a/tests/data/test_modules/example_module/__manifest__.py
+++ b/tests/data/test_modules/example_module/__manifest__.py
@@ -16,14 +16,14 @@
     "dependencies": {
         "python": ["loguru"],
         "bin": ["bash"],
-        },
-    # configurations that this module takes. These are argparse-compliant dicationaries, that are 
+    },
+    # configurations that this module takes. These are argparse-compliant dicationaries, that are
     # used to create command line arguments when the programme is run.
     # The full name of the config option will become: `module_name.config_name`
     "configs": {
-            "csv_file": {"default": "db.csv", "help": "CSV file name"},
-            "required_field": {"required": True, "help": "required field in the CSV file"},
-        },
+        "csv_file": {"default": "db.csv", "help": "CSV file name"},
+        "required_field": {"required": True, "help": "required field in the CSV file"},
+    },
     # A description of the module, used for documentation
     "description": "This is an example module",
-}
\ No newline at end of file
+}
diff --git a/tests/data/test_modules/example_module/example_module.py b/tests/data/test_modules/example_module/example_module.py
index 7def054..392abe0 100644
--- a/tests/data/test_modules/example_module/example_module.py
+++ b/tests/data/test_modules/example_module/example_module.py
@@ -1,5 +1,6 @@
 from auto_archiver.core import Extractor, Enricher, Feeder, Database, Storage, Formatter, Metadata
 
+
 class ExampleModule(Extractor, Enricher, Feeder, Database, Storage, Formatter):
     def download(self, item):
         print("download")
@@ -7,7 +8,6 @@ class ExampleModule(Extractor, Enricher, Feeder, Database, Storage, Formatter):
     def __iter__(self):
         yield Metadata().set_url("https://example.com")
 
-    
     def done(self, result):
         print("done")
 
@@ -16,13 +16,12 @@ class ExampleModule(Extractor, Enricher, Feeder, Database, Storage, Formatter):
 
     def get_cdn_url(self, media):
         return "nice_url"
-    
+
     def save(self, item):
         print("save")
-    
+
     def uploadf(self, file, key, **kwargs):
         print("uploadf")
 
-    
     def format(self, item):
         print("format")
diff --git a/tests/databases/test_api_db.py b/tests/databases/test_api_db.py
index 5d1ea84..4627425 100644
--- a/tests/databases/test_api_db.py
+++ b/tests/databases/test_api_db.py
@@ -41,9 +41,16 @@ def test_fetch(api_db, metadata, mocker):
     mock_datetime = mocker.patch("auto_archiver.core.metadata.datetime.datetime")
     mock_datetime.now.return_value = "2021-01-01T00:00:00"
     mock_get.return_value.status_code = 200
-    mock_get.return_value.json.return_value = [{"result": {}}, {"result":
-        {'media': [], 'metadata': {'_processed_at': '2021-01-01T00:00:00', 'url': 'https://example.com'},
-         'status': 'no archiver'}}]
+    mock_get.return_value.json.return_value = [
+        {"result": {}},
+        {
+            "result": {
+                "media": [],
+                "metadata": {"_processed_at": "2021-01-01T00:00:00", "url": "https://example.com"},
+                "status": "no archiver",
+            }
+        },
+    ]
     assert api_db.fetch(metadata) == metadata
 
 
@@ -52,8 +59,15 @@ def test_done_success(api_db, metadata, mocker):
     mock_post.return_value.status_code = 201
     api_db.done(metadata)
     mock_post.assert_called_once()
-    mock_post.assert_called_once_with("https://api.example.com/interop/submit-archive",
-                                      json={'author_id': 'Someone', 'url': 'https://example.com',
-                                            'public': False, 'group_id': '123', 'tags': ['[', ']'], 'result': '{"status": "no archiver", "metadata": {"_processed_at": "2021-01-01T00:00:00", "url": "https://example.com"}, "media": []}'},
-                                      headers={'Authorization': 'Bearer test-token'})
-
+    mock_post.assert_called_once_with(
+        "https://api.example.com/interop/submit-archive",
+        json={
+            "author_id": "Someone",
+            "url": "https://example.com",
+            "public": False,
+            "group_id": "123",
+            "tags": ["[", "]"],
+            "result": '{"status": "no archiver", "metadata": {"_processed_at": "2021-01-01T00:00:00", "url": "https://example.com"}, "media": []}',
+        },
+        headers={"Authorization": "Bearer test-token"},
+    )
diff --git a/tests/databases/test_atlos_db.py b/tests/databases/test_atlos_db.py
index a73f1df..6c79a53 100644
--- a/tests/databases/test_atlos_db.py
+++ b/tests/databases/test_atlos_db.py
@@ -50,9 +50,7 @@ def test_failed_with_atlos_id(atlos_db, metadata, mocker):
     post_mock = mocker.patch.object(atlos_db, "_post", return_value=fake_resp)
     atlos_db.failed(metadata, "failure reason")
     expected_endpoint = f"/api/v2/source_material/metadata/42/auto_archiver"
-    expected_json = {
-        "metadata": {"processed": True, "status": "error", "error": "failure reason"}
-    }
+    expected_json = {"metadata": {"processed": True, "status": "error", "error": "failure reason"}}
     post_mock.assert_called_once_with(expected_endpoint, json=expected_json)
 
 
diff --git a/tests/databases/test_csv_db.py b/tests/databases/test_csv_db.py
index afca0d8..bf5b7cb 100644
--- a/tests/databases/test_csv_db.py
+++ b/tests/databases/test_csv_db.py
@@ -1,4 +1,3 @@
-
 from auto_archiver.modules.csv_db import CSVDb
 from auto_archiver.core import Metadata
 
@@ -9,12 +8,21 @@ def test_store_item(tmp_path, setup_module):
     temp_db = tmp_path / "temp_db.csv"
     db = setup_module(CSVDb, {"csv_file": temp_db.as_posix()})
 
-    item = Metadata().set_url("http://example.com").set_title("Example").set_content("Example content").success("my-archiver")
+    item = (
+        Metadata()
+        .set_url("http://example.com")
+        .set_title("Example")
+        .set_content("Example content")
+        .success("my-archiver")
+    )
 
     db.done(item)
 
     with open(temp_db, "r", encoding="utf-8") as f:
-        assert f.read().strip() == f"status,metadata,media\nmy-archiver: success,\"{{'_processed_at': {repr(item.get('_processed_at'))}, 'url': 'http://example.com', 'title': 'Example', 'content': 'Example content'}}\",[]"
+        assert (
+            f.read().strip()
+            == f"status,metadata,media\nmy-archiver: success,\"{{'_processed_at': {repr(item.get('_processed_at'))}, 'url': 'http://example.com', 'title': 'Example', 'content': 'Example content'}}\",[]"
+        )
 
     # TODO: csv db doesn't have a fetch method - need to add it (?)
-    # assert db.fetch(item) == item
\ No newline at end of file
+    # assert db.fetch(item) == item
diff --git a/tests/databases/test_gsheet_db.py b/tests/databases/test_gsheet_db.py
index 2f1202d..6c9e585 100644
--- a/tests/databases/test_gsheet_db.py
+++ b/tests/databases/test_gsheet_db.py
@@ -28,6 +28,7 @@ def mock_metadata(mocker):
     metadata.get_first_image.return_value = None
     return metadata
 
+
 @pytest.fixture
 def metadata():
     metadata = Metadata()
@@ -51,6 +52,7 @@ def mock_media(mocker):
     mock_media.get.return_value = "not-calculated"
     return mock_media
 
+
 @pytest.fixture
 def gsheets_db(mock_gworksheet, setup_module, mocker):
     mocker.patch("gspread.service_account")
@@ -59,7 +61,22 @@ def gsheets_db(mock_gworksheet, setup_module, mocker):
         "sheet_id": None,
         "header": 1,
         "service_account": "test/service_account.json",
-        "columns": {'url': 'link', 'status': 'archive status', 'folder': 'destination folder', 'archive': 'archive location', 'date': 'archive date', 'thumbnail': 'thumbnail', 'timestamp': 'upload timestamp', 'title': 'upload title', 'text': 'text content', 'screenshot': 'screenshot', 'hash': 'hash', 'pdq_hash': 'perceptual hashes', 'wacz': 'wacz', 'replaywebpage': 'replaywebpage'},
+        "columns": {
+            "url": "link",
+            "status": "archive status",
+            "folder": "destination folder",
+            "archive": "archive location",
+            "date": "archive date",
+            "thumbnail": "thumbnail",
+            "timestamp": "upload timestamp",
+            "title": "upload title",
+            "text": "text content",
+            "screenshot": "screenshot",
+            "hash": "hash",
+            "pdq_hash": "perceptual hashes",
+            "wacz": "wacz",
+            "replaywebpage": "replaywebpage",
+        },
         "allow_worksheets": set(),
         "block_worksheets": set(),
         "use_sheet_names_in_stored_paths": True,
@@ -78,20 +95,21 @@ def fixed_timestamp():
 @pytest.fixture
 def expected_calls(mock_media, fixed_timestamp):
     """Fixture for the expected cell updates."""
-    return  [
-        (1, 'status', 'my-archiver: success'),
-        (1, 'archive', 'http://example.com/screenshot.png'),
-        (1, 'date', '2025-02-01T00:00:00+00:00'),
-        (1, 'title', 'Example Title'),
-        (1, 'text', 'Example Content'),
-        (1, 'timestamp', '2025-01-01T00:00:00+00:00'),
-        (1, 'hash', 'not-calculated'),
+    return [
+        (1, "status", "my-archiver: success"),
+        (1, "archive", "http://example.com/screenshot.png"),
+        (1, "date", "2025-02-01T00:00:00+00:00"),
+        (1, "title", "Example Title"),
+        (1, "text", "Example Content"),
+        (1, "timestamp", "2025-01-01T00:00:00+00:00"),
+        (1, "hash", "not-calculated"),
         # (1, 'screenshot', 'http://example.com/screenshot.png'),
         # (1, 'thumbnail', '=IMAGE("http://example.com/thumbnail.png")'),
         # (1, 'wacz', 'http://example.com/browsertrix.wacz'),
         # (1, 'replaywebpage', 'https://replayweb.page/?source=http%3A%2F%2Fexample.com%2Fbrowsertrix.wacz#view=pages&url=')
     ]
 
+
 def test_retrieve_gsheet(gsheets_db, metadata, mock_gworksheet):
     gw, row = gsheets_db._retrieve_gsheet(metadata)
     assert gw == mock_gworksheet
@@ -100,27 +118,34 @@ def test_retrieve_gsheet(gsheets_db, metadata, mock_gworksheet):
 
 def test_started(gsheets_db, mock_metadata, mock_gworksheet):
     gsheets_db.started(mock_metadata)
-    mock_gworksheet.set_cell.assert_called_once_with(1, 'status', 'Archive in progress')
+    mock_gworksheet.set_cell.assert_called_once_with(1, "status", "Archive in progress")
+
 
 def test_failed(gsheets_db, mock_metadata, mock_gworksheet):
     reason = "Test failure"
     gsheets_db.failed(mock_metadata, reason)
-    mock_gworksheet.set_cell.assert_called_once_with(1, 'status', f'Archive failed {reason}')
+    mock_gworksheet.set_cell.assert_called_once_with(1, "status", f"Archive failed {reason}")
 
 
 def test_aborted(gsheets_db, mock_metadata, mock_gworksheet):
     gsheets_db.aborted(mock_metadata)
-    mock_gworksheet.set_cell.assert_called_once_with(1, 'status', '')
+    mock_gworksheet.set_cell.assert_called_once_with(1, "status", "")
 
 
 def test_done(gsheets_db, metadata, mock_gworksheet, expected_calls, mocker):
-    mocker.patch("auto_archiver.modules.gsheet_feeder_db.gsheet_feeder_db.get_current_timestamp", return_value='2025-02-01T00:00:00+00:00')
+    mocker.patch(
+        "auto_archiver.modules.gsheet_feeder_db.gsheet_feeder_db.get_current_timestamp",
+        return_value="2025-02-01T00:00:00+00:00",
+    )
     gsheets_db.done(metadata)
     mock_gworksheet.batch_set_cell.assert_called_once_with(expected_calls)
 
 
 def test_done_cached(gsheets_db, metadata, mock_gworksheet, mocker):
-    mocker.patch("auto_archiver.modules.gsheet_feeder_db.gsheet_feeder_db.get_current_timestamp", return_value='2025-02-01T00:00:00+00:00')
+    mocker.patch(
+        "auto_archiver.modules.gsheet_feeder_db.gsheet_feeder_db.get_current_timestamp",
+        return_value="2025-02-01T00:00:00+00:00",
+    )
     gsheets_db.done(metadata, cached=True)
 
     # Verify the status message includes "[cached]"
@@ -131,15 +156,17 @@ def test_done_cached(gsheets_db, metadata, mock_gworksheet, mocker):
 def test_done_missing_media(gsheets_db, metadata, mock_gworksheet, mocker):
     # clear media from metadata
     metadata.media = []
-    mocker.patch("auto_archiver.modules.gsheet_feeder_db.gsheet_feeder_db.get_current_timestamp", return_value='2025-02-01T00:00:00+00:00')
+    mocker.patch(
+        "auto_archiver.modules.gsheet_feeder_db.gsheet_feeder_db.get_current_timestamp",
+        return_value="2025-02-01T00:00:00+00:00",
+    )
     gsheets_db.done(metadata)
     # Verify nothing media-related gets updated
     call_args = mock_gworksheet.batch_set_cell.call_args[0][0]
-    media_fields = {'archive', 'screenshot', 'thumbnail', 'wacz', 'replaywebpage'}
+    media_fields = {"archive", "screenshot", "thumbnail", "wacz", "replaywebpage"}
     assert all(call[1] not in media_fields for call in call_args)
 
+
 def test_safe_status_update(gsheets_db, metadata, mock_gworksheet):
     gsheets_db._safe_status_update(metadata, "Test status")
-    mock_gworksheet.set_cell.assert_called_once_with(1, 'status', 'Test status')
-
-
+    mock_gworksheet.set_cell.assert_called_once_with(1, "status", "Test status")
diff --git a/tests/enrichers/test_hash_enricher.py b/tests/enrichers/test_hash_enricher.py
index c2fe67a..05b3b3c 100644
--- a/tests/enrichers/test_hash_enricher.py
+++ b/tests/enrichers/test_hash_enricher.py
@@ -4,34 +4,50 @@ from auto_archiver.modules.hash_enricher import HashEnricher
 from auto_archiver.core import Metadata, Media
 from auto_archiver.core.module import ModuleFactory
 
-@pytest.mark.parametrize("algorithm, filename, expected_hash", [
-    ("SHA-256", "tests/data/testfile_1.txt", "1b4f0e9851971998e732078544c96b36c3d01cedf7caa332359d6f1d83567014"),
-    ("SHA-256", "tests/data/testfile_2.txt", "60303ae22b998861bce3b28f33eec1be758a213c86c93c076dbe9f558c11c752"),
-    ("SHA3-512", "tests/data/testfile_1.txt", "d2d8cc4f369b340130bd2b29b8b54e918b7c260c3279176da9ccaa37c96eb71735fc97568e892dc6220bf4ae0d748edb46bd75622751556393be3f482e6f794e"),
-    ("SHA3-512", "tests/data/testfile_2.txt", "e35970edaa1e0d8af7d948491b2da0450a49fd9cc1e83c5db4c6f175f9550cf341f642f6be8cfb0bfa476e4258e5088c5ad549087bf02811132ac2fa22b734c6")
-])
+
+@pytest.mark.parametrize(
+    "algorithm, filename, expected_hash",
+    [
+        ("SHA-256", "tests/data/testfile_1.txt", "1b4f0e9851971998e732078544c96b36c3d01cedf7caa332359d6f1d83567014"),
+        ("SHA-256", "tests/data/testfile_2.txt", "60303ae22b998861bce3b28f33eec1be758a213c86c93c076dbe9f558c11c752"),
+        (
+            "SHA3-512",
+            "tests/data/testfile_1.txt",
+            "d2d8cc4f369b340130bd2b29b8b54e918b7c260c3279176da9ccaa37c96eb71735fc97568e892dc6220bf4ae0d748edb46bd75622751556393be3f482e6f794e",
+        ),
+        (
+            "SHA3-512",
+            "tests/data/testfile_2.txt",
+            "e35970edaa1e0d8af7d948491b2da0450a49fd9cc1e83c5db4c6f175f9550cf341f642f6be8cfb0bfa476e4258e5088c5ad549087bf02811132ac2fa22b734c6",
+        ),
+    ],
+)
 def test_calculate_hash(algorithm, filename, expected_hash, setup_module):
     # test SHA-256
     he = setup_module(HashEnricher, {"algorithm": algorithm, "chunksize": 100})
     assert he.calculate_hash(filename) == expected_hash
 
+
 def test_default_config_values(setup_module):
     he = setup_module(HashEnricher)
     assert he.algorithm == "SHA-256"
     assert he.chunksize == 16000000
 
+
 def test_config():
     # test default config
-    c = ModuleFactory().get_module_lazy('hash_enricher').configs
+    c = ModuleFactory().get_module_lazy("hash_enricher").configs
     assert c["algorithm"]["default"] == "SHA-256"
     assert c["chunksize"]["default"] == 16000000
     assert c["algorithm"]["choices"] == ["SHA-256", "SHA3-512"]
     assert c["algorithm"]["help"] == "hash algorithm to use"
-    assert c["chunksize"]["help"] == "number of bytes to use when reading files in chunks (if this value is too large you will run out of RAM), default is 16MB"
+    assert (
+        c["chunksize"]["help"]
+        == "number of bytes to use when reading files in chunks (if this value is too large you will run out of RAM), default is 16MB"
+    )
 
 
 def test_hash_media(setup_module):
-
     he = setup_module(HashEnricher, {"algorithm": "SHA-256", "chunksize": 1})
 
     # generate metadata with two test files
@@ -46,4 +62,4 @@ def test_hash_media(setup_module):
     he.enrich(m)
 
     assert m.media[0].get("hash") == "SHA-256:1b4f0e9851971998e732078544c96b36c3d01cedf7caa332359d6f1d83567014"
-    assert m.media[1].get("hash") == "SHA-256:60303ae22b998861bce3b28f33eec1be758a213c86c93c076dbe9f558c11c752"
\ No newline at end of file
+    assert m.media[1].get("hash") == "SHA-256:60303ae22b998861bce3b28f33eec1be758a213c86c93c076dbe9f558c11c752"
diff --git a/tests/enrichers/test_meta_enricher.py b/tests/enrichers/test_meta_enricher.py
index 476e25b..906e629 100644
--- a/tests/enrichers/test_meta_enricher.py
+++ b/tests/enrichers/test_meta_enricher.py
@@ -16,6 +16,7 @@ def mock_metadata(mocker):
     mock.get_all_media.return_value = []
     return mock
 
+
 @pytest.fixture
 def mock_media(mocker):
     """Creates a mock Media object."""
@@ -59,6 +60,7 @@ def test_enrich_file_sizes(meta_enricher, metadata, tmp_path):
     assert metadata.get("total_bytes") == 3000
     assert metadata.get("total_size") == "2.9 KB"
 
+
 @pytest.mark.parametrize(
     "size, expected",
     [
@@ -74,6 +76,7 @@ def test_human_readable_bytes(size, expected):
     enricher = MetaEnricher()
     assert enricher.human_readable_bytes(size) == expected
 
+
 def test_enrich_file_sizes_no_media(meta_enricher, metadata):
     """Test that enrich_file_sizes() handles empty media list gracefully."""
     meta_enricher.enrich_file_sizes(metadata)
@@ -91,4 +94,4 @@ def test_enrich_archive_duration(meta_enricher, metadata, mocker):
     mock_datetime.now.return_value = mock_now
     meta_enricher.enrich_archive_duration(metadata)
 
-    assert metadata.get("archive_duration_seconds") == 630
\ No newline at end of file
+    assert metadata.get("archive_duration_seconds") == 630
diff --git a/tests/enrichers/test_metadata_enricher.py b/tests/enrichers/test_metadata_enricher.py
index 888837d..14cfc44 100644
--- a/tests/enrichers/test_metadata_enricher.py
+++ b/tests/enrichers/test_metadata_enricher.py
@@ -1,4 +1,3 @@
-
 import pytest
 
 from auto_archiver.core import Media
@@ -33,9 +32,7 @@ def test_get_metadata(enricher, output, expected, mocker):
 
     result = enricher.get_metadata("test.jpg")
     assert result == expected
-    mock_run.assert_called_once_with(
-        ["exiftool", "test.jpg"], capture_output=True, text=True
-    )
+    mock_run.assert_called_once_with(["exiftool", "test.jpg"], capture_output=True, text=True)
 
 
 def test_get_metadata_exiftool_not_found(enricher, mocker):
@@ -85,4 +82,3 @@ def test_metadata_pickle(enricher, unpickle, mocker):
     actual_media = metadata.media
     assert len(expected_media) == len(actual_media)
     assert actual_media[0].properties.get("metadata") == expected_media[0].properties.get("metadata")
-
diff --git a/tests/enrichers/test_pdq_hash_enricher.py b/tests/enrichers/test_pdq_hash_enricher.py
index a8470fb..daf3f0c 100644
--- a/tests/enrichers/test_pdq_hash_enricher.py
+++ b/tests/enrichers/test_pdq_hash_enricher.py
@@ -57,7 +57,7 @@ def test_enrich_handles_corrupted_image(metadata_with_images, mocker):
         ("screenshot", False),
         ("warc-file-123", False),
         ("regular-image", True),
-    ]
+    ],
 )
 def test_enrich_excludes_by_filetype(media_id, should_have_hash, mocker):
     metadata = Metadata()
@@ -73,4 +73,3 @@ def test_enrich_excludes_by_filetype(media_id, should_have_hash, mocker):
 
     media_item = metadata.media[0]
     assert (media_item.get("pdq_hash") is not None) == should_have_hash
-
diff --git a/tests/enrichers/test_screenshot_enricher.py b/tests/enrichers/test_screenshot_enricher.py
index 25ca51d..ee6c2a7 100644
--- a/tests/enrichers/test_screenshot_enricher.py
+++ b/tests/enrichers/test_screenshot_enricher.py
@@ -19,9 +19,11 @@ def mock_selenium_env(mocker):
     mock_popen = mocker.patch("subprocess.Popen")
     mock_is_connectable = mocker.patch("selenium.webdriver.common.service.Service.is_connectable", return_value=True)
     mock_firefox_options = mocker.patch("selenium.webdriver.FirefoxOptions")
+
     # Define side effect for `shutil.which`
     def mock_which_side_effect(dep):
         return "/mock/geckodriver" if dep == "geckodriver" else None
+
     mock_which.side_effect = mock_which_side_effect
 
     # Mock binary paths
@@ -104,13 +106,7 @@ def test_enrich_adds_screenshot(
     ],
 )
 def test_enrich_auth_wall(
-    screenshot_enricher,
-    metadata_with_video,
-    mock_selenium_env,
-    common_patches,
-    url,
-    is_auth,
-    mocker
+    screenshot_enricher, metadata_with_video, mock_selenium_env, common_patches, url, is_auth, mocker
 ):
     # Testing with and without is_auth_wall
     mock_driver, mock_driver_class, _ = mock_selenium_env
@@ -128,9 +124,7 @@ def test_enrich_auth_wall(
         assert metadata_with_video.media[1].properties.get("id") == "screenshot"
 
 
-def test_handle_timeout_exception(
-    screenshot_enricher, metadata_with_video, mock_selenium_env, mocker
-):
+def test_handle_timeout_exception(screenshot_enricher, metadata_with_video, mock_selenium_env, mocker):
     mock_driver, mock_driver_class, mock_options_instance = mock_selenium_env
 
     mock_driver.get.side_effect = TimeoutException
@@ -140,9 +134,7 @@ def test_handle_timeout_exception(
     assert len(metadata_with_video.media) == 1
 
 
-def test_handle_general_exception(
-    screenshot_enricher, metadata_with_video, mock_selenium_env, mocker
-):
+def test_handle_general_exception(screenshot_enricher, metadata_with_video, mock_selenium_env, mocker):
     """Test proper handling of unexpected general exceptions"""
     mock_driver, mock_driver_class, mock_options_instance = mock_selenium_env
     # Simulate a generic exception when save_screenshot is called
@@ -152,9 +144,7 @@ def test_handle_general_exception(
     mock_log = mocker.patch("loguru.logger.error")
     screenshot_enricher.enrich(metadata_with_video)
     # Verify that the exception was logged with the log
-    mock_log.assert_called_once_with(
-        "Got error while loading webdriver for screenshot enricher: Unexpected Error"
-    )
+    mock_log.assert_called_once_with("Got error while loading webdriver for screenshot enricher: Unexpected Error")
     # And no new media was added due to the error
     assert len(metadata_with_video.media) == 1
 
diff --git a/tests/enrichers/test_ssl_enricher.py b/tests/enrichers/test_ssl_enricher.py
index eb7ba6b..cd118ad 100644
--- a/tests/enrichers/test_ssl_enricher.py
+++ b/tests/enrichers/test_ssl_enricher.py
@@ -51,4 +51,3 @@ def test_ssl_error_handling(enricher, metadata, mocker):
     mocker.patch("ssl.get_server_certificate", side_effect=ssl.SSLError("SSL error"))
     with pytest.raises(ssl.SSLError, match="SSL error"):
         enricher.enrich(metadata)
-
diff --git a/tests/enrichers/test_thumbnail_enricher.py b/tests/enrichers/test_thumbnail_enricher.py
index effc25e..3ebc798 100644
--- a/tests/enrichers/test_thumbnail_enricher.py
+++ b/tests/enrichers/test_thumbnail_enricher.py
@@ -25,7 +25,7 @@ def mock_ffmpeg_environment(mocker):
     # Mocking all the ffmpeg calls in one place
     mock_ffmpeg_input = mocker.patch("ffmpeg.input")
     mock_makedirs = mocker.patch("os.makedirs")
-    mocker.patch.object(Media, "is_video", return_value=True),
+    (mocker.patch.object(Media, "is_video", return_value=True),)
     mock_probe = mocker.patch(
         "ffmpeg.probe",
         return_value={
@@ -35,9 +35,7 @@ def mock_ffmpeg_environment(mocker):
         },
     )
     mock_output = mocker.MagicMock()
-    mock_ffmpeg_input.return_value.filter.return_value.output.return_value = (
-        mock_output
-    )
+    mock_ffmpeg_input.return_value.filter.return_value.output.return_value = mock_output
 
     return {
         "mock_ffmpeg_input": mock_ffmpeg_input,
@@ -47,14 +45,21 @@ def mock_ffmpeg_environment(mocker):
     }
 
 
-@pytest.mark.parametrize("thumbnails_per_minute, max_thumbnails, expected_count", [
-    (10, 5, 5),  # Capped at max_thumbnails
-    (1, 10, 2),  # Less than max_thumbnails
-    (60, 7, 7),  # Matches exactly
-])
+@pytest.mark.parametrize(
+    "thumbnails_per_minute, max_thumbnails, expected_count",
+    [
+        (10, 5, 5),  # Capped at max_thumbnails
+        (1, 10, 2),  # Less than max_thumbnails
+        (60, 7, 7),  # Matches exactly
+    ],
+)
 def test_enrich_thumbnail_limits(
-    thumbnail_enricher, metadata_with_video, mock_ffmpeg_environment,
-    thumbnails_per_minute, max_thumbnails, expected_count
+    thumbnail_enricher,
+    metadata_with_video,
+    mock_ffmpeg_environment,
+    thumbnails_per_minute,
+    max_thumbnails,
+    expected_count,
 ):
     thumbnail_enricher.thumbnails_per_minute = thumbnails_per_minute
     thumbnail_enricher.max_thumbnails = max_thumbnails
@@ -65,8 +70,8 @@ def test_enrich_thumbnail_limits(
     thumbnails = metadata_with_video.media[0].get("thumbnails")
     assert len(thumbnails) == expected_count
 
-def test_enrich_handles_probe_failure(thumbnail_enricher, metadata_with_video, mocker):
 
+def test_enrich_handles_probe_failure(thumbnail_enricher, metadata_with_video, mocker):
     mocker.patch("ffmpeg.probe", side_effect=Exception("Probe error"))
     mocker.patch("os.makedirs")
     mock_logger = mocker.patch("loguru.logger.error")
@@ -74,36 +79,43 @@ def test_enrich_handles_probe_failure(thumbnail_enricher, metadata_with_video, m
 
     thumbnail_enricher.enrich(metadata_with_video)
     # Ensure error was logged
-    mock_logger.assert_called_with(
-        f"error getting duration of video video.mp4: Probe error"
-    )
+    mock_logger.assert_called_with(f"error getting duration of video video.mp4: Probe error")
     # Ensure no thumbnails were created
     thumbnails = metadata_with_video.media[0].get("thumbnails")
     assert thumbnails is None
 
 
 def test_enrich_skips_non_video_files(thumbnail_enricher, metadata_with_video, mocker):
-        mocker.patch.object(Media, "is_video", return_value=False)
-        mock_ffmpeg = mocker.patch("ffmpeg.input")
-        thumbnail_enricher.enrich(metadata_with_video)
-        mock_ffmpeg.assert_not_called()
+    mocker.patch.object(Media, "is_video", return_value=False)
+    mock_ffmpeg = mocker.patch("ffmpeg.input")
+    thumbnail_enricher.enrich(metadata_with_video)
+    mock_ffmpeg.assert_not_called()
 
 
-@pytest.mark.parametrize("thumbnails_per_minute,max_thumbnails,expected_count", [
-    (60, 5, 5), # caught by max
-    (60, 20, 10), # caught by t/min
-    (0, 20, 1), # test min caught (1)
-    (11, 20, 1), # test min caught (1)
-    (12, 20, 2), # test caught by t/min
-])
+@pytest.mark.parametrize(
+    "thumbnails_per_minute,max_thumbnails,expected_count",
+    [
+        (60, 5, 5),  # caught by max
+        (60, 20, 10),  # caught by t/min
+        (0, 20, 1),  # test min caught (1)
+        (11, 20, 1),  # test min caught (1)
+        (12, 20, 2),  # test caught by t/min
+    ],
+)
 def test_enrich_handles_short_video(
-    thumbnail_enricher, metadata_with_video, mock_ffmpeg_environment, thumbnails_per_minute, max_thumbnails, expected_count, mocker
+    thumbnail_enricher,
+    metadata_with_video,
+    mock_ffmpeg_environment,
+    thumbnails_per_minute,
+    max_thumbnails,
+    expected_count,
+    mocker,
 ):
     # override mock duration
     fake_duration = 10
     mocker.patch(
         "ffmpeg.probe",
-        return_value={ "streams": [{"codec_type": "video", "duration": str(fake_duration)}]},
+        return_value={"streams": [{"codec_type": "video", "duration": str(fake_duration)}]},
     )
     thumbnail_enricher.thumbnails_per_minute = thumbnails_per_minute
     thumbnail_enricher.max_thumbnails = max_thumbnails
@@ -114,9 +126,7 @@ def test_enrich_handles_short_video(
     assert len(thumbnails) == expected_count
 
 
-def test_uses_existing_duration(
-    thumbnail_enricher, metadata_with_video, mock_ffmpeg_environment
-):
+def test_uses_existing_duration(thumbnail_enricher, metadata_with_video, mock_ffmpeg_environment):
     metadata_with_video.media[0].set("duration", 60)
     thumbnail_enricher.enrich(metadata_with_video)
     mock_ffmpeg_environment["mock_probe"].assert_not_called()
@@ -125,7 +135,7 @@ def test_uses_existing_duration(
 
 def test_enrich_metadata_structure(thumbnail_enricher, metadata_with_video, mock_ffmpeg_environment, mocker):
     fake_duration = 120
-    mocker.patch("ffmpeg.probe", return_value={'streams': [{'codec_type': 'video', 'duration': str(fake_duration)}]})
+    mocker.patch("ffmpeg.probe", return_value={"streams": [{"codec_type": "video", "duration": str(fake_duration)}]})
     thumbnail_enricher.thumbnails_per_minute = 2
     thumbnail_enricher.max_thumbnails = 4
 
diff --git a/tests/enrichers/test_wayback_enricher.py b/tests/enrichers/test_wayback_enricher.py
index 5406e39..796c805 100644
--- a/tests/enrichers/test_wayback_enricher.py
+++ b/tests/enrichers/test_wayback_enricher.py
@@ -8,34 +8,43 @@ from auto_archiver.core import Metadata
 @pytest.fixture
 def mock_is_auth_wall(mocker):
     """Fixture to mock is_auth_wall behavior."""
+
     def _mock_is_auth_wall(return_value: bool):
         return mocker.patch("auto_archiver.utils.url.is_auth_wall", return_value=return_value)
+
     return _mock_is_auth_wall
 
+
 @pytest.fixture
 def mock_post_success(mocker):
     """Fixture to mock POST requests with a successful response."""
+
     def _mock_post(json_data: dict = None, status_code: int = 200):
         json_data = {"job_id": "job123"} if json_data is None else json_data
         resp = mocker.Mock(status_code=status_code)
         resp.json.return_value = json_data
         return mocker.patch("requests.post", return_value=resp)
+
     return _mock_post
 
+
 @pytest.fixture
 def mock_get_success(mocker):
     """Fixture to mock GET requests returning a completed archive status."""
+
     def _mock_get(json_data: dict = None, status_code: int = 200):
         json_data = json_data or {
             "status": "success",
             "timestamp": "20250101010101",
-            "original_url": "https://example.com"
+            "original_url": "https://example.com",
         }
         resp = mocker.Mock(status_code=status_code)
         resp.json.return_value = json_data
         return mocker.patch("requests.get", return_value=resp)
+
     return _mock_get
 
+
 @pytest.fixture
 def wayback_extractor_enricher(setup_module) -> WaybackExtractorEnricher:
     configs: dict = {
@@ -49,12 +58,7 @@ def wayback_extractor_enricher(setup_module) -> WaybackExtractorEnricher:
     return setup_module("wayback_extractor_enricher", configs)
 
 
-def test_download_success(
-    wayback_extractor_enricher,
-    mock_is_auth_wall,
-    mock_post_success,
-    mock_get_success
-):
+def test_download_success(wayback_extractor_enricher, mock_is_auth_wall, mock_post_success, mock_get_success):
     mock_is_auth_wall(False)
     mock_post_success()
     mock_get_success()
@@ -63,34 +67,28 @@ def test_download_success(
     result = wayback_extractor_enricher.download(metadata)
     assert result.get("wayback") == "https://web.archive.org/web/20250101010101/https://example.com"
 
+
 def test_enrich_auth_wall(wayback_extractor_enricher, metadata, mock_is_auth_wall):
     mock_is_auth_wall(True)
     result = wayback_extractor_enricher.enrich(metadata)
     assert result is None
 
+
 def test_enrich_already_enriched(wayback_extractor_enricher, metadata):
     metadata.set("wayback", "existing")
     result = wayback_extractor_enricher.enrich(metadata)
     assert result is True
 
-def test_enrich_post_failure(
-    wayback_extractor_enricher,
-        metadata,
-    mock_is_auth_wall,
-    mock_post_success
-):
+
+def test_enrich_post_failure(wayback_extractor_enricher, metadata, mock_is_auth_wall, mock_post_success):
     mock_is_auth_wall(False)
     mock_post_success(json_data={"error": "server error"}, status_code=500)
     result = wayback_extractor_enricher.enrich(metadata)
     assert result is False
     assert "Internet archive failed with status of 500" in metadata.get("wayback")
 
-def test_enrich_post_json_decode_error(
-    wayback_extractor_enricher,
-        metadata,
-    mock_is_auth_wall,
-    mocker
-):
+
+def test_enrich_post_json_decode_error(wayback_extractor_enricher, metadata, mock_is_auth_wall, mocker):
     mock_is_auth_wall(False)
     resp = mocker.Mock(status_code=200)
     resp.json.side_effect = json.decoder.JSONDecodeError("msg", "doc", 0)
@@ -98,22 +96,15 @@ def test_enrich_post_json_decode_error(
     mocker.patch("requests.post", return_value=resp)
     assert wayback_extractor_enricher.enrich(metadata) is False
 
-def test_enrich_no_job_id(
-    wayback_extractor_enricher,
-        metadata,
-    mock_is_auth_wall,
-    mock_post_success
-):
+
+def test_enrich_no_job_id(wayback_extractor_enricher, metadata, mock_is_auth_wall, mock_post_success):
     mock_is_auth_wall(False)
     mock_post_success(json_data={})
     assert wayback_extractor_enricher.enrich(metadata) is False
 
+
 def test_enrich_get_success(
-    wayback_extractor_enricher,
-        metadata,
-    mock_is_auth_wall,
-    mock_post_success,
-    mock_get_success
+    wayback_extractor_enricher, metadata, mock_is_auth_wall, mock_post_success, mock_get_success
 ):
     mock_is_auth_wall(False)
     mock_post_success()
@@ -122,24 +113,18 @@ def test_enrich_get_success(
     assert metadata.get("wayback") == "https://web.archive.org/web/20250101010101/https://example.com"
     assert metadata.get("check wayback") == "https://web.archive.org/web/*/https://example.com"
 
+
 def test_enrich_get_failure(
-    wayback_extractor_enricher,
-        metadata,
-    mock_is_auth_wall,
-    mock_post_success,
-    mock_get_success
+    wayback_extractor_enricher, metadata, mock_is_auth_wall, mock_post_success, mock_get_success
 ):
     mock_is_auth_wall(False)
     mock_post_success()
     mock_get_success(json_data={"status": "failed"}, status_code=400)
     assert wayback_extractor_enricher.enrich(metadata) is False
 
+
 def test_enrich_get_request_exception(
-    wayback_extractor_enricher,
-        metadata,
-    mock_is_auth_wall,
-    mock_post_success,
-    mocker
+    wayback_extractor_enricher, metadata, mock_is_auth_wall, mock_post_success, mocker
 ):
     mock_is_auth_wall(False)
     mock_post_success()
@@ -149,12 +134,9 @@ def test_enrich_get_request_exception(
     assert wayback_extractor_enricher.enrich(metadata) is True
     assert metadata.get("wayback").get("job_id") == "job123"
 
+
 def test_enrich_get_json_decode_error(
-    wayback_extractor_enricher,
-        metadata,
-    mock_is_auth_wall,
-    mock_post_success,
-    mocker
+    wayback_extractor_enricher, metadata, mock_is_auth_wall, mock_post_success, mocker
 ):
     mock_is_auth_wall(False)
     mock_post_success()
diff --git a/tests/enrichers/test_whisper_enricher.py b/tests/enrichers/test_whisper_enricher.py
index ee1844a..0278e91 100644
--- a/tests/enrichers/test_whisper_enricher.py
+++ b/tests/enrichers/test_whisper_enricher.py
@@ -16,7 +16,7 @@ def enricher(mocker):
         "include_srt": False,
         "timeout": 5,
         "action": "translate",
-        "steps": {"storages": ["s3_storage"]}
+        "steps": {"storages": ["s3_storage"]},
     }
     mock_s3 = mocker.MagicMock(spec=S3Storage)
     mock_s3.get_cdn_url.return_value = TEST_S3_URL
@@ -25,7 +25,7 @@ def enricher(mocker):
     instance.display_name = "Whisper Enricher"
     instance.config_setup({instance.name: config})
     # bypassing the setup method and mocking S3 setup
-    instance.stores = config['steps']['storages']
+    instance.stores = config["steps"]["storages"]
     instance.s3 = mock_s3
     yield instance, mock_s3
 
@@ -63,19 +63,14 @@ def test_successful_job_submission(enricher, metadata, mock_requests, mocker):
     # Mock the complete API interaction chain
     mock_status_response = mocker.MagicMock()
     mock_status_response.status_code = 200
-    mock_status_response.json.return_value = {
-        "status": "success",
-        "meta": {}
-    }
+    mock_status_response.json.return_value = {"status": "success", "meta": {}}
     mock_artifacts_response = mocker.MagicMock()
     mock_artifacts_response.status_code = 200
-    mock_artifacts_response.json.return_value = [{
-        "data": [{"start": 0, "end": 5, "text": "test transcript"}]
-    }]
+    mock_artifacts_response.json.return_value = [{"data": [{"start": 0, "end": 5, "text": "test transcript"}]}]
     # Set up mock response sequence
     mock_requests.get.side_effect = [
         mock_status_response,  # First call: status check
-        mock_artifacts_response  # Second call: artifacts check
+        mock_artifacts_response,  # Second call: artifacts check
     ]
 
     # Run enrichment (without opening file)
@@ -84,15 +79,17 @@ def test_successful_job_submission(enricher, metadata, mock_requests, mocker):
     mock_requests.post.assert_called_once_with(
         "http://testapi/jobs",
         json={"url": "http://cdn.example.com/test.mp4", "type": "translate"},
-        headers={"Authorization": "Bearer whisper-key"}
+        headers={"Authorization": "Bearer whisper-key"},
     )
     # Verify job status checks
     assert mock_requests.get.call_count == 2
     assert "artifact_0_text" in metadata.media[0].get("whisper_model")
-    assert metadata.media[0].get("whisper_model") == {'artifact_0_text': 'test transcript',
-                                                      'job_artifacts_check': 'http://testapi/jobs/job123/artifacts',
-                                                      'job_id': 'job123',
-                                                      'job_status_check': 'http://testapi/jobs/job123'}
+    assert metadata.media[0].get("whisper_model") == {
+        "artifact_0_text": "test transcript",
+        "job_artifacts_check": "http://testapi/jobs/job123/artifacts",
+        "job_id": "job123",
+        "job_status_check": "http://testapi/jobs/job123",
+    }
 
 
 def test_submit_job(enricher, mocker):
diff --git a/tests/extractors/test_extractor_base.py b/tests/extractors/test_extractor_base.py
index 6e77ec3..0240529 100644
--- a/tests/extractors/test_extractor_base.py
+++ b/tests/extractors/test_extractor_base.py
@@ -7,7 +7,6 @@ from auto_archiver.core.extractor import Extractor
 
 
 class TestExtractorBase(object):
-
     extractor_module: str = None
     config: dict = None
 
@@ -17,7 +16,7 @@ class TestExtractorBase(object):
         assert self.config is not None, "self.config must be a dict set on the subclass"
 
         self.extractor: Type[Extractor] = setup_module(self.extractor_module, self.config)
-    
+
     def assertValidResponseMetadata(self, test_response: Metadata, title: str, timestamp: str, status: str = ""):
         assert test_response is not False
 
diff --git a/tests/extractors/test_generic_extractor.py b/tests/extractors/test_generic_extractor.py
index 33f35b7..ec7aae4 100644
--- a/tests/extractors/test_generic_extractor.py
+++ b/tests/extractors/test_generic_extractor.py
@@ -9,26 +9,28 @@ import pytest
 from auto_archiver.modules.generic_extractor.generic_extractor import GenericExtractor
 from .test_extractor_base import TestExtractorBase
 
-CI=os.getenv("GITHUB_ACTIONS", '') == 'true'
+CI = os.getenv("GITHUB_ACTIONS", "") == "true"
+
+
 class TestGenericExtractor(TestExtractorBase):
-    """Tests Generic Extractor
-    """
-    extractor_module = 'generic_extractor'
+    """Tests Generic Extractor"""
+
+    extractor_module = "generic_extractor"
     extractor: GenericExtractor
 
     config = {
-        'subtitles': False,
-        'comments': False,
-        'livestreams': False,
-        'live_from_start': False,
-        'end_means_success': True,
-        'allow_playlist': False,
-        'max_downloads': "inf",
-        'proxy': None,
-        'cookies_from_browser': False,
-        'cookie_file': None,
-        }
-    
+        "subtitles": False,
+        "comments": False,
+        "livestreams": False,
+        "live_from_start": False,
+        "end_means_success": True,
+        "allow_playlist": False,
+        "max_downloads": "inf",
+        "proxy": None,
+        "cookies_from_browser": False,
+        "cookie_file": None,
+    }
+
     def test_load_dropin(self):
         # test loading dropins that are in the generic_archiver package
         package = "auto_archiver.modules.generic_extractor"
@@ -38,21 +40,26 @@ class TestGenericExtractor(TestExtractorBase):
         path = os.path.join(dirname(dirname(__file__)), "data/")
         assert self.extractor.dropin_for_name("dropin", additional_paths=[path])
 
-
-
-    @pytest.mark.parametrize("url, is_suitable", [
-        ("https://www.youtube.com/watch?v=5qap5aO4i9A", True),
-        ("https://www.tiktok.com/@funnycats0ftiktok/video/7345101300750748970?lang=en", True),
-        ("https://www.instagram.com/p/CU1J9JYJ9Zz/", True),
-        ("https://www.facebook.com/nytimes/videos/10160796550110716", True),
-        ("https://www.twitch.tv/videos/1167226570", True),
-        ("https://bellingcat.com/news/2021/10/08/ukrainian-soldiers-are-being-killed-by-landmines-in-the-donbas/", True),
-        ("https://google.com", True)])
+    @pytest.mark.parametrize(
+        "url, is_suitable",
+        [
+            ("https://www.youtube.com/watch?v=5qap5aO4i9A", True),
+            ("https://www.tiktok.com/@funnycats0ftiktok/video/7345101300750748970?lang=en", True),
+            ("https://www.instagram.com/p/CU1J9JYJ9Zz/", True),
+            ("https://www.facebook.com/nytimes/videos/10160796550110716", True),
+            ("https://www.twitch.tv/videos/1167226570", True),
+            (
+                "https://bellingcat.com/news/2021/10/08/ukrainian-soldiers-are-being-killed-by-landmines-in-the-donbas/",
+                True,
+            ),
+            ("https://google.com", True),
+        ],
+    )
     def test_suitable_urls(self, make_item, url, is_suitable):
         """
-            Note: expected behaviour is to return True for all URLs, as YoutubeDLArchiver should be able to handle all URLs
-            This behaviour may be changed in the future (e.g. if we want the youtubedl archiver to just handle URLs it has extractors for,
-            and then if and only if all archivers fails, does it fall back to the generic archiver)
+        Note: expected behaviour is to return True for all URLs, as YoutubeDLArchiver should be able to handle all URLs
+        This behaviour may be changed in the future (e.g. if we want the youtubedl archiver to just handle URLs it has extractors for,
+        and then if and only if all archivers fails, does it fall back to the generic archiver)
         """
         assert self.extractor.suitable(url) == is_suitable
 
@@ -63,11 +70,14 @@ class TestGenericExtractor(TestExtractorBase):
         assert result.get_url() == "https://www.tiktok.com/@funnycats0ftiktok/video/7345101300750748970"
 
     @pytest.mark.download
-    @pytest.mark.parametrize("url", [
-        "https://bsky.app/profile/colborne.bsky.social/post/3lcxcpgt6j42l",
-        "twitter.com/bellingcat/status/123",
-        "https://www.youtube.com/watch?v=1"
-    ])
+    @pytest.mark.parametrize(
+        "url",
+        [
+            "https://bsky.app/profile/colborne.bsky.social/post/3lcxcpgt6j42l",
+            "twitter.com/bellingcat/status/123",
+            "https://www.youtube.com/watch?v=1",
+        ],
+    )
     def test_download_nonexistent_media(self, make_item, url):
         """
         Test to make sure that the extractor doesn't break on non-existend posts/media
@@ -78,7 +88,10 @@ class TestGenericExtractor(TestExtractorBase):
         result = self.extractor.download(item)
         assert not result
 
-    @pytest.mark.skipif(CI, reason="Currently no way to authenticate when on CI. Youtube (yt-dlp) doesn't support logging in with username/password.")
+    @pytest.mark.skipif(
+        CI,
+        reason="Currently no way to authenticate when on CI. Youtube (yt-dlp) doesn't support logging in with username/password.",
+    )
     @pytest.mark.download
     def test_youtube_download(self, make_item):
         # url https://www.youtube.com/watch?v=5qap5aO4i9A
@@ -87,7 +100,10 @@ class TestGenericExtractor(TestExtractorBase):
         result = self.extractor.download(item)
         assert result.get_url() == "https://www.youtube.com/watch?v=J---aiyznGQ"
         assert result.get_title() == "Keyboard Cat! - THE ORIGINAL!"
-        assert result.get('description') == "Buy NEW Keyboard Cat Merch! https://keyboardcat.creator-spring.com\n\nxo Keyboard Cat memes make your day better!\nhttp://www.keyboardcatstore.com/\nhttps://www.facebook.com/thekeyboardcat\nhttp://www.charlieschmidt.com/"
+        assert (
+            result.get("description")
+            == "Buy NEW Keyboard Cat Merch! https://keyboardcat.creator-spring.com\n\nxo Keyboard Cat memes make your day better!\nhttp://www.keyboardcatstore.com/\nhttps://www.facebook.com/thekeyboardcat\nhttp://www.charlieschmidt.com/"
+        )
         assert len(result.media) == 2
         assert Path(result.media[0].filename).name == "J---aiyznGQ.webm"
         assert Path(result.media[1].filename).name == "hqdefault.jpg"
@@ -103,7 +119,7 @@ class TestGenericExtractor(TestExtractorBase):
         item = make_item("https://bsky.app/profile/bellingcat.com/post/3lfn3hbcxgc2q")
         result = self.extractor.download(item)
         assert result is not False
-    
+
     @pytest.mark.download
     def test_bluesky_download_no_media(self, make_item):
         item = make_item("https://bsky.app/profile/bellingcat.com/post/3lfphwmcs4c2z")
@@ -115,7 +131,7 @@ class TestGenericExtractor(TestExtractorBase):
         item = make_item("https://bsky.app/profile/bellingcat.com/post/3le2l4gsxlk2i")
         result = self.extractor.download(item)
         assert result is not False
-    
+
     @pytest.mark.skipif(CI, reason="Truth social blocks GH actions.")
     @pytest.mark.download
     def test_truthsocial_download_video(self, make_item):
@@ -130,14 +146,14 @@ class TestGenericExtractor(TestExtractorBase):
         item = make_item("https://truthsocial.com/@bbcnewa/posts/109598702184774628")
         result = self.extractor.download(item)
         assert result is not False
-    
+
     @pytest.mark.skipif(CI, reason="Truth social blocks GH actions.")
     @pytest.mark.download
     def test_truthsocial_download_poll(self, make_item):
         item = make_item("https://truthsocial.com/@CNN_US/posts/113724326568555098")
         result = self.extractor.download(item)
         assert result is not False
-    
+
     @pytest.mark.skipif(CI, reason="Truth social blocks GH actions.")
     @pytest.mark.download
     def test_truthsocial_download_single_image(self, make_item):
@@ -159,7 +175,7 @@ class TestGenericExtractor(TestExtractorBase):
         url = "https://x.com/Bellingcat/status/17197025860711058"
         response = self.extractor.download(make_item(url))
         assert not response
-    
+
     @pytest.mark.download
     def test_twitter_download_malformed_tweetid(self, make_item):
         # this tweet does not exist
@@ -169,7 +185,6 @@ class TestGenericExtractor(TestExtractorBase):
 
     @pytest.mark.download
     def test_twitter_download_tweet_no_media(self, make_item):
-        
         item = make_item("https://twitter.com/MeCookieMonster/status/1617921633456640001?s=20&t=3d0g4ZQis7dCbSDg-mE7-w")
         post = self.extractor.download(item)
 
@@ -177,9 +192,9 @@ class TestGenericExtractor(TestExtractorBase):
             post,
             "Onion rings are just vegetable donuts.",
             datetime.datetime(2023, 1, 24, 16, 25, 51, tzinfo=datetime.timezone.utc),
-            "yt-dlp_Twitter: success"
+            "yt-dlp_Twitter: success",
         )
-    
+
     @pytest.mark.download
     def test_twitter_download_video(self, make_item):
         url = "https://x.com/bellingcat/status/1871552600346415571"
@@ -187,26 +202,46 @@ class TestGenericExtractor(TestExtractorBase):
         self.assertValidResponseMetadata(
             post,
             "Bellingcat - This month's Bellingchat Premium is with @KolinaKoltai. She reveals how she investigated a platform allowing users to create AI-generated child sexual abuse material and explains why it's crucial to investigate the people behind these services",
-            datetime.datetime(2024, 12, 24, 13, 44, 46, tzinfo=datetime.timezone.utc)
+            datetime.datetime(2024, 12, 24, 13, 44, 46, tzinfo=datetime.timezone.utc),
         )
 
-    @pytest.mark.xfail(reason="Currently failing, sensitive content requires logged in users/cookies - not yet implemented")
+    @pytest.mark.xfail(
+        reason="Currently failing, sensitive content requires logged in users/cookies - not yet implemented"
+    )
     @pytest.mark.download
-    @pytest.mark.parametrize("url, title, timestamp, image_hash", [
-            ("https://x.com/SozinhoRamalho/status/1876710769913450647", "ignore tweet, testing sensitivity warning nudity", datetime.datetime(2024, 12, 31, 14, 18, 33, tzinfo=datetime.timezone.utc), "image_hash"),
-            ("https://x.com/SozinhoRamalho/status/1876710875475681357", "ignore tweet, testing sensitivity warning violence", datetime.datetime(2024, 12, 31, 14, 18, 33, tzinfo=datetime.timezone.utc), "image_hash"),
-            ("https://x.com/SozinhoRamalho/status/1876711053813227618", "ignore tweet, testing sensitivity warning sensitive", datetime.datetime(2024, 12, 31, 14, 18, 33, tzinfo=datetime.timezone.utc), "image_hash"),
-            ("https://x.com/SozinhoRamalho/status/1876711141314801937", "ignore tweet, testing sensitivity warning nudity, violence, sensitivity", datetime.datetime(2024, 12, 31, 14, 18, 33, tzinfo=datetime.timezone.utc), "image_hash"),
-        ])
+    @pytest.mark.parametrize(
+        "url, title, timestamp, image_hash",
+        [
+            (
+                "https://x.com/SozinhoRamalho/status/1876710769913450647",
+                "ignore tweet, testing sensitivity warning nudity",
+                datetime.datetime(2024, 12, 31, 14, 18, 33, tzinfo=datetime.timezone.utc),
+                "image_hash",
+            ),
+            (
+                "https://x.com/SozinhoRamalho/status/1876710875475681357",
+                "ignore tweet, testing sensitivity warning violence",
+                datetime.datetime(2024, 12, 31, 14, 18, 33, tzinfo=datetime.timezone.utc),
+                "image_hash",
+            ),
+            (
+                "https://x.com/SozinhoRamalho/status/1876711053813227618",
+                "ignore tweet, testing sensitivity warning sensitive",
+                datetime.datetime(2024, 12, 31, 14, 18, 33, tzinfo=datetime.timezone.utc),
+                "image_hash",
+            ),
+            (
+                "https://x.com/SozinhoRamalho/status/1876711141314801937",
+                "ignore tweet, testing sensitivity warning nudity, violence, sensitivity",
+                datetime.datetime(2024, 12, 31, 14, 18, 33, tzinfo=datetime.timezone.utc),
+                "image_hash",
+            ),
+        ],
+    )
     def test_twitter_download_sensitive_media(self, url, title, timestamp, image_hash, make_item):
-
         """Download tweets with sensitive media"""
 
         post = self.extractor.download(make_item(url))
-        self.assertValidResponseMetadata(
-            post,
-            title,
-            timestamp
-        )
+        self.assertValidResponseMetadata(post, title, timestamp)
         assert len(post.media) == 1
-        assert post.media[0].hash == image_hash
\ No newline at end of file
+        assert post.media[0].hash == image_hash
diff --git a/tests/extractors/test_instagram_api_extractor.py b/tests/extractors/test_instagram_api_extractor.py
index 7eba8e9..d8a1cc0 100644
--- a/tests/extractors/test_instagram_api_extractor.py
+++ b/tests/extractors/test_instagram_api_extractor.py
@@ -15,10 +15,11 @@ def mock_user_response():
             "username": "test_user",
             "full_name": "Test User",
             "profile_pic_url_hd": "http://example.com/profile.jpg",
-            "profile_pic_url": "http://example.com/profile_lowres.jpg"
+            "profile_pic_url": "http://example.com/profile_lowres.jpg",
         }
     }
 
+
 @pytest.fixture
 def mock_post_response():
     return {
@@ -27,16 +28,14 @@ def mock_post_response():
         "caption_text": "Test Caption",
         "taken_at": datetime.now().timestamp(),
         "video_url": "http://example.com/video.mp4",
-        "thumbnail_url": "http://example.com/thumbnail.jpg"
+        "thumbnail_url": "http://example.com/thumbnail.jpg",
     }
 
+
 @pytest.fixture
 def mock_story_response():
-    return [{
-        "id": "story_123",
-        "taken_at": datetime.now().timestamp(),
-        "video_url": "http://example.com/story.mp4"
-    }]
+    return [{"id": "story_123", "taken_at": datetime.now().timestamp(), "video_url": "http://example.com/story.mp4"}]
+
 
 @pytest.fixture
 def mock_highlight_response():
@@ -46,11 +45,13 @@ def mock_highlight_response():
                 "highlight:123": {
                     "id": "123",
                     "title": "Test Highlight",
-                    "items": [{
-                        "id": "item_123",
-                        "taken_at": datetime.now().timestamp(),
-                        "video_url": "http://example.com/highlight.mp4"
-                    }]
+                    "items": [
+                        {
+                            "id": "item_123",
+                            "taken_at": datetime.now().timestamp(),
+                            "video_url": "http://example.com/highlight.mp4",
+                        }
+                    ],
                 }
             }
         }
@@ -81,24 +82,30 @@ class TestInstagramAPIExtractor(TestExtractorBase):
         m.set("netloc", "instagram.com")
         return m
 
-    @pytest.mark.parametrize("url,expected", [
-        ("https://instagram.com/user", [("", "user", "")]),
-        ("https://instagr.am/p/post_id", []),
-        ("https://youtube.com", []),
-        ("https://www.instagram.com/reel/reel_id", [("reel", "reel_id", "")]),
-        ("https://instagram.com/stories/highlights/123", [("stories/highlights", "123", "")]),
-        ("https://instagram.com/stories/user/123", [("stories", "user", "123")]),
-    ])
+    @pytest.mark.parametrize(
+        "url,expected",
+        [
+            ("https://instagram.com/user", [("", "user", "")]),
+            ("https://instagr.am/p/post_id", []),
+            ("https://youtube.com", []),
+            ("https://www.instagram.com/reel/reel_id", [("reel", "reel_id", "")]),
+            ("https://instagram.com/stories/highlights/123", [("stories/highlights", "123", "")]),
+            ("https://instagram.com/stories/user/123", [("stories", "user", "123")]),
+        ],
+    )
     def test_url_parsing(self, url, expected):
         assert self.extractor.valid_url.findall(url) == expected
 
     def test_initialize(self):
         assert self.extractor.api_endpoint[-1] != "/"
 
-    @pytest.mark.parametrize("input_dict,expected", [
-        ({"x": 0, "valid": "data"}, {"valid": "data"}),
-        ({"nested": {"y": None, "valid": [{}]}}, {"nested": {"valid": [{}]}}),
-    ])
+    @pytest.mark.parametrize(
+        "input_dict,expected",
+        [
+            ({"x": 0, "valid": "data"}, {"valid": "data"}),
+            ({"nested": {"y": None, "valid": [{}]}}, {"nested": {"valid": [{}]}}),
+        ],
+    )
     def test_cleanup_dict(self, input_dict, expected):
         assert self.extractor.cleanup_dict(input_dict) == expected
 
@@ -114,8 +121,8 @@ class TestInstagramAPIExtractor(TestExtractorBase):
 
     def test_download_profile_basic(self, metadata, mock_user_response, mocker):
         """Test basic profile download without full_profile"""
-        mock_call = mocker.patch.object(self.extractor, 'call_api')
-        mock_download = mocker.patch.object(self.extractor, 'download_from_url')
+        mock_call = mocker.patch.object(self.extractor, "call_api")
+        mock_download = mocker.patch.object(self.extractor, "download_from_url")
         # Mock API responses
         mock_call.return_value = mock_user_response
         mock_download.return_value = "profile.jpg"
@@ -132,17 +139,14 @@ class TestInstagramAPIExtractor(TestExtractorBase):
 
     def test_download_profile_full(self, metadata, mock_user_response, mock_story_response, mocker):
         """Test full profile download with stories/posts"""
-        mock_call = mocker.patch.object(self.extractor, 'call_api')
-        mock_posts = mocker.patch.object(self.extractor, 'download_all_posts')
-        mock_highlights = mocker.patch.object(self.extractor, 'download_all_highlights')
-        mock_tagged = mocker.patch.object(self.extractor, 'download_all_tagged')
-        mock_stories = mocker.patch.object(self.extractor, '_download_stories_reusable')
+        mock_call = mocker.patch.object(self.extractor, "call_api")
+        mock_posts = mocker.patch.object(self.extractor, "download_all_posts")
+        mock_highlights = mocker.patch.object(self.extractor, "download_all_highlights")
+        mock_tagged = mocker.patch.object(self.extractor, "download_all_tagged")
+        mock_stories = mocker.patch.object(self.extractor, "_download_stories_reusable")
 
         self.extractor.full_profile = True
-        mock_call.side_effect = [
-            mock_user_response,
-            mock_story_response
-        ]
+        mock_call.side_effect = [mock_user_response, mock_story_response]
         mock_highlights.return_value = None
         mock_stories.return_value = mock_story_response
         mock_posts.return_value = None
@@ -155,7 +159,7 @@ class TestInstagramAPIExtractor(TestExtractorBase):
 
     def test_download_profile_not_found(self, metadata, mocker):
         """Test profile not found error"""
-        mock_call = mocker.patch.object(self.extractor, 'call_api')
+        mock_call = mocker.patch.object(self.extractor, "call_api")
         mock_call.return_value = {"user": None}
         with pytest.raises(AssertionError) as exc_info:
             self.extractor.download_profile(metadata, "invalid_user")
@@ -163,18 +167,14 @@ class TestInstagramAPIExtractor(TestExtractorBase):
 
     def test_download_profile_error_handling(self, metadata, mock_user_response, mocker):
         """Test error handling in full profile mode"""
-        mock_call = mocker.patch.object(self.extractor, 'call_api')
-        mock_highlights = mocker.patch.object(self.extractor, 'download_all_highlights')
-        mock_tagged = mocker.patch.object(self.extractor, 'download_all_tagged')
-        stories_tagged = mocker.patch.object(self.extractor, '_download_stories_reusable')
-        mock_posts = mocker.patch.object(self.extractor, 'download_all_posts')
+        mock_call = mocker.patch.object(self.extractor, "call_api")
+        mock_highlights = mocker.patch.object(self.extractor, "download_all_highlights")
+        mock_tagged = mocker.patch.object(self.extractor, "download_all_tagged")
+        stories_tagged = mocker.patch.object(self.extractor, "_download_stories_reusable")
+        mock_posts = mocker.patch.object(self.extractor, "download_all_posts")
 
         self.extractor.full_profile = True
-        mock_call.side_effect = [
-            mock_user_response,
-            Exception("Stories API failed"),
-            Exception("Posts API failed")
-        ]
+        mock_call.side_effect = [mock_user_response, Exception("Stories API failed"), Exception("Posts API failed")]
         mock_highlights.return_value = None
         mock_tagged.return_value = None
         stories_tagged.return_value = None
@@ -182,4 +182,4 @@ class TestInstagramAPIExtractor(TestExtractorBase):
         result = self.extractor.download_profile(metadata, "test_user")
 
         assert result.is_success()
-        assert "Error downloading stories for test_user" in result.metadata["errors"]
\ No newline at end of file
+        assert "Error downloading stories for test_user" in result.metadata["errors"]
diff --git a/tests/extractors/test_instagram_extractor.py b/tests/extractors/test_instagram_extractor.py
index 647cab4..0cafa2b 100644
--- a/tests/extractors/test_instagram_extractor.py
+++ b/tests/extractors/test_instagram_extractor.py
@@ -5,8 +5,7 @@ from auto_archiver.modules.instagram_extractor import InstagramExtractor
 
 @pytest.fixture
 def instagram_extractor(setup_module, mocker):
-
-    extractor_module: str = 'instagram_extractor'
+    extractor_module: str = "instagram_extractor"
     config: dict = {
         "username": "user_name",
         "password": "password123",
@@ -17,20 +16,26 @@ def instagram_extractor(setup_module, mocker):
     fake_loader.load_session_from_file.return_value = None
     fake_loader.login.return_value = None
     fake_loader.save_session_to_file.return_value = None
-    mocker.patch("instaloader.Instaloader", return_value=fake_loader,)
+    mocker.patch(
+        "instaloader.Instaloader",
+        return_value=fake_loader,
+    )
     return setup_module(extractor_module, config)
 
 
-@pytest.mark.parametrize("url", [
-    "https://www.instagram.com/p/",
-    "https://www.instagram.com/p/1234567890/",
-    "https://www.instagram.com/reel/1234567890/",
-    "https://www.instagram.com/username/",
-    "https://www.instagram.com/username/stories/",
-    "https://www.instagram.com/username/highlights/",
-])
+@pytest.mark.parametrize(
+    "url",
+    [
+        "https://www.instagram.com/p/",
+        "https://www.instagram.com/p/1234567890/",
+        "https://www.instagram.com/reel/1234567890/",
+        "https://www.instagram.com/username/",
+        "https://www.instagram.com/username/stories/",
+        "https://www.instagram.com/username/highlights/",
+    ],
+)
 def test_regex_matches(url: str, instagram_extractor: InstagramExtractor) -> None:
     """
     Ensure that the valid_url regex matches all provided Instagram URLs.
     """
-    assert instagram_extractor.valid_url.match(url)
\ No newline at end of file
+    assert instagram_extractor.valid_url.match(url)
diff --git a/tests/extractors/test_instagram_tbot_extractor.py b/tests/extractors/test_instagram_tbot_extractor.py
index f274728..1b31d07 100644
--- a/tests/extractors/test_instagram_tbot_extractor.py
+++ b/tests/extractors/test_instagram_tbot_extractor.py
@@ -9,8 +9,8 @@ from tests.extractors.test_extractor_base import TestExtractorBase
 
 @pytest.fixture
 def patch_extractor_methods(request, setup_module, mocker):
-    mocker.patch.object(InstagramTbotExtractor, '_prepare_session_file', return_value=None)
-    mocker.patch.object(InstagramTbotExtractor, '_initialize_telegram_client', return_value=None)
+    mocker.patch.object(InstagramTbotExtractor, "_prepare_session_file", return_value=None)
+    mocker.patch.object(InstagramTbotExtractor, "_initialize_telegram_client", return_value=None)
     yield
 
 
@@ -35,12 +35,7 @@ def mock_telegram_client(mocker):
 @pytest.fixture
 def extractor(setup_module, patch_extractor_methods, mocker):
     extractor_module = "instagram_tbot_extractor"
-    config = {
-        "api_id": 12345,
-        "api_hash": "test_api_hash",
-        "session_file": "test_session",
-        "timeout": 4
-    }
+    config = {"api_id": 12345, "api_hash": "test_api_hash", "session_file": "test_session", "timeout": 4}
     extractor = setup_module(extractor_module, config)
     extractor.client = mocker.MagicMock()
     extractor.session_file = "test_session"
@@ -79,21 +74,30 @@ class TestInstagramTbotExtractorReal(TestExtractorBase):
         "session_file": "secrets/anon-insta",
     }
 
-    @pytest.mark.parametrize("url, expected_status, message, len_media", [
-        ("https://www.instagram.com/p/C4QgLbrIKXG", "insta-via-bot: success",
-         "Are you new to Bellingcat? - The way we share our investigations is different. 💭\nWe want you to read our story but also learn ou",
-         6),
-        ("https://www.instagram.com/reel/DEVLK8qoIbg/", "insta-via-bot: success",
-         "Our volunteer community is at the centre of many incredible Bellingcat investigations and tools. Stephanie Ladel is one such vol",
-         3),
-        # instagram tbot not working (potentially intermittently?) for stories - replace with a live story to retest
-        # ("https://www.instagram.com/stories/bellingcatofficial/3556336382743057476/", False, "Media not found or unavailable"),
-        # Seems to be working intermittently for highlights
-        # ("https://www.instagram.com/stories/highlights/17868810693068139/", "insta-via-bot: success", None, 50),
-        # Marking invalid url as success
-        ("https://www.instagram.com/p/INVALID", "insta-via-bot: success", "Media not found or unavailable", 0),
-        ("https://www.youtube.com/watch?v=ymCMy8OffHM", False, None, 0),
-    ])
+    @pytest.mark.parametrize(
+        "url, expected_status, message, len_media",
+        [
+            (
+                "https://www.instagram.com/p/C4QgLbrIKXG",
+                "insta-via-bot: success",
+                "Are you new to Bellingcat? - The way we share our investigations is different. 💭\nWe want you to read our story but also learn ou",
+                6,
+            ),
+            (
+                "https://www.instagram.com/reel/DEVLK8qoIbg/",
+                "insta-via-bot: success",
+                "Our volunteer community is at the centre of many incredible Bellingcat investigations and tools. Stephanie Ladel is one such vol",
+                3,
+            ),
+            # instagram tbot not working (potentially intermittently?) for stories - replace with a live story to retest
+            # ("https://www.instagram.com/stories/bellingcatofficial/3556336382743057476/", False, "Media not found or unavailable"),
+            # Seems to be working intermittently for highlights
+            # ("https://www.instagram.com/stories/highlights/17868810693068139/", "insta-via-bot: success", None, 50),
+            # Marking invalid url as success
+            ("https://www.instagram.com/p/INVALID", "insta-via-bot: success", "Media not found or unavailable", 0),
+            ("https://www.youtube.com/watch?v=ymCMy8OffHM", False, None, 0),
+        ],
+    )
     def test_download(self, url, expected_status, message, len_media, metadata_sample):
         """Test the `download()` method with various Instagram URLs."""
         metadata_sample.set_url(url)
diff --git a/tests/extractors/test_twitter_api_extractor.py b/tests/extractors/test_twitter_api_extractor.py
index 26394ac..0664f49 100644
--- a/tests/extractors/test_twitter_api_extractor.py
+++ b/tests/extractors/test_twitter_api_extractor.py
@@ -10,8 +10,7 @@ from auto_archiver.modules.twitter_api_extractor import TwitterApiExtractor
 
 @pytest.mark.incremental
 class TestTwitterApiExtractor(TestExtractorBase):
-
-    extractor_module = 'twitter_api_extractor'
+    extractor_module = "twitter_api_extractor"
 
     config = {
         "bearer_tokens": [],
@@ -22,41 +21,79 @@ class TestTwitterApiExtractor(TestExtractorBase):
         "access_secret": os.environ.get("TWITTER_ACCESS_SECRET"),
     }
 
-    @pytest.mark.parametrize("url, expected", [
-        ("https://x.com/bellingcat/status/1874097816571961839", "https://x.com/bellingcat/status/1874097816571961839"), # x.com urls unchanged
-        ("https://twitter.com/bellingcat/status/1874097816571961839", "https://twitter.com/bellingcat/status/1874097816571961839"), # twitter urls unchanged
-        ("https://twitter.com/bellingcat/status/1874097816571961839?s=20&t=3d0g4ZQis7dCbSDg-mE7-w", "https://twitter.com/bellingcat/status/1874097816571961839?s=20&t=3d0g4ZQis7dCbSDg-mE7-w"), # don't strip params from twitter urls (changed Jan 2025)
-        ("https://www.bellingcat.com/category/resources/", "https://www.bellingcat.com/category/resources/"), # non-twitter/x urls unchanged
-        ("https://www.bellingcat.com/category/resources/?s=20&t=3d0g4ZQis7dCbSDg-mE7-w", "https://www.bellingcat.com/category/resources/?s=20&t=3d0g4ZQis7dCbSDg-mE7-w"), # shouldn't strip params from non-twitter/x URLs
-    ])
+    @pytest.mark.parametrize(
+        "url, expected",
+        [
+            (
+                "https://x.com/bellingcat/status/1874097816571961839",
+                "https://x.com/bellingcat/status/1874097816571961839",
+            ),  # x.com urls unchanged
+            (
+                "https://twitter.com/bellingcat/status/1874097816571961839",
+                "https://twitter.com/bellingcat/status/1874097816571961839",
+            ),  # twitter urls unchanged
+            (
+                "https://twitter.com/bellingcat/status/1874097816571961839?s=20&t=3d0g4ZQis7dCbSDg-mE7-w",
+                "https://twitter.com/bellingcat/status/1874097816571961839?s=20&t=3d0g4ZQis7dCbSDg-mE7-w",
+            ),  # don't strip params from twitter urls (changed Jan 2025)
+            (
+                "https://www.bellingcat.com/category/resources/",
+                "https://www.bellingcat.com/category/resources/",
+            ),  # non-twitter/x urls unchanged
+            (
+                "https://www.bellingcat.com/category/resources/?s=20&t=3d0g4ZQis7dCbSDg-mE7-w",
+                "https://www.bellingcat.com/category/resources/?s=20&t=3d0g4ZQis7dCbSDg-mE7-w",
+            ),  # shouldn't strip params from non-twitter/x URLs
+        ],
+    )
     def test_sanitize_url(self, url, expected):
         assert expected == self.extractor.sanitize_url(url)
 
     @pytest.mark.download
     def test_sanitize_url_download(self):
-        assert "https://www.bellingcat.com/category/resources/" == self.extractor.sanitize_url("https://t.co/yl3oOJatFp")
+        assert "https://www.bellingcat.com/category/resources/" == self.extractor.sanitize_url(
+            "https://t.co/yl3oOJatFp"
+        )
 
-    @pytest.mark.parametrize("url, exptected_username, exptected_tweetid", [
-        ("https://twitter.com/bellingcat/status/1874097816571961839", "bellingcat", "1874097816571961839"),
-        ("https://x.com/bellingcat/status/1874097816571961839", "bellingcat", "1874097816571961839"),
-        ("https://www.bellingcat.com/category/resources/", False, False)
-        ])
+    @pytest.mark.parametrize(
+        "url, exptected_username, exptected_tweetid",
+        [
+            ("https://twitter.com/bellingcat/status/1874097816571961839", "bellingcat", "1874097816571961839"),
+            ("https://x.com/bellingcat/status/1874097816571961839", "bellingcat", "1874097816571961839"),
+            ("https://www.bellingcat.com/category/resources/", False, False),
+        ],
+    )
     def test_get_username_tweet_id_from_url(self, url, exptected_username, exptected_tweetid):
-    
         username, tweet_id = self.extractor.get_username_tweet_id(url)
         assert exptected_username == username
         assert exptected_tweetid == tweet_id
 
     def test_choose_variants(self):
         # taken from the response for url https://x.com/bellingcat/status/1871552600346415571
-        variant_list = [MediaVariant(content_type='application/x-mpegURL', url='https://video.twimg.com/ext_tw_video/1871551993677852672/pu/pl/ovWo7ux-bKROwYIC.m3u8?tag=12&v=e1b'),
-                        MediaVariant(bit_rate=256000, content_type='video/mp4', url='https://video.twimg.com/ext_tw_video/1871551993677852672/pu/vid/avc1/480x270/OqZIrKV0LFswMvxS.mp4?tag=12'),
-                        MediaVariant(bit_rate=832000, content_type='video/mp4', url='https://video.twimg.com/ext_tw_video/1871551993677852672/pu/vid/avc1/640x360/uiDZDSmZ8MZn9hsi.mp4?tag=12'),
-                        MediaVariant(bit_rate=2176000, content_type='video/mp4', url='https://video.twimg.com/ext_tw_video/1871551993677852672/pu/vid/avc1/1280x720/6Y340Esh568WZnRZ.mp4?tag=12')
-                        ]
+        variant_list = [
+            MediaVariant(
+                content_type="application/x-mpegURL",
+                url="https://video.twimg.com/ext_tw_video/1871551993677852672/pu/pl/ovWo7ux-bKROwYIC.m3u8?tag=12&v=e1b",
+            ),
+            MediaVariant(
+                bit_rate=256000,
+                content_type="video/mp4",
+                url="https://video.twimg.com/ext_tw_video/1871551993677852672/pu/vid/avc1/480x270/OqZIrKV0LFswMvxS.mp4?tag=12",
+            ),
+            MediaVariant(
+                bit_rate=832000,
+                content_type="video/mp4",
+                url="https://video.twimg.com/ext_tw_video/1871551993677852672/pu/vid/avc1/640x360/uiDZDSmZ8MZn9hsi.mp4?tag=12",
+            ),
+            MediaVariant(
+                bit_rate=2176000,
+                content_type="video/mp4",
+                url="https://video.twimg.com/ext_tw_video/1871551993677852672/pu/vid/avc1/1280x720/6Y340Esh568WZnRZ.mp4?tag=12",
+            ),
+        ]
         chosen_variant = self.extractor.choose_variant(variant_list)
         assert chosen_variant == variant_list[3]
-    
+
     @pytest.mark.skipif(not os.environ.get("TWITTER_BEARER_TOKEN"), reason="No Twitter bearer token provided")
     @pytest.mark.download
     def test_download_nonexistent_tweet(self, make_item):
@@ -76,7 +113,6 @@ class TestTwitterApiExtractor(TestExtractorBase):
     @pytest.mark.skipif(not os.environ.get("TWITTER_BEARER_TOKEN"), reason="No Twitter bearer token provided")
     @pytest.mark.download
     def test_download_tweet_no_media(self, make_item):
-        
         item = make_item("https://twitter.com/MeCookieMonster/status/1617921633456640001?s=20&t=3d0g4ZQis7dCbSDg-mE7-w")
         post = self.extractor.download(item)
 
@@ -84,7 +120,7 @@ class TestTwitterApiExtractor(TestExtractorBase):
             post,
             "Onion rings are just vegetable donuts.",
             datetime.datetime(2023, 1, 24, 16, 25, 51, tzinfo=datetime.timezone.utc),
-            "twitter-api: success"
+            "twitter-api: success",
         )
 
     @pytest.mark.skipif(not os.environ.get("TWITTER_BEARER_TOKEN"), reason="No Twitter bearer token provided")
@@ -95,27 +131,41 @@ class TestTwitterApiExtractor(TestExtractorBase):
         self.assertValidResponseMetadata(
             post,
             "This month's Bellingchat Premium is with @KolinaKoltai. She reveals how she investigated a platform allowing users to create AI-generated child sexual abuse material and explains why it's crucial to investigate the people behind these services https://t.co/SfBUq0hSD0 https://t.co/rIHx0WlKp8",
-            datetime.datetime(2024, 12, 24, 13, 44, 46, tzinfo=datetime.timezone.utc)
+            datetime.datetime(2024, 12, 24, 13, 44, 46, tzinfo=datetime.timezone.utc),
         )
 
     @pytest.mark.skipif(not os.environ.get("TWITTER_BEARER_TOKEN"), reason="No Twitter bearer token provided")
-    @pytest.mark.parametrize("url, title, timestamp", [
-            ("https://x.com/SozinhoRamalho/status/1876710769913450647", "ignore tweet, testing sensitivity warning nudity https://t.co/t3u0hQsSB1", datetime.datetime(2024, 12, 31, 14, 18, 33, tzinfo=datetime.timezone.utc)),
-            ("https://x.com/SozinhoRamalho/status/1876710875475681357", "ignore tweet, testing sensitivity warning violence https://t.co/syYDSkpjZD", datetime.datetime(2024, 12, 31, 14, 18, 33, tzinfo=datetime.timezone.utc)),
-            ("https://x.com/SozinhoRamalho/status/1876711053813227618", "ignore tweet, testing sensitivity warning sensitive https://t.co/XE7cRdjzYq", datetime.datetime(2024, 12, 31, 14, 18, 33, tzinfo=datetime.timezone.utc)),
-            ("https://x.com/SozinhoRamalho/status/1876711141314801937", "ignore tweet, testing sensitivity warning nudity, violence, sensitivity https://t.co/YxCFbbhYE3", datetime.datetime(2024, 12, 31, 14, 18, 33, tzinfo=datetime.timezone.utc)),
-        ])
+    @pytest.mark.parametrize(
+        "url, title, timestamp",
+        [
+            (
+                "https://x.com/SozinhoRamalho/status/1876710769913450647",
+                "ignore tweet, testing sensitivity warning nudity https://t.co/t3u0hQsSB1",
+                datetime.datetime(2024, 12, 31, 14, 18, 33, tzinfo=datetime.timezone.utc),
+            ),
+            (
+                "https://x.com/SozinhoRamalho/status/1876710875475681357",
+                "ignore tweet, testing sensitivity warning violence https://t.co/syYDSkpjZD",
+                datetime.datetime(2024, 12, 31, 14, 18, 33, tzinfo=datetime.timezone.utc),
+            ),
+            (
+                "https://x.com/SozinhoRamalho/status/1876711053813227618",
+                "ignore tweet, testing sensitivity warning sensitive https://t.co/XE7cRdjzYq",
+                datetime.datetime(2024, 12, 31, 14, 18, 33, tzinfo=datetime.timezone.utc),
+            ),
+            (
+                "https://x.com/SozinhoRamalho/status/1876711141314801937",
+                "ignore tweet, testing sensitivity warning nudity, violence, sensitivity https://t.co/YxCFbbhYE3",
+                datetime.datetime(2024, 12, 31, 14, 18, 33, tzinfo=datetime.timezone.utc),
+            ),
+        ],
+    )
     @pytest.mark.download
     def test_download_sensitive_media(self, url, title, timestamp, check_hash, make_item):
-
         """Download tweets with sensitive media"""
 
         post = self.extractor.download(make_item(url))
-        self.assertValidResponseMetadata(
-            post,
-            title,
-            timestamp
-        )
+        self.assertValidResponseMetadata(post, title, timestamp)
         assert len(post.media) == 1
         # check the SHA1 hash (quick) of the media, to make sure it's valid
-        check_hash(post.media[0].filename, "3eea9c03b2dcedd1eb9a169d8bfd1cf877996fab4961de019a96eb9d32d2d733")
\ No newline at end of file
+        check_hash(post.media[0].filename, "3eea9c03b2dcedd1eb9a169d8bfd1cf877996fab4961de019a96eb9d32d2d733")
diff --git a/tests/extractors/test_vk_extractor.py b/tests/extractors/test_vk_extractor.py
index 80eb9dd..040e5f7 100644
--- a/tests/extractors/test_vk_extractor.py
+++ b/tests/extractors/test_vk_extractor.py
@@ -9,6 +9,7 @@ def mock_vk_scraper(mocker):
     """Fixture to mock VkScraper."""
     return mocker.patch("auto_archiver.modules.vk_extractor.vk_extractor.VkScraper")
 
+
 @pytest.fixture
 def vk_extractor(setup_module, mock_vk_scraper) -> VkExtractor:
     """Fixture to initialize VkExtractor with mocked VkScraper."""
@@ -39,7 +40,7 @@ def test_vk_url_but_scrape_returns_empty(vk_extractor, metadata):
 def test_successful_scrape_and_download(vk_extractor, metadata, mocker):
     mock_scrapes = [
         {"text": "Post Title", "datetime": "2023-01-01T00:00:00", "id": 1},
-        {"text": "Another Post", "datetime": "2023-01-02T00:00:00", "id": 2}
+        {"text": "Another Post", "datetime": "2023-01-02T00:00:00", "id": 2},
     ]
     mock_filenames = ["image1.jpg", "image2.png"]
     vk_extractor.vks.scrape.return_value = mock_scrapes
@@ -56,16 +57,16 @@ def test_successful_scrape_and_download(vk_extractor, metadata, mocker):
     assert len(result.media) == 2
     assert result.media[0].filename == "image1.jpg"
     assert result.media[1].filename == "image2.png"
-    vk_extractor.vks.download_media.assert_called_once_with(
-        mock_scrapes, vk_extractor.tmp_dir
-    )
+    vk_extractor.vks.download_media.assert_called_once_with(mock_scrapes, vk_extractor.tmp_dir)
 
 
 def test_adds_first_title_and_timestamp(vk_extractor):
     metadata = Metadata().set_url("https://vk.com/no-metadata")
     metadata.set_url("https://vk.com/no-metadata")
-    mock_scrapes = [{"text": "value", "datetime": "2023-01-01T00:00:00"},
-                    {"text": "value2", "datetime": "2023-01-02T00:00:00"}]
+    mock_scrapes = [
+        {"text": "value", "datetime": "2023-01-01T00:00:00"},
+        {"text": "value2", "datetime": "2023-01-02T00:00:00"},
+    ]
     vk_extractor.vks.scrape.return_value = mock_scrapes
     vk_extractor.vks.download_media.return_value = []
     result = vk_extractor.download(metadata)
@@ -73,4 +74,4 @@ def test_adds_first_title_and_timestamp(vk_extractor):
     assert result.get_title() == "value"
     # formatted timestamp
     assert result.get_timestamp() == "2023-01-01T00:00:00+00:00"
-    assert result.is_success()
\ No newline at end of file
+    assert result.is_success()
diff --git a/tests/feeders/test_atlos_feeder.py b/tests/feeders/test_atlos_feeder.py
index 1ef9fab..f423136 100644
--- a/tests/feeders/test_atlos_feeder.py
+++ b/tests/feeders/test_atlos_feeder.py
@@ -36,29 +36,45 @@ def atlos_feeder(setup_module, mocker) -> AtlosFeeder:
 @pytest.fixture
 def mock_atlos_api(atlos_feeder):
     """Fixture to update the atlos_feeder.session.get side_effect."""
+
     def _mock_responses(responses):
         atlos_feeder.session.get.side_effect = [FakeAPIResponse(data) for data in responses]
+
     return _mock_responses
 
 
 def test_atlos_feeder_iter_yields_valid_metadata(atlos_feeder, mock_atlos_api):
     """Test valid items are yielded and invalid ones ignored."""
-    mock_atlos_api([
-        {
-            "next": None,
-            "results": [
-                {"source_url": "http://example.com", "id": 1,
-                 "metadata": {"auto_archiver": {"processed": False}},
-                 "visibility": "visible", "status": "complete"},
-                {"source_url": "", "id": 2,
-                 "metadata": {"auto_archiver": {"processed": False}},
-                 "visibility": "visible", "status": "complete"},
-                {"source_url": "http://example.org", "id": 3,
-                 "metadata": {"auto_archiver": {"processed": True}},
-                 "visibility": "visible", "status": "complete"},
-            ],
-        }
-    ])
+    mock_atlos_api(
+        [
+            {
+                "next": None,
+                "results": [
+                    {
+                        "source_url": "http://example.com",
+                        "id": 1,
+                        "metadata": {"auto_archiver": {"processed": False}},
+                        "visibility": "visible",
+                        "status": "complete",
+                    },
+                    {
+                        "source_url": "",
+                        "id": 2,
+                        "metadata": {"auto_archiver": {"processed": False}},
+                        "visibility": "visible",
+                        "status": "complete",
+                    },
+                    {
+                        "source_url": "http://example.org",
+                        "id": 3,
+                        "metadata": {"auto_archiver": {"processed": True}},
+                        "visibility": "visible",
+                        "status": "complete",
+                    },
+                ],
+            }
+        ]
+    )
 
     items = list(atlos_feeder)
     assert len(items) == 1
@@ -68,24 +84,34 @@ def test_atlos_feeder_iter_yields_valid_metadata(atlos_feeder, mock_atlos_api):
 
 def test_atlos_feeder_multiple_pages(atlos_feeder, mock_atlos_api):
     """Test iteration over multiple pages with valid items."""
-    mock_atlos_api([
-        {
-            "next": "cursor2",
-            "results": [
-                {"source_url": "http://example1.com", "id": 10,
-                 "metadata": {"auto_archiver": {"processed": False}},
-                 "visibility": "visible", "status": "complete"},
-            ],
-        },
-        {
-            "next": None,
-            "results": [
-                {"source_url": "http://example2.com", "id": 20,
-                 "metadata": {"auto_archiver": {"processed": False}},
-                 "visibility": "visible", "status": "complete"},
-            ],
-        },
-    ])
+    mock_atlos_api(
+        [
+            {
+                "next": "cursor2",
+                "results": [
+                    {
+                        "source_url": "http://example1.com",
+                        "id": 10,
+                        "metadata": {"auto_archiver": {"processed": False}},
+                        "visibility": "visible",
+                        "status": "complete",
+                    },
+                ],
+            },
+            {
+                "next": None,
+                "results": [
+                    {
+                        "source_url": "http://example2.com",
+                        "id": 20,
+                        "metadata": {"auto_archiver": {"processed": False}},
+                        "visibility": "visible",
+                        "status": "complete",
+                    },
+                ],
+            },
+        ]
+    )
 
     items = list(atlos_feeder)
     assert len(items) == 2
diff --git a/tests/feeders/test_csv_feeder.py b/tests/feeders/test_csv_feeder.py
index 546c3a7..965f8ad 100644
--- a/tests/feeders/test_csv_feeder.py
+++ b/tests/feeders/test_csv_feeder.py
@@ -1,13 +1,16 @@
 import pytest
 
+
 @pytest.fixture
 def headerless_csv_file():
     return "tests/data/csv_no_headers.csv"
 
+
 @pytest.fixture
 def header_csv_file():
     return "tests/data/csv_with_headers.csv"
 
+
 @pytest.fixture
 def header_csv_file_non_default_column():
     return "tests/data/csv_with_headers_non_default_column.csv"
@@ -23,6 +26,7 @@ def test_csv_feeder_no_headers(headerless_csv_file, setup_module):
     assert urls[0].get_url() == "https://example.com/1/"
     assert urls[1].get_url() == "https://example.com/2/"
 
+
 def test_csv_feeder_with_headers(header_csv_file, setup_module):
     from auto_archiver.modules.csv_feeder.csv_feeder import CSVFeeder
 
@@ -33,10 +37,10 @@ def test_csv_feeder_with_headers(header_csv_file, setup_module):
     assert urls[0].get_url() == "https://example.com/1/"
     assert urls[1].get_url() == "https://example.com/2/"
 
+
 def test_csv_feeder_wrong_column(header_csv_file, setup_module, caplog):
     from auto_archiver.modules.csv_feeder.csv_feeder import CSVFeeder
 
-
     with caplog.at_level("WARNING"):
         feeder = setup_module(CSVFeeder, {"files": [header_csv_file], "column": 1})
         urls = list(feeder)
@@ -54,4 +58,4 @@ def test_csv_feeder_column_by_name(header_csv_file, setup_module):
     urls = list(feeder)
     assert len(urls) == 2
     assert urls[0].get_url() == "https://example.com/1/"
-    assert urls[1].get_url() == "https://example.com/2/"
\ No newline at end of file
+    assert urls[1].get_url() == "https://example.com/2/"
diff --git a/tests/feeders/test_gsheet_feeder.py b/tests/feeders/test_gsheet_feeder.py
index 9ca81b0..0fc2681 100644
--- a/tests/feeders/test_gsheet_feeder.py
+++ b/tests/feeders/test_gsheet_feeder.py
@@ -19,35 +19,32 @@ def test_setup_without_sheet_and_sheet_id(setup_module, mocker):
 @pytest.fixture
 def gsheet_feeder(setup_module, mocker) -> GsheetsFeederDB:
     config: dict = {
-                "service_account": "dummy.json",
-                "sheet": "test-auto-archiver",
-                "sheet_id": None,
-                "header": 1,
-                "columns": {
-                    "url": "link",
-                    "status": "archive status",
-                    "folder": "destination folder",
-                    "archive": "archive location",
-                    "date": "archive date",
-                    "thumbnail": "thumbnail",
-                    "timestamp": "upload timestamp",
-                    "title": "upload title",
-                    "text": "text content",
-                    "screenshot": "screenshot",
-                    "hash": "hash",
-                    "pdq_hash": "perceptual hashes",
-                    "wacz": "wacz",
-                    "replaywebpage": "replaywebpage",
-                },
-                "allow_worksheets": set(),
-                "block_worksheets": set(),
-                "use_sheet_names_in_stored_paths": True,
-            }
+        "service_account": "dummy.json",
+        "sheet": "test-auto-archiver",
+        "sheet_id": None,
+        "header": 1,
+        "columns": {
+            "url": "link",
+            "status": "archive status",
+            "folder": "destination folder",
+            "archive": "archive location",
+            "date": "archive date",
+            "thumbnail": "thumbnail",
+            "timestamp": "upload timestamp",
+            "title": "upload title",
+            "text": "text content",
+            "screenshot": "screenshot",
+            "hash": "hash",
+            "pdq_hash": "perceptual hashes",
+            "wacz": "wacz",
+            "replaywebpage": "replaywebpage",
+        },
+        "allow_worksheets": set(),
+        "block_worksheets": set(),
+        "use_sheet_names_in_stored_paths": True,
+    }
     mocker.patch("gspread.service_account")
-    feeder = setup_module(
-        "gsheet_feeder_db",
-        config
-    )
+    feeder = setup_module("gsheet_feeder_db", config)
     feeder.gsheets_client = mocker.MagicMock()
     return feeder
 
@@ -128,9 +125,7 @@ def test__set_metadata_with_folder(gsheet_feeder: GsheetsFeederDB):
         (None, "ABC123", "open_by_key", "ABC123", "opening by sheet ID"),
     ],
 )
-def test_open_sheet_with_name_or_id(
-    setup_module, sheet, sheet_id, expected_method, expected_arg, description, mocker
-):
+def test_open_sheet_with_name_or_id(setup_module, sheet, sheet_id, expected_method, expected_arg, description, mocker):
     """Ensure open_sheet() correctly opens by name or ID based on configuration."""
     mock_service_account = mocker.patch("gspread.service_account")
     mock_client = mocker.MagicMock()
@@ -145,9 +140,7 @@ def test_open_sheet_with_name_or_id(
     )
     sheet_result = feeder.open_sheet()
     # Validate the correct method was called
-    getattr(mock_client, expected_method).assert_called_once_with(
-        expected_arg
-    ), f"Failed: {description}"
+    getattr(mock_client, expected_method).assert_called_once_with(expected_arg), f"Failed: {description}"
     assert sheet_result == "MockSheet", f"Failed: {description}"
 
 
@@ -220,9 +213,7 @@ class TestGSheetsFeederReal:
 
     @pytest.fixture(autouse=True)
     def setup_feeder(self, setup_module):
-        assert (
-            self.module_name is not None
-        ), "self.module_name must be set on the subclass"
+        assert self.module_name is not None, "self.module_name must be set on the subclass"
         assert self.config is not None, "self.config must be a dict set on the subclass"
         self.feeder: Type[Feeder] = setup_module(self.module_name, self.config)
 
@@ -241,9 +232,7 @@ class TestGSheetsFeederReal:
         """Ensure open_sheet() connects to a real Google Sheets instance."""
         sheet = self.feeder.open_sheet()
         assert sheet is not None, "open_sheet() should return a valid sheet instance"
-        assert hasattr(
-            sheet, "worksheets"
-        ), "Returned object should have worksheets method"
+        assert hasattr(sheet, "worksheets"), "Returned object should have worksheets method"
 
     def test_iter_yields_metadata_real_data(self):
         """Ensure __iter__() yields Metadata objects for real test sheet data."""
diff --git a/tests/feeders/test_gworksheet.py b/tests/feeders/test_gworksheet.py
index b6a0b5c..c817be3 100644
--- a/tests/feeders/test_gworksheet.py
+++ b/tests/feeders/test_gworksheet.py
@@ -81,40 +81,27 @@ class TestGWorksheet:
             (False, ""),
         ],
     )
-    def test_get_cell_or_default_handles_empty_values(
-        self, mock_worksheet, when_empty, expected
-    ):
+    def test_get_cell_or_default_handles_empty_values(self, mock_worksheet, when_empty, expected):
         mock_worksheet.get_values.return_value[1][0] = ""  # Empty URL cell
         g = GWorksheet(mock_worksheet)
-        assert (
-            g.get_cell_or_default(
-                2, "url", default="default", when_empty_use_default=when_empty
-            )
-            == expected
-        )
+        assert g.get_cell_or_default(2, "url", default="default", when_empty_use_default=when_empty) == expected
 
     def test_get_cell_or_default_handles_missing_columns(self, gworksheet):
-        assert (
-            gworksheet.get_cell_or_default(1, "invalid_col", default="safe") == "safe"
-        )
+        assert gworksheet.get_cell_or_default(1, "invalid_col", default="safe") == "safe"
 
     # Test write operations
     def test_set_cell_updates_correct_position(self, mock_worksheet, gworksheet):
         gworksheet.set_cell(2, "url", "new_url")
         mock_worksheet.update_cell.assert_called_once_with(2, 1, "new_url")
 
-    def test_batch_set_cell_formats_requests_correctly(
-        self, mock_worksheet, gworksheet
-    ):
+    def test_batch_set_cell_formats_requests_correctly(self, mock_worksheet, gworksheet):
         updates = [(2, "url", "new_url"), (3, "status", "processed")]
         gworksheet.batch_set_cell(updates)
         expected_batch = [
             {"range": "A2", "values": [["new_url"]]},
             {"range": "B3", "values": [["processed"]]},
         ]
-        mock_worksheet.batch_update.assert_called_once_with(
-            expected_batch, value_input_option="USER_ENTERED"
-        )
+        mock_worksheet.batch_update.assert_called_once_with(expected_batch, value_input_option="USER_ENTERED")
 
     def test_batch_set_cell_truncates_long_values(self, mock_worksheet, gworksheet):
         long_value = "x" * 50000
diff --git a/tests/formatters/test_html_formatter.py b/tests/formatters/test_html_formatter.py
index 60abaa7..502e231 100644
--- a/tests/formatters/test_html_formatter.py
+++ b/tests/formatters/test_html_formatter.py
@@ -5,13 +5,13 @@ from auto_archiver.core import Metadata, Media
 def test_format(setup_module):
     formatter = setup_module(HtmlFormatter)
 
-    metadata = Metadata().set("content", "Hello, world!").set_url('https://example.com')
+    metadata = Metadata().set("content", "Hello, world!").set_url("https://example.com")
 
     final_media = formatter.format(metadata)
     assert isinstance(final_media, Media)
     assert ".html" in final_media.filename
-    with open (final_media.filename, "r", encoding="utf-8") as f:
+    with open(final_media.filename, "r", encoding="utf-8") as f:
         content = f.read()
         assert "Hello, world!" in content
     assert final_media.mimetype == "text/html"
-    assert "SHA-256:" in final_media.get('hash')
\ No newline at end of file
+    assert "SHA-256:" in final_media.get("hash")
diff --git a/tests/storages/test_S3_storage.py b/tests/storages/test_S3_storage.py
index fe60329..b40709f 100644
--- a/tests/storages/test_S3_storage.py
+++ b/tests/storages/test_S3_storage.py
@@ -8,6 +8,7 @@ class TestS3Storage:
     """
     Test suite for S3Storage.
     """
+
     module_name: str = "s3_storage"
     storage: Type[S3Storage]
     config: dict = {
@@ -32,10 +33,10 @@ class TestS3Storage:
         """Test that S3 client is initialized with correct parameters"""
 
         assert self.storage.s3 is not None
-        assert self.storage.s3.meta.region_name == 'test-region'
+        assert self.storage.s3.meta.region_name == "test-region"
 
     def test_get_cdn_url_generation(self):
-        """Test CDN URL formatting """
+        """Test CDN URL formatting"""
         media = Media("test.txt")
         media.key = "path/to/file.txt"
         url = self.storage.get_cdn_url(media)
@@ -46,14 +47,14 @@ class TestS3Storage:
     def test_uploadf_sets_acl_public(self, mocker):
         media = Media("test.txt")
         mock_file = mocker.MagicMock()
-        mock_s3_upload = mocker.patch.object(self.storage.s3, 'upload_fileobj')
-        mocker.patch.object(self.storage, 'is_upload_needed', return_value=True)
+        mock_s3_upload = mocker.patch.object(self.storage.s3, "upload_fileobj")
+        mocker.patch.object(self.storage, "is_upload_needed", return_value=True)
         self.storage.uploadf(mock_file, media)
         mock_s3_upload.assert_called_once_with(
             mock_file,
-            Bucket='test-bucket',
+            Bucket="test-bucket",
             Key=media.key,
-            ExtraArgs={'ACL': 'public-read', 'ContentType': 'text/plain'}
+            ExtraArgs={"ACL": "public-read", "ContentType": "text/plain"},
         )
 
     def test_upload_decision_logic(self, mocker):
@@ -61,23 +62,31 @@ class TestS3Storage:
         media = Media("test.txt")
         assert self.storage.is_upload_needed(media) is True
         self.storage.random_no_duplicate = True
-        mock_calc_hash = mocker.patch('auto_archiver.modules.s3_storage.s3_storage.calculate_file_hash', return_value='beepboop123beepboop123beepboop123')
-        mock_file_in_folder = mocker.patch.object(self.storage, 'file_in_folder', return_value='existing_key.txt')
+        mock_calc_hash = mocker.patch(
+            "auto_archiver.modules.s3_storage.s3_storage.calculate_file_hash",
+            return_value="beepboop123beepboop123beepboop123",
+        )
+        mock_file_in_folder = mocker.patch.object(self.storage, "file_in_folder", return_value="existing_key.txt")
         assert self.storage.is_upload_needed(media) is False
-        assert media.key == 'existing_key.txt'
-        mock_file_in_folder.assert_called_with('no-dups/beepboop123beepboop123be')
+        assert media.key == "existing_key.txt"
+        mock_file_in_folder.assert_called_with("no-dups/beepboop123beepboop123be")
 
     def test_skips_upload_when_duplicate_exists(self, mocker):
         """Test that upload skips when file_in_folder finds existing object"""
         self.storage.random_no_duplicate = True
-        mock_file_in_folder = mocker.patch.object(S3Storage, 'file_in_folder', return_value="existing_folder/existing_file.txt")
+        mock_file_in_folder = mocker.patch.object(
+            S3Storage, "file_in_folder", return_value="existing_folder/existing_file.txt"
+        )
         media = Media("test.txt")
         media.key = "original_path.txt"
-        mock_calculate_hash = mocker.patch('auto_archiver.modules.s3_storage.s3_storage.calculate_file_hash', return_value="beepboop123beepboop123beepboop123")
+        mock_calculate_hash = mocker.patch(
+            "auto_archiver.modules.s3_storage.s3_storage.calculate_file_hash",
+            return_value="beepboop123beepboop123beepboop123",
+        )
         assert self.storage.is_upload_needed(media) is False
         assert media.key == "existing_folder/existing_file.txt"
         assert media.get("previously archived") is True
-        mock_upload = mocker.patch.object(self.storage.s3, 'upload_fileobj')
+        mock_upload = mocker.patch.object(self.storage.s3, "upload_fileobj")
         result = self.storage.uploadf(None, media)
         mock_upload.assert_not_called()
         assert result is True
@@ -85,21 +94,20 @@ class TestS3Storage:
     def test_uploads_with_correct_parameters(self, mocker):
         media = Media("test.txt")
         media.key = "original_key.txt"
-        mocker.patch.object(S3Storage, 'is_upload_needed', return_value=True)
-        media.mimetype = 'image/png'
+        mocker.patch.object(S3Storage, "is_upload_needed", return_value=True)
+        media.mimetype = "image/png"
         mock_file = mocker.MagicMock()
-        mock_upload = mocker.patch.object(self.storage.s3, 'upload_fileobj')
+        mock_upload = mocker.patch.object(self.storage.s3, "upload_fileobj")
         self.storage.uploadf(mock_file, media)
         mock_upload.assert_called_once_with(
             mock_file,
-            Bucket='test-bucket',
-            Key='original_key.txt',
-            ExtraArgs={
-                'ACL': 'public-read',
-                'ContentType': 'image/png'
-            }
+            Bucket="test-bucket",
+            Key="original_key.txt",
+            ExtraArgs={"ACL": "public-read", "ContentType": "image/png"},
         )
 
     def test_file_in_folder_exists(self, mocker):
-        mock_list_objects = mocker.patch.object(self.storage.s3, 'list_objects', return_value={'Contents': [{'Key': 'path/to/file.txt'}]})
-        assert self.storage.file_in_folder('path/to/') == 'path/to/file.txt'
+        mock_list_objects = mocker.patch.object(
+            self.storage.s3, "list_objects", return_value={"Contents": [{"Key": "path/to/file.txt"}]}
+        )
+        assert self.storage.file_in_folder("path/to/") == "path/to/file.txt"
diff --git a/tests/storages/test_atlos_storage.py b/tests/storages/test_atlos_storage.py
index bcd8f18..6d8444d 100644
--- a/tests/storages/test_atlos_storage.py
+++ b/tests/storages/test_atlos_storage.py
@@ -101,7 +101,9 @@ def test_upload_not_uploaded(tmp_path, atlos_storage: AtlosStorage, metadata: Me
     assert file_tuple[0] == os.path.basename(media.filename)
 
 
-def test_upload_post_http_error(tmp_path, atlos_storage: AtlosStorage, metadata: Metadata, media: Media, mocker) -> None:
+def test_upload_post_http_error(
+    tmp_path, atlos_storage: AtlosStorage, metadata: Metadata, media: Media, mocker
+) -> None:
     """Test upload() propagates HTTP error during POST."""
     metadata.set("atlos_id", 303)
     fake_get_response = {"result": {"artifacts": []}}
@@ -109,4 +111,3 @@ def test_upload_post_http_error(tmp_path, atlos_storage: AtlosStorage, metadata:
     mocker.patch.object(atlos_storage, "_post", side_effect=Exception("HTTP error"))
     with pytest.raises(Exception, match="HTTP error"):
         atlos_storage.upload(media, metadata)
-
diff --git a/tests/storages/test_gdrive_storage.py b/tests/storages/test_gdrive_storage.py
index f5ff87c..48b3029 100644
--- a/tests/storages/test_gdrive_storage.py
+++ b/tests/storages/test_gdrive_storage.py
@@ -12,26 +12,28 @@ from tests.storages.test_storage_base import TestStorageBase
 def gdrive_storage(setup_module, mocker):
     module_name: str = "gdrive_storage"
     storage: GDriveStorage
-    config: dict = {'path_generator': 'url',
-            'filename_generator': 'static',
-            'root_folder_id': "fake_root_folder_id",
-            'oauth_token': None,
-            'service_account': 'fake_service_account.json'
-                    }
-    mocker.patch('google.oauth2.service_account.Credentials.from_service_account_file')
+    config: dict = {
+        "path_generator": "url",
+        "filename_generator": "static",
+        "root_folder_id": "fake_root_folder_id",
+        "oauth_token": None,
+        "service_account": "fake_service_account.json",
+    }
+    mocker.patch("google.oauth2.service_account.Credentials.from_service_account_file")
     return setup_module(module_name, config)
 
 
 def test_initialize_fails_with_non_existent_creds(setup_module):
     """Test that the Google Drive service raises a FileNotFoundError when the service account file does not exist.
-        (and isn't mocked)
+    (and isn't mocked)
     """
-    config: dict = {'path_generator': 'url',
-                    'filename_generator': 'static',
-                    'root_folder_id': "fake_root_folder_id",
-                    'oauth_token': None,
-                    'service_account': 'fake_service_account.json'
-                    }
+    config: dict = {
+        "path_generator": "url",
+        "filename_generator": "static",
+        "root_folder_id": "fake_root_folder_id",
+        "oauth_token": None,
+        "service_account": "fake_service_account.json",
+    }
     with pytest.raises(FileNotFoundError) as exc_info:
         setup_module("gdrive_storage", config)
     assert "No such file or directory" in str(exc_info.value)
@@ -48,12 +50,12 @@ def test_get_id_from_parent_and_name(gdrive_storage, mocker):
     result = gdrive_storage._get_id_from_parent_and_name("parent", "mock", retries=1, use_mime_type=False)
     assert result == "123"
 
+
 def test_path_parts():
     media = Media(filename="test.jpg")
     media.key = "folder1/folder2/test.jpg"
 
 
-
 @pytest.mark.skip(reason="Requires real credentials")
 @pytest.mark.download
 class TestGDriveStorageConnected(TestStorageBase):
@@ -63,19 +65,17 @@ class TestGDriveStorageConnected(TestStorageBase):
 
     module_name: str = "gdrive_storage"
     storage: Type[GDriveStorage]
-    config: dict = {'path_generator': 'url',
-            'filename_generator': 'static',
-            # TODO: replace with real root folder id
-            'root_folder_id': "1TVY_oJt95_dmRSEdP9m5zFy7l50TeCSk",
-            'oauth_token': None,
-            'service_account': 'secrets/service_account.json'
-                    }
-
+    config: dict = {
+        "path_generator": "url",
+        "filename_generator": "static",
+        # TODO: replace with real root folder id
+        "root_folder_id": "1TVY_oJt95_dmRSEdP9m5zFy7l50TeCSk",
+        "oauth_token": None,
+        "service_account": "secrets/service_account.json",
+    }
 
     def test_initialize_with_real_credentials(self):
         """
         Test that the Google Drive service can be initialized with real credentials.
         """
         assert self.storage.service is not None
-
-
diff --git a/tests/storages/test_local_storage.py b/tests/storages/test_local_storage.py
index 7617867..6fe30b5 100644
--- a/tests/storages/test_local_storage.py
+++ b/tests/storages/test_local_storage.py
@@ -1,4 +1,3 @@
-
 import os
 from pathlib import Path
 
@@ -34,13 +33,13 @@ def test_get_cdn_url_relative(local_storage):
     assert local_storage.get_cdn_url(media) == expected
 
 
-
 def test_get_cdn_url_absolute(local_storage):
     media = Media(key="test.txt", filename="dummy.txt")
     local_storage.save_absolute = True
     expected = os.path.abspath(os.path.join(local_storage.save_to, media.key))
     assert local_storage.get_cdn_url(media) == expected
 
+
 def test_upload_file_contents_and_metadata(local_storage, sample_media):
     dest = os.path.join(local_storage.save_to, sample_media.key)
     assert local_storage.upload(sample_media) is True
@@ -51,5 +50,3 @@ def test_upload_nonexistent_source(local_storage):
     media = Media(key="missing.txt", filename="nonexistent.txt")
     with pytest.raises(FileNotFoundError):
         local_storage.upload(media)
-
-
diff --git a/tests/storages/test_storage_base.py b/tests/storages/test_storage_base.py
index 7578acd..1acbcdd 100644
--- a/tests/storages/test_storage_base.py
+++ b/tests/storages/test_storage_base.py
@@ -7,16 +7,11 @@ from auto_archiver.core.storage import Storage
 
 
 class TestStorageBase(object):
-
     module_name: str = None
     config: dict = None
 
     @pytest.fixture(autouse=True)
     def setup_storage(self, setup_module):
-        assert (
-            self.module_name is not None
-        ), "self.module_name must be set on the subclass"
+        assert self.module_name is not None, "self.module_name must be set on the subclass"
         assert self.config is not None, "self.config must be a dict set on the subclass"
-        self.storage: Type[Storage] = setup_module(
-            self.module_name, self.config
-        )
+        self.storage: Type[Storage] = setup_module(self.module_name, self.config)
diff --git a/tests/test_config.py b/tests/test_config.py
index 75fe515..0de4f16 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -3,39 +3,46 @@ from auto_archiver.core import config
 from ruamel.yaml.scanner import ScannerError
 from ruamel.yaml.comments import CommentedMap
 
+
 def test_return_default_config_for_nonexistent_file():
     assert config.read_yaml("nonexistent_file.yaml") == config.EMPTY_CONFIG
 
+
 def test_return_default_config_for_empty_file(tmp_path):
     empty_file = tmp_path / "empty_file.yaml"
     empty_file.write_text("")
     assert config.read_yaml(empty_file) == config.EMPTY_CONFIG
 
+
 def test_raise_error_on_invalid_yaml(tmp_path):
     invalid_yaml = tmp_path / "invalid_yaml.yaml"
-    invalid_yaml.write_text("key: \"value_without_end_quote")
+    invalid_yaml.write_text('key: "value_without_end_quote')
     # make sure it raises ScannerError
     with pytest.raises(ScannerError):
         config.read_yaml(invalid_yaml)
 
+
 def test_write_yaml(tmp_path):
     yaml_file = tmp_path / "write_yaml.yaml"
     config.store_yaml(config.EMPTY_CONFIG, yaml_file.as_posix())
     assert "steps:\n" in yaml_file.read_text()
 
+
 def test_round_trip_comments(tmp_path):
     yaml_file = tmp_path / "round_trip_comments.yaml"
 
     with open(yaml_file, "w") as f:
-        f.write("generic_extractor:\n  facebook_cookie: abc # end of line comment\n  subtitles: true\n  # comments: false\n  # livestreams: false\n  list_type:\n    - value1\n    - value2")
+        f.write(
+            "generic_extractor:\n  facebook_cookie: abc # end of line comment\n  subtitles: true\n  # comments: false\n  # livestreams: false\n  list_type:\n    - value1\n    - value2"
+        )
 
     loaded = config.read_yaml(yaml_file)
     # check the comments are preserved
-    assert loaded['generic_extractor']['facebook_cookie'] == "abc"
-    assert loaded['generic_extractor'].ca.items['facebook_cookie'][2].value == "# end of line comment\n"
+    assert loaded["generic_extractor"]["facebook_cookie"] == "abc"
+    assert loaded["generic_extractor"].ca.items["facebook_cookie"][2].value == "# end of line comment\n"
 
     # add some more items to my_settings
-    loaded['generic_extractor']['list_type'].append("bellingcat")
+    loaded["generic_extractor"]["list_type"].append("bellingcat")
     config.store_yaml(loaded, yaml_file.as_posix())
 
     assert "# comments: false" in yaml_file.read_text()
@@ -43,14 +50,17 @@ def test_round_trip_comments(tmp_path):
     assert "abc # end of line comment" in yaml_file.read_text()
     assert "- value2\n  - bellingcat" in yaml_file.read_text()
 
+
 def test_merge_dicts():
     yaml_dict = config.EMPTY_CONFIG
-    yaml_dict['settings'] = CommentedMap(**{
+    yaml_dict["settings"] = CommentedMap(
+        **{
             "key1": ["a"],
             "key2": "old_value",
             "key3": ["a", "b", "c"],
             "key5": "value5",
-        })
+        }
+    )
 
     dotdict = {
         "settings.key1": ["b", "c"],
@@ -77,6 +87,7 @@ def test_check_types():
     assert config.is_dict_type([]) == False
     assert config.is_dict_type("") == False
 
+
 def test_from_dot_notation():
     dotdict = {
         "settings.key1": ["a", "b", "c"],
@@ -88,16 +99,17 @@ def test_from_dot_notation():
     assert normal_dict["settings"]["key2"] == "new_value"
     assert normal_dict["settings"]["key3"]["key4"] == "value"
 
+
 def test_to_dot_notation():
     yaml_dict = config.EMPTY_CONFIG
-    yaml_dict['settings'] = {
+    yaml_dict["settings"] = {
         "key1": ["a", "b", "c"],
         "key2": "new_value",
         "key3": {
             "key4": "value",
-        }
+        },
     }
     dotdict = config.to_dot_notation(yaml_dict)
     assert dotdict["settings.key1"] == ["a", "b", "c"]
     assert dotdict["settings.key2"] == "new_value"
-    assert dotdict["settings.key3.key4"] == "value"
\ No newline at end of file
+    assert dotdict["settings.key3.key4"] == "value"
diff --git a/tests/test_implementation.py b/tests/test_implementation.py
index 51e9d79..973d53d 100644
--- a/tests/test_implementation.py
+++ b/tests/test_implementation.py
@@ -10,21 +10,23 @@ def orchestration_file_path(tmp_path):
     folder.mkdir(exist_ok=True)
     return (folder / "example_orch.yaml").as_posix()
 
+
 @pytest.fixture
 def orchestration_file(orchestration_file_path):
-    def _orchestration_file(content=''):
+    def _orchestration_file(content=""):
         with open(orchestration_file_path, "w") as f:
             f.write(content)
         return orchestration_file_path
-    
+
     return _orchestration_file
 
+
 @pytest.fixture
 def autoarchiver(tmp_path, monkeypatch, request):
     def _autoarchiver(args=[]):
-
         def cleanup():
             from loguru import logger
+
             if not logger._core.handlers.get(0):
                 logger._core.handlers_count = 0
                 logger.add(sys.stderr)
@@ -47,6 +49,7 @@ def test_run_auto_archiver_no_args(caplog, autoarchiver):
 
     assert "provide at least one URL via the command line, or set up an alternative feeder" in caplog.text
 
+
 def test_run_auto_archiver_invalid_file(caplog, autoarchiver):
     # exec 'auto-archiver' on the command lin
     with pytest.raises(SystemExit):
@@ -54,6 +57,7 @@ def test_run_auto_archiver_invalid_file(caplog, autoarchiver):
 
     assert "Make sure the file exists and try again, or run without th" in caplog.text
 
+
 def test_run_auto_archiver_empty_file(caplog, autoarchiver, orchestration_file):
     # create a valid (empty) orchestration file
     path = orchestration_file(content="")
@@ -64,6 +68,7 @@ def test_run_auto_archiver_empty_file(caplog, autoarchiver, orchestration_file):
     # should treat an empty file as if there is no file at all
     assert " No URLs provided. Please provide at least one URL via the com" in caplog.text
 
+
 def test_call_autoarchiver_main(caplog, monkeypatch, tmp_path):
     from auto_archiver.__main__ import main
 
@@ -75,4 +80,4 @@ def test_call_autoarchiver_main(caplog, monkeypatch, tmp_path):
         with pytest.raises(SystemExit):
             main()
 
-    assert "No URLs provided. Please provide at least one" in caplog.text
\ No newline at end of file
+    assert "No URLs provided. Please provide at least one" in caplog.text
diff --git a/tests/test_metadata.py b/tests/test_metadata.py
index e1f7797..e838979 100644
--- a/tests/test_metadata.py
+++ b/tests/test_metadata.py
@@ -62,18 +62,8 @@ def test_simple_merge(basic_metadata):
 
 
 def test_left_merge():
-    left = (
-        Metadata()
-        .set("tags", ["a"])
-        .set("stats", {"views": 10})
-        .set("status", "success")
-    )
-    right = (
-        Metadata()
-        .set("tags", ["b"])
-        .set("stats", {"likes": 5})
-        .set("status", "no archiver")
-    )
+    left = Metadata().set("tags", ["a"]).set("stats", {"views": 10}).set("status", "success")
+    right = Metadata().set("tags", ["b"]).set("stats", {"likes": 5}).set("status", "no archiver")
 
     left.merge(right, overwrite_left=True)
     assert left.get("status") == "no archiver"
@@ -120,6 +110,7 @@ def test_is_empty():
 def test_store():
     pass
 
+
 # Test Media operations
 
 
@@ -176,6 +167,7 @@ def test_choose_most_complete():
     res = Metadata.choose_most_complete([m_more, m_less])
     assert res.metadata.get("title") == "Title 1"
 
+
 def test_choose_most_complete_from_pickles(unpickle):
     # test most complete from pickles before and after an enricher has run
     # Only compares length of media, not the actual media
diff --git a/tests/test_modules.py b/tests/test_modules.py
index 7a2b14d..1ff4f45 100644
--- a/tests/test_modules.py
+++ b/tests/test_modules.py
@@ -3,6 +3,7 @@ import pytest
 from auto_archiver.core.module import ModuleFactory, LazyBaseModule
 from auto_archiver.core.base_module import BaseModule
 
+
 @pytest.fixture
 def example_module():
     import auto_archiver
@@ -14,12 +15,14 @@ def example_module():
 
     return module_factory.get_module_lazy("example_module")
 
+
 def test_get_module_lazy(example_module):
     assert example_module.name == "example_module"
     assert example_module.display_name == "Example Module"
 
     assert example_module.manifest is not None
 
+
 def test_python_dependency_check(example_module):
     # example_module requires loguru, which is not installed
     # monkey patch the manifest to include a nonexistnet dependency
@@ -30,11 +33,13 @@ def test_python_dependency_check(example_module):
 
     assert load_error.value.code == 1
 
+
 def test_binary_dependency_check(example_module):
     # example_module requires ffmpeg, which is not installed
     # monkey patch the manifest to include a nonexistnet dependency
     example_module.manifest["dependencies"]["binary"] = ["does_not_exist"]
 
+
 def test_module_dependency_check_loads_module(example_module):
     # example_module requires cli_feeder, which is not installed
     # monkey patch the manifest to include a nonexistnet dependency
@@ -49,19 +54,20 @@ def test_module_dependency_check_loads_module(example_module):
     assert module_factory._lazy_modules["hash_enricher"] is not None
     assert module_factory._lazy_modules["hash_enricher"]._instance is not None
 
-def test_load_module(example_module):
 
+def test_load_module(example_module):
     # setup the module, and check that config is set to the default values
     loaded_module = example_module.load({})
     assert loaded_module is not None
     assert isinstance(loaded_module, BaseModule)
     assert loaded_module.name == "example_module"
     assert loaded_module.display_name == "Example Module"
-    assert loaded_module.config["example_module"] ==  {"csv_file" : "db.csv"}
+    assert loaded_module.config["example_module"] == {"csv_file": "db.csv"}
 
     # check that the vlaue is set on the module itself
     assert loaded_module.csv_file == "db.csv"
 
+
 @pytest.mark.parametrize("module_name", ["local_storage", "generic_extractor", "html_formatter", "csv_db"])
 def test_load_modules(module_name):
     # test that specific modules can be loaded
@@ -96,5 +102,3 @@ def test_lazy_base_module(module_name):
     assert len(lazy_module.configs) > 0
     assert len(lazy_module.description) > 0
     assert len(lazy_module.version) > 0
-
-
diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py
index 72f4949..64d84d8 100644
--- a/tests/test_orchestrator.py
+++ b/tests/test_orchestrator.py
@@ -9,49 +9,63 @@ from auto_archiver.core import Metadata
 TEST_ORCHESTRATION = "tests/data/test_orchestration.yaml"
 TEST_MODULES = "tests/data/test_modules/"
 
+
 @pytest.fixture
 def test_args():
-    return ["--config", TEST_ORCHESTRATION,
-            "--module_paths", TEST_MODULES,
-            "--example_module.required_field", "some_value"] # just set this for normal testing, we will remove it later
+    return [
+        "--config",
+        TEST_ORCHESTRATION,
+        "--module_paths",
+        TEST_MODULES,
+        "--example_module.required_field",
+        "some_value",
+    ]  # just set this for normal testing, we will remove it later
+
 
 @pytest.fixture
 def orchestrator():
     return ArchivingOrchestrator()
 
+
 @pytest.fixture
 def basic_parser(orchestrator) -> ArgumentParser:
     return orchestrator.setup_basic_parser()
 
+
 def test_setup_orchestrator(orchestrator):
     assert orchestrator is not None
 
+
 def test_parse_config():
     pass
 
+
 def test_parse_basic(basic_parser):
     args = basic_parser.parse_args(["--config", TEST_ORCHESTRATION])
     assert args.config_file == TEST_ORCHESTRATION
 
+
 @pytest.mark.parametrize("mode", ["simple", "full"])
 def test_mode(basic_parser, mode):
     args = basic_parser.parse_args(["--mode", mode])
     assert args.mode == mode
 
+
 def test_mode_invalid(basic_parser, capsys):
     with pytest.raises(SystemExit) as exit_error:
         basic_parser.parse_args(["--mode", "invalid"])
     assert exit_error.value.code == 2
     assert "invalid choice" in capsys.readouterr().err
 
+
 def test_version(basic_parser, capsys):
     with pytest.raises(SystemExit) as exit_error:
         basic_parser.parse_args(["--version"])
     assert exit_error.value.code == 0
     assert capsys.readouterr().out == f"{__version__}\n"
 
-def test_help(orchestrator, basic_parser, capsys):
 
+def test_help(orchestrator, basic_parser, capsys):
     args = basic_parser.parse_args(["--help"])
     assert args.help == True
 
@@ -83,14 +97,17 @@ def test_help(orchestrator, basic_parser, capsys):
 
 def test_add_custom_modules_path(orchestrator, test_args):
     orchestrator.setup_config(test_args)
-    
+
     import auto_archiver
+
     assert "tests/data/test_modules/" in auto_archiver.modules.__path__
 
-def test_add_custom_modules_path_invalid(orchestrator, caplog, test_args):
 
-    orchestrator.setup_config(test_args +  # we still need to load the real path to get the example_module 
-                          ["--module_paths", "tests/data/invalid_test_modules/"])
+def test_add_custom_modules_path_invalid(orchestrator, caplog, test_args):
+    orchestrator.setup_config(
+        test_args  # we still need to load the real path to get the example_module
+        + ["--module_paths", "tests/data/invalid_test_modules/"]
+    )
 
     assert caplog.records[0].message == "Path 'tests/data/invalid_test_modules/' does not exist. Skipping..."
 
@@ -104,11 +121,11 @@ def test_check_required_values(orchestrator, caplog, test_args):
 
     assert caplog.records[1].message == "the following arguments are required: --example_module.required_field"
 
-def test_get_required_values_from_config(orchestrator, test_args, tmp_path):
 
+def test_get_required_values_from_config(orchestrator, test_args, tmp_path):
     # load the default example yaml, add a required field, then run the orchestrator
     test_yaml = read_yaml(TEST_ORCHESTRATION)
-    test_yaml['example_module'] = {'required_field': 'some_value'}
+    test_yaml["example_module"] = {"required_field": "some_value"}
     # write it to a temp file
     tmp_file = (tmp_path / "temp_config.yaml").as_posix()
     store_yaml(test_yaml, tmp_file)
@@ -117,27 +134,42 @@ def test_get_required_values_from_config(orchestrator, test_args, tmp_path):
     config = orchestrator.setup_config(["--config", tmp_file, "--module_paths", TEST_MODULES])
     assert config is not None
 
-def test_load_authentication_string(orchestrator, test_args):
 
-    config = orchestrator.setup_config(test_args + ["--authentication", '{"facebook.com": {"username": "my_username", "password": "my_password"}}'])
-    assert config['authentication'] == {"facebook.com": {"username": "my_username", "password": "my_password"}}
+def test_load_authentication_string(orchestrator, test_args):
+    config = orchestrator.setup_config(
+        test_args + ["--authentication", '{"facebook.com": {"username": "my_username", "password": "my_password"}}']
+    )
+    assert config["authentication"] == {"facebook.com": {"username": "my_username", "password": "my_password"}}
+
 
 def test_load_authentication_string_concat_site(orchestrator, test_args):
-    
     config = orchestrator.setup_config(test_args + ["--authentication", '{"x.com,twitter.com": {"api_key": "my_key"}}'])
-    assert config['authentication'] == {"x.com": {"api_key": "my_key"},
-                                                     "twitter.com": {"api_key": "my_key"}}
+    assert config["authentication"] == {"x.com": {"api_key": "my_key"}, "twitter.com": {"api_key": "my_key"}}
+
 
 def test_load_invalid_authentication_string(orchestrator, test_args):
     with pytest.raises(ArgumentTypeError):
-        orchestrator.setup_config(test_args + ["--authentication", "{\''invalid_json"])
+        orchestrator.setup_config(test_args + ["--authentication", "{''invalid_json"])
+
 
 def test_load_authentication_invalid_dict(orchestrator, test_args):
     with pytest.raises(ArgumentTypeError):
         orchestrator.setup_config(test_args + ["--authentication", "[true, false]"])
 
+
 def test_load_modules_from_commandline(orchestrator, test_args):
-    args = test_args + ["--feeders", "example_module", "--extractors", "example_module", "--databases", "example_module", "--enrichers", "example_module", "--formatters", "example_module"]
+    args = test_args + [
+        "--feeders",
+        "example_module",
+        "--extractors",
+        "example_module",
+        "--databases",
+        "example_module",
+        "--enrichers",
+        "example_module",
+        "--formatters",
+        "example_module",
+    ]
 
     orchestrator.setup(args)
 
@@ -153,27 +185,37 @@ def test_load_modules_from_commandline(orchestrator, test_args):
     assert orchestrator.enrichers[0].name == "example_module"
     assert orchestrator.formatters[0].name == "example_module"
 
+
 def test_load_settings_for_module_from_commandline(orchestrator, test_args):
-    args = test_args + ["--feeders", "gsheet_feeder_db", "--gsheet_feeder_db.sheet_id", "123", "--gsheet_feeder_db.service_account", "tests/data/test_service_account.json"]
+    args = test_args + [
+        "--feeders",
+        "gsheet_feeder_db",
+        "--gsheet_feeder_db.sheet_id",
+        "123",
+        "--gsheet_feeder_db.service_account",
+        "tests/data/test_service_account.json",
+    ]
 
     orchestrator.setup(args)
 
     assert len(orchestrator.feeders) == 1
     assert orchestrator.feeders[0].name == "gsheet_feeder_db"
-    assert orchestrator.config['gsheet_feeder_db']['sheet_id'] == "123"
+    assert orchestrator.config["gsheet_feeder_db"]["sheet_id"] == "123"
 
 
 def test_multiple_orchestrator(test_args):
-
-    o1_args = test_args + ["--feeders", "gsheet_feeder_db", "--gsheet_feeder_db.service_account", "tests/data/test_service_account.json"]
+    o1_args = test_args + [
+        "--feeders",
+        "gsheet_feeder_db",
+        "--gsheet_feeder_db.service_account",
+        "tests/data/test_service_account.json",
+    ]
     o1 = ArchivingOrchestrator()
 
     with pytest.raises(ValueError) as exit_error:
         # this should fail because the gsheet_feeder_db requires a sheet_id / sheet
         o1.setup(o1_args)
 
-
-
     o2_args = test_args + ["--feeders", "example_module"]
     o2 = ArchivingOrchestrator()
     o2.setup(o2_args)
@@ -182,4 +224,4 @@ def test_multiple_orchestrator(test_args):
 
     output: Metadata = list(o2.feed())
     assert len(output) == 1
-    assert output[0].get_url() == "https://example.com"
\ No newline at end of file
+    assert output[0].get_url() == "https://example.com"
diff --git a/tests/utils/test_misc.py b/tests/utils/test_misc.py
index 0023077..844d3d2 100644
--- a/tests/utils/test_misc.py
+++ b/tests/utils/test_misc.py
@@ -14,7 +14,7 @@ from auto_archiver.utils.misc import (
     update_nested_dict,
     calculate_file_hash,
     random_str,
-    get_timestamp
+    get_timestamp,
 )
 
 
@@ -38,40 +38,46 @@ class TestDirectoryUtils:
         mkdir_if_not_exists(existing_dir)
         assert existing_dir.exists()
 
+
 class TestURLExpansion:
-    @pytest.mark.parametrize("input_url,expected", [
-        ("https://example.com", "https://example.com"),
-        ("https://t.co/test", "https://expanded.url")
-    ])
+    @pytest.mark.parametrize(
+        "input_url,expected",
+        [("https://example.com", "https://example.com"), ("https://t.co/test", "https://expanded.url")],
+    )
     def test_expand_url(self, input_url, expected, mocker):
         mock_response = mocker.Mock()
         mock_response.url = "https://expanded.url"
-        mocker.patch('requests.get', return_value=mock_response)
+        mocker.patch("requests.get", return_value=mock_response)
         result = expand_url(input_url)
         assert result == expected
 
     def test_expand_url_handles_errors(self, caplog, mocker):
-        mocker.patch('requests.get', side_effect=Exception("Connection error"))
+        mocker.patch("requests.get", side_effect=Exception("Connection error"))
         url = "https://t.co/error"
         result = expand_url(url)
         assert result == url
         assert f"Failed to expand url {url}" in caplog.text
 
+
 class TestAttributeHandling:
     class Sample:
         exists = "value"
         none = None
 
-    @pytest.mark.parametrize("obj,attr,default,expected", [
-        (Sample(), "exists", "default", "value"),
-        (Sample(), "none", "default", "default"),
-        (Sample(), "missing", "default", "default"),
-        (None, "anything", "fallback", "fallback"),
-    ])
+    @pytest.mark.parametrize(
+        "obj,attr,default,expected",
+        [
+            (Sample(), "exists", "default", "value"),
+            (Sample(), "none", "default", "default"),
+            (Sample(), "missing", "default", "default"),
+            (None, "anything", "fallback", "fallback"),
+        ],
+    )
     def test_getattr_or(self, obj, attr, default, expected):
         # Test gets attribute or returns a default value
         assert getattr_or(obj, attr, default) == expected
 
+
 class TestDateTimeHandling:
     def test_datetime_encoder(self, sample_datetime):
         result = json.dumps({"dt": sample_datetime}, cls=DateTimeEncoder)
@@ -83,11 +89,14 @@ class TestDateTimeHandling:
         result = dump_payload(payload)
         assert str(sample_datetime) in result
 
-    @pytest.mark.parametrize("dt_str,fmt,expected", [
-        ("2023-01-01 12:00:00+00:00", None, datetime(2023, 1, 1, 12, 0, tzinfo=timezone.utc)),
-        ("20230101 120000", "%Y%m%d %H%M%S", datetime(2023, 1, 1, 12, 0)),
-        ("invalid", None, None),
-    ])
+    @pytest.mark.parametrize(
+        "dt_str,fmt,expected",
+        [
+            ("2023-01-01 12:00:00+00:00", None, datetime(2023, 1, 1, 12, 0, tzinfo=timezone.utc)),
+            ("20230101 120000", "%Y%m%d %H%M%S", datetime(2023, 1, 1, 12, 0)),
+            ("invalid", None, None),
+        ],
+    )
     def test_datetime_from_string(self, dt_str, fmt, expected):
         result = get_datetime_from_str(dt_str, fmt)
         if expected is None:
@@ -95,16 +104,21 @@ class TestDateTimeHandling:
         else:
             assert result == expected.replace(tzinfo=result.tzinfo)
 
+
 class TestDictUtils:
-    @pytest.mark.parametrize("original,update,expected", [
-        ({"a": 1}, {"b": 2}, {"a": 1, "b": 2}),
-        ({"nested": {"a": 1}}, {"nested": {"b": 2}}, {"nested": {"a": 1, "b": 2}}),
-        ({"a": {"b": {"c": 1}}}, {"a": {"b": {"c": 2}}}, {"a": {"b": {"c": 2}}}),
-    ])
+    @pytest.mark.parametrize(
+        "original,update,expected",
+        [
+            ({"a": 1}, {"b": 2}, {"a": 1, "b": 2}),
+            ({"nested": {"a": 1}}, {"nested": {"b": 2}}, {"nested": {"a": 1, "b": 2}}),
+            ({"a": {"b": {"c": 1}}}, {"a": {"b": {"c": 2}}}, {"a": {"b": {"c": 2}}}),
+        ],
+    )
     def test_update_nested_dict(self, original, update, expected):
         update_nested_dict(original, update)
         assert original == expected
 
+
 class TestHashingUtils:
     def test_file_hashing(self, sample_file):
         expected = hashlib.sha256(b"test content").hexdigest()
@@ -118,6 +132,7 @@ class TestHashingUtils:
         expected = hashlib.sha256(content).hexdigest()
         assert calculate_file_hash(str(file_path)) == expected
 
+
 class TestMiscUtils:
     def test_random_str_length(self):
         for length in [8, 16, 32]:
@@ -131,14 +146,17 @@ class TestMiscUtils:
     def test_random_str_uniqueness(self):
         assert random_str() != random_str()
 
-    @pytest.mark.parametrize("ts_input,utc,iso,expected_type", [
-        (datetime.now(), True, True, str),
-        ("2023-01-01T12:00:00+00:00", False, False, datetime),
-        (1672574400, True, True, str),
-    ])
+    @pytest.mark.parametrize(
+        "ts_input,utc,iso,expected_type",
+        [
+            (datetime.now(), True, True, str),
+            ("2023-01-01T12:00:00+00:00", False, False, datetime),
+            (1672574400, True, True, str),
+        ],
+    )
     def test_timestamp_parsing(self, ts_input, utc, iso, expected_type):
         result = get_timestamp(ts_input, utc=utc, iso=iso)
         assert isinstance(result, expected_type)
 
     def test_invalid_timestamp_returns_none(self):
-        assert get_timestamp("invalid-date") is None
\ No newline at end of file
+        assert get_timestamp("invalid-date") is None

From ca44a40b88f75fb2b078cf2e92057bfd937788dd Mon Sep 17 00:00:00 2001
From: erinhmclark <erinhannahmary.clark@gmail.com>
Date: Mon, 10 Mar 2025 19:03:45 +0000
Subject: [PATCH 14/55] Ruff fix on src.

---
 .pre-commit-config.yaml                          |  7 +++++++
 src/auto_archiver/core/base_module.py            |  4 ++--
 src/auto_archiver/core/config.py                 |  3 +--
 src/auto_archiver/core/extractor.py              |  3 ---
 src/auto_archiver/core/metadata.py               |  2 +-
 src/auto_archiver/core/orchestrator.py           |  2 +-
 .../modules/gdrive_storage/gdrive_storage.py     |  2 +-
 .../generic_extractor/generic_extractor.py       |  5 +++--
 .../modules/generic_extractor/twitter.py         |  6 ++++--
 .../modules/gsheet_feeder_db/gsheet_feeder_db.py |  2 +-
 .../modules/html_formatter/html_formatter.py     |  4 +++-
 .../instagram_api_extractor.py                   | 16 ++++++++--------
 .../instagram_extractor/instagram_extractor.py   |  8 +++++---
 .../instagram_tbot_extractor.py                  |  4 ++--
 .../screenshot_enricher/screenshot_enricher.py   |  3 ++-
 .../modules/ssl_enricher/ssl_enricher.py         |  3 ++-
 .../telegram_extractor/telegram_extractor.py     |  4 +++-
 .../telethon_extractor/telethon_extractor.py     |  8 +++++---
 .../thumbnail_enricher/thumbnail_enricher.py     |  3 ++-
 .../wacz_extractor_enricher.py                   |  5 +++--
 .../wayback_extractor_enricher.py                |  7 ++++---
 .../modules/whisper_enricher/whisper_enricher.py |  5 +++--
 src/auto_archiver/utils/webdriver.py             |  6 +++---
 23 files changed, 66 insertions(+), 46 deletions(-)
 create mode 100644 .pre-commit-config.yaml

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
new file mode 100644
index 0000000..0fdf695
--- /dev/null
+++ b/.pre-commit-config.yaml
@@ -0,0 +1,7 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.9.10
+    hooks:
+      - id: ruff
+#        args: [ --fix ]
+      - id: ruff-format
diff --git a/src/auto_archiver/core/base_module.py b/src/auto_archiver/core/base_module.py
index d809e59..d717e4b 100644
--- a/src/auto_archiver/core/base_module.py
+++ b/src/auto_archiver/core/base_module.py
@@ -1,8 +1,8 @@
 from __future__ import annotations
 
-from typing import Mapping, Any, Type, TYPE_CHECKING
+from typing import Mapping, Any, TYPE_CHECKING
 from abc import ABC
-from copy import deepcopy, copy
+from copy import deepcopy
 from tempfile import TemporaryDirectory
 from auto_archiver.utils import url as UrlUtil
 from auto_archiver.core.consts import MODULE_TYPES as CONF_MODULE_TYPES
diff --git a/src/auto_archiver/core/config.py b/src/auto_archiver/core/config.py
index 8122809..f9e8c17 100644
--- a/src/auto_archiver/core/config.py
+++ b/src/auto_archiver/core/config.py
@@ -6,7 +6,7 @@ flexible setup in various environments.
 """
 
 import argparse
-from ruamel.yaml import YAML, CommentedMap, add_representer
+from ruamel.yaml import YAML, CommentedMap
 import json
 
 from loguru import logger
@@ -14,7 +14,6 @@ from loguru import logger
 from copy import deepcopy
 from auto_archiver.core.consts import MODULE_TYPES
 
-from typing import Any, List, Type, Tuple
 
 _yaml: YAML = YAML()
 
diff --git a/src/auto_archiver/core/extractor.py b/src/auto_archiver/core/extractor.py
index 8ad13f5..cf42f1e 100644
--- a/src/auto_archiver/core/extractor.py
+++ b/src/auto_archiver/core/extractor.py
@@ -7,12 +7,9 @@ Factory method to initialize an extractor instance based on its name.
 """
 
 from __future__ import annotations
-from pathlib import Path
 from abc import abstractmethod
-from dataclasses import dataclass
 import mimetypes
 import os
-import mimetypes
 import requests
 from loguru import logger
 from retrying import retry
diff --git a/src/auto_archiver/core/metadata.py b/src/auto_archiver/core/metadata.py
index 9c696a2..7961981 100644
--- a/src/auto_archiver/core/metadata.py
+++ b/src/auto_archiver/core/metadata.py
@@ -13,7 +13,7 @@ from __future__ import annotations
 import hashlib
 from typing import Any, List, Union, Dict
 from dataclasses import dataclass, field
-from dataclasses_json import dataclass_json, config
+from dataclasses_json import dataclass_json
 import datetime
 from urllib.parse import urlparse
 from dateutil.parser import parse as parse_dt
diff --git a/src/auto_archiver/core/orchestrator.py b/src/auto_archiver/core/orchestrator.py
index ba00995..6200b0a 100644
--- a/src/auto_archiver/core/orchestrator.py
+++ b/src/auto_archiver/core/orchestrator.py
@@ -123,7 +123,7 @@ Here's how that would look: \n\nsteps:\n  {module_type}s:\n  - [your_{module_typ
                     )
                 if module_type == "extractor" and config["steps"].get("archivers"):
                     raise SetupError(
-                        f"As of version 0.13.0 of Auto Archiver, the 'archivers' step name has been changed to 'extractors'. Change this in your configuration file and try again. \
+                        "As of version 0.13.0 of Auto Archiver, the 'archivers' step name has been changed to 'extractors'. Change this in your configuration file and try again. \
 Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_here]\n  enrichers:...\n"
                     )
                 raise SetupError(
diff --git a/src/auto_archiver/modules/gdrive_storage/gdrive_storage.py b/src/auto_archiver/modules/gdrive_storage/gdrive_storage.py
index f01ea4e..02ec427 100644
--- a/src/auto_archiver/modules/gdrive_storage/gdrive_storage.py
+++ b/src/auto_archiver/modules/gdrive_storage/gdrive_storage.py
@@ -135,7 +135,7 @@ class GDriveStorage(Storage):
         debug_header: str = f"[searching {name=} in {parent_id=}]"
         query_string = f"'{parent_id}' in parents and name = '{name}' and trashed = false "
         if use_mime_type:
-            query_string += f" and mimeType='application/vnd.google-apps.folder' "
+            query_string += " and mimeType='application/vnd.google-apps.folder' "
 
         for attempt in range(retries):
             results = (
diff --git a/src/auto_archiver/modules/generic_extractor/generic_extractor.py b/src/auto_archiver/modules/generic_extractor/generic_extractor.py
index 107ce93..08118ad 100644
--- a/src/auto_archiver/modules/generic_extractor/generic_extractor.py
+++ b/src/auto_archiver/modules/generic_extractor/generic_extractor.py
@@ -1,4 +1,5 @@
-import datetime, os
+import datetime
+import os
 import importlib
 import subprocess
 from typing import Generator, Type
@@ -386,7 +387,7 @@ class GenericExtractor(Extractor):
             item.set("replaced_url", url)
 
         ydl_options = {
-            "outtmpl": os.path.join(self.tmp_dir, f"%(id)s.%(ext)s"),
+            "outtmpl": os.path.join(self.tmp_dir, "%(id)s.%(ext)s"),
             "quiet": False,
             "noplaylist": not self.allow_playlist,
             "writesubtitles": self.subtitles,
diff --git a/src/auto_archiver/modules/generic_extractor/twitter.py b/src/auto_archiver/modules/generic_extractor/twitter.py
index 5b8468c..e4cbe74 100644
--- a/src/auto_archiver/modules/generic_extractor/twitter.py
+++ b/src/auto_archiver/modules/generic_extractor/twitter.py
@@ -1,4 +1,6 @@
-import re, mimetypes, json
+import re
+import mimetypes
+import json
 from datetime import datetime
 
 from loguru import logger
@@ -35,7 +37,7 @@ class Twitter(GenericDropin):
         result = Metadata()
         try:
             if not tweet.get("user") or not tweet.get("created_at"):
-                raise ValueError(f"Error retreiving post. Are you sure it exists?")
+                raise ValueError("Error retreiving post. Are you sure it exists?")
             timestamp = datetime.strptime(tweet["created_at"], "%a %b %d %H:%M:%S %z %Y")
         except (ValueError, KeyError) as ex:
             logger.warning(f"Unable to parse tweet: {str(ex)}\nRetreived tweet data: {tweet}")
diff --git a/src/auto_archiver/modules/gsheet_feeder_db/gsheet_feeder_db.py b/src/auto_archiver/modules/gsheet_feeder_db/gsheet_feeder_db.py
index 4a9c9b3..109be3f 100644
--- a/src/auto_archiver/modules/gsheet_feeder_db/gsheet_feeder_db.py
+++ b/src/auto_archiver/modules/gsheet_feeder_db/gsheet_feeder_db.py
@@ -20,7 +20,7 @@ from slugify import slugify
 from auto_archiver.core import Feeder, Database, Media
 from auto_archiver.core import Metadata
 from auto_archiver.modules.gsheet_feeder_db import GWorksheet
-from auto_archiver.utils.misc import calculate_file_hash, get_current_timestamp
+from auto_archiver.utils.misc import get_current_timestamp
 
 
 class GsheetsFeederDB(Feeder, Database):
diff --git a/src/auto_archiver/modules/html_formatter/html_formatter.py b/src/auto_archiver/modules/html_formatter/html_formatter.py
index 88a9eca..f5da1d8 100644
--- a/src/auto_archiver/modules/html_formatter/html_formatter.py
+++ b/src/auto_archiver/modules/html_formatter/html_formatter.py
@@ -1,5 +1,7 @@
 from __future__ import annotations
-import mimetypes, os, pathlib
+import mimetypes
+import os
+import pathlib
 from jinja2 import Environment, FileSystemLoader
 from urllib.parse import quote
 from loguru import logger
diff --git a/src/auto_archiver/modules/instagram_api_extractor/instagram_api_extractor.py b/src/auto_archiver/modules/instagram_api_extractor/instagram_api_extractor.py
index bb37df2..bae06bc 100644
--- a/src/auto_archiver/modules/instagram_api_extractor/instagram_api_extractor.py
+++ b/src/auto_archiver/modules/instagram_api_extractor/instagram_api_extractor.py
@@ -95,7 +95,7 @@ class InstagramAPIExtractor(Extractor):
         result.set_title(user.get("full_name", username)).set("data", user)
         if pic_url := user.get("profile_pic_url_hd", user.get("profile_pic_url")):
             filename = self.download_from_url(pic_url)
-            result.add_media(Media(filename=filename), id=f"profile_picture")
+            result.add_media(Media(filename=filename), id="profile_picture")
 
         if self.full_profile:
             user_id = user.get("pk")
@@ -133,7 +133,7 @@ class InstagramAPIExtractor(Extractor):
 
     def download_all_highlights(self, result, username, user_id):
         count_highlights = 0
-        highlights = self.call_api(f"v1/user/highlights", {"user_id": user_id})
+        highlights = self.call_api("v1/user/highlights", {"user_id": user_id})
         for h in highlights:
             try:
                 h_info = self._download_highlights_reusable(result, h.get("pk"))
@@ -151,9 +151,9 @@ class InstagramAPIExtractor(Extractor):
 
     def download_post(self, result: Metadata, code: str = None, id: str = None, context: str = None) -> Metadata:
         if id:
-            post = self.call_api(f"v1/media/by/id", {"id": id})
+            post = self.call_api("v1/media/by/id", {"id": id})
         else:
-            post = self.call_api(f"v1/media/by/code", {"code": code})
+            post = self.call_api("v1/media/by/code", {"code": code})
         assert post, f"Post {id or code} not found"
 
         if caption_text := post.get("caption_text"):
@@ -173,7 +173,7 @@ class InstagramAPIExtractor(Extractor):
         return result.success("insta highlights")
 
     def _download_highlights_reusable(self, result: Metadata, id: str) -> dict:
-        full_h = self.call_api(f"v2/highlight/by/id", {"id": id})
+        full_h = self.call_api("v2/highlight/by/id", {"id": id})
         h_info = full_h.get("response", {}).get("reels", {}).get(f"highlight:{id}")
         assert h_info, f"Highlight {id} not found: {full_h=}"
 
@@ -200,7 +200,7 @@ class InstagramAPIExtractor(Extractor):
         return result.success(f"insta stories {now}")
 
     def _download_stories_reusable(self, result: Metadata, username: str) -> list[dict]:
-        stories = self.call_api(f"v1/user/stories/by/username", {"username": username})
+        stories = self.call_api("v1/user/stories/by/username", {"username": username})
         if not stories or not len(stories):
             return []
         stories = stories[::-1]  # newest to oldest
@@ -219,7 +219,7 @@ class InstagramAPIExtractor(Extractor):
 
         post_count = 0
         while end_cursor != "":
-            posts = self.call_api(f"v1/user/medias/chunk", {"user_id": user_id, "end_cursor": end_cursor})
+            posts = self.call_api("v1/user/medias/chunk", {"user_id": user_id, "end_cursor": end_cursor})
             if not len(posts) or not type(posts) == list or len(posts) != 2:
                 break
             posts, end_cursor = posts[0], posts[1]
@@ -244,7 +244,7 @@ class InstagramAPIExtractor(Extractor):
 
         tagged_count = 0
         while next_page_id != None:
-            resp = self.call_api(f"v2/user/tag/medias", {"user_id": user_id, "page_id": next_page_id})
+            resp = self.call_api("v2/user/tag/medias", {"user_id": user_id, "page_id": next_page_id})
             posts = resp.get("response", {}).get("items", [])
             if not len(posts):
                 break
diff --git a/src/auto_archiver/modules/instagram_extractor/instagram_extractor.py b/src/auto_archiver/modules/instagram_extractor/instagram_extractor.py
index f310771..294b4e7 100644
--- a/src/auto_archiver/modules/instagram_extractor/instagram_extractor.py
+++ b/src/auto_archiver/modules/instagram_extractor/instagram_extractor.py
@@ -4,7 +4,9 @@ highlights, and tagged posts. Authentication is required via username/password o
 
 """
 
-import re, os, shutil
+import re
+import os
+import shutil
 import instaloader
 from loguru import logger
 
@@ -36,9 +38,9 @@ class InstagramExtractor(Extractor):
         )
         try:
             self.insta.load_session_from_file(self.username, self.session_file)
-        except Exception as e:
+        except Exception:
             try:
-                logger.debug(f"Session file failed", exc_info=True)
+                logger.debug("Session file failed", exc_info=True)
                 logger.info("No valid session file found - Attempting login with use and password.")
                 self.insta.login(self.username, self.password)
                 self.insta.save_session_to_file(self.session_file)
diff --git a/src/auto_archiver/modules/instagram_tbot_extractor/instagram_tbot_extractor.py b/src/auto_archiver/modules/instagram_tbot_extractor/instagram_tbot_extractor.py
index 39ed893..81d2bf6 100644
--- a/src/auto_archiver/modules/instagram_tbot_extractor/instagram_tbot_extractor.py
+++ b/src/auto_archiver/modules/instagram_tbot_extractor/instagram_tbot_extractor.py
@@ -51,7 +51,7 @@ class InstagramTbotExtractor(Extractor):
         """Initializes the Telegram client."""
         try:
             self.client = TelegramClient(self.session_file, self.api_id, self.api_hash)
-        except OperationalError as e:
+        except OperationalError:
             logger.error(
                 f"Unable to access the {self.session_file} session. "
                 "Ensure that you don't use the same session file here and in telethon_extractor. "
@@ -68,7 +68,7 @@ class InstagramTbotExtractor(Extractor):
 
     def download(self, item: Metadata) -> Metadata:
         url = item.get_url()
-        if not "instagram.com" in url:
+        if "instagram.com" not in url:
             return False
 
         result = Metadata()
diff --git a/src/auto_archiver/modules/screenshot_enricher/screenshot_enricher.py b/src/auto_archiver/modules/screenshot_enricher/screenshot_enricher.py
index 9fa2d62..491bd51 100644
--- a/src/auto_archiver/modules/screenshot_enricher/screenshot_enricher.py
+++ b/src/auto_archiver/modules/screenshot_enricher/screenshot_enricher.py
@@ -1,5 +1,6 @@
 from loguru import logger
-import time, os
+import time
+import os
 import base64
 
 from selenium.common.exceptions import TimeoutException
diff --git a/src/auto_archiver/modules/ssl_enricher/ssl_enricher.py b/src/auto_archiver/modules/ssl_enricher/ssl_enricher.py
index 74d80ce..3ab1389 100644
--- a/src/auto_archiver/modules/ssl_enricher/ssl_enricher.py
+++ b/src/auto_archiver/modules/ssl_enricher/ssl_enricher.py
@@ -1,4 +1,5 @@
-import ssl, os
+import ssl
+import os
 from slugify import slugify
 from urllib.parse import urlparse
 from loguru import logger
diff --git a/src/auto_archiver/modules/telegram_extractor/telegram_extractor.py b/src/auto_archiver/modules/telegram_extractor/telegram_extractor.py
index 5184024..e63fb8d 100644
--- a/src/auto_archiver/modules/telegram_extractor/telegram_extractor.py
+++ b/src/auto_archiver/modules/telegram_extractor/telegram_extractor.py
@@ -1,4 +1,6 @@
-import requests, re, html
+import requests
+import re
+import html
 from bs4 import BeautifulSoup
 from loguru import logger
 
diff --git a/src/auto_archiver/modules/telethon_extractor/telethon_extractor.py b/src/auto_archiver/modules/telethon_extractor/telethon_extractor.py
index be878a2..b06962e 100644
--- a/src/auto_archiver/modules/telethon_extractor/telethon_extractor.py
+++ b/src/auto_archiver/modules/telethon_extractor/telethon_extractor.py
@@ -10,7 +10,9 @@ from telethon.errors.rpcerrorlist import (
 )
 from loguru import logger
 from tqdm import tqdm
-import re, time, os
+import re
+import time
+import os
 
 from auto_archiver.core import Extractor
 from auto_archiver.core import Metadata, Media
@@ -63,11 +65,11 @@ class TelethonExtractor(Extractor):
                                 logger.warning(
                                     f"please add the property id='{ent.id}' to the 'channel_invites' configuration where {invite=}, not doing so can lead to a minutes-long setup time due to telegram's rate limiting."
                                 )
-                        except ValueError as e:
+                        except ValueError:
                             logger.info(f"joining new channel {invite=}")
                             try:
                                 self.client(ImportChatInviteRequest(match.group(2)))
-                            except UserAlreadyParticipantError as e:
+                            except UserAlreadyParticipantError:
                                 logger.info(f"already joined {invite=}")
                             except InviteRequestSentError:
                                 logger.warning(f"already sent a join request with {invite} still no answer")
diff --git a/src/auto_archiver/modules/thumbnail_enricher/thumbnail_enricher.py b/src/auto_archiver/modules/thumbnail_enricher/thumbnail_enricher.py
index 2f50c6b..1543cec 100644
--- a/src/auto_archiver/modules/thumbnail_enricher/thumbnail_enricher.py
+++ b/src/auto_archiver/modules/thumbnail_enricher/thumbnail_enricher.py
@@ -7,7 +7,8 @@ and identify important moments without watching the entire video.
 
 """
 
-import ffmpeg, os
+import ffmpeg
+import os
 from loguru import logger
 
 from auto_archiver.core import Enricher
diff --git a/src/auto_archiver/modules/wacz_extractor_enricher/wacz_extractor_enricher.py b/src/auto_archiver/modules/wacz_extractor_enricher/wacz_extractor_enricher.py
index ec61572..975d49a 100644
--- a/src/auto_archiver/modules/wacz_extractor_enricher/wacz_extractor_enricher.py
+++ b/src/auto_archiver/modules/wacz_extractor_enricher/wacz_extractor_enricher.py
@@ -1,6 +1,8 @@
 import jsonlines
 import mimetypes
-import os, shutil, subprocess
+import os
+import shutil
+import subprocess
 from zipfile import ZipFile
 from loguru import logger
 from warcio.archiveiterator import ArchiveIterator
@@ -186,7 +188,6 @@ class WaczExtractorEnricher(Enricher, Extractor):
         # get media out of .warc
         counter = 0
         seen_urls = set()
-        import json
 
         with open(warc_filename, "rb") as warc_stream:
             for record in ArchiveIterator(warc_stream):
diff --git a/src/auto_archiver/modules/wayback_extractor_enricher/wayback_extractor_enricher.py b/src/auto_archiver/modules/wayback_extractor_enricher/wayback_extractor_enricher.py
index 2dc3545..f06effd 100644
--- a/src/auto_archiver/modules/wayback_extractor_enricher/wayback_extractor_enricher.py
+++ b/src/auto_archiver/modules/wayback_extractor_enricher/wayback_extractor_enricher.py
@@ -1,6 +1,7 @@
 import json
 from loguru import logger
-import time, requests
+import time
+import requests
 
 from auto_archiver.core import Extractor, Enricher
 from auto_archiver.utils import url as UrlUtil
@@ -57,7 +58,7 @@ class WaybackExtractorEnricher(Enricher, Extractor):
             if not job_id:
                 logger.error(f"Wayback failed with {r.json()}")
                 return False
-        except json.decoder.JSONDecodeError as e:
+        except json.decoder.JSONDecodeError:
             logger.error(f"Expected a JSON with job_id from Wayback and got {r.text}")
             return False
 
@@ -80,7 +81,7 @@ class WaybackExtractorEnricher(Enricher, Extractor):
             except requests.exceptions.RequestException as e:
                 logger.warning(f"RequestException: fetching status for {url=} due to: {e}")
                 break
-            except json.decoder.JSONDecodeError as e:
+            except json.decoder.JSONDecodeError:
                 logger.error(f"Expected a JSON from Wayback and got {r.text} for {url=}")
                 break
             except Exception as e:
diff --git a/src/auto_archiver/modules/whisper_enricher/whisper_enricher.py b/src/auto_archiver/modules/whisper_enricher/whisper_enricher.py
index 0c884bb..d2205e2 100644
--- a/src/auto_archiver/modules/whisper_enricher/whisper_enricher.py
+++ b/src/auto_archiver/modules/whisper_enricher/whisper_enricher.py
@@ -1,5 +1,6 @@
 import traceback
-import requests, time
+import requests
+import time
 from loguru import logger
 
 from auto_archiver.core import Enricher
@@ -16,7 +17,7 @@ class WhisperEnricher(Enricher):
     def setup(self) -> None:
         self.stores = self.config["steps"]["storages"]
         self.s3 = self.module_factory.get_module("s3_storage", self.config)
-        if not "s3_storage" in self.stores:
+        if "s3_storage" not in self.stores:
             logger.error(
                 "WhisperEnricher: To use the WhisperEnricher you need to use S3Storage so files are accessible publicly to the whisper service being called."
             )
diff --git a/src/auto_archiver/utils/webdriver.py b/src/auto_archiver/utils/webdriver.py
index 0690ab5..2ba185e 100644
--- a/src/auto_archiver/utils/webdriver.py
+++ b/src/auto_archiver/utils/webdriver.py
@@ -66,15 +66,15 @@ class CookieSettingDriver(webdriver.Firefox):
 
         if self.facebook_accept_cookies:
             try:
-                logger.debug(f"Trying fb click accept cookie popup.")
+                logger.debug("Trying fb click accept cookie popup.")
                 super(CookieSettingDriver, self).get("http://www.facebook.com")
                 essential_only = self.find_element(By.XPATH, "//span[contains(text(), 'Decline optional cookies')]")
                 essential_only.click()
-                logger.debug(f"fb click worked")
+                logger.debug("fb click worked")
                 # linux server needs a sleep otherwise facebook cookie won't have worked and we'll get a popup on next page
                 time.sleep(2)
             except Exception as e:
-                logger.warning(f"Failed on fb accept cookies.", e)
+                logger.warning("Failed on fb accept cookies.", e)
 
         # now get the actual URL
         super(CookieSettingDriver, self).get(url)

From e7fa88f1c7ffc781e9336205c8d9c59b0c3477e8 Mon Sep 17 00:00:00 2001
From: erinhmclark <erinhannahmary.clark@gmail.com>
Date: Mon, 10 Mar 2025 21:45:30 +0000
Subject: [PATCH 15/55] Implementing ruff suggestions.

---
 pyproject.toml                                |  2 +-
 src/auto_archiver/core/config.py              |  2 +-
 src/auto_archiver/core/media.py               |  2 +-
 src/auto_archiver/core/metadata.py            | 27 +++++++++----------
 src/auto_archiver/core/module.py              |  4 +--
 src/auto_archiver/core/orchestrator.py        |  2 +-
 .../modules/cli_feeder/__manifest__.py        |  1 -
 .../modules/csv_feeder/__manifest__.py        |  1 -
 .../modules/gsheet_feeder_db/__manifest__.py  | 17 +++---------
 .../modules/gsheet_feeder_db/gworksheet.py    |  4 +--
 .../instagram_api_extractor.py                |  8 +++---
 .../telegram_extractor/telegram_extractor.py  |  2 +-
 .../twitter_api_extractor.py                  |  2 +-
 .../wacz_extractor_enricher/__manifest__.py   |  4 ++-
 .../whisper_enricher/whisper_enricher.py      |  2 +-
 src/auto_archiver/utils/misc.py               |  4 +--
 16 files changed, 36 insertions(+), 48 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index 44cf89b..68ac238 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -108,7 +108,7 @@ ignore = []
 
 [tool.ruff.lint.per-file-ignores]
 # Ignore import violations in __init__.py files
-"__init__.py" = ["F401"]
+"__init__.py" = ["F401", "F403"]
 
 [tool.ruff.format]
 docstring-code-format = false
diff --git a/src/auto_archiver/core/config.py b/src/auto_archiver/core/config.py
index f9e8c17..6c4300f 100644
--- a/src/auto_archiver/core/config.py
+++ b/src/auto_archiver/core/config.py
@@ -80,7 +80,7 @@ class AuthenticationJsonParseAction(argparse.Action):
                         auth_dict = auth_dict["authentication"]
                     auth_dict["load_from_file"] = path
                     return auth_dict
-            except:
+            except Exception:
                 return None
 
         if isinstance(auth_dict, dict) and auth_dict.get("from_file"):
diff --git a/src/auto_archiver/core/media.py b/src/auto_archiver/core/media.py
index 439e056..d593502 100644
--- a/src/auto_archiver/core/media.py
+++ b/src/auto_archiver/core/media.py
@@ -123,6 +123,6 @@ class Media:
             try:
                 fsize = os.path.getsize(self.filename)
                 return fsize > 20_000
-            except:
+            except Exception as e:
                 pass
         return True
diff --git a/src/auto_archiver/core/metadata.py b/src/auto_archiver/core/metadata.py
index 7961981..bbb124d 100644
--- a/src/auto_archiver/core/metadata.py
+++ b/src/auto_archiver/core/metadata.py
@@ -48,15 +48,16 @@ class Metadata:
                 self.status = right.status
             self._context.update(right._context)
             for k, v in right.metadata.items():
-                assert k not in self.metadata or type(v) == type(self.get(k))
-                if type(v) not in [dict, list, set] or k not in self.metadata:
+                assert k not in self.metadata or type(v) is type(self.get(k))
+                if not isinstance(v, (dict, list, set)) or k not in self.metadata:
                     self.set(k, v)
                 else:  # key conflict
-                    if type(v) in [dict, set]:
+                    if isinstance(v, (dict, set)):
                         self.set(k, self.get(k) | v)
-                    elif type(v) == list:
+                    elif type(v) is list:
                         self.set(k, self.get(k) + v)
             self.media.extend(right.media)
+
         else:  # invert and do same logic
             return right.merge(self)
         return self
@@ -126,28 +127,26 @@ class Metadata:
         return self.get("title")
 
     def set_timestamp(self, timestamp: datetime.datetime) -> Metadata:
-        if type(timestamp) == str:
+        if isinstance(timestamp, str):
             timestamp = parse_dt(timestamp)
-        assert type(timestamp) == datetime.datetime, "set_timestamp expects a datetime instance"
+        assert isinstance(timestamp, datetime.datetime), "set_timestamp expects a datetime instance"
         return self.set("timestamp", timestamp)
 
-    def get_timestamp(self, utc=True, iso=True) -> datetime.datetime:
+    def get_timestamp(self, utc=True, iso=True) -> datetime.datetime | str | None:
         ts = self.get("timestamp")
         if not ts:
-            return
+            return None
         try:
-            if type(ts) == str:
+            if isinstance(ts, str):
                 ts = datetime.datetime.fromisoformat(ts)
-            if type(ts) == float:
+            elif isinstance(ts, float):
                 ts = datetime.datetime.fromtimestamp(ts)
             if utc:
                 ts = ts.replace(tzinfo=datetime.timezone.utc)
-            if iso:
-                return ts.isoformat()
-            return ts
+            return ts.isoformat() if iso else ts
         except Exception as e:
             logger.error(f"Unable to parse timestamp {ts}: {e}")
-            return
+            return None
 
     def add_media(self, media: Media, id: str = None) -> Metadata:
         # adds a new media, optionally including an id
diff --git a/src/auto_archiver/core/module.py b/src/auto_archiver/core/module.py
index 3e2110a..6eac968 100644
--- a/src/auto_archiver/core/module.py
+++ b/src/auto_archiver/core/module.py
@@ -47,7 +47,7 @@ class ModuleFactory:
 
             # see odoo/module/module.py -> initialize_sys_path
             if path not in auto_archiver.modules.__path__:
-                if HAS_SETUP_PATHS == True:
+                if HAS_SETUP_PATHS:
                     logger.warning(
                         f"You are attempting to re-initialise the module paths with: '{path}' for a 2nd time. \
                                        This could lead to unexpected behaviour. It is recommended to only use a single modules path. \
@@ -228,7 +228,7 @@ class LazyBaseModule:
                     # we must now load this module and set it up with the config
                     m.load(config)
                     return True
-                except:
+                except Exception:
                     logger.error(f"Unable to setup module '{dep}' for use in module '{self.name}'")
                     return False
             except IndexError:
diff --git a/src/auto_archiver/core/orchestrator.py b/src/auto_archiver/core/orchestrator.py
index 6200b0a..4c583e7 100644
--- a/src/auto_archiver/core/orchestrator.py
+++ b/src/auto_archiver/core/orchestrator.py
@@ -531,7 +531,7 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
         except Exception as e:
             logger.error(f"Got unexpected error on item {item}: {e}\n{traceback.format_exc()}")
             for d in self.databases:
-                if type(e) == AssertionError:
+                if isinstance(e, AssertionError):
                     d.failed(item, str(e))
                 else:
                     d.failed(item, reason="unexpected error")
diff --git a/src/auto_archiver/modules/cli_feeder/__manifest__.py b/src/auto_archiver/modules/cli_feeder/__manifest__.py
index 218f7d0..f874405 100644
--- a/src/auto_archiver/modules/cli_feeder/__manifest__.py
+++ b/src/auto_archiver/modules/cli_feeder/__manifest__.py
@@ -3,7 +3,6 @@
     "type": ["feeder"],
     "entry_point": "cli_feeder::CLIFeeder",
     "requires_setup": False,
-    "description": "Feeds URLs to orchestrator from the command line",
     "configs": {
         "urls": {
             "default": None,
diff --git a/src/auto_archiver/modules/csv_feeder/__manifest__.py b/src/auto_archiver/modules/csv_feeder/__manifest__.py
index ffb4e24..d6e8caa 100644
--- a/src/auto_archiver/modules/csv_feeder/__manifest__.py
+++ b/src/auto_archiver/modules/csv_feeder/__manifest__.py
@@ -1,7 +1,6 @@
 {
     "name": "CSV Feeder",
     "type": ["feeder"],
-    "requires_setup": False,
     "dependencies": {"python": ["loguru"], "bin": [""]},
     "requires_setup": True,
     "entry_point": "csv_feeder::CSVFeeder",
diff --git a/src/auto_archiver/modules/gsheet_feeder_db/__manifest__.py b/src/auto_archiver/modules/gsheet_feeder_db/__manifest__.py
index 6c9d071..6547233 100644
--- a/src/auto_archiver/modules/gsheet_feeder_db/__manifest__.py
+++ b/src/auto_archiver/modules/gsheet_feeder_db/__manifest__.py
@@ -12,7 +12,9 @@
             "default": None,
             "help": "the id of the sheet to archive (alternative to 'sheet' config)",
         },
-        "header": {"default": 1, "type": "int", "help": "index of the header row (starts at 1)", "type": "int"},
+        "header": {"default": 1,
+                   "help": "index of the header row (starts at 1)",
+                   "type": "int"},
         "service_account": {
             "default": "secrets/service_account.json",
             "help": "service account JSON file path. Learn how to create one: https://gspread.readthedocs.io/en/latest/oauth2.html",
@@ -51,19 +53,6 @@
             "help": "if True the stored files path will include 'workbook_name/worksheet_name/...'",
             "type": "bool",
         },
-        "allow_worksheets": {
-            "default": set(),
-            "help": "(CSV) only worksheets whose name is included in allow are included (overrides worksheet_block), leave empty so all are allowed",
-        },
-        "block_worksheets": {
-            "default": set(),
-            "help": "(CSV) explicitly block some worksheets from being processed",
-        },
-        "use_sheet_names_in_stored_paths": {
-            "default": True,
-            "type": "bool",
-            "help": "if True the stored files path will include 'workbook_name/worksheet_name/...'",
-        },
     },
     "description": """
     GsheetsFeederDatabase
diff --git a/src/auto_archiver/modules/gsheet_feeder_db/gworksheet.py b/src/auto_archiver/modules/gsheet_feeder_db/gworksheet.py
index 84cd45e..6dac059 100644
--- a/src/auto_archiver/modules/gsheet_feeder_db/gworksheet.py
+++ b/src/auto_archiver/modules/gsheet_feeder_db/gworksheet.py
@@ -68,7 +68,7 @@ class GWorksheet:
 
         if fresh:
             return self.wks.cell(row, col_index + 1).value
-        if type(row) == int:
+        if isinstance(row, int):
             row = self.get_row(row)
 
         if col_index >= len(row):
@@ -84,7 +84,7 @@ class GWorksheet:
             if when_empty_use_default and val.strip() == "":
                 return default
             return val
-        except:
+        except Exception:
             return default
 
     def set_cell(self, row: int, col: str, val):
diff --git a/src/auto_archiver/modules/instagram_api_extractor/instagram_api_extractor.py b/src/auto_archiver/modules/instagram_api_extractor/instagram_api_extractor.py
index bae06bc..5f13ecf 100644
--- a/src/auto_archiver/modules/instagram_api_extractor/instagram_api_extractor.py
+++ b/src/auto_archiver/modules/instagram_api_extractor/instagram_api_extractor.py
@@ -74,9 +74,9 @@ class InstagramAPIExtractor(Extractor):
         # repeats 3 times to remove nested empty values
         if not self.minimize_json_output:
             return d
-        if type(d) == list:
+        if isinstance(d, list):
             return [self.cleanup_dict(v) for v in d]
-        if type(d) != dict:
+        if not isinstance(d, dict):
             return d
         return {
             k: clean_v
@@ -220,7 +220,7 @@ class InstagramAPIExtractor(Extractor):
         post_count = 0
         while end_cursor != "":
             posts = self.call_api("v1/user/medias/chunk", {"user_id": user_id, "end_cursor": end_cursor})
-            if not len(posts) or not type(posts) == list or len(posts) != 2:
+            if not posts or not isinstance(posts, list) or len(posts) != 2:
                 break
             posts, end_cursor = posts[0], posts[1]
             logger.info(f"parsing {len(posts)} posts, next {end_cursor=}")
@@ -243,7 +243,7 @@ class InstagramAPIExtractor(Extractor):
         pbar = tqdm(desc="downloading tagged posts")
 
         tagged_count = 0
-        while next_page_id != None:
+        while next_page_id is not None:
             resp = self.call_api("v2/user/tag/medias", {"user_id": user_id, "page_id": next_page_id})
             posts = resp.get("response", {}).get("items", [])
             if not len(posts):
diff --git a/src/auto_archiver/modules/telegram_extractor/telegram_extractor.py b/src/auto_archiver/modules/telegram_extractor/telegram_extractor.py
index e63fb8d..e70198d 100644
--- a/src/auto_archiver/modules/telegram_extractor/telegram_extractor.py
+++ b/src/auto_archiver/modules/telegram_extractor/telegram_extractor.py
@@ -61,7 +61,7 @@ class TelegramExtractor(Extractor):
                 else:
                     duration = float(duration)
                 m_video.set("duration", duration)
-            except:
+            except Exception:
                 pass
             result.add_media(m_video)
 
diff --git a/src/auto_archiver/modules/twitter_api_extractor/twitter_api_extractor.py b/src/auto_archiver/modules/twitter_api_extractor/twitter_api_extractor.py
index a7af607..1c08235 100644
--- a/src/auto_archiver/modules/twitter_api_extractor/twitter_api_extractor.py
+++ b/src/auto_archiver/modules/twitter_api_extractor/twitter_api_extractor.py
@@ -46,7 +46,7 @@ class TwitterApiExtractor(Extractor):
                 r = requests.get(url, timeout=30)
                 logger.debug(f"Expanded url {url} to {r.url}")
                 url = r.url
-            except:
+            except Exception:
                 logger.error(f"Failed to expand url {url}")
         return url
 
diff --git a/src/auto_archiver/modules/wacz_extractor_enricher/__manifest__.py b/src/auto_archiver/modules/wacz_extractor_enricher/__manifest__.py
index d2477b4..7916049 100644
--- a/src/auto_archiver/modules/wacz_extractor_enricher/__manifest__.py
+++ b/src/auto_archiver/modules/wacz_extractor_enricher/__manifest__.py
@@ -14,7 +14,9 @@
             "help": "browsertrix-profile (for profile generation see https://github.com/webrecorder/browsertrix-crawler#creating-and-using-browser-profiles).",
         },
         "docker_commands": {"default": None, "help": "if a custom docker invocation is needed"},
-        "timeout": {"default": 120, "type": "int", "help": "timeout for WACZ generation in seconds", "type": "int"},
+        "timeout": {"default": 120,
+                    "help": "timeout for WACZ generation in seconds",
+                    "type": "int"},
         "extract_media": {
             "default": False,
             "type": "bool",
diff --git a/src/auto_archiver/modules/whisper_enricher/whisper_enricher.py b/src/auto_archiver/modules/whisper_enricher/whisper_enricher.py
index d2205e2..063bd26 100644
--- a/src/auto_archiver/modules/whisper_enricher/whisper_enricher.py
+++ b/src/auto_archiver/modules/whisper_enricher/whisper_enricher.py
@@ -88,7 +88,7 @@ class WhisperEnricher(Enricher):
         while not all_completed and (time.time() - start_time) <= self.timeout:
             all_completed = True
             for job_id in job_results:
-                if job_results[job_id] != False:
+                if job_results[job_id] is not False:
                     continue
                 all_completed = False  # at least one not ready
                 try:
diff --git a/src/auto_archiver/utils/misc.py b/src/auto_archiver/utils/misc.py
index 379bff5..fe1864b 100644
--- a/src/auto_archiver/utils/misc.py
+++ b/src/auto_archiver/utils/misc.py
@@ -21,7 +21,7 @@ def expand_url(url):
             r = requests.get(url)
             logger.debug(f"Expanded url {url} to {r.url}")
             return r.url
-        except:
+        except Exception:
             logger.error(f"Failed to expand url {url}")
     return url
 
@@ -32,7 +32,7 @@ def getattr_or(o: object, prop: str, default=None):
         if res is None:
             raise
         return res
-    except:
+    except Exception:
         return default
 
 

From 81aa343f21360f55a3f0d18e0aa99befc6c27c96 Mon Sep 17 00:00:00 2001
From: erinhmclark <erinhannahmary.clark@gmail.com>
Date: Tue, 11 Mar 2025 10:45:07 +0000
Subject: [PATCH 16/55] Merge main.

---
 src/auto_archiver/core/orchestrator.py        | 19 ++++++++++++-------
 src/auto_archiver/core/storage.py             |  2 --
 .../modules/local_storage/local_storage.py    |  2 +-
 .../modules/s3_storage/s3_storage.py          |  3 ++-
 4 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/src/auto_archiver/core/orchestrator.py b/src/auto_archiver/core/orchestrator.py
index 6a95046..dca2f4a 100644
--- a/src/auto_archiver/core/orchestrator.py
+++ b/src/auto_archiver/core/orchestrator.py
@@ -79,7 +79,7 @@ class ArchivingOrchestrator:
                     raise SetupError(f"It appears you have '{module_type}' set under 'steps' in your configuration file, but as of version 0.13.0 of Auto Archiver, you must use '{module_type}s'. Change this in your configuration file and try again. \
 Here's how that would look: \n\nsteps:\n  {module_type}s:\n  - [your_{module_type}_name_here]\n  {'extractors:...' if module_type == 'feeder' else '...'}\n")
                 if module_type == 'extractor' and config['steps'].get('archivers'):
-                    raise SetupError(f"As of version 0.13.0 of Auto Archiver, the 'archivers' step name has been changed to 'extractors'. Change this in your configuration file and try again. \
+                    raise SetupError("As of version 0.13.0 of Auto Archiver, the 'archivers' step name has been changed to 'extractors'. Change this in your configuration file and try again. \
 Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_here]\n  enrichers:...\n")
                 raise SetupError(f"No {module_type}s were configured. Make sure to set at least one {module_type} in your configuration file or on the command line (using --{module_type}s)")
 
@@ -438,7 +438,7 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
         except Exception as e:
             logger.error(f'Got unexpected error on item {item}: {e}\n{traceback.format_exc()}')
             for d in self.databases:
-                if type(e) == AssertionError:
+                if isinstance(e, AssertionError):
                     d.failed(item, str(e))
                 else:
                     d.failed(item, reason="unexpected error")
@@ -473,7 +473,8 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
             url = a.sanitize_url(url)
 
         result.set_url(url)
-        if original_url != url: result.set("original_url", original_url)
+        if original_url != url:
+            result.set("original_url", original_url)
 
         # 2 - notify start to DBs, propagate already archived if feature enabled in DBs
         cached_result = None
@@ -484,7 +485,8 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
         if cached_result:
             logger.debug("Found previously archived entry")
             for d in self.databases:
-                try: d.done(cached_result, cached=True)
+                try:
+                    d.done(cached_result, cached=True)
                 except Exception as e:
                     logger.error(f"ERROR database {d.name}: {e}: {traceback.format_exc()}")
             return cached_result
@@ -494,13 +496,15 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
             logger.info(f"Trying extractor {a.name} for {url}")
             try:
                 result.merge(a.download(result))
-                if result.is_success(): break
+                if result.is_success():
+                    break
             except Exception as e:
                 logger.error(f"ERROR archiver {a.name}: {e}: {traceback.format_exc()}")
 
         # 4 - call enrichers to work with archived content
         for e in self.enrichers:
-            try: e.enrich(result)
+            try:
+                e.enrich(result)
             except Exception as exc:
                 logger.error(f"ERROR enricher {e.name}: {exc}: {traceback.format_exc()}")
 
@@ -518,7 +522,8 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
 
         # signal completion to databases and archivers
         for d in self.databases:
-            try: d.done(result)
+            try:
+                d.done(result)
             except Exception as e:
                 logger.error(f"ERROR database {d.name}: {e}: {traceback.format_exc()}")
 
diff --git a/src/auto_archiver/core/storage.py b/src/auto_archiver/core/storage.py
index 63ccf8d..a13aa89 100644
--- a/src/auto_archiver/core/storage.py
+++ b/src/auto_archiver/core/storage.py
@@ -23,7 +23,6 @@ from __future__ import annotations
 from abc import abstractmethod
 from typing import IO
 import os
-import platform
 
 from loguru import logger
 from slugify import slugify
@@ -31,7 +30,6 @@ from slugify import slugify
 from auto_archiver.utils.misc import random_str
 
 from auto_archiver.core import Media, BaseModule, Metadata
-from auto_archiver.modules.hash_enricher.hash_enricher import HashEnricher
 
 class Storage(BaseModule):
     
diff --git a/src/auto_archiver/modules/local_storage/local_storage.py b/src/auto_archiver/modules/local_storage/local_storage.py
index 2b1a101..54f4a0e 100644
--- a/src/auto_archiver/modules/local_storage/local_storage.py
+++ b/src/auto_archiver/modules/local_storage/local_storage.py
@@ -13,7 +13,7 @@ class LocalStorage(Storage):
 
     def setup(self) -> None:
         if len(self.save_to) > 200:
-            raise SetupError(f"Your save_to path is too long, this will cause issues saving files on your computer. Please use a shorter path.")
+            raise SetupError("Your save_to path is too long, this will cause issues saving files on your computer. Please use a shorter path.")
 
     def get_cdn_url(self, media: Media) -> str:
         dest = media.key
diff --git a/src/auto_archiver/modules/s3_storage/s3_storage.py b/src/auto_archiver/modules/s3_storage/s3_storage.py
index 183a944..bb87812 100644
--- a/src/auto_archiver/modules/s3_storage/s3_storage.py
+++ b/src/auto_archiver/modules/s3_storage/s3_storage.py
@@ -28,7 +28,8 @@ class S3Storage(Storage):
         return self.cdn_url.format(bucket=self.bucket, region=self.region, key=media.key)
 
     def uploadf(self, file: IO[bytes], media: Media, **kwargs: dict) -> None:
-        if not self.is_upload_needed(media): return True
+        if not self.is_upload_needed(media):
+            return True
 
         extra_args = kwargs.get("extra_args", {})
         if not self.private and 'ACL' not in extra_args:

From 7a81ab617a665a7768e6d9984a16b9ee8b77baa2 Mon Sep 17 00:00:00 2001
From: Patrick Robertson <robertson.patrick@gmail.com>
Date: Tue, 11 Mar 2025 11:57:25 +0000
Subject: [PATCH 17/55] Better checking of cookies to add to webdriver

---
 src/auto_archiver/utils/webdriver.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/auto_archiver/utils/webdriver.py b/src/auto_archiver/utils/webdriver.py
index cb4e2a9..af3b7dd 100644
--- a/src/auto_archiver/utils/webdriver.py
+++ b/src/auto_archiver/utils/webdriver.py
@@ -3,6 +3,7 @@ from __future__ import annotations
 
 import os
 import time
+import re
 
 #import domain_for_url
 from urllib.parse import urlparse, urlunparse
@@ -48,8 +49,9 @@ class CookieSettingDriver(webdriver.Firefox):
                         self.driver.add_cookie({'name': name, 'value': value})
             elif self.cookiejar:
                 domain = urlparse(url).netloc.lstrip("www.")
+                regex = re.compile(f"(www)?\.?{domain}$")
                 for cookie in self.cookiejar:
-                    if domain in cookie.domain:
+                    if regex.match(cookie.domain):
                         try:
                             self.add_cookie({
                                 'name': cookie.name,
@@ -60,7 +62,7 @@ class CookieSettingDriver(webdriver.Firefox):
                                 'expiry': cookie.expires
                             })
                         except Exception as e:
-                            logger.warning(f"Failed to add cookie to webdriver: {e}")
+                            logger.warning(f"Failed to add cookie ({cookie.domain}) to webdriver for url {domain}: {e}")
         
         if self.facebook_accept_cookies:
             try:
@@ -81,7 +83,7 @@ class CookieSettingDriver(webdriver.Firefox):
             # try and click the 'close' button on the 'login' window to close it
             try:
                 xpath = "//div[@role='dialog']//div[@aria-label='Close']"
-                WebDriverWait(self, 5).until(EC.element_to_be_clickable((By.XPATH, xpath))).click()
+                WebDriverWait(self, 2).until(EC.element_to_be_clickable((By.XPATH, xpath))).click()
             except selenium_exceptions.NoSuchElementException:
                 logger.warning("Unable to find the 'close' button on the facebook login window")
                 pass

From 37eac64442c581fabf599710efd47f4d35a483c9 Mon Sep 17 00:00:00 2001
From: Patrick Robertson <robertson.patrick@gmail.com>
Date: Tue, 11 Mar 2025 17:10:44 +0000
Subject: [PATCH 18/55] Remove desc

---
 .../opentimestamps_enricher/__manifest__.py   | 41 +------------------
 1 file changed, 2 insertions(+), 39 deletions(-)

diff --git a/src/auto_archiver/modules/opentimestamps_enricher/__manifest__.py b/src/auto_archiver/modules/opentimestamps_enricher/__manifest__.py
index cfed1fb..645a04d 100644
--- a/src/auto_archiver/modules/opentimestamps_enricher/__manifest__.py
+++ b/src/auto_archiver/modules/opentimestamps_enricher/__manifest__.py
@@ -1,50 +1,13 @@
 {
     "name": "Opentimestamps Enricher",
     "type": ["enricher"],
-    "requires_setup": False,
+    "requires_setup": True,
     "dependencies": {
         "python": [
             "loguru",
-            "opentimestamps",
+            "opentimestamps-client",
         ],
     },
-    "configs": {
-        "tsa_urls": {
-            "default": [
-                    # [Adobe Approved Trust List] and [Windows Cert Store]
-                    "http://timestamp.digicert.com",
-                    "http://timestamp.identrust.com",
-                    # "https://timestamp.entrust.net/TSS/RFC3161sha2TS", # not valid for timestamping
-                    # "https://timestamp.sectigo.com", # wait 15 seconds between each request.
-
-                    # [Adobe: European Union Trusted Lists].
-                    # "https://timestamp.sectigo.com/qualified", # wait 15 seconds between each request.
-
-                    # [Windows Cert Store]
-                    "http://timestamp.globalsign.com/tsa/r6advanced1",
-                    # [Adobe: European Union Trusted Lists] and [Windows Cert Store]
-                    # "http://ts.quovadisglobal.com/eu", # not valid for timestamping
-                    # "http://tsa.belgium.be/connect", # self-signed certificate in certificate chain
-                    # "https://timestamp.aped.gov.gr/qtss", # self-signed certificate in certificate chain
-                    # "http://tsa.sep.bg", # self-signed certificate in certificate chain
-                    # "http://tsa.izenpe.com", #unable to get local issuer certificate
-                    # "http://kstamp.keynectis.com/KSign", # unable to get local issuer certificate
-                    "http://tss.accv.es:8318/tsa",
-                ],
-            "help": "List of RFC3161 Time Stamp Authorities to use, separate with commas if passed via the command line.",
-        }
-    },
     "description": """
-    Generates RFC3161-compliant timestamp tokens using Time Stamp Authorities (TSA) for archived files.
-
-    ### Features
-    - Creates timestamp tokens to prove the existence of files at a specific time, useful for legal and authenticity purposes.
-    - Aggregates file hashes into a text file and timestamps the concatenated data.
-    - Uses multiple Time Stamp Authorities (TSAs) to ensure reliability and redundancy.
-    - Validates timestamping certificates against trusted Certificate Authorities (CAs) using the `certifi` trust store.
-
-    ### Notes
-    - Should be run after the `hash_enricher` to ensure file hashes are available.
-    - Requires internet access to interact with the configured TSAs.
     """
 }

From 28c5396b749a0ed07189cb75041ab2d811392f8b Mon Sep 17 00:00:00 2001
From: erinhmclark <erinhannahmary.clark@gmail.com>
Date: Tue, 11 Mar 2025 17:25:24 +0000
Subject: [PATCH 19/55] Move ruff to dev dependencies.

---
 poetry.lock    | 4 ++--
 pyproject.toml | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/poetry.lock b/poetry.lock
index 1580583..b679542 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -2252,7 +2252,7 @@ version = "0.9.10"
 description = "An extremely fast Python linter and code formatter, written in Rust."
 optional = false
 python-versions = ">=3.7"
-groups = ["main"]
+groups = ["dev"]
 files = [
     {file = "ruff-0.9.10-py3-none-linux_armv6l.whl", hash = "sha256:eb4d25532cfd9fe461acc83498361ec2e2252795b4f40b17e80692814329e42d"},
     {file = "ruff-0.9.10-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:188a6638dab1aa9bb6228a7302387b2c9954e455fb25d6b4470cb0641d16759d"},
@@ -3213,4 +3213,4 @@ test = ["pytest (>=8.1,<9.0)", "pytest-rerunfailures (>=14.0,<15.0)"]
 [metadata]
 lock-version = "2.1"
 python-versions = ">=3.10,<3.13"
-content-hash = "57907984411be7c5bb9c9f0628476156c9ab59ba32de6e0e42a51b396e8b696d"
+content-hash = "0feae518c3a51717bd80e90eea3cd3ed53925af656f00b662c856bae38a742bb"
diff --git a/pyproject.toml b/pyproject.toml
index 68ac238..cbcfb61 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -57,7 +57,6 @@ dependencies = [
     "certvalidator (>=0.0.0)",
     "rich-argparse (>=1.6.0,<2.0.0)",
     "ruamel-yaml (>=0.18.10,<0.19.0)",
-    "ruff (>=0.9.10,<0.10.0)",
 ]
 
 [tool.poetry.group.dev.dependencies]
@@ -65,6 +64,7 @@ pytest = "^8.3.4"
 autopep8 = "^2.3.1"
 pytest-loguru = "^0.4.0"
 pytest-mock = "^3.14.0"
+ruff = "^0.9.10"
 
 [tool.poetry.group.docs.dependencies]
 sphinx = "^8.1.3"

From b70ed97ffd79bfdf87fbb286968c97faf21a7146 Mon Sep 17 00:00:00 2001
From: Patrick Robertson <robertson.patrick@gmail.com>
Date: Tue, 11 Mar 2025 17:28:28 +0000
Subject: [PATCH 20/55] Create opentimestamps module

---
 .../opentimestamps_enricher/__manifest__.py   |  45 ++++-
 .../opentimestamps_enricher.py                | 174 ++++++++++++++++++
 2 files changed, 216 insertions(+), 3 deletions(-)

diff --git a/src/auto_archiver/modules/opentimestamps_enricher/__manifest__.py b/src/auto_archiver/modules/opentimestamps_enricher/__manifest__.py
index 645a04d..849bb3d 100644
--- a/src/auto_archiver/modules/opentimestamps_enricher/__manifest__.py
+++ b/src/auto_archiver/modules/opentimestamps_enricher/__manifest__.py
@@ -1,13 +1,52 @@
 {
-    "name": "Opentimestamps Enricher",
+    "name": "OpenTimestamps Enricher",
     "type": ["enricher"],
     "requires_setup": True,
     "dependencies": {
         "python": [
             "loguru",
-            "opentimestamps-client",
+            "opentimestamps",
+            "slugify",
         ],
     },
+    "configs": {
+        "use_calendars": {
+            "default": True,
+            "help": "Whether to connect to OpenTimestamps calendar servers to create timestamps. If false, creates local timestamp proofs only.",
+            "type": "bool"
+        },
+        "calendar_urls": {
+            "default": [
+                "https://alice.btc.calendar.opentimestamps.org",
+                "https://bob.btc.calendar.opentimestamps.org",
+                "https://finney.calendar.eternitywall.com"
+            ],
+            "help": "List of OpenTimestamps calendar servers to use for timestamping.",
+            "type": "list"
+        },
+        "calendar_whitelist": {
+            "default": [],
+            "help": "Optional whitelist of calendar servers. If empty, all calendar servers are allowed.",
+            "type": "list"
+        },
+        "verify_timestamps": {
+            "default": True,
+            "help": "Whether to verify timestamps after creating them.",
+            "type": "bool"
+        }
+    },
     "description": """
+    Creates OpenTimestamps proofs for archived files, providing blockchain-backed evidence of file existence at a specific time.
+
+    ### Features
+    - Creates cryptographic timestamp proofs that link files to the Bitcoin blockchain
+    - Verifies existing timestamp proofs to confirm the time a file existed
+    - Uses multiple calendar servers to ensure reliability and redundancy
+    - Stores timestamp proofs alongside original files for future verification
+
+    ### Notes
+    - Can work offline to create timestamp proofs that can be upgraded later
+    - Verification checks if timestamps have been confirmed in the Bitcoin blockchain
+    - Should run after files have been archived and hashed
     """
-}
+}
\ No newline at end of file
diff --git a/src/auto_archiver/modules/opentimestamps_enricher/opentimestamps_enricher.py b/src/auto_archiver/modules/opentimestamps_enricher/opentimestamps_enricher.py
index e69de29..01e8964 100644
--- a/src/auto_archiver/modules/opentimestamps_enricher/opentimestamps_enricher.py
+++ b/src/auto_archiver/modules/opentimestamps_enricher/opentimestamps_enricher.py
@@ -0,0 +1,174 @@
+import os
+import hashlib
+from importlib.metadata import version
+
+from slugify import slugify
+from loguru import logger
+import opentimestamps
+from opentimestamps.calendar import RemoteCalendar, DEFAULT_CALENDAR_WHITELIST
+from opentimestamps.core.timestamp import Timestamp, DetachedTimestampFile
+from opentimestamps.core.notary import PendingAttestation, BitcoinBlockHeaderAttestation
+from auto_archiver.core import Enricher
+from auto_archiver.core import Metadata, Media
+from auto_archiver.version import __version__
+
+
+class OpentimestampsEnricher(Enricher):
+    """
+    Uses OpenTimestamps to create and verify timestamps for files. OpenTimestamps is a service that 
+    timestamps data using the Bitcoin blockchain, providing a decentralized and secure way to prove 
+    that data existed at a certain point in time.
+
+    The enricher hashes files in the archive and creates timestamp proofs that can later be verified.
+    These proofs are stored alongside the original files and can be used to verify the timestamp
+    even if the OpenTimestamps calendar servers are unavailable.
+    """
+
+    def setup(self):
+        # Initialize any resources needed
+        pass
+
+    def cleanup(self) -> None:
+        # Clean up any resources used
+        pass
+
+    def enrich(self, to_enrich: Metadata) -> None:
+        url = to_enrich.get_url()
+        logger.debug(f"OpenTimestamps timestamping files for {url=}")
+
+        # Get the media files to timestamp
+        media_files = [m for m in to_enrich.media if m.get("filename") and not m.get("opentimestamps")]
+        
+        if not media_files:
+            logger.warning(f"No files found to timestamp in {url=}")
+            return
+
+        timestamp_files = []
+        for media in media_files:
+            try:
+                # Get the file path from the media
+                file_path = media.get("filename")
+                if not os.path.exists(file_path):
+                    logger.warning(f"File not found: {file_path}")
+                    continue
+                
+                # Create timestamp for the file
+                logger.debug(f"Creating timestamp for {file_path}")
+                
+                # Hash the file
+                with open(file_path, 'rb') as f:
+                    file_bytes = f.read()
+                file_hash = hashlib.sha256(file_bytes).digest()
+                
+                # Create a timestamp with the file hash
+                timestamp = Timestamp(file_hash)
+                
+                # Create a detached timestamp file with the timestamp
+                detached_timestamp = DetachedTimestampFile(timestamp)
+                
+                # Submit to calendar servers
+                if self.use_calendars:
+                    logger.debug(f"Submitting timestamp to calendar servers for {file_path}")
+                    calendars = []
+                    whitelist = DEFAULT_CALENDAR_WHITELIST
+                    
+                    if self.calendar_whitelist:
+                        whitelist = set(self.calendar_whitelist)
+                    
+                    # Create calendar instances
+                    for url in self.calendar_urls:
+                        if url in whitelist:
+                            calendars.append(RemoteCalendar(url))
+                    
+                    # Submit the hash to each calendar
+                    for calendar in calendars:
+                        try:
+                            calendar_timestamp = calendar.submit(file_hash)
+                            timestamp.merge(calendar_timestamp)
+                            logger.debug(f"Successfully submitted to calendar: {calendar.url}")
+                        except Exception as e:
+                            logger.warning(f"Failed to submit to calendar {calendar.url}: {e}")
+                else:
+                    logger.info("Skipping calendar submission as per configuration")
+                
+                # Save the timestamp proof to a file
+                timestamp_path = os.path.join(self.tmp_dir, f"{os.path.basename(file_path)}.ots")
+                with open(timestamp_path, 'wb') as f:
+                    detached_timestamp.serialize(f)
+                
+                # Create media for the timestamp file
+                timestamp_media = Media(filename=timestamp_path)
+                timestamp_media.set("source_file", os.path.basename(file_path))
+                timestamp_media.set("opentimestamps_version", opentimestamps.__version__)
+                
+                # Verify the timestamp if needed
+                if self.verify_timestamps:
+                    verification_info = self.verify_timestamp(detached_timestamp)
+                    for key, value in verification_info.items():
+                        timestamp_media.set(key, value)
+                
+                timestamp_files.append(timestamp_media)
+                
+                # Update the original media to indicate it's been timestamped
+                media.set("opentimestamps", True)
+                media.set("opentimestamp_file", timestamp_path)
+                
+            except Exception as e:
+                logger.warning(f"Error while timestamping {media.get('filename')}: {e}")
+        
+        # Add timestamp files to the metadata
+        if timestamp_files:
+            for ts_media in timestamp_files:
+                to_enrich.add_media(ts_media)
+            
+            to_enrich.set("opentimestamped", True)
+            to_enrich.set("opentimestamps_count", len(timestamp_files))
+            logger.success(f"{len(timestamp_files)} OpenTimestamps proofs created for {url=}")
+        else:
+            logger.warning(f"No successful timestamps created for {url=}")
+    
+    def verify_timestamp(self, detached_timestamp):
+        """
+        Verify a timestamp and extract verification information.
+        
+        Args:
+            detached_timestamp: The detached timestamp to verify.
+            
+        Returns:
+            dict: Information about the verification result.
+        """
+        result = {}
+        
+        # Check if we have attestations
+        attestations = list(detached_timestamp.timestamp.all_attestations())
+        result["attestation_count"] = len(attestations)
+        
+        if attestations:
+            attestation_info = []
+            for msg, attestation in attestations:
+                info = {}
+                
+                # Process different types of attestations
+                if isinstance(attestation, PendingAttestation):
+                    info["type"] = "pending"
+                    info["uri"] = attestation.uri.decode('utf-8')
+                
+                elif isinstance(attestation, BitcoinBlockHeaderAttestation):
+                    info["type"] = "bitcoin"
+                    info["block_height"] = attestation.height
+                
+                attestation_info.append(info)
+            
+            result["attestations"] = attestation_info
+            
+            # For at least one confirmed attestation
+            if any(a.get("type") == "bitcoin" for a in attestation_info):
+                result["verified"] = True
+            else:
+                result["verified"] = False
+                result["pending"] = True
+        else:
+            result["verified"] = False
+            result["pending"] = False
+        
+        return result
\ No newline at end of file

From 28041d94d97d0a9fc88272a72cdcf8a623501812 Mon Sep 17 00:00:00 2001
From: Patrick Robertson <robertson.patrick@gmail.com>
Date: Tue, 11 Mar 2025 17:33:54 +0000
Subject: [PATCH 21/55] Add unit tests for opentimestamps enricher

---
 .../enrichers/test_opentimestamps_enricher.py | 242 ++++++++++++++++++
 1 file changed, 242 insertions(+)
 create mode 100644 tests/enrichers/test_opentimestamps_enricher.py

diff --git a/tests/enrichers/test_opentimestamps_enricher.py b/tests/enrichers/test_opentimestamps_enricher.py
new file mode 100644
index 0000000..5681561
--- /dev/null
+++ b/tests/enrichers/test_opentimestamps_enricher.py
@@ -0,0 +1,242 @@
+from pathlib import Path
+import pytest
+import os
+import tempfile
+import hashlib
+
+from opentimestamps.core.timestamp import Timestamp, DetachedTimestampFile
+from opentimestamps.calendar import RemoteCalendar
+from opentimestamps.core.notary import PendingAttestation, BitcoinBlockHeaderAttestation
+
+from auto_archiver.core import Metadata, Media
+
+@pytest.fixture
+def sample_file_path():
+    with tempfile.NamedTemporaryFile(delete=False) as tmp:
+        tmp.write(b"This is a test file content for OpenTimestamps")
+        return tmp.name
+
+@pytest.fixture
+def detached_timestamp_file():
+    """Create a simple detached timestamp file for testing"""
+    file_hash = hashlib.sha256(b"Test content").digest()
+    timestamp = Timestamp(file_hash)
+    
+    # Add a pending attestation
+    pending = PendingAttestation(b"https://example.calendar.com")
+    timestamp.attestations.add(pending)
+    
+    # Add a bitcoin attestation
+    bitcoin = BitcoinBlockHeaderAttestation(783000)  # Some block height
+    timestamp.attestations.add(bitcoin)
+    
+    return DetachedTimestampFile(timestamp)
+
+@pytest.fixture
+def verified_timestamp_file():
+    """Create a timestamp file with a Bitcoin attestation"""
+    file_hash = hashlib.sha256(b"Verified content").digest()
+    timestamp = Timestamp(file_hash)
+    
+    # Add only a Bitcoin attestation
+    bitcoin = BitcoinBlockHeaderAttestation(783000)  # Some block height
+    timestamp.attestations.add(bitcoin)
+    
+    return DetachedTimestampFile(timestamp)
+
+@pytest.fixture
+def pending_timestamp_file():
+    """Create a timestamp file with only pending attestations"""
+    file_hash = hashlib.sha256(b"Pending content").digest()
+    timestamp = Timestamp(file_hash)
+    
+    # Add only pending attestations
+    pending1 = PendingAttestation(b"https://example1.calendar.com")
+    pending2 = PendingAttestation(b"https://example2.calendar.com")
+    timestamp.attestations.add(pending1)
+    timestamp.attestations.add(pending2)
+    
+    return DetachedTimestampFile(timestamp)
+
+@pytest.mark.download
+def test_download_tsr(setup_module, mocker):
+    """Test submitting a hash to calendar servers"""
+    # Mock the RemoteCalendar submit method
+    mock_submit = mocker.patch.object(RemoteCalendar, 'submit')
+    test_timestamp = Timestamp(hashlib.sha256(b"test").digest())
+    mock_submit.return_value = test_timestamp
+    
+    # Setup enricher
+    ots = setup_module("opentimestamps_enricher")
+    
+    # Create a calendar
+    calendar = RemoteCalendar("https://alice.btc.calendar.opentimestamps.org")
+    
+    # Test submission
+    file_hash = hashlib.sha256(b"Test file content").digest()
+    result = calendar.submit(file_hash)
+    
+    assert mock_submit.called
+    assert isinstance(result, Timestamp)
+    assert result == test_timestamp
+
+def test_verify_timestamp(setup_module, detached_timestamp_file):
+    """Test the verification of timestamp attestations"""
+    ots = setup_module("opentimestamps_enricher")
+    
+    # Test verification
+    verification_info = ots.verify_timestamp(detached_timestamp_file)
+    
+    # Check verification results
+    assert verification_info["attestation_count"] == 2
+    assert verification_info["verified"] == True
+    assert len(verification_info["attestations"]) == 2
+    
+    # Check attestation types
+    assertion_types = [a["type"] for a in verification_info["attestations"]]
+    assert "pending" in assertion_types
+    assert "bitcoin" in assertion_types
+    
+    # Check Bitcoin attestation details
+    bitcoin_attestation = next(a for a in verification_info["attestations"] if a["type"] == "bitcoin")
+    assert bitcoin_attestation["block_height"] == 783000
+
+def test_verify_pending_only(setup_module, pending_timestamp_file):
+    """Test verification of timestamps with only pending attestations"""
+    ots = setup_module("opentimestamps_enricher")
+    
+    verification_info = ots.verify_timestamp(pending_timestamp_file)
+    
+    assert verification_info["attestation_count"] == 2
+    assert verification_info["verified"] == False
+    assert verification_info["pending"] == True
+    
+    # All attestations should be of type "pending"
+    assert all(a["type"] == "pending" for a in verification_info["attestations"])
+    
+    # Check URIs of pending attestations
+    uris = [a["uri"] for a in verification_info["attestations"]]
+    assert "https://example1.calendar.com" in uris
+    assert "https://example2.calendar.com" in uris
+
+def test_verify_bitcoin_completed(setup_module, verified_timestamp_file):
+    """Test verification of timestamps with completed Bitcoin attestations"""
+    ots = setup_module("opentimestamps_enricher")
+    
+    verification_info = ots.verify_timestamp(verified_timestamp_file)
+    
+    assert verification_info["attestation_count"] == 1
+    assert verification_info["verified"] == True
+    assert "pending" not in verification_info
+    
+    # Check that the attestation is a Bitcoin attestation
+    attestation = verification_info["attestations"][0]
+    assert attestation["type"] == "bitcoin"
+    assert attestation["block_height"] == 783000
+
+def test_full_enriching(setup_module, sample_file_path, sample_media, mocker):
+    """Test the complete enrichment process"""
+    # Mock the calendar submission to avoid network requests
+    mock_calendar = mocker.patch.object(RemoteCalendar, 'submit')
+    test_timestamp = Timestamp(hashlib.sha256(b"test").digest())
+    # Add a bitcoin attestation to the test timestamp
+    bitcoin = BitcoinBlockHeaderAttestation(783000)
+    test_timestamp.attestations.add(bitcoin)
+    mock_calendar.return_value = test_timestamp
+    
+    # Setup enricher
+    ots = setup_module("opentimestamps_enricher")
+    
+    # Create test metadata with sample file
+    metadata = Metadata().set_url("https://example.com")
+    sample_media.set("filename", sample_file_path)
+    metadata.add_media(sample_media)
+    
+    # Run enrichment
+    ots.enrich(metadata)
+    
+    # Verify results
+    assert metadata.get("opentimestamped") == True
+    assert metadata.get("opentimestamps_count") == 1
+    
+    # Check that we have two media items: the original and the timestamp
+    assert len(metadata.media) == 2
+    
+    # Check that the original media was updated
+    assert metadata.media[0].get("opentimestamps") == True
+    assert metadata.media[0].get("opentimestamp_file") is not None
+    
+    # Check the timestamp file media
+    timestamp_media = metadata.media[1]
+    assert timestamp_media.get("source_file") == os.path.basename(sample_file_path)
+    assert timestamp_media.get("opentimestamps_version") is not None
+    
+    # Check verification results on the timestamp media
+    assert timestamp_media.get("verified") == True
+    assert timestamp_media.get("attestation_count") == 1
+
+def test_full_enriching_no_calendars(setup_module, sample_file_path, sample_media, mocker):
+    """Test enrichment process with calendars disabled"""
+    # Setup enricher with calendars disabled
+    ots = setup_module("opentimestamps_enricher", {"use_calendars": False})
+    
+    # Create test metadata with sample file
+    metadata = Metadata().set_url("https://example.com")
+    sample_media.set("filename", sample_file_path)
+    metadata.add_media(sample_media)
+    
+    # Run enrichment
+    ots.enrich(metadata)
+    
+    # Verify results
+    assert metadata.get("opentimestamped") == True
+    assert metadata.get("opentimestamps_count") == 1
+    
+    # Check the timestamp file media
+    timestamp_media = metadata.media[1]
+    assert timestamp_media.get("source_file") == os.path.basename(sample_file_path)
+    
+    # Verify status should be false since we didn't use calendars
+    assert timestamp_media.get("verified") == False
+    assert timestamp_media.get("attestation_count") == 0
+
+def test_full_enriching_calendar_error(setup_module, sample_file_path, sample_media, mocker):
+    """Test enrichment when calendar servers return errors"""
+    # Mock the calendar submission to raise an exception
+    mock_calendar = mocker.patch.object(RemoteCalendar, 'submit')
+    mock_calendar.side_effect = Exception("Calendar server error")
+    
+    # Setup enricher
+    ots = setup_module("opentimestamps_enricher")
+    
+    # Create test metadata with sample file
+    metadata = Metadata().set_url("https://example.com")
+    sample_media.set("filename", sample_file_path)
+    metadata.add_media(sample_media)
+    
+    # Run enrichment (should complete despite calendar errors)
+    ots.enrich(metadata)
+    
+    # Verify results
+    assert metadata.get("opentimestamped") == True
+    assert metadata.get("opentimestamps_count") == 1
+    
+    # Verify status should be false since calendar submissions failed
+    timestamp_media = metadata.media[1]
+    assert timestamp_media.get("verified") == False
+    assert timestamp_media.get("attestation_count") == 0
+
+def test_no_files_to_stamp(setup_module):
+    """Test enrichment with no files to timestamp"""
+    # Setup enricher
+    ots = setup_module("opentimestamps_enricher")
+    
+    # Create empty metadata
+    metadata = Metadata().set_url("https://example.com")
+    
+    # Run enrichment
+    ots.enrich(metadata)
+    
+    # Verify no timestamping occurred
+    assert metadata.get("opentimestamped") is None
+    assert len(metadata.media) == 0
\ No newline at end of file

From 8ca7698fa0e770322edf0044f38dbe01013788cd Mon Sep 17 00:00:00 2001
From: erinhmclark <erinhannahmary.clark@gmail.com>
Date: Tue, 11 Mar 2025 19:58:02 +0000
Subject: [PATCH 22/55] Move Makefile and fix import error with unused import.

---
 docs/Makefile => Makefile           |  4 ++--
 src/auto_archiver/core/storage.py   |  1 +
 tests/storages/test_storage_base.py | 19 ++++++++++++++++++-
 3 files changed, 21 insertions(+), 3 deletions(-)
 rename docs/Makefile => Makefile (91%)

diff --git a/docs/Makefile b/Makefile
similarity index 91%
rename from docs/Makefile
rename to Makefile
index 92dd33a..988ac2c 100644
--- a/docs/Makefile
+++ b/Makefile
@@ -5,8 +5,8 @@
 # from the environment for the first two.
 SPHINXOPTS    ?=
 SPHINXBUILD   ?= sphinx-build
-SOURCEDIR     = source
-BUILDDIR      = _build
+SOURCEDIR     = docs/source
+BUILDDIR      = docs/_build
 
 # Put it first so that "make" without argument is like "make help".
 help:
diff --git a/src/auto_archiver/core/storage.py b/src/auto_archiver/core/storage.py
index a13aa89..c73e29c 100644
--- a/src/auto_archiver/core/storage.py
+++ b/src/auto_archiver/core/storage.py
@@ -30,6 +30,7 @@ from slugify import slugify
 from auto_archiver.utils.misc import random_str
 
 from auto_archiver.core import Media, BaseModule, Metadata
+from auto_archiver.modules.hash_enricher.hash_enricher import HashEnricher
 
 class Storage(BaseModule):
     
diff --git a/tests/storages/test_storage_base.py b/tests/storages/test_storage_base.py
index 53dfbd7..62f2ddc 100644
--- a/tests/storages/test_storage_base.py
+++ b/tests/storages/test_storage_base.py
@@ -92,4 +92,21 @@ def test_really_long_name(storage_base, dummy_file):
     url = f"https://example.com/{'file'*100}"
     media = Media(filename=dummy_file)
     storage.set_key(media, url, Metadata())
-    assert media.key == f"https-example-com-{'file'*13}/6ae8a75555209fd6c44157c0.txt"
\ No newline at end of file
+    assert media.key == f"https-example-com-{'file'*13}/6ae8a75555209fd6c44157c0.txt"
+
+
+def test_storage_loads_hash_enricher(storage_base, dummy_file):
+    """Ensure 'hash_enricher' is properly loaded without an explicit import."""
+    config = {"path_generator": "url", "filename_generator": "static"}
+    storage = storage_base(config)
+
+    url = "https://example.com/file/"
+    media = Media(filename=dummy_file)
+    metadata = Metadata()
+
+    try:
+        storage.set_key(media, url, metadata)
+    except Exception as e:
+        pytest.fail(f"Storage failed to dynamically load hash_enricher: {e}")
+
+    assert media.key is not None, "Expected media.key to be set, but it was None"

From 1423c103631480371898d57e2c358f85be2238bd Mon Sep 17 00:00:00 2001
From: Patrick Robertson <robertson.patrick@gmail.com>
Date: Wed, 12 Mar 2025 10:24:57 +0000
Subject: [PATCH 23/55] Finish off timestamping module

---
 poetry.lock                                   | 30 +++++++-
 pyproject.toml                                |  1 +
 src/auto_archiver/core/module.py              |  4 +-
 .../opentimestamps_enricher/__manifest__.py   | 12 ++-
 .../opentimestamps_enricher.py                | 76 +++++++++++--------
 .../timestamping_enricher.py                  |  2 +-
 .../enrichers/test_opentimestamps_enricher.py | 68 +++++++++++------
 7 files changed, 130 insertions(+), 63 deletions(-)

diff --git a/poetry.lock b/poetry.lock
index f59d5c9..17365ac 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1361,6 +1361,22 @@ rsa = ["cryptography (>=3.0.0)"]
 signals = ["blinker (>=1.4.0)"]
 signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"]
 
+[[package]]
+name = "opentimestamps"
+version = "0.4.5"
+description = "Create and verify OpenTimestamps proofs"
+optional = false
+python-versions = "*"
+groups = ["main"]
+files = [
+    {file = "opentimestamps-0.4.5-py3-none-any.whl", hash = "sha256:a4912b3bd1b612a3ef5fac925b9137889e6c5cb91cc9e76c8202a2bf8abe26b5"},
+    {file = "opentimestamps-0.4.5.tar.gz", hash = "sha256:56726ccde97fb67f336a7f237ce36808e5593c3089d68d900b1c83d0ebf9dcfa"},
+]
+
+[package.dependencies]
+pycryptodomex = ">=3.3.1"
+python-bitcoinlib = ">=0.9.0,<0.13.0"
+
 [[package]]
 name = "oscrypto"
 version = "1.3.0"
@@ -1834,6 +1850,18 @@ pytest = ">=6.2.5"
 [package.extras]
 dev = ["pre-commit", "pytest-asyncio", "tox"]
 
+[[package]]
+name = "python-bitcoinlib"
+version = "0.12.2"
+description = "The Swiss Army Knife of the Bitcoin protocol."
+optional = false
+python-versions = "*"
+groups = ["main"]
+files = [
+    {file = "python-bitcoinlib-0.12.2.tar.gz", hash = "sha256:c65ab61427c77c38d397bfc431f71d86fd355b453a536496ec3fcb41bd10087d"},
+    {file = "python_bitcoinlib-0.12.2-py3-none-any.whl", hash = "sha256:2f29a9f475f21c12169b3a6cc8820f34f11362d7ff1200a5703dce3e4e903a44"},
+]
+
 [[package]]
 name = "python-dateutil"
 version = "2.9.0.post0"
@@ -3185,4 +3213,4 @@ test = ["pytest (>=8.1,<9.0)", "pytest-rerunfailures (>=14.0,<15.0)"]
 [metadata]
 lock-version = "2.1"
 python-versions = ">=3.10,<3.13"
-content-hash = "2d0a953383901fe12e97f6f56a76a9d8008788695425792eedbf739a18585188"
+content-hash = "e42f3bc122fe5d98deb6aa224ddf531b6f45a50b7c61213721ff5c8258e424e3"
diff --git a/pyproject.toml b/pyproject.toml
index 30693d5..b6b2aa8 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -57,6 +57,7 @@ dependencies = [
     "certvalidator (>=0.0.0)",
     "rich-argparse (>=1.6.0,<2.0.0)",
     "ruamel-yaml (>=0.18.10,<0.19.0)",
+    "opentimestamps (>=0.4.5,<0.5.0)",
 ]
 
 [tool.poetry.group.dev.dependencies]
diff --git a/src/auto_archiver/core/module.py b/src/auto_archiver/core/module.py
index 2c6617d..96d5420 100644
--- a/src/auto_archiver/core/module.py
+++ b/src/auto_archiver/core/module.py
@@ -6,7 +6,7 @@ by handling user configuration, validating the steps properties, and implementin
 from __future__ import annotations
 
 from dataclasses import dataclass
-from typing import List, TYPE_CHECKING
+from typing import List, TYPE_CHECKING, Type
 import shutil
 import ast
 import copy
@@ -57,7 +57,7 @@ class ModuleFactory:
 
         HAS_SETUP_PATHS = True
 
-    def get_module(self, module_name: str, config: dict) -> BaseModule:
+    def get_module(self, module_name: str, config: dict) -> Type[BaseModule]:
         """
         Gets and sets up a module using the provided config
         
diff --git a/src/auto_archiver/modules/opentimestamps_enricher/__manifest__.py b/src/auto_archiver/modules/opentimestamps_enricher/__manifest__.py
index 849bb3d..136c0c2 100644
--- a/src/auto_archiver/modules/opentimestamps_enricher/__manifest__.py
+++ b/src/auto_archiver/modules/opentimestamps_enricher/__manifest__.py
@@ -6,7 +6,6 @@
         "python": [
             "loguru",
             "opentimestamps",
-            "slugify",
         ],
     },
     "configs": {
@@ -19,14 +18,16 @@
             "default": [
                 "https://alice.btc.calendar.opentimestamps.org",
                 "https://bob.btc.calendar.opentimestamps.org",
-                "https://finney.calendar.eternitywall.com"
+                "https://finney.calendar.eternitywall.com",
+                # "https://ots.btc.catallaxy.com/", # ipv4 only
             ],
-            "help": "List of OpenTimestamps calendar servers to use for timestamping.",
+            "help": "List of OpenTimestamps calendar servers to use for timestamping. See here for a list of calendars maintained by opentimestamps:\
+https://opentimestamps.org/#calendars",
             "type": "list"
         },
         "calendar_whitelist": {
             "default": [],
-            "help": "Optional whitelist of calendar servers. If empty, all calendar servers are allowed.",
+            "help": "Optional whitelist of calendar servers. Override this if you are using your own calendar servers. e.g. ['https://mycalendar.com']",
             "type": "list"
         },
         "verify_timestamps": {
@@ -38,6 +39,9 @@
     "description": """
     Creates OpenTimestamps proofs for archived files, providing blockchain-backed evidence of file existence at a specific time.
 
+    Uses OpenTimestamps – a service that timestamps data using the Bitcoin blockchain, providing a decentralized 
+    and secure way to prove that data existed at a certain point in time.
+
     ### Features
     - Creates cryptographic timestamp proofs that link files to the Bitcoin blockchain
     - Verifies existing timestamp proofs to confirm the time a file existed
diff --git a/src/auto_archiver/modules/opentimestamps_enricher/opentimestamps_enricher.py b/src/auto_archiver/modules/opentimestamps_enricher/opentimestamps_enricher.py
index 01e8964..2b74ee6 100644
--- a/src/auto_archiver/modules/opentimestamps_enricher/opentimestamps_enricher.py
+++ b/src/auto_archiver/modules/opentimestamps_enricher/opentimestamps_enricher.py
@@ -1,36 +1,19 @@
 import os
 import hashlib
-from importlib.metadata import version
+from typing import TYPE_CHECKING
 
-from slugify import slugify
 from loguru import logger
 import opentimestamps
 from opentimestamps.calendar import RemoteCalendar, DEFAULT_CALENDAR_WHITELIST
 from opentimestamps.core.timestamp import Timestamp, DetachedTimestampFile
 from opentimestamps.core.notary import PendingAttestation, BitcoinBlockHeaderAttestation
+from opentimestamps.core.op import OpSHA256
+from opentimestamps.core import serialize
 from auto_archiver.core import Enricher
 from auto_archiver.core import Metadata, Media
-from auto_archiver.version import __version__
-
+from auto_archiver.utils.misc import calculate_file_hash
 
 class OpentimestampsEnricher(Enricher):
-    """
-    Uses OpenTimestamps to create and verify timestamps for files. OpenTimestamps is a service that 
-    timestamps data using the Bitcoin blockchain, providing a decentralized and secure way to prove 
-    that data existed at a certain point in time.
-
-    The enricher hashes files in the archive and creates timestamp proofs that can later be verified.
-    These proofs are stored alongside the original files and can be used to verify the timestamp
-    even if the OpenTimestamps calendar servers are unavailable.
-    """
-
-    def setup(self):
-        # Initialize any resources needed
-        pass
-
-    def cleanup(self) -> None:
-        # Clean up any resources used
-        pass
 
     def enrich(self, to_enrich: Metadata) -> None:
         url = to_enrich.get_url()
@@ -38,7 +21,7 @@ class OpentimestampsEnricher(Enricher):
 
         # Get the media files to timestamp
         media_files = [m for m in to_enrich.media if m.get("filename") and not m.get("opentimestamps")]
-        
+
         if not media_files:
             logger.warning(f"No files found to timestamp in {url=}")
             return
@@ -52,21 +35,26 @@ class OpentimestampsEnricher(Enricher):
                     logger.warning(f"File not found: {file_path}")
                     continue
                 
-                # Create timestamp for the file
+                # Create timestamp for the file - hash is SHA256
+                # Note: ONLY SHA256 is used/supported here. Opentimestamps supports other hashes, but not SHA3-512
+                # see opentimestamps.core.op
                 logger.debug(f"Creating timestamp for {file_path}")
-                
-                # Hash the file
+                file_hash = None
                 with open(file_path, 'rb') as f:
-                    file_bytes = f.read()
-                file_hash = hashlib.sha256(file_bytes).digest()
+                    file_hash = OpSHA256().hash_fd(f)
+
+                if not file_hash:
+                    logger.warning(f"Failed to hash file for timestamping, skipping: {file_path}")
+                    continue
                 
                 # Create a timestamp with the file hash
                 timestamp = Timestamp(file_hash)
                 
-                # Create a detached timestamp file with the timestamp
-                detached_timestamp = DetachedTimestampFile(timestamp)
+                # Create a detached timestamp file with the hash operation and timestamp
+                detached_timestamp = DetachedTimestampFile(OpSHA256(), timestamp)
                 
                 # Submit to calendar servers
+                submitted_to_calendar = False
                 if self.use_calendars:
                     logger.debug(f"Submitting timestamp to calendar servers for {file_path}")
                     calendars = []
@@ -76,9 +64,11 @@ class OpentimestampsEnricher(Enricher):
                         whitelist = set(self.calendar_whitelist)
                     
                     # Create calendar instances
+                    calendar_urls = []
                     for url in self.calendar_urls:
                         if url in whitelist:
                             calendars.append(RemoteCalendar(url))
+                            calendar_urls.append(url)
                     
                     # Submit the hash to each calendar
                     for calendar in calendars:
@@ -86,15 +76,35 @@ class OpentimestampsEnricher(Enricher):
                             calendar_timestamp = calendar.submit(file_hash)
                             timestamp.merge(calendar_timestamp)
                             logger.debug(f"Successfully submitted to calendar: {calendar.url}")
+                            submitted_to_calendar = True
                         except Exception as e:
                             logger.warning(f"Failed to submit to calendar {calendar.url}: {e}")
+                    
+                    # If all calendar submissions failed, add pending attestations
+                    if not submitted_to_calendar and not timestamp.attestations:
+                        logger.info("All calendar submissions failed, creating pending attestations")
+                        for url in calendar_urls:
+                            pending = PendingAttestation(url)
+                            timestamp.attestations.add(pending)
                 else:
                     logger.info("Skipping calendar submission as per configuration")
+                    
+                    # Add dummy pending attestation for testing when calendars are disabled
+                    for url in self.calendar_urls:
+                        pending = PendingAttestation(url)
+                        timestamp.attestations.add(pending)
                 
                 # Save the timestamp proof to a file
                 timestamp_path = os.path.join(self.tmp_dir, f"{os.path.basename(file_path)}.ots")
-                with open(timestamp_path, 'wb') as f:
-                    detached_timestamp.serialize(f)
+                try:
+                    with open(timestamp_path, 'wb') as f:
+                        # Create a serialization context and write to the file
+                        ctx = serialize.BytesSerializationContext()
+                        detached_timestamp.serialize(ctx)
+                        f.write(ctx.getbytes())
+                except Exception as e:
+                    logger.warning(f"Failed to serialize timestamp file: {e}")
+                    continue
                 
                 # Create media for the timestamp file
                 timestamp_media = Media(filename=timestamp_path)
@@ -106,6 +116,8 @@ class OpentimestampsEnricher(Enricher):
                     verification_info = self.verify_timestamp(detached_timestamp)
                     for key, value in verification_info.items():
                         timestamp_media.set(key, value)
+                else:
+                    logger.warning(f"Not verifying the timestamp for media file {file_path}")
                 
                 timestamp_files.append(timestamp_media)
                 
@@ -151,7 +163,7 @@ class OpentimestampsEnricher(Enricher):
                 # Process different types of attestations
                 if isinstance(attestation, PendingAttestation):
                     info["type"] = "pending"
-                    info["uri"] = attestation.uri.decode('utf-8')
+                    info["uri"] = attestation.uri
                 
                 elif isinstance(attestation, BitcoinBlockHeaderAttestation):
                     info["type"] = "bitcoin"
diff --git a/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py b/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py
index 078c1ba..b83e86c 100644
--- a/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py
+++ b/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py
@@ -30,7 +30,7 @@ class TimestampingEnricher(Enricher):
         if not len(hashes):
             logger.warning(f"No hashes found in {url=}")
             return
-        
+
         tmp_dir = self.tmp_dir
         hashes_fn = os.path.join(tmp_dir, "hashes.txt")
 
diff --git a/tests/enrichers/test_opentimestamps_enricher.py b/tests/enrichers/test_opentimestamps_enricher.py
index 5681561..db171e5 100644
--- a/tests/enrichers/test_opentimestamps_enricher.py
+++ b/tests/enrichers/test_opentimestamps_enricher.py
@@ -10,53 +10,69 @@ from opentimestamps.core.notary import PendingAttestation, BitcoinBlockHeaderAtt
 
 from auto_archiver.core import Metadata, Media
 
+
+# TODO: Remove once timestamping overhaul is merged
 @pytest.fixture
-def sample_file_path():
-    with tempfile.NamedTemporaryFile(delete=False) as tmp:
-        tmp.write(b"This is a test file content for OpenTimestamps")
-        return tmp.name
+def sample_media(tmp_path) -> Media:
+    """Fixture creating a Media object with temporary source file"""
+    src_file = tmp_path / "source.txt"
+    src_file.write_text("test content")
+    return Media(_key="subdir/test.txt", filename=str(src_file))
+
+
+@pytest.fixture
+def sample_file_path(tmp_path):
+    tmp_file = tmp_path / "test.txt"
+    tmp_file.write_text("This is a test file content for OpenTimestamps")
+    return str(tmp_file)
 
 @pytest.fixture
 def detached_timestamp_file():
     """Create a simple detached timestamp file for testing"""
     file_hash = hashlib.sha256(b"Test content").digest()
+    from opentimestamps.core.op import OpSHA256
+    file_hash_op = OpSHA256()
     timestamp = Timestamp(file_hash)
     
     # Add a pending attestation
-    pending = PendingAttestation(b"https://example.calendar.com")
+    pending = PendingAttestation("https://example.calendar.com")
     timestamp.attestations.add(pending)
     
     # Add a bitcoin attestation
     bitcoin = BitcoinBlockHeaderAttestation(783000)  # Some block height
     timestamp.attestations.add(bitcoin)
     
-    return DetachedTimestampFile(timestamp)
+    return DetachedTimestampFile(file_hash_op, timestamp)
 
 @pytest.fixture
 def verified_timestamp_file():
     """Create a timestamp file with a Bitcoin attestation"""
     file_hash = hashlib.sha256(b"Verified content").digest()
+    from opentimestamps.core.op import OpSHA256
+    file_hash_op = OpSHA256()
     timestamp = Timestamp(file_hash)
     
     # Add only a Bitcoin attestation
     bitcoin = BitcoinBlockHeaderAttestation(783000)  # Some block height
     timestamp.attestations.add(bitcoin)
     
-    return DetachedTimestampFile(timestamp)
+    return DetachedTimestampFile(file_hash_op, timestamp)
 
 @pytest.fixture
 def pending_timestamp_file():
     """Create a timestamp file with only pending attestations"""
     file_hash = hashlib.sha256(b"Pending content").digest()
+    from opentimestamps.core.op import OpSHA256
+    file_hash_op = OpSHA256()
     timestamp = Timestamp(file_hash)
     
     # Add only pending attestations
-    pending1 = PendingAttestation(b"https://example1.calendar.com")
-    pending2 = PendingAttestation(b"https://example2.calendar.com")
+    pending1 = PendingAttestation("https://example1.calendar.com")
+    pending2 = PendingAttestation("https://example2.calendar.com")
     timestamp.attestations.add(pending1)
     timestamp.attestations.add(pending2)
     
-    return DetachedTimestampFile(timestamp)
+    return DetachedTimestampFile(file_hash_op, timestamp)
 
 @pytest.mark.download
 def test_download_tsr(setup_module, mocker):
@@ -66,7 +82,7 @@ def test_download_tsr(setup_module, mocker):
     test_timestamp = Timestamp(hashlib.sha256(b"test").digest())
     mock_submit.return_value = test_timestamp
     
-    # Setup enricher
+
     ots = setup_module("opentimestamps_enricher")
     
     # Create a calendar
@@ -121,6 +137,7 @@ def test_verify_pending_only(setup_module, pending_timestamp_file):
 
 def test_verify_bitcoin_completed(setup_module, verified_timestamp_file):
     """Test verification of timestamps with completed Bitcoin attestations"""
+
     ots = setup_module("opentimestamps_enricher")
     
     verification_info = ots.verify_timestamp(verified_timestamp_file)
@@ -136,15 +153,21 @@ def test_verify_bitcoin_completed(setup_module, verified_timestamp_file):
 
 def test_full_enriching(setup_module, sample_file_path, sample_media, mocker):
     """Test the complete enrichment process"""
+
     # Mock the calendar submission to avoid network requests
     mock_calendar = mocker.patch.object(RemoteCalendar, 'submit')
-    test_timestamp = Timestamp(hashlib.sha256(b"test").digest())
-    # Add a bitcoin attestation to the test timestamp
-    bitcoin = BitcoinBlockHeaderAttestation(783000)
-    test_timestamp.attestations.add(bitcoin)
-    mock_calendar.return_value = test_timestamp
     
-    # Setup enricher
+    # Create a function that returns a new timestamp for each call
+    def side_effect(digest):
+        test_timestamp = Timestamp(digest)
+        # Add a bitcoin attestation to the test timestamp
+        bitcoin = BitcoinBlockHeaderAttestation(783000)
+        test_timestamp.attestations.add(bitcoin)
+        return test_timestamp
+    
+    mock_calendar.side_effect = side_effect
+    
+
     ots = setup_module("opentimestamps_enricher")
     
     # Create test metadata with sample file
@@ -176,8 +199,6 @@ def test_full_enriching(setup_module, sample_file_path, sample_media, mocker):
     assert timestamp_media.get("attestation_count") == 1
 
 def test_full_enriching_no_calendars(setup_module, sample_file_path, sample_media, mocker):
-    """Test enrichment process with calendars disabled"""
-    # Setup enricher with calendars disabled
     ots = setup_module("opentimestamps_enricher", {"use_calendars": False})
     
     # Create test metadata with sample file
@@ -198,7 +219,8 @@ def test_full_enriching_no_calendars(setup_module, sample_file_path, sample_medi
     
     # Verify status should be false since we didn't use calendars
     assert timestamp_media.get("verified") == False
-    assert timestamp_media.get("attestation_count") == 0
+    # We expect 3 pending attestations (one for each calendar URL)
+    assert timestamp_media.get("attestation_count") == 3
 
 def test_full_enriching_calendar_error(setup_module, sample_file_path, sample_media, mocker):
     """Test enrichment when calendar servers return errors"""
@@ -206,7 +228,7 @@ def test_full_enriching_calendar_error(setup_module, sample_file_path, sample_me
     mock_calendar = mocker.patch.object(RemoteCalendar, 'submit')
     mock_calendar.side_effect = Exception("Calendar server error")
     
-    # Setup enricher
+
     ots = setup_module("opentimestamps_enricher")
     
     # Create test metadata with sample file
@@ -224,11 +246,11 @@ def test_full_enriching_calendar_error(setup_module, sample_file_path, sample_me
     # Verify status should be false since calendar submissions failed
     timestamp_media = metadata.media[1]
     assert timestamp_media.get("verified") == False
-    assert timestamp_media.get("attestation_count") == 0
+    # We expect 3 pending attestations (one for each calendar URL that's enabled by default in __manifest__)
+    assert timestamp_media.get("attestation_count") == 3
 
 def test_no_files_to_stamp(setup_module):
     """Test enrichment with no files to timestamp"""
-    # Setup enricher
     ots = setup_module("opentimestamps_enricher")
     
     # Create empty metadata

From abc90b19d5777804a829a29a4561d7eebbd8b9f8 Mon Sep 17 00:00:00 2001
From: erinhmclark <erinhannahmary.clark@gmail.com>
Date: Wed, 12 Mar 2025 10:35:56 +0000
Subject: [PATCH 24/55] Update pyproject.toml

---
 pyproject.toml | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index cbcfb61..86c8db8 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -95,6 +95,9 @@ markers = [
 [tool.ruff]
 #exclude = ["docs"]
 line-length = 120
+# Remove this for a more detailed lint report
+output-format = "concise"
+
 
 [tool.ruff.lint]
 #add bugbear?
@@ -103,8 +106,9 @@ line-length = 120
 # ANN : annotations
 #extend-select = ["B"]
 
-# E701 - multiple statements on one line (I vote to keep this but I notice it's used quite a lot!)
-ignore = []
+# Ignore unused imports as some are currently required for lazy loading
+# This can be removed for a `lint check` run which is manually reviewed
+ignore = ["F401"]
 
 [tool.ruff.lint.per-file-ignores]
 # Ignore import violations in __init__.py files

From 94aeee8313f13b02ac2fca38094064d6d5d70f52 Mon Sep 17 00:00:00 2001
From: erinhmclark <erinhannahmary.clark@gmail.com>
Date: Wed, 12 Mar 2025 10:37:30 +0000
Subject: [PATCH 25/55] Move Makefile to the root of the project and add
 commands for tests, linting and running docker.

---
 Makefile | 47 ++++++++++++++++++++++++++++++++++++++---------
 1 file changed, 38 insertions(+), 9 deletions(-)

diff --git a/Makefile b/Makefile
index 988ac2c..72f2058 100644
--- a/Makefile
+++ b/Makefile
@@ -1,20 +1,49 @@
-# Minimal makefile for Sphinx documentation
-#
-
-# You can set these variables from the command line, and also
-# from the environment for the first two.
+# Variables
 SPHINXOPTS    ?=
 SPHINXBUILD   ?= sphinx-build
 SOURCEDIR     = docs/source
 BUILDDIR      = docs/_build
 
-# Put it first so that "make" without argument is like "make help".
+.PHONY: help
 help:
 	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+	@echo "Additional Commands:"
+	@echo "  make test         - Run all tests in 'tests/' with pytest"
+	@echo "  make lint         - Run ruff linter and auto-fix issues"
+	@echo "  make docs         - Generate documentation (same as 'make html')"
+	@echo "  make clean_docs   - Remove generated docs"
+	@echo "  make docker-run   - Run the Docker container"
 
-.PHONY: help Makefile
+.PHONY: test
+test:
+	@echo "Running tests..."
+	@pytest tests --disable-warnings
 
-# Catch-all target: route all unknown targets to Sphinx using the new
-# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+.PHONY: lint
+lint:
+	@echo "Linting with ruff..."
+	@ruff check --fix .
+
+.PHONY: docs
+docs:
+	@echo "Building documentation..."
+	@$(SPHINXBUILD) -M html "$(SOURCEDIR)" "$(BUILDDIR)"
+
+.PHONY: clean_docs
+clean_docs:
+	@echo "Cleaning up generated documentation files..."
+	@$(SPHINXBUILD) -M clean "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+	@rm -rf "$(SOURCEDIR)/autoapi/" "$(SOURCEDIR)/modules/autogen/"
+	@echo "Cleanup complete."
+
+
+# Run Docker with default settings
+.PHONY: docker-run
+docker-run:
+	@echo "Running Auto Archiver Docker container..."
+	@docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver
+
+# Catch-all for Sphinx commands
+.PHONY: Makefile
 %: Makefile
 	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

From 79f576be1dee4448c9b78a6424aef132dff877ca Mon Sep 17 00:00:00 2001
From: erinhmclark <erinhannahmary.clark@gmail.com>
Date: Wed, 12 Mar 2025 10:38:16 +0000
Subject: [PATCH 26/55] Run fix on tests.

---
 tests/databases/test_atlos_db.py           | 4 ++--
 tests/enrichers/test_thumbnail_enricher.py | 2 +-
 tests/storages/test_atlos_storage.py       | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/databases/test_atlos_db.py b/tests/databases/test_atlos_db.py
index 6c79a53..15f4b55 100644
--- a/tests/databases/test_atlos_db.py
+++ b/tests/databases/test_atlos_db.py
@@ -49,7 +49,7 @@ def test_failed_with_atlos_id(atlos_db, metadata, mocker):
     fake_resp = FakeAPIResponse({}, raise_error=False)
     post_mock = mocker.patch.object(atlos_db, "_post", return_value=fake_resp)
     atlos_db.failed(metadata, "failure reason")
-    expected_endpoint = f"/api/v2/source_material/metadata/42/auto_archiver"
+    expected_endpoint = "/api/v2/source_material/metadata/42/auto_archiver"
     expected_json = {"metadata": {"processed": True, "status": "error", "error": "failure reason"}}
     post_mock.assert_called_once_with(expected_endpoint, json=expected_json)
 
@@ -85,7 +85,7 @@ def test_done_with_atlos_id(atlos_db, metadata, mocker):
     fake_resp = FakeAPIResponse({}, raise_error=False)
     post_mock = mocker.patch.object(atlos_db, "_post", return_value=fake_resp)
     atlos_db.done(metadata)
-    expected_endpoint = f"/api/v2/source_material/metadata/99/auto_archiver"
+    expected_endpoint = "/api/v2/source_material/metadata/99/auto_archiver"
     expected_results = metadata.metadata.copy()
     expected_results["timestamp"] = now.isoformat()
     expected_json = {
diff --git a/tests/enrichers/test_thumbnail_enricher.py b/tests/enrichers/test_thumbnail_enricher.py
index 3ebc798..fdc28b7 100644
--- a/tests/enrichers/test_thumbnail_enricher.py
+++ b/tests/enrichers/test_thumbnail_enricher.py
@@ -79,7 +79,7 @@ def test_enrich_handles_probe_failure(thumbnail_enricher, metadata_with_video, m
 
     thumbnail_enricher.enrich(metadata_with_video)
     # Ensure error was logged
-    mock_logger.assert_called_with(f"error getting duration of video video.mp4: Probe error")
+    mock_logger.assert_called_with("error getting duration of video video.mp4: Probe error")
     # Ensure no thumbnails were created
     thumbnails = metadata_with_video.media[0].get("thumbnails")
     assert thumbnails is None
diff --git a/tests/storages/test_atlos_storage.py b/tests/storages/test_atlos_storage.py
index 81cae70..7268c8d 100644
--- a/tests/storages/test_atlos_storage.py
+++ b/tests/storages/test_atlos_storage.py
@@ -90,7 +90,7 @@ def test_upload_not_uploaded(tmp_path, atlos_storage: AtlosStorage, metadata: Me
 
     get_mock.assert_called_once()
     post_mock.assert_called_once()
-    expected_endpoint = f"/api/v2/source_material/upload/202"
+    expected_endpoint = "/api/v2/source_material/upload/202"
     call_args = post_mock.call_args[0]
     assert call_args[0] == expected_endpoint
     call_kwargs = post_mock.call_args[1]

From 394b8b2dd18aea008e4e742bb2e987c7b519c646 Mon Sep 17 00:00:00 2001
From: Patrick Robertson <robertson.patrick@gmail.com>
Date: Wed, 12 Mar 2025 11:45:13 +0000
Subject: [PATCH 27/55] Improvements to opentimestamps enricher - make OTS file
 a sub-file of original media

---
 scripts/settings/src/schema.json              | 596 ++++++++----------
 .../modules/gdrive_storage/__manifest__.py    |   2 +-
 .../modules/local_storage/__manifest__.py     |   2 +-
 .../opentimestamps_enricher/__manifest__.py   |   7 +
 .../opentimestamps_enricher.py                |  20 +-
 .../modules/s3_storage/__manifest__.py        |   2 +-
 .../enrichers/test_opentimestamps_enricher.py |  27 +-
 7 files changed, 309 insertions(+), 347 deletions(-)

diff --git a/scripts/settings/src/schema.json b/scripts/settings/src/schema.json
index 64a903a..70eb71b 100644
--- a/scripts/settings/src/schema.json
+++ b/scripts/settings/src/schema.json
@@ -1,151 +1,25 @@
 {
     "modules": {
-        "gsheet_feeder": {
-            "name": "gsheet_feeder",
-            "display_name": "Google Sheets Feeder",
+        "atlos_feeder_db_storage": {
+            "name": "atlos_feeder_db_storage",
+            "display_name": "Atlos Feeder Database Storage",
             "manifest": {
-                "name": "Google Sheets Feeder",
+                "name": "Atlos Feeder Database Storage",
                 "author": "Bellingcat",
                 "type": [
-                    "feeder"
+                    "feeder",
+                    "database",
+                    "storage"
                 ],
                 "requires_setup": true,
-                "description": "\n    GsheetsFeeder \n    A Google Sheets-based feeder for the Auto Archiver.\n\n    This reads data from Google Sheets and filters rows based on user-defined rules.\n    The filtered rows are processed into `Metadata` objects.\n\n    ### Features\n    - Validates the sheet structure and filters rows based on input configurations.\n    - Processes only worksheets allowed by the `allow_worksheets` and `block_worksheets` configurations.\n    - Ensures only rows with valid URLs and unprocessed statuses are included for archival.\n    - Supports organizing stored files into folder paths based on sheet and worksheet names.\n\n    ### Setup\n    - Requires a Google Service Account JSON file for authentication, which should be stored in `secrets/gsheets_service_account.json`.\n    To set up a service account, follow the instructions [here](https://gspread.readthedocs.io/en/latest/oauth2.html).\n    - Define the `sheet` or `sheet_id` configuration to specify the sheet to archive.\n    - Customize the column names in your Google sheet using the `columns` configuration.\n    ",
-                "dependencies": {
-                    "python": [
-                        "loguru",
-                        "gspread",
-                        "slugify"
-                    ]
-                },
-                "entry_point": "gsheet_feeder::GsheetsFeeder",
-                "version": "1.0",
-                "configs": {
-                    "sheet": {
-                        "default": null,
-                        "help": "name of the sheet to archive"
-                    },
-                    "sheet_id": {
-                        "default": null,
-                        "help": "the id of the sheet to archive (alternative to 'sheet' config)"
-                    },
-                    "header": {
-                        "default": 1,
-                        "type": "int",
-                        "help": "index of the header row (starts at 1)"
-                    },
-                    "service_account": {
-                        "default": "secrets/service_account.json",
-                        "help": "service account JSON file path. Learn how to create one: https://gspread.readthedocs.io/en/latest/oauth2.html",
-                        "required": true
-                    },
-                    "columns": {
-                        "default": {
-                            "url": "link",
-                            "status": "archive status",
-                            "folder": "destination folder",
-                            "archive": "archive location",
-                            "date": "archive date",
-                            "thumbnail": "thumbnail",
-                            "timestamp": "upload timestamp",
-                            "title": "upload title",
-                            "text": "text content",
-                            "screenshot": "screenshot",
-                            "hash": "hash",
-                            "pdq_hash": "perceptual hashes",
-                            "wacz": "wacz",
-                            "replaywebpage": "replaywebpage"
-                        },
-                        "help": "Custom names for the columns in your Google sheet. If you don't want to use the default column names, change them with this setting",
-                        "type": "json_loader"
-                    },
-                    "allow_worksheets": {
-                        "default": [],
-                        "help": "A list of worksheet names that should be processed (overrides worksheet_block), leave empty so all are allowed"
-                    },
-                    "block_worksheets": {
-                        "default": [],
-                        "help": "A list of worksheet names for worksheets that should be explicitly blocked from being processed"
-                    },
-                    "use_sheet_names_in_stored_paths": {
-                        "default": true,
-                        "help": "if True the stored files path will include 'workbook_name/worksheet_name/...'",
-                        "type": "bool"
-                    }
-                }
-            },
-            "configs": {
-                "sheet": {
-                    "default": null,
-                    "help": "name of the sheet to archive"
-                },
-                "sheet_id": {
-                    "default": null,
-                    "help": "the id of the sheet to archive (alternative to 'sheet' config)"
-                },
-                "header": {
-                    "default": 1,
-                    "type": "int",
-                    "help": "index of the header row (starts at 1)"
-                },
-                "service_account": {
-                    "default": "secrets/service_account.json",
-                    "help": "service account JSON file path. Learn how to create one: https://gspread.readthedocs.io/en/latest/oauth2.html",
-                    "required": true
-                },
-                "columns": {
-                    "default": {
-                        "url": "link",
-                        "status": "archive status",
-                        "folder": "destination folder",
-                        "archive": "archive location",
-                        "date": "archive date",
-                        "thumbnail": "thumbnail",
-                        "timestamp": "upload timestamp",
-                        "title": "upload title",
-                        "text": "text content",
-                        "screenshot": "screenshot",
-                        "hash": "hash",
-                        "pdq_hash": "perceptual hashes",
-                        "wacz": "wacz",
-                        "replaywebpage": "replaywebpage"
-                    },
-                    "help": "Custom names for the columns in your Google sheet. If you don't want to use the default column names, change them with this setting",
-                    "type": "json_loader"
-                },
-                "allow_worksheets": {
-                    "default": [],
-                    "help": "A list of worksheet names that should be processed (overrides worksheet_block), leave empty so all are allowed"
-                },
-                "block_worksheets": {
-                    "default": [],
-                    "help": "A list of worksheet names for worksheets that should be explicitly blocked from being processed"
-                },
-                "use_sheet_names_in_stored_paths": {
-                    "default": true,
-                    "help": "if True the stored files path will include 'workbook_name/worksheet_name/...'",
-                    "type": "bool"
-                }
-            }
-        },
-        "atlos_feeder": {
-            "name": "atlos_feeder",
-            "display_name": "Atlos Feeder",
-            "manifest": {
-                "name": "Atlos Feeder",
-                "author": "Bellingcat",
-                "type": [
-                    "feeder"
-                ],
-                "requires_setup": true,
-                "description": "\n    AtlosFeeder: A feeder module that integrates with the Atlos API to fetch source material URLs for archival.\n\n    ### Features\n    - Connects to the Atlos API to retrieve a list of source material URLs.\n    - Filters source materials based on visibility, processing status, and metadata.\n    - Converts filtered source materials into `Metadata` objects with the relevant `atlos_id` and URL.\n    - Iterates through paginated results using a cursor for efficient API interaction.\n\n    ### Notes\n    - Requires an Atlos API endpoint and a valid API token for authentication.\n    - Ensures only unprocessed, visible, and ready-to-archive URLs are returned.\n    - Handles pagination transparently when retrieving data from the Atlos API.\n    ",
+                "description": "\n    A module that integrates with the Atlos API to fetch source material URLs for archival, uplaod extracted media,\n    \n    [Atlos](https://www.atlos.org/) is a visual investigation and archiving platform designed for investigative research, journalism, and open-source intelligence (OSINT). \n    It helps users organize, analyze, and store media from various sources, making it easier to track and investigate digital evidence.\n    \n    To get started create a new project and obtain an API token from the settings page. You can group event's into Atlos's 'incidents'.\n    Here you can add 'source material' by URLn and the Atlos feeder will fetch these URLs for archival.\n    \n    You can use Atlos only as a 'feeder', however you can also implement the 'database' and 'storage' features to store the media files in Atlos which is recommended.\n    The Auto Archiver will retain the Atlos ID for each item, ensuring that the media and database outputs are uplaoded back into the relevant media item.\n    \n    \n    ### Features\n    - Connects to the Atlos API to retrieve a list of source material URLs.\n    - Iterates through the URLs from all source material items which are unprocessed, visible, and ready to archive.\n    - If the storage option is selected, it will store the media files alongside the original source material item in Atlos.\n    - Is the database option is selected it will output the results to the media item, as well as updating failure status with error details when archiving fails.\n    - Skips Storege/ database upload for items without an Atlos ID - restricting that you must use the Atlos feeder so that it has the Atlos ID to store the results with.\n\n    ### Notes\n    - Requires an Atlos account with a project and a valid API token for authentication.\n    - Ensures only unprocessed, visible, and ready-to-archive URLs are returned.\n    - Feches any media items within an Atlos project, regardless of separation into incidents.\n    ",
                 "dependencies": {
                     "python": [
                         "loguru",
                         "requests"
                     ]
                 },
-                "entry_point": "",
+                "entry_point": "atlos_feeder_db_storage::AtlosFeederDbStorage",
                 "version": "1.0",
                 "configs": {
                     "api_token": {
@@ -222,6 +96,135 @@
                 }
             }
         },
+        "gsheet_feeder_db": {
+            "name": "gsheet_feeder_db",
+            "display_name": "Google Sheets Feeder Database",
+            "manifest": {
+                "name": "Google Sheets Feeder Database",
+                "author": "Bellingcat",
+                "type": [
+                    "feeder",
+                    "database"
+                ],
+                "requires_setup": true,
+                "description": "\n    GsheetsFeederDatabase\n    A Google Sheets-based feeder and optional database for the Auto Archiver.\n\n    This reads data from Google Sheets and filters rows based on user-defined rules.\n    The filtered rows are processed into `Metadata` objects.\n\n    ### Features\n    - Validates the sheet structure and filters rows based on input configurations.\n    - Processes only worksheets allowed by the `allow_worksheets` and `block_worksheets` configurations.\n    - Ensures only rows with valid URLs and unprocessed statuses are included for archival.\n    - Supports organizing stored files into folder paths based on sheet and worksheet names.\n    - If the database is enabled, this updates the Google Sheet with the status of the archived URLs, including in progress, success or failure, and method used.\n    - Saves metadata such as title, text, timestamp, hashes, screenshots, and media URLs to designated columns.\n    - Formats media-specific metadata, such as thumbnails and PDQ hashes for the sheet.\n    - Skips redundant updates for empty or invalid data fields.\n\n    ### Setup\n    - Requires a Google Service Account JSON file for authentication, which should be stored in `secrets/gsheets_service_account.json`.\n    To set up a service account, follow the instructions [here](https://gspread.readthedocs.io/en/latest/oauth2.html).\n    - Define the `sheet` or `sheet_id` configuration to specify the sheet to archive.\n    - Customize the column names in your Google sheet using the `columns` configuration.\n    - The Google Sheet can be used soley as a feeder or as a feeder and database, but note you can't currently feed into the database from an alternate feeder.\n    ",
+                "dependencies": {
+                    "python": [
+                        "loguru",
+                        "gspread",
+                        "slugify"
+                    ]
+                },
+                "entry_point": "gsheet_feeder_db::GsheetsFeederDB",
+                "version": "1.0",
+                "configs": {
+                    "sheet": {
+                        "default": null,
+                        "help": "name of the sheet to archive"
+                    },
+                    "sheet_id": {
+                        "default": null,
+                        "help": "the id of the sheet to archive (alternative to 'sheet' config)"
+                    },
+                    "header": {
+                        "default": 1,
+                        "type": "int",
+                        "help": "index of the header row (starts at 1)"
+                    },
+                    "service_account": {
+                        "default": "secrets/service_account.json",
+                        "help": "service account JSON file path. Learn how to create one: https://gspread.readthedocs.io/en/latest/oauth2.html",
+                        "required": true
+                    },
+                    "columns": {
+                        "default": {
+                            "url": "link",
+                            "status": "archive status",
+                            "folder": "destination folder",
+                            "archive": "archive location",
+                            "date": "archive date",
+                            "thumbnail": "thumbnail",
+                            "timestamp": "upload timestamp",
+                            "title": "upload title",
+                            "text": "text content",
+                            "screenshot": "screenshot",
+                            "hash": "hash",
+                            "pdq_hash": "perceptual hashes",
+                            "wacz": "wacz",
+                            "replaywebpage": "replaywebpage"
+                        },
+                        "help": "Custom names for the columns in your Google sheet. If you don't want to use the default column names, change them with this setting",
+                        "type": "json_loader"
+                    },
+                    "allow_worksheets": {
+                        "default": [],
+                        "help": "(CSV) only worksheets whose name is included in allow are included (overrides worksheet_block), leave empty so all are allowed"
+                    },
+                    "block_worksheets": {
+                        "default": [],
+                        "help": "(CSV) explicitly block some worksheets from being processed"
+                    },
+                    "use_sheet_names_in_stored_paths": {
+                        "default": true,
+                        "type": "bool",
+                        "help": "if True the stored files path will include 'workbook_name/worksheet_name/...'"
+                    }
+                }
+            },
+            "configs": {
+                "sheet": {
+                    "default": null,
+                    "help": "name of the sheet to archive"
+                },
+                "sheet_id": {
+                    "default": null,
+                    "help": "the id of the sheet to archive (alternative to 'sheet' config)"
+                },
+                "header": {
+                    "default": 1,
+                    "type": "int",
+                    "help": "index of the header row (starts at 1)"
+                },
+                "service_account": {
+                    "default": "secrets/service_account.json",
+                    "help": "service account JSON file path. Learn how to create one: https://gspread.readthedocs.io/en/latest/oauth2.html",
+                    "required": true
+                },
+                "columns": {
+                    "default": {
+                        "url": "link",
+                        "status": "archive status",
+                        "folder": "destination folder",
+                        "archive": "archive location",
+                        "date": "archive date",
+                        "thumbnail": "thumbnail",
+                        "timestamp": "upload timestamp",
+                        "title": "upload title",
+                        "text": "text content",
+                        "screenshot": "screenshot",
+                        "hash": "hash",
+                        "pdq_hash": "perceptual hashes",
+                        "wacz": "wacz",
+                        "replaywebpage": "replaywebpage"
+                    },
+                    "help": "Custom names for the columns in your Google sheet. If you don't want to use the default column names, change them with this setting",
+                    "type": "json_loader"
+                },
+                "allow_worksheets": {
+                    "default": [],
+                    "help": "(CSV) only worksheets whose name is included in allow are included (overrides worksheet_block), leave empty so all are allowed"
+                },
+                "block_worksheets": {
+                    "default": [],
+                    "help": "(CSV) explicitly block some worksheets from being processed"
+                },
+                "use_sheet_names_in_stored_paths": {
+                    "default": true,
+                    "type": "bool",
+                    "help": "if True the stored files path will include 'workbook_name/worksheet_name/...'"
+                }
+            }
+        },
         "cli_feeder": {
             "name": "cli_feeder",
             "display_name": "Command Line Feeder",
@@ -470,7 +473,7 @@
                     "extractor"
                 ],
                 "requires_setup": true,
-                "description": "\n    Uses the [Instaloader library](https://instaloader.github.io/as-module.html) to download content from Instagram. This class handles both individual posts\n    and user profiles, downloading as much information as possible, including images, videos, text, stories,\n    highlights, and tagged posts. \n    Authentication is required via username/password or a session file.\n                    \n                    ",
+                "description": "\n    Uses the [Instaloader library](https://instaloader.github.io/as-module.html) to download content from Instagram. \n    \n      > \u26a0\ufe0f **Warning**  \n      > This module is not actively maintained due to known issues with blocking.  \n      > Prioritise usage of the [Instagram Tbot Extractor](./instagram_tbot_extractor.md) and [Instagram API Extractor](./instagram_api_extractor.md)\n  \n    This class handles both individual posts and user profiles, downloading as much information as possible, including images, videos, text, stories,\n    highlights, and tagged posts. \n    Authentication is required via username/password or a session file.\n                    \n                    ",
                 "dependencies": {
                     "python": [
                         "instaloader",
@@ -482,38 +485,38 @@
                 "configs": {
                     "username": {
                         "required": true,
-                        "help": "a valid Instagram username"
+                        "help": "A valid Instagram username."
                     },
                     "password": {
                         "required": true,
-                        "help": "the corresponding Instagram account password"
+                        "help": "The corresponding Instagram account password."
                     },
                     "download_folder": {
                         "default": "instaloader",
-                        "help": "name of a folder to temporarily download content to"
+                        "help": "Name of a folder to temporarily download content to."
                     },
                     "session_file": {
                         "default": "secrets/instaloader.session",
-                        "help": "path to the instagram session which saves session credentials"
+                        "help": "Path to the instagram session file which saves session credentials. If one doesn't exist this gives the path to store a new one."
                     }
                 }
             },
             "configs": {
                 "username": {
                     "required": true,
-                    "help": "a valid Instagram username"
+                    "help": "A valid Instagram username."
                 },
                 "password": {
                     "required": true,
-                    "help": "the corresponding Instagram account password"
+                    "help": "The corresponding Instagram account password."
                 },
                 "download_folder": {
                     "default": "instaloader",
-                    "help": "name of a folder to temporarily download content to"
+                    "help": "Name of a folder to temporarily download content to."
                 },
                 "session_file": {
                     "default": "secrets/instaloader.session",
-                    "help": "path to the instagram session which saves session credentials"
+                    "help": "Path to the instagram session file which saves session credentials. If one doesn't exist this gives the path to store a new one."
                 }
             }
         },
@@ -661,7 +664,7 @@
                     "extractor"
                 ],
                 "requires_setup": false,
-                "description": "\nThis is the generic extractor used by auto-archiver, which uses `yt-dlp` under the hood.\n\nThis module is responsible for downloading and processing media content from platforms\nsupported by `yt-dlp`, such as YouTube, Facebook, and others. It provides functionality\nfor retrieving videos, subtitles, comments, and other metadata, and it integrates with\nthe broader archiving framework.\n\n### Features\n- Supports downloading videos and playlists.\n- Retrieves metadata like titles, descriptions, upload dates, and durations.\n- Downloads subtitles and comments when enabled.\n- Configurable options for handling live streams, proxies, and more.\n- Supports authentication of websites using the 'authentication' settings from your orchestration.\n\n### Dropins\n- For websites supported by `yt-dlp` that also contain posts in addition to videos\n (e.g. Facebook, Twitter, Bluesky), dropins can be created to extract post data and create \n metadata objects. Some dropins are included in this generic_archiver by default, but\ncustom dropins can be created to handle additional websites and passed to the archiver\nvia the command line using the `--dropins` option (TODO!).\n",
+                "description": "\nThis is the generic extractor used by auto-archiver, which uses `yt-dlp` under the hood.\n\nThis module is responsible for downloading and processing media content from platforms\nsupported by `yt-dlp`, such as YouTube, Facebook, and others. It provides functionality\nfor retrieving videos, subtitles, comments, and other metadata, and it integrates with\nthe broader archiving framework.\n\n### Features\n- Supports downloading videos and playlists.\n- Retrieves metadata like titles, descriptions, upload dates, and durations.\n- Downloads subtitles and comments when enabled.\n- Configurable options for handling live streams, proxies, and more.\n- Supports authentication of websites using the 'authentication' settings from your orchestration.\n\n### Dropins\n- For websites supported by `yt-dlp` that also contain posts in addition to videos\n (e.g. Facebook, Twitter, Bluesky), dropins can be created to extract post data and create \n metadata objects. Some dropins are included in this generic_archiver by default, but\ncustom dropins can be created to handle additional websites and passed to the archiver\nvia the command line using the `--dropins` option (TODO!).\n\n### Auto-Updates\n\nThe Generic Extractor will also automatically check for updates to `yt-dlp` (every 5 days by default).\nThis can be configured using the `ytdlp_update_interval` setting (or disabled by setting it to -1).\nIf you are having issues with the extractor, you can review the version of `yt-dlp` being used with `yt-dlp --version`.\n\n",
                 "dependencies": {
                     "python": [
                         "yt_dlp",
@@ -710,6 +713,11 @@
                     "max_downloads": {
                         "default": "inf",
                         "help": "Use to limit the number of videos to download when a channel or long page is being extracted. 'inf' means no limit."
+                    },
+                    "ytdlp_update_interval": {
+                        "default": 5,
+                        "help": "How often to check for yt-dlp updates (days). If positive, will check and update yt-dlp every [num] days. Set it to -1 to disable, or 0 to always update on every run.",
+                        "type": "int"
                     }
                 }
             },
@@ -751,9 +759,38 @@
                 "max_downloads": {
                     "default": "inf",
                     "help": "Use to limit the number of videos to download when a channel or long page is being extracted. 'inf' means no limit."
+                },
+                "ytdlp_update_interval": {
+                    "default": 5,
+                    "help": "How often to check for yt-dlp updates (days). If positive, will check and update yt-dlp every [num] days. Set it to -1 to disable, or 0 to always update on every run.",
+                    "type": "int"
                 }
             }
         },
+        "tiktok_tikwm_extractor": {
+            "name": "tiktok_tikwm_extractor",
+            "display_name": "Tiktok Tikwm Extractor",
+            "manifest": {
+                "name": "Tiktok Tikwm Extractor",
+                "author": "Bellingcat",
+                "type": [
+                    "extractor"
+                ],
+                "requires_setup": false,
+                "description": "\n    Uses an unofficial TikTok video download platform's API to download videos: https://tikwm.com/\n\t\n\tThis extractor complements the generic_extractor which can already get TikTok videos, but this one can extract special videos like those marked as sensitive.\n\n    ### Features\n    - Downloads the video and, if possible, also the video cover.\n\t- Stores extra metadata about the post like author information, and more as returned by tikwm.com. \n\n    ### Notes\n    - If tikwm.com is down, this extractor will not work.\n\t- If tikwm.com changes their API, this extractor may break.\n\t- If no video is found, this extractor will consider the extraction failed.\n    ",
+                "dependencies": {
+                    "python": [
+                        "loguru",
+                        "requests"
+                    ],
+                    "bin": []
+                },
+                "entry_point": "",
+                "version": "1.0",
+                "configs": {}
+            },
+            "configs": null
+        },
         "telegram_extractor": {
             "name": "telegram_extractor",
             "display_name": "Telegram Extractor",
@@ -1054,7 +1091,7 @@
                         "help": "width of the screenshots"
                     },
                     "height": {
-                        "default": 720,
+                        "default": 1024,
                         "type": "int",
                         "help": "height of the screenshots"
                     },
@@ -1091,7 +1128,7 @@
                     "help": "width of the screenshots"
                 },
                 "height": {
-                    "default": 720,
+                    "default": 1024,
                     "type": "int",
                     "help": "height of the screenshots"
                 },
@@ -1201,6 +1238,79 @@
                 }
             }
         },
+        "opentimestamps_enricher": {
+            "name": "opentimestamps_enricher",
+            "display_name": "OpenTimestamps Enricher",
+            "manifest": {
+                "name": "OpenTimestamps Enricher",
+                "author": "Bellingcat",
+                "type": [
+                    "enricher"
+                ],
+                "requires_setup": true,
+                "description": "\n    Creates OpenTimestamps proofs for archived files, providing blockchain-backed evidence of file existence at a specific time.\n\n    Uses OpenTimestamps \u2013 a service that timestamps data using the Bitcoin blockchain, providing a decentralized \n    and secure way to prove that data existed at a certain point in time.\n\n    ### Features\n    - Creates cryptographic timestamp proofs that link files to the Bitcoin blockchain\n    - Verifies existing timestamp proofs to confirm the time a file existed\n    - Uses multiple calendar servers to ensure reliability and redundancy\n    - Stores timestamp proofs alongside original files for future verification\n\n    ### Notes\n    - Can work offline to create timestamp proofs that can be upgraded later\n    - Verification checks if timestamps have been confirmed in the Bitcoin blockchain\n    - Should run after files have been archived and hashed\n\n    ### Verifying Timestamps Later\n    If you wish to verify a timestamp (ots) file later, you can install the opentimestamps-client command line tool and use the `ots verify` command.\n    Example: `ots verify my_file.ots`\n\n    Note: if you're using local storage with a filename_generator set to 'static' (a hash) or random, the files will be renamed when they are saved to the\n    final location meaning you will need to specify the original filename when verifying the timestamp with `ots verify -f original_filename my_file.ots`.\n    ",
+                "dependencies": {
+                    "python": [
+                        "loguru",
+                        "opentimestamps"
+                    ]
+                },
+                "entry_point": "",
+                "version": "1.0",
+                "configs": {
+                    "use_calendars": {
+                        "default": true,
+                        "help": "Whether to connect to OpenTimestamps calendar servers to create timestamps. If false, creates local timestamp proofs only.",
+                        "type": "bool"
+                    },
+                    "calendar_urls": {
+                        "default": [
+                            "https://alice.btc.calendar.opentimestamps.org",
+                            "https://bob.btc.calendar.opentimestamps.org",
+                            "https://finney.calendar.eternitywall.com"
+                        ],
+                        "help": "List of OpenTimestamps calendar servers to use for timestamping. See here for a list of calendars maintained by opentimestamps:https://opentimestamps.org/#calendars",
+                        "type": "list"
+                    },
+                    "calendar_whitelist": {
+                        "default": [],
+                        "help": "Optional whitelist of calendar servers. Override this if you are using your own calendar servers. e.g. ['https://mycalendar.com']",
+                        "type": "list"
+                    },
+                    "verify_timestamps": {
+                        "default": true,
+                        "help": "Whether to verify timestamps after creating them.",
+                        "type": "bool"
+                    }
+                }
+            },
+            "configs": {
+                "use_calendars": {
+                    "default": true,
+                    "help": "Whether to connect to OpenTimestamps calendar servers to create timestamps. If false, creates local timestamp proofs only.",
+                    "type": "bool"
+                },
+                "calendar_urls": {
+                    "default": [
+                        "https://alice.btc.calendar.opentimestamps.org",
+                        "https://bob.btc.calendar.opentimestamps.org",
+                        "https://finney.calendar.eternitywall.com"
+                    ],
+                    "help": "List of OpenTimestamps calendar servers to use for timestamping. See here for a list of calendars maintained by opentimestamps:https://opentimestamps.org/#calendars",
+                    "type": "list"
+                },
+                "calendar_whitelist": {
+                    "default": [],
+                    "help": "Optional whitelist of calendar servers. Override this if you are using your own calendar servers. e.g. ['https://mycalendar.com']",
+                    "type": "list"
+                },
+                "verify_timestamps": {
+                    "default": true,
+                    "help": "Whether to verify timestamps after creating them.",
+                    "type": "bool"
+                }
+            }
+        },
         "thumbnail_enricher": {
             "name": "thumbnail_enricher",
             "display_name": "Thumbnail Enricher",
@@ -1381,56 +1491,6 @@
                 }
             }
         },
-        "atlos_db": {
-            "name": "atlos_db",
-            "display_name": "Atlos Database",
-            "manifest": {
-                "name": "Atlos Database",
-                "author": "Bellingcat",
-                "type": [
-                    "database"
-                ],
-                "requires_setup": true,
-                "description": "\nHandles integration with the Atlos platform for managing archival results.\n\n### Features\n- Outputs archival results to the Atlos API for storage and tracking.\n- Updates failure status with error details when archiving fails.\n- Processes and formats metadata, including ISO formatting for datetime fields.\n- Skips processing for items without an Atlos ID.\n\n### Setup\nRequired configs:\n- atlos_url: Base URL for the Atlos API.\n- api_token: Authentication token for API access.\n",
-                "dependencies": {
-                    "python": [
-                        "loguru",
-                        ""
-                    ],
-                    "bin": [
-                        ""
-                    ]
-                },
-                "entry_point": "atlos_db::AtlosDb",
-                "version": "1.0",
-                "configs": {
-                    "api_token": {
-                        "default": null,
-                        "help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/",
-                        "required": true,
-                        "type": "str"
-                    },
-                    "atlos_url": {
-                        "default": "https://platform.atlos.org",
-                        "help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.",
-                        "type": "str"
-                    }
-                }
-            },
-            "configs": {
-                "api_token": {
-                    "default": null,
-                    "help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/",
-                    "required": true,
-                    "type": "str"
-                },
-                "atlos_url": {
-                    "default": "https://platform.atlos.org",
-                    "help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.",
-                    "type": "str"
-                }
-            }
-        },
         "api_db": {
             "name": "api_db",
             "display_name": "Auto Archiver API Database",
@@ -1473,9 +1533,9 @@
                         "help": "which group of users have access to the archive in case public=false as author"
                     },
                     "use_api_cache": {
-                        "default": true,
+                        "default": false,
                         "type": "bool",
-                        "help": "if False then the API database will be queried prior to any archiving operations and stop if the link has already been archived"
+                        "help": "if True then the API database will be queried prior to any archiving operations and stop if the link has already been archived"
                     },
                     "store_results": {
                         "default": true,
@@ -1511,9 +1571,9 @@
                     "help": "which group of users have access to the archive in case public=false as author"
                 },
                 "use_api_cache": {
-                    "default": true,
+                    "default": false,
                     "type": "bool",
-                    "help": "if False then the API database will be queried prior to any archiving operations and stop if the link has already been archived"
+                    "help": "if True then the API database will be queried prior to any archiving operations and stop if the link has already been archived"
                 },
                 "store_results": {
                     "default": true,
@@ -1526,58 +1586,6 @@
                 }
             }
         },
-        "gsheet_db": {
-            "name": "gsheet_db",
-            "display_name": "Google Sheets Database",
-            "manifest": {
-                "name": "Google Sheets Database",
-                "author": "Bellingcat",
-                "type": [
-                    "database"
-                ],
-                "requires_setup": true,
-                "description": "\n    GsheetsDatabase:\n    Handles integration with Google Sheets for tracking archival tasks.\n\n### Features\n- Updates a Google Sheet with the status of the archived URLs, including in progress, success or failure, and method used.\n- Saves metadata such as title, text, timestamp, hashes, screenshots, and media URLs to designated columns.\n- Formats media-specific metadata, such as thumbnails and PDQ hashes for the sheet.\n- Skips redundant updates for empty or invalid data fields.\n\n### Notes\n- Currently works only with metadata provided by GsheetFeeder. \n- Requires configuration of a linked Google Sheet and appropriate API credentials.\n    ",
-                "dependencies": {
-                    "python": [
-                        "loguru",
-                        "gspread",
-                        "slugify"
-                    ]
-                },
-                "entry_point": "gsheet_db::GsheetsDb",
-                "version": "1.0",
-                "configs": {
-                    "allow_worksheets": {
-                        "default": [],
-                        "help": "(CSV) only worksheets whose name is included in allow are included (overrides worksheet_block), leave empty so all are allowed"
-                    },
-                    "block_worksheets": {
-                        "default": [],
-                        "help": "(CSV) explicitly block some worksheets from being processed"
-                    },
-                    "use_sheet_names_in_stored_paths": {
-                        "default": true,
-                        "type": "bool",
-                        "help": "if True the stored files path will include 'workbook_name/worksheet_name/...'"
-                    }
-                }
-            },
-            "configs": {
-                "allow_worksheets": {
-                    "default": [],
-                    "help": "(CSV) only worksheets whose name is included in allow are included (overrides worksheet_block), leave empty so all are allowed"
-                },
-                "block_worksheets": {
-                    "default": [],
-                    "help": "(CSV) explicitly block some worksheets from being processed"
-                },
-                "use_sheet_names_in_stored_paths": {
-                    "default": true,
-                    "type": "bool",
-                    "help": "if True the stored files path will include 'workbook_name/worksheet_name/...'"
-                }
-            }
-        },
         "console_db": {
             "name": "console_db",
             "display_name": "Console Database",
@@ -1664,7 +1672,7 @@
                     },
                     "filename_generator": {
                         "default": "static",
-                        "help": "how to name stored files: 'random' creates a random string; 'static' uses a replicable strategy such as a hash.",
+                        "help": "how to name stored files: 'random' creates a random string; 'static' uses a hash, with the settings of the 'hash_enricher' module (defaults to SHA256 if not enabled).",
                         "choices": [
                             "random",
                             "static"
@@ -1696,7 +1704,7 @@
                 },
                 "filename_generator": {
                     "default": "static",
-                    "help": "how to name stored files: 'random' creates a random string; 'static' uses a replicable strategy such as a hash.",
+                    "help": "how to name stored files: 'random' creates a random string; 'static' uses a hash, with the settings of the 'hash_enricher' module (defaults to SHA256 if not enabled).",
                     "choices": [
                         "random",
                         "static"
@@ -1716,54 +1724,6 @@
                 }
             }
         },
-        "atlos_storage": {
-            "name": "atlos_storage",
-            "display_name": "Atlos Storage",
-            "manifest": {
-                "name": "Atlos Storage",
-                "author": "Bellingcat",
-                "type": [
-                    "storage"
-                ],
-                "requires_setup": true,
-                "description": "\n    Stores media files in a [Atlos](https://www.atlos.org/).\n\n    ### Features\n    - Saves media files to Atlos, organizing them into folders based on the provided path structure.\n\n    ### Notes\n    - Requires setup with Atlos credentials.\n    - Files are uploaded to the specified `root_folder_id` and organized by the `media.key` structure.\n    ",
-                "dependencies": {
-                    "python": [
-                        "loguru",
-                        "boto3"
-                    ],
-                    "bin": []
-                },
-                "entry_point": "",
-                "version": "1.0",
-                "configs": {
-                    "api_token": {
-                        "default": null,
-                        "help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/",
-                        "required": true,
-                        "type": "str"
-                    },
-                    "atlos_url": {
-                        "default": "https://platform.atlos.org",
-                        "help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.",
-                        "type": "str"
-                    }
-                }
-            },
-            "configs": {
-                "api_token": {
-                    "default": null,
-                    "help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/",
-                    "required": true,
-                    "type": "str"
-                },
-                "atlos_url": {
-                    "default": "https://platform.atlos.org",
-                    "help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.",
-                    "type": "str"
-                }
-            }
-        },
         "s3_storage": {
             "name": "s3_storage",
             "display_name": "S3 Storage",
@@ -1796,7 +1756,7 @@
                     },
                     "filename_generator": {
                         "default": "static",
-                        "help": "how to name stored files: 'random' creates a random string; 'static' uses a replicable strategy such as a hash.",
+                        "help": "how to name stored files: 'random' creates a random string; 'static' uses a hash, with the settings of the 'hash_enricher' module (defaults to SHA256 if not enabled).",
                         "choices": [
                             "random",
                             "static"
@@ -1850,7 +1810,7 @@
                 },
                 "filename_generator": {
                     "default": "static",
-                    "help": "how to name stored files: 'random' creates a random string; 'static' uses a replicable strategy such as a hash.",
+                    "help": "how to name stored files: 'random' creates a random string; 'static' uses a hash, with the settings of the 'hash_enricher' module (defaults to SHA256 if not enabled).",
                     "choices": [
                         "random",
                         "static"
@@ -1922,7 +1882,7 @@
                     },
                     "filename_generator": {
                         "default": "static",
-                        "help": "how to name stored files: 'random' creates a random string; 'static' uses a replicable strategy such as a hash.",
+                        "help": "how to name stored files: 'random' creates a random string; 'static' uses a hash, with the settings of the 'hash_enricher' module (defaults to SHA256 if not enabled)",
                         "choices": [
                             "random",
                             "static"
@@ -1951,7 +1911,7 @@
                 },
                 "filename_generator": {
                     "default": "static",
-                    "help": "how to name stored files: 'random' creates a random string; 'static' uses a replicable strategy such as a hash.",
+                    "help": "how to name stored files: 'random' creates a random string; 'static' uses a hash, with the settings of the 'hash_enricher' module (defaults to SHA256 if not enabled)",
                     "choices": [
                         "random",
                         "static"
@@ -2029,9 +1989,9 @@
     "steps": {
         "feeders": [
             "cli_feeder",
-            "gsheet_feeder",
-            "atlos_feeder",
-            "csv_feeder"
+            "atlos_feeder_db_storage",
+            "csv_feeder",
+            "gsheet_feeder_db"
         ],
         "extractors": [
             "wayback_extractor_enricher",
@@ -2039,6 +1999,7 @@
             "instagram_api_extractor",
             "instagram_tbot_extractor",
             "generic_extractor",
+            "tiktok_tikwm_extractor",
             "twitter_api_extractor",
             "instagram_extractor",
             "telethon_extractor",
@@ -2055,20 +2016,21 @@
             "meta_enricher",
             "pdq_hash_enricher",
             "whisper_enricher",
+            "opentimestamps_enricher",
             "ssl_enricher",
             "hash_enricher"
         ],
         "databases": [
             "console_db",
-            "atlos_db",
             "api_db",
             "csv_db",
-            "gsheet_db"
+            "atlos_feeder_db_storage",
+            "gsheet_feeder_db"
         ],
         "storages": [
             "local_storage",
             "gdrive_storage",
-            "atlos_storage",
+            "atlos_feeder_db_storage",
             "s3_storage"
         ],
         "formatters": [
@@ -2077,9 +2039,9 @@
         ]
     },
     "configs": [
-        "gsheet_feeder",
-        "atlos_feeder",
+        "atlos_feeder_db_storage",
         "csv_feeder",
+        "gsheet_feeder_db",
         "cli_feeder",
         "instagram_api_extractor",
         "instagram_tbot_extractor",
@@ -2093,15 +2055,13 @@
         "timestamping_enricher",
         "screenshot_enricher",
         "whisper_enricher",
+        "opentimestamps_enricher",
         "thumbnail_enricher",
         "ssl_enricher",
         "hash_enricher",
-        "atlos_db",
         "api_db",
-        "gsheet_db",
         "csv_db",
         "gdrive_storage",
-        "atlos_storage",
         "s3_storage",
         "local_storage",
         "html_formatter"
diff --git a/src/auto_archiver/modules/gdrive_storage/__manifest__.py b/src/auto_archiver/modules/gdrive_storage/__manifest__.py
index 73784b8..46e4fa1 100644
--- a/src/auto_archiver/modules/gdrive_storage/__manifest__.py
+++ b/src/auto_archiver/modules/gdrive_storage/__manifest__.py
@@ -19,7 +19,7 @@
         },
         "filename_generator": {
             "default": "static",
-            "help": "how to name stored files: 'random' creates a random string; 'static' uses a replicable strategy such as a hash.",
+            "help": "how to name stored files: 'random' creates a random string; 'static' uses a hash, with the settings of the 'hash_enricher' module (defaults to SHA256 if not enabled).",
             "choices": ["random", "static"],
         },
         "root_folder_id": {"required": True,
diff --git a/src/auto_archiver/modules/local_storage/__manifest__.py b/src/auto_archiver/modules/local_storage/__manifest__.py
index 8ad6381..72f59d1 100644
--- a/src/auto_archiver/modules/local_storage/__manifest__.py
+++ b/src/auto_archiver/modules/local_storage/__manifest__.py
@@ -13,7 +13,7 @@
         },
         "filename_generator": {
             "default": "static",
-            "help": "how to name stored files: 'random' creates a random string; 'static' uses a replicable strategy such as a hash.",
+            "help": "how to name stored files: 'random' creates a random string; 'static' uses a hash, with the settings of the 'hash_enricher' module (defaults to SHA256 if not enabled)",
             "choices": ["random", "static"],
         },
         "save_to": {"default": "./local_archive", "help": "folder where to save archived content"},
diff --git a/src/auto_archiver/modules/opentimestamps_enricher/__manifest__.py b/src/auto_archiver/modules/opentimestamps_enricher/__manifest__.py
index 136c0c2..ff038e1 100644
--- a/src/auto_archiver/modules/opentimestamps_enricher/__manifest__.py
+++ b/src/auto_archiver/modules/opentimestamps_enricher/__manifest__.py
@@ -52,5 +52,12 @@ https://opentimestamps.org/#calendars",
     - Can work offline to create timestamp proofs that can be upgraded later
     - Verification checks if timestamps have been confirmed in the Bitcoin blockchain
     - Should run after files have been archived and hashed
+
+    ### Verifying Timestamps Later
+    If you wish to verify a timestamp (ots) file later, you can install the opentimestamps-client command line tool and use the `ots verify` command.
+    Example: `ots verify my_file.ots`
+
+    Note: if you're using local storage with a filename_generator set to 'static' (a hash) or random, the files will be renamed when they are saved to the
+    final location meaning you will need to specify the original filename when verifying the timestamp with `ots verify -f original_filename my_file.ots`.
     """
 }
\ No newline at end of file
diff --git a/src/auto_archiver/modules/opentimestamps_enricher/opentimestamps_enricher.py b/src/auto_archiver/modules/opentimestamps_enricher/opentimestamps_enricher.py
index 2b74ee6..cdeb78d 100644
--- a/src/auto_archiver/modules/opentimestamps_enricher/opentimestamps_enricher.py
+++ b/src/auto_archiver/modules/opentimestamps_enricher/opentimestamps_enricher.py
@@ -20,8 +20,7 @@ class OpentimestampsEnricher(Enricher):
         logger.debug(f"OpenTimestamps timestamping files for {url=}")
 
         # Get the media files to timestamp
-        media_files = [m for m in to_enrich.media if m.get("filename") and not m.get("opentimestamps")]
-
+        media_files = [m for m in to_enrich.media if m.filename and not m.get("opentimestamps")]
         if not media_files:
             logger.warning(f"No files found to timestamp in {url=}")
             return
@@ -30,7 +29,7 @@ class OpentimestampsEnricher(Enricher):
         for media in media_files:
             try:
                 # Get the file path from the media
-                file_path = media.get("filename")
+                file_path = media.filename
                 if not os.path.exists(file_path):
                     logger.warning(f"File not found: {file_path}")
                     continue
@@ -108,7 +107,8 @@ class OpentimestampsEnricher(Enricher):
                 
                 # Create media for the timestamp file
                 timestamp_media = Media(filename=timestamp_path)
-                timestamp_media.set("source_file", os.path.basename(file_path))
+                # explicitly set the mimetype, normally .ots files are 'application/vnd.oasis.opendocument.spreadsheet-template'
+                media.mimetype = "application/vnd.opentimestamps"
                 timestamp_media.set("opentimestamps_version", opentimestamps.__version__)
                 
                 # Verify the timestamp if needed
@@ -119,20 +119,16 @@ class OpentimestampsEnricher(Enricher):
                 else:
                     logger.warning(f"Not verifying the timestamp for media file {file_path}")
                 
-                timestamp_files.append(timestamp_media)
-                
+                media.set("opentimestamp_files", [timestamp_media])
+                timestamp_files.append(timestamp_media.filename)
                 # Update the original media to indicate it's been timestamped
                 media.set("opentimestamps", True)
-                media.set("opentimestamp_file", timestamp_path)
                 
             except Exception as e:
-                logger.warning(f"Error while timestamping {media.get('filename')}: {e}")
+                logger.warning(f"Error while timestamping {media.filename}: {e}")
         
         # Add timestamp files to the metadata
         if timestamp_files:
-            for ts_media in timestamp_files:
-                to_enrich.add_media(ts_media)
-            
             to_enrich.set("opentimestamped", True)
             to_enrich.set("opentimestamps_count", len(timestamp_files))
             logger.success(f"{len(timestamp_files)} OpenTimestamps proofs created for {url=}")
@@ -162,7 +158,7 @@ class OpentimestampsEnricher(Enricher):
                 
                 # Process different types of attestations
                 if isinstance(attestation, PendingAttestation):
-                    info["type"] = "pending"
+                    info["type"] = f"pending (as of {attestation.date})"
                     info["uri"] = attestation.uri
                 
                 elif isinstance(attestation, BitcoinBlockHeaderAttestation):
diff --git a/src/auto_archiver/modules/s3_storage/__manifest__.py b/src/auto_archiver/modules/s3_storage/__manifest__.py
index bf032e7..156f562 100644
--- a/src/auto_archiver/modules/s3_storage/__manifest__.py
+++ b/src/auto_archiver/modules/s3_storage/__manifest__.py
@@ -13,7 +13,7 @@
         },
         "filename_generator": {
             "default": "static",
-            "help": "how to name stored files: 'random' creates a random string; 'static' uses a replicable strategy such as a hash.",
+            "help": "how to name stored files: 'random' creates a random string; 'static' uses a hash, with the settings of the 'hash_enricher' module (defaults to SHA256 if not enabled).",
             "choices": ["random", "static"],
         },
         "bucket": {"default": None, "help": "S3 bucket name"},
diff --git a/tests/enrichers/test_opentimestamps_enricher.py b/tests/enrichers/test_opentimestamps_enricher.py
index db171e5..391fb06 100644
--- a/tests/enrichers/test_opentimestamps_enricher.py
+++ b/tests/enrichers/test_opentimestamps_enricher.py
@@ -172,7 +172,7 @@ def test_full_enriching(setup_module, sample_file_path, sample_media, mocker):
     
     # Create test metadata with sample file
     metadata = Metadata().set_url("https://example.com")
-    sample_media.set("filename", sample_file_path)
+    sample_media.filename = sample_file_path
     metadata.add_media(sample_media)
     
     # Run enrichment
@@ -182,16 +182,17 @@ def test_full_enriching(setup_module, sample_file_path, sample_media, mocker):
     assert metadata.get("opentimestamped") == True
     assert metadata.get("opentimestamps_count") == 1
     
-    # Check that we have two media items: the original and the timestamp
-    assert len(metadata.media) == 2
+    # Check that we have one parent media item: the original
+    assert len(metadata.media) == 1
     
     # Check that the original media was updated
     assert metadata.media[0].get("opentimestamps") == True
-    assert metadata.media[0].get("opentimestamp_file") is not None
     
-    # Check the timestamp file media
-    timestamp_media = metadata.media[1]
-    assert timestamp_media.get("source_file") == os.path.basename(sample_file_path)
+    # Check the timestamp file media is a child of the original
+    assert len(metadata.media[0].get("opentimestamp_files")) == 1
+
+    timestamp_media = metadata.media[0].get("opentimestamp_files")[0]
+
     assert timestamp_media.get("opentimestamps_version") is not None
     
     # Check verification results on the timestamp media
@@ -203,7 +204,7 @@ def test_full_enriching_no_calendars(setup_module, sample_file_path, sample_medi
     
     # Create test metadata with sample file
     metadata = Metadata().set_url("https://example.com")
-    sample_media.set("filename", sample_file_path)
+    sample_media.filename = sample_file_path
     metadata.add_media(sample_media)
     
     # Run enrichment
@@ -212,10 +213,8 @@ def test_full_enriching_no_calendars(setup_module, sample_file_path, sample_medi
     # Verify results
     assert metadata.get("opentimestamped") == True
     assert metadata.get("opentimestamps_count") == 1
-    
-    # Check the timestamp file media
-    timestamp_media = metadata.media[1]
-    assert timestamp_media.get("source_file") == os.path.basename(sample_file_path)
+
+    timestamp_media = metadata.media[0].get("opentimestamp_files")[0]
     
     # Verify status should be false since we didn't use calendars
     assert timestamp_media.get("verified") == False
@@ -233,7 +232,7 @@ def test_full_enriching_calendar_error(setup_module, sample_file_path, sample_me
     
     # Create test metadata with sample file
     metadata = Metadata().set_url("https://example.com")
-    sample_media.set("filename", sample_file_path)
+    sample_media.filename = sample_file_path
     metadata.add_media(sample_media)
     
     # Run enrichment (should complete despite calendar errors)
@@ -244,7 +243,7 @@ def test_full_enriching_calendar_error(setup_module, sample_file_path, sample_me
     assert metadata.get("opentimestamps_count") == 1
     
     # Verify status should be false since calendar submissions failed
-    timestamp_media = metadata.media[1]
+    timestamp_media = metadata.media[0].get("opentimestamp_files")[0]
     assert timestamp_media.get("verified") == False
     # We expect 3 pending attestations (one for each calendar URL that's enabled by default in __manifest__)
     assert timestamp_media.get("attestation_count") == 3

From 1d664524eb61557311a26246354ed0ca16a9191b Mon Sep 17 00:00:00 2001
From: Patrick Robertson <robertson.patrick@gmail.com>
Date: Wed, 12 Mar 2025 11:54:25 +0000
Subject: [PATCH 28/55] Add info on last check/last updated to the metadata

---
 .../opentimestamps_enricher/opentimestamps_enricher.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/auto_archiver/modules/opentimestamps_enricher/opentimestamps_enricher.py b/src/auto_archiver/modules/opentimestamps_enricher/opentimestamps_enricher.py
index cdeb78d..cf110a2 100644
--- a/src/auto_archiver/modules/opentimestamps_enricher/opentimestamps_enricher.py
+++ b/src/auto_archiver/modules/opentimestamps_enricher/opentimestamps_enricher.py
@@ -1,6 +1,5 @@
 import os
-import hashlib
-from typing import TYPE_CHECKING
+import datetime
 
 from loguru import logger
 import opentimestamps
@@ -108,7 +107,7 @@ class OpentimestampsEnricher(Enricher):
                 # Create media for the timestamp file
                 timestamp_media = Media(filename=timestamp_path)
                 # explicitly set the mimetype, normally .ots files are 'application/vnd.oasis.opendocument.spreadsheet-template'
-                media.mimetype = "application/vnd.opentimestamps"
+                timestamp_media.mimetype = "application/vnd.opentimestamps"
                 timestamp_media.set("opentimestamps_version", opentimestamps.__version__)
                 
                 # Verify the timestamp if needed
@@ -158,12 +157,14 @@ class OpentimestampsEnricher(Enricher):
                 
                 # Process different types of attestations
                 if isinstance(attestation, PendingAttestation):
-                    info["type"] = f"pending (as of {attestation.date})"
+                    info["type"] = f"pending"
                     info["uri"] = attestation.uri
                 
                 elif isinstance(attestation, BitcoinBlockHeaderAttestation):
                     info["type"] = "bitcoin"
                     info["block_height"] = attestation.height
+
+                info["last_check"] = datetime.datetime.now().isoformat()[:-7]
                 
                 attestation_info.append(info)
             
@@ -178,5 +179,6 @@ class OpentimestampsEnricher(Enricher):
         else:
             result["verified"] = False
             result["pending"] = False
+        result["last_updated"] = datetime.datetime.now().isoformat()[:-7]
         
         return result
\ No newline at end of file

From 753c3c62141228d0ac9c03c41573b71cf67013a7 Mon Sep 17 00:00:00 2001
From: erinhmclark <erinhannahmary.clark@gmail.com>
Date: Wed, 12 Mar 2025 14:27:45 +0000
Subject: [PATCH 29/55] Linting tests

---
 docs/scripts/scripts.py                        |  2 +-
 docs/source/installation/setup.md              |  1 +
 pyproject.toml                                 |  4 ++--
 scripts/create_update_gdrive_oauth_token.py    |  3 ++-
 .../tiktok_tikwm_extractor.py                  |  2 +-
 tests/enrichers/test_meta_enricher.py          |  1 -
 tests/enrichers/test_screenshot_enricher.py    |  9 ++++-----
 .../extractors/test_tiktok_tikwm_extractor.py  | 10 +++++-----
 tests/feeders/test_gsheet_feeder.py            |  6 +++---
 tests/storages/test_S3_storage.py              |  8 ++++----
 tests/storages/test_atlos_storage.py           |  1 -
 tests/storages/test_gdrive_storage.py          |  3 +--
 tests/test_config.py                           | 18 +++++++++---------
 tests/test_modules.py                          |  6 +++---
 tests/test_orchestrator.py                     |  8 ++++----
 15 files changed, 40 insertions(+), 42 deletions(-)

diff --git a/docs/scripts/scripts.py b/docs/scripts/scripts.py
index ca1d348..bfddd29 100644
--- a/docs/scripts/scripts.py
+++ b/docs/scripts/scripts.py
@@ -52,7 +52,7 @@ def generate_module_docs():
         for type in manifest["type"]:
             modules_by_type.setdefault(type, []).append(module)
 
-        description = "\n".join(l.lstrip() for l in manifest["description"].split("\n"))
+        description = "\n".join(line.lstrip() for line in manifest["description"].split("\n"))
         types = ", ".join(type_color[t] for t in manifest["type"])
         readme_str = f"""
 # {manifest["name"]}
diff --git a/docs/source/installation/setup.md b/docs/source/installation/setup.md
index e5c96a6..f5b6e9d 100644
--- a/docs/source/installation/setup.md
+++ b/docs/source/installation/setup.md
@@ -51,6 +51,7 @@ The invocations below will run the auto-archiver Docker image using a configurat
 docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver
 
 # uses the same configuration, but with the `gsheet_feeder`, a header on row 2 and with some different column names
+# Note this expects you to have followed the [Google Sheets setup](how_to/google_sheets.md) and added your service_account.json to the `secrets/` folder
 # notice that columns is a dictionary so you need to pass it as JSON and it will override only the values provided
 docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver --feeders=gsheet_feeder --gsheet_feeder.sheet="use it on another sheets doc" --gsheet_feeder.header=2 --gsheet_feeder.columns='{"url": "link"}'
 # Runs auto-archiver for the first time, but in 'full' mode, enabling all modules to get a full settings file
diff --git a/pyproject.toml b/pyproject.toml
index 86c8db8..90ac56f 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -96,7 +96,7 @@ markers = [
 #exclude = ["docs"]
 line-length = 120
 # Remove this for a more detailed lint report
-output-format = "concise"
+#output-format = "concise"
 
 
 [tool.ruff.lint]
@@ -104,7 +104,7 @@ output-format = "concise"
 # I : isort
 # UP : upgrade, e.g. use fstrings
 # ANN : annotations
-#extend-select = ["B"]
+extend-select = ["B"]
 
 # Ignore unused imports as some are currently required for lazy loading
 # This can be removed for a `lint check` run which is manually reviewed
diff --git a/scripts/create_update_gdrive_oauth_token.py b/scripts/create_update_gdrive_oauth_token.py
index a57043e..edd2565 100644
--- a/scripts/create_update_gdrive_oauth_token.py
+++ b/scripts/create_update_gdrive_oauth_token.py
@@ -1,5 +1,6 @@
 import os.path
-import click, json
+import click
+import json
 
 from google.auth.transport.requests import Request
 from google.oauth2.credentials import Credentials
diff --git a/src/auto_archiver/modules/tiktok_tikwm_extractor/tiktok_tikwm_extractor.py b/src/auto_archiver/modules/tiktok_tikwm_extractor/tiktok_tikwm_extractor.py
index 8b07775..e7ed91a 100644
--- a/src/auto_archiver/modules/tiktok_tikwm_extractor/tiktok_tikwm_extractor.py
+++ b/src/auto_archiver/modules/tiktok_tikwm_extractor/tiktok_tikwm_extractor.py
@@ -14,7 +14,7 @@ class TiktokTikwmExtractor(Extractor):
     """
     TIKWM_ENDPOINT = "https://www.tikwm.com/api/?url={url}"
 
-    def download(self, item: Metadata) -> Metadata:
+    def download(self, item: Metadata) -> bool | Metadata:
         url = item.get_url()
         
         if not re.match(TikTokIE._VALID_URL, url):
diff --git a/tests/enrichers/test_meta_enricher.py b/tests/enrichers/test_meta_enricher.py
index 906e629..fe0d737 100644
--- a/tests/enrichers/test_meta_enricher.py
+++ b/tests/enrichers/test_meta_enricher.py
@@ -1,4 +1,3 @@
-import datetime
 from datetime import datetime, timedelta, timezone
 
 import pytest
diff --git a/tests/enrichers/test_screenshot_enricher.py b/tests/enrichers/test_screenshot_enricher.py
index ee6c2a7..b86bb17 100644
--- a/tests/enrichers/test_screenshot_enricher.py
+++ b/tests/enrichers/test_screenshot_enricher.py
@@ -15,9 +15,9 @@ def mock_selenium_env(mocker):
     mock_which = mocker.patch("shutil.which")
     mock_driver_class = mocker.patch("auto_archiver.utils.webdriver.CookieSettingDriver")
     mock_binary_paths = mocker.patch("selenium.webdriver.common.selenium_manager.SeleniumManager.binary_paths")
-    mock_is_file = mocker.patch("pathlib.Path.is_file", return_value=True)
+    mocker.patch("pathlib.Path.is_file", return_value=True)
     mock_popen = mocker.patch("subprocess.Popen")
-    mock_is_connectable = mocker.patch("selenium.webdriver.common.service.Service.is_connectable", return_value=True)
+    mocker.patch("selenium.webdriver.common.service.Service.is_connectable", return_value=True)
     mock_firefox_options = mocker.patch("selenium.webdriver.FirefoxOptions")
 
     # Define side effect for `shutil.which`
@@ -157,13 +157,12 @@ def test_pdf_creation(mocker, screenshot_enricher, metadata_with_video, mock_sel
     # Mock the print_page method to return base64-encoded content
     mock_driver.print_page.return_value = base64.b64encode(b"fake_pdf_content").decode("utf-8")
     # Patch functions with mocker
-    mock_os_path_join = mocker.patch("os.path.join", side_effect=lambda *args: f"{args[-1]}")
-    mock_random_str = mocker.patch(
+    mocker.patch("os.path.join", side_effect=lambda *args: f"{args[-1]}")
+    mocker.patch(
         "auto_archiver.modules.screenshot_enricher.screenshot_enricher.random_str",
         return_value="fixed123",
     )
     mock_open = mocker.patch("builtins.open", new_callable=mocker.mock_open)
-    mock_log_error = mocker.patch("loguru.logger.error")
 
     screenshot_enricher.enrich(metadata_with_video)
     # Verify screenshot and PDF creation
diff --git a/tests/extractors/test_tiktok_tikwm_extractor.py b/tests/extractors/test_tiktok_tikwm_extractor.py
index e8ad8df..f675ac0 100644
--- a/tests/extractors/test_tiktok_tikwm_extractor.py
+++ b/tests/extractors/test_tiktok_tikwm_extractor.py
@@ -39,7 +39,7 @@ class TestTiktokTikwmExtractor(TestExtractorBase):
         mock_get, mock_logger = self.get_mockers(mocker)
         if valid_url:
             mock_get.return_value.status_code = 404
-        assert self.extractor.download(make_item(url)) == False
+        assert self.extractor.download(make_item(url)) is False
         assert mock_get.call_count == int(valid_url)
         assert mock_logger.error.call_count == int(valid_url)
 
@@ -47,7 +47,7 @@ class TestTiktokTikwmExtractor(TestExtractorBase):
         mock_get, mock_logger = self.get_mockers(mocker)
         mock_get.return_value.status_code = 200
         mock_get.return_value.json.side_effect = ValueError
-        assert self.extractor.download(make_item(self.VALID_EXAMPLE_URL)) == False
+        assert self.extractor.download(make_item(self.VALID_EXAMPLE_URL)) is False
         mock_get.assert_called_once()
         mock_get.return_value.json.assert_called_once()
         mock_logger.error.assert_called_once()
@@ -68,7 +68,7 @@ class TestTiktokTikwmExtractor(TestExtractorBase):
         mock_get, mock_logger = self.get_mockers(mocker)
         mock_get.return_value.status_code = 200
         mock_get.return_value.json.return_value = response
-        assert self.extractor.download(make_item(self.VALID_EXAMPLE_URL)) == False
+        assert self.extractor.download(make_item(self.VALID_EXAMPLE_URL)) is False
         mock_get.assert_called_once()
         mock_get.return_value.json.assert_called_once()
         mock_logger.error.assert_called_once()
@@ -86,7 +86,7 @@ class TestTiktokTikwmExtractor(TestExtractorBase):
 
         result = self.extractor.download(make_item(self.VALID_EXAMPLE_URL))
         if not has_vid:
-            assert result == False
+            assert result is False
         else:
             assert result.is_success()
             assert len(result.media) == 1
@@ -99,7 +99,7 @@ class TestTiktokTikwmExtractor(TestExtractorBase):
         else:
             mock_logger.error.assert_not_called()
 
-    def test_correct_extraction(self, mocker, make_item):
+    def test_correct_data_extracted(self, mocker, make_item):
         mock_get, _ = self.get_mockers(mocker)
         mock_get.return_value.status_code = 200
         mock_get.return_value.json.return_value = {"msg": "success", "data": {
diff --git a/tests/feeders/test_gsheet_feeder.py b/tests/feeders/test_gsheet_feeder.py
index 0fc2681..bf34757 100644
--- a/tests/feeders/test_gsheet_feeder.py
+++ b/tests/feeders/test_gsheet_feeder.py
@@ -172,10 +172,10 @@ def test_should_process_sheet(setup_module, mocker):
             "block_worksheets": {"Sheet3"},
         },
     )
-    assert gdb.should_process_sheet("TestSheet") == True
-    assert gdb.should_process_sheet("Sheet3") == False
+    assert gdb.should_process_sheet("TestSheet") is True
+    assert gdb.should_process_sheet("Sheet3") is False
     # False if allow_worksheets is set
-    assert gdb.should_process_sheet("AnotherSheet") == False
+    assert gdb.should_process_sheet("AnotherSheet") is False
 
 
 @pytest.mark.skip(reason="Requires a real connection")
diff --git a/tests/storages/test_S3_storage.py b/tests/storages/test_S3_storage.py
index abf9763..9e27b3f 100644
--- a/tests/storages/test_S3_storage.py
+++ b/tests/storages/test_S3_storage.py
@@ -61,7 +61,7 @@ class TestS3Storage:
         media = Media("test.txt")
         assert self.storage.is_upload_needed(media) is True
         self.storage.random_no_duplicate = True
-        mock_calc_hash = mocker.patch('auto_archiver.modules.s3_storage.s3_storage.calculate_file_hash', return_value='beepboop123beepboop123beepboop123')
+        mocker.patch('auto_archiver.modules.s3_storage.s3_storage.calculate_file_hash', return_value='beepboop123beepboop123beepboop123')
         mock_file_in_folder = mocker.patch.object(self.storage, 'file_in_folder', return_value='existing_key.txt')
         assert self.storage.is_upload_needed(media) is False
         assert media.key == 'existing_key.txt'
@@ -70,10 +70,10 @@ class TestS3Storage:
     def test_skips_upload_when_duplicate_exists(self, mocker):
         """Test that upload skips when file_in_folder finds existing object"""
         self.storage.random_no_duplicate = True
-        mock_file_in_folder = mocker.patch.object(S3Storage, 'file_in_folder', return_value="existing_folder/existing_file.txt")
+        mocker.patch.object(S3Storage, 'file_in_folder', return_value="existing_folder/existing_file.txt")
         media = Media("test.txt")
         media._key = "original_path.txt"
-        mock_calculate_hash = mocker.patch('auto_archiver.modules.s3_storage.s3_storage.calculate_file_hash', return_value="beepboop123beepboop123beepboop123")
+        mocker.patch('auto_archiver.modules.s3_storage.s3_storage.calculate_file_hash', return_value="beepboop123beepboop123beepboop123")
         assert self.storage.is_upload_needed(media) is False
         assert media.key == "existing_folder/existing_file.txt"
         assert media.get("previously archived") is True
@@ -101,5 +101,5 @@ class TestS3Storage:
         )
 
     def test_file_in_folder_exists(self, mocker):
-        mock_list_objects = mocker.patch.object(self.storage.s3, 'list_objects', return_value={'Contents': [{'Key': 'path/to/file.txt'}]})
+        mocker.patch.object(self.storage.s3, 'list_objects', return_value={'Contents': [{'Key': 'path/to/file.txt'}]})
         assert self.storage.file_in_folder('path/to/') == 'path/to/file.txt'
diff --git a/tests/storages/test_atlos_storage.py b/tests/storages/test_atlos_storage.py
index 7268c8d..f273c7e 100644
--- a/tests/storages/test_atlos_storage.py
+++ b/tests/storages/test_atlos_storage.py
@@ -94,7 +94,6 @@ def test_upload_not_uploaded(tmp_path, atlos_storage: AtlosStorage, metadata: Me
     call_args = post_mock.call_args[0]
     assert call_args[0] == expected_endpoint
     call_kwargs = post_mock.call_args[1]
-    expected_headers = {"Authorization": f"Bearer {atlos_storage.api_token}"}
     expected_params = {"title": media.properties}
     assert call_kwargs["params"] == expected_params
     file_tuple = call_kwargs["files"]["file"]
diff --git a/tests/storages/test_gdrive_storage.py b/tests/storages/test_gdrive_storage.py
index 1e418f0..08c516f 100644
--- a/tests/storages/test_gdrive_storage.py
+++ b/tests/storages/test_gdrive_storage.py
@@ -9,9 +9,8 @@ from tests.storages.test_storage_base import TestStorageBase
 
 
 @pytest.fixture
-def gdrive_storage(setup_module, mocker):
+def gdrive_storage(setup_module, mocker) -> GDriveStorage:
     module_name: str = "gdrive_storage"
-    storage: GDriveStorage
     config: dict = {
         "path_generator": "url",
         "filename_generator": "static",
diff --git a/tests/test_config.py b/tests/test_config.py
index 0de4f16..03b06e7 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -77,15 +77,15 @@ def test_merge_dicts():
 
 
 def test_check_types():
-    assert config.is_list_type([]) == True
-    assert config.is_list_type(()) == True
-    assert config.is_list_type(set()) == True
-    assert config.is_list_type({}) == False
-    assert config.is_list_type("") == False
-    assert config.is_dict_type({}) == True
-    assert config.is_dict_type(CommentedMap()) == True
-    assert config.is_dict_type([]) == False
-    assert config.is_dict_type("") == False
+    assert config.is_list_type([]) is True
+    assert config.is_list_type(()) is True
+    assert config.is_list_type(set()) is True
+    assert config.is_list_type({}) is False
+    assert config.is_list_type("") is False
+    assert config.is_dict_type({}) is True
+    assert config.is_dict_type(CommentedMap()) is True
+    assert config.is_dict_type([]) is False
+    assert config.is_dict_type("") is False
 
 
 def test_from_dot_notation():
diff --git a/tests/test_modules.py b/tests/test_modules.py
index 1ff4f45..b6018da 100644
--- a/tests/test_modules.py
+++ b/tests/test_modules.py
@@ -9,10 +9,8 @@ def example_module():
     import auto_archiver
 
     module_factory = ModuleFactory()
-
-    previous_path = auto_archiver.modules.__path__
+    # previous_path = auto_archiver.modules.__path__
     auto_archiver.modules.__path__.append("tests/data/test_modules/")
-
     return module_factory.get_module_lazy("example_module")
 
 
@@ -84,6 +82,8 @@ def test_load_modules(module_name):
     # check that default settings are applied
     default_config = module.configs
     assert loaded_module.name in loaded_module.config.keys()
+    defaults = {k: v.get("default") for k, v in default_config.items()}
+    assert loaded_module.config[module_name] == defaults
 
 
 @pytest.mark.parametrize("module_name", ["local_storage", "generic_extractor", "html_formatter", "csv_db"])
diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py
index 64d84d8..a79aa70 100644
--- a/tests/test_orchestrator.py
+++ b/tests/test_orchestrator.py
@@ -67,7 +67,7 @@ def test_version(basic_parser, capsys):
 
 def test_help(orchestrator, basic_parser, capsys):
     args = basic_parser.parse_args(["--help"])
-    assert args.help == True
+    assert args.help is True
 
     # test the show_help() on orchestrator
     with pytest.raises(SystemExit) as exit_error:
@@ -116,8 +116,8 @@ def test_check_required_values(orchestrator, caplog, test_args):
     # drop the example_module.required_field from the test_args
     test_args = test_args[:-2]
 
-    with pytest.raises(SystemExit) as exit_error:
-        config = orchestrator.setup_config(test_args)
+    with pytest.raises(SystemExit):
+        orchestrator.setup_config(test_args)
 
     assert caplog.records[1].message == "the following arguments are required: --example_module.required_field"
 
@@ -212,7 +212,7 @@ def test_multiple_orchestrator(test_args):
     ]
     o1 = ArchivingOrchestrator()
 
-    with pytest.raises(ValueError) as exit_error:
+    with pytest.raises(ValueError):
         # this should fail because the gsheet_feeder_db requires a sheet_id / sheet
         o1.setup(o1_args)
 

From 6e52a534e7cbb2efa991a1504d77eba1ecfea234 Mon Sep 17 00:00:00 2001
From: erinhmclark <erinhannahmary.clark@gmail.com>
Date: Wed, 12 Mar 2025 16:07:05 +0000
Subject: [PATCH 30/55] More fixes from Bugbear suggestions

---
 src/auto_archiver/core/base_module.py | 5 +++--
 src/auto_archiver/core/config.py      | 2 +-
 src/auto_archiver/core/module.py      | 2 +-
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/auto_archiver/core/base_module.py b/src/auto_archiver/core/base_module.py
index d717e4b..642b8ee 100644
--- a/src/auto_archiver/core/base_module.py
+++ b/src/auto_archiver/core/base_module.py
@@ -59,7 +59,8 @@ class BaseModule(ABC):
             setattr(self, key, val)
 
     def setup(self):
-        # For any additional setup required by modules, e.g. autehntication
+        # For any additional setup required by modules outside of the configs in the manifesst,
+        # e.g. authentication
         pass
 
     def auth_for_site(self, site: str, extract_cookies=True) -> Mapping[str, Any]:
@@ -88,7 +89,7 @@ class BaseModule(ABC):
         # TODO: think about if/how we can deal with sites that have multiple domains (main one is x.com/twitter.com)
         # for now the user must enter them both, like "x.com,twitter.com" in their config. Maybe we just hard-code?
 
-        site = UrlUtil.domain_for_url(site).lstrip("www.")
+        site = UrlUtil.domain_for_url(site).removeprefix("www.")
         # add the 'www' version of the site to the list of sites to check
         authdict = {}
 
diff --git a/src/auto_archiver/core/config.py b/src/auto_archiver/core/config.py
index 6c4300f..59c1eec 100644
--- a/src/auto_archiver/core/config.py
+++ b/src/auto_archiver/core/config.py
@@ -65,7 +65,7 @@ class AuthenticationJsonParseAction(argparse.Action):
             auth_dict = json.loads(values)
             setattr(namespace, self.dest, auth_dict)
         except json.JSONDecodeError as e:
-            raise argparse.ArgumentTypeError(f"Invalid JSON input for argument '{self.dest}': {e}")
+            raise argparse.ArgumentTypeError(f"Invalid JSON input for argument '{self.dest}': {e}") from e
 
         def load_from_file(path):
             try:
diff --git a/src/auto_archiver/core/module.py b/src/auto_archiver/core/module.py
index 6eac968..94b4e48 100644
--- a/src/auto_archiver/core/module.py
+++ b/src/auto_archiver/core/module.py
@@ -194,7 +194,7 @@ class LazyBaseModule:
             try:
                 manifest.update(ast.literal_eval(f.read()))
             except (ValueError, TypeError, SyntaxError, MemoryError, RecursionError) as e:
-                raise ValueError(f"Error loading manifest from file {self.path}/{MANIFEST_FILE}: {e}")
+                raise ValueError(f"Error loading manifest from file {self.path}/{MANIFEST_FILE}: {e}") from e
 
         self._manifest = manifest
         self._entry_point = manifest["entry_point"]

From e76551ba227b1666a68b266f57628bcdae8a9e51 Mon Sep 17 00:00:00 2001
From: erinhmclark <erinhannahmary.clark@gmail.com>
Date: Thu, 13 Mar 2025 13:21:32 +0000
Subject: [PATCH 31/55] Add documentation, pre-commit hook, more make commands
 and

---
 .pre-commit-config.yaml                       |   7 +-
 Makefile                                      |  36 ++-
 .../development/developer_guidelines.md       |   1 +
 docs/source/development/style_guide.md        |  39 +++
 poetry.lock                                   | 129 +++++++-
 pyproject.toml                                |  17 +-
 src/auto_archiver/core/consts.py              |  28 +-
 src/auto_archiver/core/orchestrator.py        | 288 ++++++++++++------
 src/auto_archiver/core/storage.py             |  21 +-
 .../modules/gsheet_feeder_db/__manifest__.py  |   4 +-
 .../modules/local_storage/local_storage.py    |  18 +-
 .../modules/s3_storage/s3_storage.py          |  37 +--
 .../tiktok_tikwm_extractor/__init__.py        |   2 +-
 .../tiktok_tikwm_extractor/__manifest__.py    |   7 +-
 .../tiktok_tikwm_extractor.py                 |   7 +-
 .../wacz_extractor_enricher/__manifest__.py   |   4 +-
 tests/conftest.py                             |   4 +-
 .../extractors/test_tiktok_tikwm_extractor.py |  85 +++---
 tests/storages/test_S3_storage.py             |  52 ++--
 tests/storages/test_local_storage.py          |  12 +-
 tests/storages/test_storage_base.py           |  30 +-
 21 files changed, 558 insertions(+), 270 deletions(-)
 create mode 100644 docs/source/development/style_guide.md

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 0fdf695..78421d7 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,7 +1,10 @@
+# Run Ruff formatter on commits.
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
     rev: v0.9.10
     hooks:
-      - id: ruff
-#        args: [ --fix ]
       - id: ruff-format
+
+      # Runs Ruff linting - just checks without fixing, but blocks commit if errors are found.
+#      - id: ruff
+#        args: ["--output-format=concise"]
\ No newline at end of file
diff --git a/Makefile b/Makefile
index 72f2058..c59f272 100644
--- a/Makefile
+++ b/Makefile
@@ -9,34 +9,54 @@ help:
 	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
 	@echo "Additional Commands:"
 	@echo "  make test         - Run all tests in 'tests/' with pytest"
-	@echo "  make lint         - Run ruff linter and auto-fix issues"
+	@echo "  make ruff-check   - Run Ruff linting and formatting checks (safe)"
+	@echo "  make ruff-clean   - Auto-fix Ruff linting and formatting issues"
 	@echo "  make docs         - Generate documentation (same as 'make html')"
-	@echo "  make clean_docs   - Remove generated docs"
+	@echo "  make clean-docs   - Remove generated docs"
 	@echo "  make docker-run   - Run the Docker container"
+	@echo "  make show-docs    - Build and open the documentation in a browser"
+
+
 
 .PHONY: test
 test:
 	@echo "Running tests..."
 	@pytest tests --disable-warnings
 
-.PHONY: lint
-lint:
-	@echo "Linting with ruff..."
-	@ruff check --fix .
+
+.PHONY: ruff-check
+ruff-check:
+	@echo "Checking code style with Ruff (safe)..."
+	@ruff check .
+
+
+.PHONY: ruff-clean
+ruff-clean:
+	@echo "Fixing lint and formatting issues with Ruff..."
+	@ruff check . --fix
+	@ruff format .
+
 
 .PHONY: docs
 docs:
 	@echo "Building documentation..."
 	@$(SPHINXBUILD) -M html "$(SOURCEDIR)" "$(BUILDDIR)"
 
-.PHONY: clean_docs
-clean_docs:
+
+.PHONY: clean-docs
+clean-docs:
 	@echo "Cleaning up generated documentation files..."
 	@$(SPHINXBUILD) -M clean "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
 	@rm -rf "$(SOURCEDIR)/autoapi/" "$(SOURCEDIR)/modules/autogen/"
 	@echo "Cleanup complete."
 
 
+.PHONY: show-docs
+show-docs:
+	@echo "Opening documentation in browser..."
+	@open "$(BUILDDIR)/html/index.html"
+
+
 # Run Docker with default settings
 .PHONY: docker-run
 docker-run:
diff --git a/docs/source/development/developer_guidelines.md b/docs/source/development/developer_guidelines.md
index 0014d8f..dd94c57 100644
--- a/docs/source/development/developer_guidelines.md
+++ b/docs/source/development/developer_guidelines.md
@@ -32,4 +32,5 @@ testing
 docs
 release
 settings_page
+style_guide
 ```
\ No newline at end of file
diff --git a/docs/source/development/style_guide.md b/docs/source/development/style_guide.md
new file mode 100644
index 0000000..a73a6fc
--- /dev/null
+++ b/docs/source/development/style_guide.md
@@ -0,0 +1,39 @@
+### Style Guide
+
+The project uses [ruff](https://docs.astral.sh/ruff/) for linting and  formatting.
+Our style configurations are set in the `pyproject.toml` file.
+
+We have a pre-commit hook to run the formatter before you commit, but Ruff can also be [integrated with most editors](https://docs.astral.sh/ruff/editors/setup/) to run automatically.
+
+We recommend you also run the linter before pushing code. 
+
+# Running the linter
+
+We have Makefile commands to run common tasks (Note if you're on Windows you might need to install `make` first, or you can use ruff directly):
+
+This outputs a report of any issues found:
+```shell
+make ruff-check
+```
+
+This command will attempt to fix any issues it can:
+
+⚠️ Warning: This can cause breaking changes. ⚠️
+
+Ensure you check any modifications by this before committing them.
+```shell
+make ruff-fix
+```
+
+**Note:** If you're on Windows you might not have `make` installed by default.
+This is included with [Git for Windows](https://gitforwindows.org/) or you can install make via [Chocolatey](https://chocolatey.org/):
+```shell
+choco install make
+```
+
+**Running directly with ruff**
+
+Alternatively, you can run the commands directly with ruff.
+
+Our rules are quite lenient for general usage, but if you want to explore more rigorous checks you can explore the [ruff documentation](https://docs.astral.sh/ruff/configuration/).
+You can then run checks to see more nuanced errors which you can review manually.
\ No newline at end of file
diff --git a/poetry.lock b/poetry.lock
index b679542..d385bc2 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -481,6 +481,18 @@ files = [
 [package.dependencies]
 pycparser = "*"
 
+[[package]]
+name = "cfgv"
+version = "3.4.0"
+description = "Validate configuration and produce human readable error messages."
+optional = false
+python-versions = ">=3.8"
+groups = ["dev"]
+files = [
+    {file = "cfgv-3.4.0-py2.py3-none-any.whl", hash = "sha256:b7265b1f29fd3316bfcd2b330d63d024f2bfd8bcb8b0272f8e19a504856c48f9"},
+    {file = "cfgv-3.4.0.tar.gz", hash = "sha256:e52591d4c5f5dead8e0f673fb16db7949d2cfb3f7da4582893288f0ded8fe560"},
+]
+
 [[package]]
 name = "charset-normalizer"
 version = "3.4.1"
@@ -696,6 +708,18 @@ calendars = ["convertdate (>=2.2.1)", "hijridate"]
 fasttext = ["fasttext (>=0.9.1)", "numpy (>=1.19.3,<2)"]
 langdetect = ["langdetect (>=1.0.0)"]
 
+[[package]]
+name = "distlib"
+version = "0.3.9"
+description = "Distribution utilities"
+optional = false
+python-versions = "*"
+groups = ["dev"]
+files = [
+    {file = "distlib-0.3.9-py2.py3-none-any.whl", hash = "sha256:47f8c22fd27c27e25a65601af709b38e4f0a45ea4fc2e710f65755fa8caaaf87"},
+    {file = "distlib-0.3.9.tar.gz", hash = "sha256:a60f20dea646b8a33f3e7772f74dc0b2d0772d2837ee1342a00645c81edf9403"},
+]
+
 [[package]]
 name = "docutils"
 version = "0.21.2"
@@ -742,6 +766,23 @@ future = "*"
 [package.extras]
 dev = ["Sphinx (==2.1.0)", "future (==0.17.1)", "numpy (==1.16.4)", "pytest (==4.6.1)", "pytest-mock (==1.10.4)", "tox (==3.12.1)"]
 
+[[package]]
+name = "filelock"
+version = "3.17.0"
+description = "A platform independent file lock."
+optional = false
+python-versions = ">=3.9"
+groups = ["dev"]
+files = [
+    {file = "filelock-3.17.0-py3-none-any.whl", hash = "sha256:533dc2f7ba78dc2f0f531fc6c4940addf7b70a481e269a5a3b93be94ffbe8338"},
+    {file = "filelock-3.17.0.tar.gz", hash = "sha256:ee4e77401ef576ebb38cd7f13b9b28893194acc20a8e68e18730ba9c0e54660e"},
+]
+
+[package.extras]
+docs = ["furo (>=2024.8.6)", "sphinx (>=8.1.3)", "sphinx-autodoc-typehints (>=3)"]
+testing = ["covdefaults (>=2.3)", "coverage (>=7.6.10)", "diff-cover (>=9.2.1)", "pytest (>=8.3.4)", "pytest-asyncio (>=0.25.2)", "pytest-cov (>=6)", "pytest-mock (>=3.14)", "pytest-timeout (>=2.3.1)", "virtualenv (>=20.28.1)"]
+typing = ["typing-extensions (>=4.12.2)"]
+
 [[package]]
 name = "future"
 version = "1.0.0"
@@ -919,6 +960,21 @@ files = [
 [package.dependencies]
 pyparsing = {version = ">=2.4.2,<3.0.0 || >3.0.0,<3.0.1 || >3.0.1,<3.0.2 || >3.0.2,<3.0.3 || >3.0.3,<4", markers = "python_version > \"3.0\""}
 
+[[package]]
+name = "identify"
+version = "2.6.9"
+description = "File identification library for Python"
+optional = false
+python-versions = ">=3.9"
+groups = ["dev"]
+files = [
+    {file = "identify-2.6.9-py2.py3-none-any.whl", hash = "sha256:c98b4322da415a8e5a70ff6e51fbc2d2932c015532d77e9f8537b4ba7813b150"},
+    {file = "identify-2.6.9.tar.gz", hash = "sha256:d40dfe3142a1421d8518e3d3985ef5ac42890683e32306ad614a29490abeb6bf"},
+]
+
+[package.extras]
+license = ["ukkonen"]
+
 [[package]]
 name = "idna"
 version = "3.10"
@@ -1260,6 +1316,18 @@ rtd = ["ipython", "sphinx (>=7)", "sphinx-autodoc2 (>=0.5.0,<0.6.0)", "sphinx-bo
 testing = ["beautifulsoup4", "coverage[toml]", "defusedxml", "pygments (<2.19)", "pytest (>=8,<9)", "pytest-cov", "pytest-param-files (>=0.6.0,<0.7.0)", "pytest-regressions", "sphinx-pytest"]
 testing-docutils = ["pygments", "pytest (>=8,<9)", "pytest-param-files (>=0.6.0,<0.7.0)"]
 
+[[package]]
+name = "nodeenv"
+version = "1.9.1"
+description = "Node.js virtual environment builder"
+optional = false
+python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7"
+groups = ["dev"]
+files = [
+    {file = "nodeenv-1.9.1-py2.py3-none-any.whl", hash = "sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9"},
+    {file = "nodeenv-1.9.1.tar.gz", hash = "sha256:6ec12890a2dab7946721edbfbcd91f3319c6ccc9aec47be7c7e6b7011ee6645f"},
+]
+
 [[package]]
 name = "numpy"
 version = "2.1.3"
@@ -1513,6 +1581,23 @@ tests = ["check-manifest", "coverage (>=7.4.2)", "defusedxml", "markdown2", "ole
 typing = ["typing-extensions"]
 xmp = ["defusedxml"]
 
+[[package]]
+name = "platformdirs"
+version = "4.3.6"
+description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`."
+optional = false
+python-versions = ">=3.8"
+groups = ["dev"]
+files = [
+    {file = "platformdirs-4.3.6-py3-none-any.whl", hash = "sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb"},
+    {file = "platformdirs-4.3.6.tar.gz", hash = "sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907"},
+]
+
+[package.extras]
+docs = ["furo (>=2024.8.6)", "proselint (>=0.14)", "sphinx (>=8.0.2)", "sphinx-autodoc-typehints (>=2.4)"]
+test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=8.3.2)", "pytest-cov (>=5)", "pytest-mock (>=3.14)"]
+type = ["mypy (>=1.11.2)"]
+
 [[package]]
 name = "pluggy"
 version = "1.5.0"
@@ -1529,6 +1614,25 @@ files = [
 dev = ["pre-commit", "tox"]
 testing = ["pytest", "pytest-benchmark"]
 
+[[package]]
+name = "pre-commit"
+version = "4.1.0"
+description = "A framework for managing and maintaining multi-language pre-commit hooks."
+optional = false
+python-versions = ">=3.9"
+groups = ["dev"]
+files = [
+    {file = "pre_commit-4.1.0-py2.py3-none-any.whl", hash = "sha256:d29e7cb346295bcc1cc75fc3e92e343495e3ea0196c9ec6ba53f49f10ab6ae7b"},
+    {file = "pre_commit-4.1.0.tar.gz", hash = "sha256:ae3f018575a588e30dfddfab9a05448bfbd6b73d78709617b5a2b853549716d4"},
+]
+
+[package.dependencies]
+cfgv = ">=2.0.0"
+identify = ">=1.0.0"
+nodeenv = ">=0.11.1"
+pyyaml = ">=5.1"
+virtualenv = ">=20.10.0"
+
 [[package]]
 name = "proto-plus"
 version = "1.26.0"
@@ -1902,7 +2006,7 @@ version = "6.0.2"
 description = "YAML parser and emitter for Python"
 optional = false
 python-versions = ">=3.8"
-groups = ["docs"]
+groups = ["dev", "docs"]
 files = [
     {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"},
     {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"},
@@ -2911,6 +3015,27 @@ typing-extensions = {version = ">=4.0", markers = "python_version < \"3.11\""}
 [package.extras]
 standard = ["colorama (>=0.4)", "httptools (>=0.6.3)", "python-dotenv (>=0.13)", "pyyaml (>=5.1)", "uvloop (>=0.14.0,!=0.15.0,!=0.15.1)", "watchfiles (>=0.13)", "websockets (>=10.4)"]
 
+[[package]]
+name = "virtualenv"
+version = "20.29.3"
+description = "Virtual Python Environment builder"
+optional = false
+python-versions = ">=3.8"
+groups = ["dev"]
+files = [
+    {file = "virtualenv-20.29.3-py3-none-any.whl", hash = "sha256:3e3d00f5807e83b234dfb6122bf37cfadf4be216c53a49ac059d02414f819170"},
+    {file = "virtualenv-20.29.3.tar.gz", hash = "sha256:95e39403fcf3940ac45bc717597dba16110b74506131845d9b687d5e73d947ac"},
+]
+
+[package.dependencies]
+distlib = ">=0.3.7,<1"
+filelock = ">=3.12.2,<4"
+platformdirs = ">=3.9.1,<5"
+
+[package.extras]
+docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.2,!=7.3)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=23.6)"]
+test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23.1)", "pytest (>=7.4)", "pytest-env (>=0.8.2)", "pytest-freezer (>=0.4.8)", "pytest-mock (>=3.11.1)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)", "setuptools (>=68)", "time-machine (>=2.10)"]
+
 [[package]]
 name = "vk-api"
 version = "11.9.9"
@@ -3213,4 +3338,4 @@ test = ["pytest (>=8.1,<9.0)", "pytest-rerunfailures (>=14.0,<15.0)"]
 [metadata]
 lock-version = "2.1"
 python-versions = ">=3.10,<3.13"
-content-hash = "0feae518c3a51717bd80e90eea3cd3ed53925af656f00b662c856bae38a742bb"
+content-hash = "fbd6cdff4eb38021115a8cd361df7c292733028822f92f45cb667971c4bce901"
diff --git a/pyproject.toml b/pyproject.toml
index 90ac56f..2defdb2 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -65,6 +65,7 @@ autopep8 = "^2.3.1"
 pytest-loguru = "^0.4.0"
 pytest-mock = "^3.14.0"
 ruff = "^0.9.10"
+pre-commit = "^4.1.0"
 
 [tool.poetry.group.docs.dependencies]
 sphinx = "^8.1.3"
@@ -96,23 +97,23 @@ markers = [
 #exclude = ["docs"]
 line-length = 120
 # Remove this for a more detailed lint report
-#output-format = "concise"
+output-format = "concise"
 
 
 [tool.ruff.lint]
-#add bugbear?
-# I : isort
-# UP : upgrade, e.g. use fstrings
-# ANN : annotations
-extend-select = ["B"]
+# Extend the rules to check for by adding them to this option:
+# See documentation for more details: https://docs.astral.sh/ruff/rules/
+#extend-select = ["B"]
 
 # Ignore unused imports as some are currently required for lazy loading
-# This can be removed for a `lint check` run which is manually reviewed
-ignore = ["F401"]
+# This can be removed for a `ruff check` run which is manually reviewed
+#ignore = ["F401"]
 
 [tool.ruff.lint.per-file-ignores]
 # Ignore import violations in __init__.py files
 "__init__.py" = ["F401", "F403"]
+# Ignore 'useless expression' in manifest files.
+"__manifest__.py" = ["B018"]
 
 [tool.ruff.format]
 docstring-code-format = false
diff --git a/src/auto_archiver/core/consts.py b/src/auto_archiver/core/consts.py
index 9a5e1e3..3b99496 100644
--- a/src/auto_archiver/core/consts.py
+++ b/src/auto_archiver/core/consts.py
@@ -1,25 +1,19 @@
 class SetupError(ValueError):
     pass
 
-MODULE_TYPES = [
-    'feeder',
-    'extractor',
-    'enricher',
-    'database',
-    'storage',
-    'formatter'
-]
+
+MODULE_TYPES = ["feeder", "extractor", "enricher", "database", "storage", "formatter"]
 
 MANIFEST_FILE = "__manifest__.py"
 
 DEFAULT_MANIFEST = {
-    'name': '', # the display name of the module
-    'author': 'Bellingcat', # creator of the module, leave this as Bellingcat or set your own name!
-    'type': [], # the type of the module, can be one or more of MODULE_TYPES
-    'requires_setup': True, # whether or not this module requires additional setup such as setting API Keys or installing additional software
-    'description': '', # a description of the module
-    'dependencies': {}, # external dependencies, e.g. python packages or binaries, in dictionary format
-    'entry_point': '', # the entry point for the module, in the format 'module_name::ClassName'. This can be left blank to use the default entry point of module_name::ModuleName
-    'version': '1.0', # the version of the module
-    'configs': {} # any configuration options this module has, these will be exposed to the user in the config file or via the command line
+    "name": "",  # the display name of the module
+    "author": "Bellingcat",  # creator of the module, leave this as Bellingcat or set your own name!
+    "type": [],  # the type of the module, can be one or more of MODULE_TYPES
+    "requires_setup": True,  # whether or not this module requires additional setup such as setting API Keys or installing additional software
+    "description": "",  # a description of the module
+    "dependencies": {},  # external dependencies, e.g. python packages or binaries, in dictionary format
+    "entry_point": "",  # the entry point for the module, in the format 'module_name::ClassName'. This can be left blank to use the default entry point of module_name::ModuleName
+    "version": "1.0",  # the version of the module
+    "configs": {},  # any configuration options this module has, these will be exposed to the user in the config file or via the command line
 }
diff --git a/src/auto_archiver/core/orchestrator.py b/src/auto_archiver/core/orchestrator.py
index dca2f4a..8c7d112 100644
--- a/src/auto_archiver/core/orchestrator.py
+++ b/src/auto_archiver/core/orchestrator.py
@@ -1,6 +1,6 @@
-""" Orchestrates all archiving steps, including feeding items,
-    archiving them with specific archivers, enrichment, storage,
-    formatting, database operations and clean up.
+"""Orchestrates all archiving steps, including feeding items,
+archiving them with specific archivers, enrichment, storage,
+formatting, database operations and clean up.
 
 """
 
@@ -19,8 +19,17 @@ import requests
 
 from .metadata import Metadata, Media
 from auto_archiver.version import __version__
-from .config import read_yaml, store_yaml, to_dot_notation, merge_dicts, is_valid_config, \
-    DefaultValidatingParser, UniqueAppendAction, AuthenticationJsonParseAction, DEFAULT_CONFIG_FILE
+from .config import (
+    read_yaml,
+    store_yaml,
+    to_dot_notation,
+    merge_dicts,
+    is_valid_config,
+    DefaultValidatingParser,
+    UniqueAppendAction,
+    AuthenticationJsonParseAction,
+    DEFAULT_CONFIG_FILE,
+)
 from .module import ModuleFactory, LazyBaseModule
 from . import validators, Feeder, Extractor, Database, Storage, Formatter, Enricher
 from .consts import MODULE_TYPES, SetupError
@@ -30,8 +39,8 @@ if TYPE_CHECKING:
     from .base_module import BaseModule
     from .module import LazyBaseModule
 
-class ArchivingOrchestrator:
 
+class ArchivingOrchestrator:
     # instance variables
     module_factory: ModuleFactory
     setup_finished: bool
@@ -61,30 +70,63 @@ class ArchivingOrchestrator:
             epilog="Check the code at https://github.com/bellingcat/auto-archiver",
             formatter_class=RichHelpFormatter,
         )
-        parser.add_argument('--help', '-h', action='store_true', dest='help', help='show a full help message and exit')
-        parser.add_argument('--version', action='version', version=__version__)
-        parser.add_argument('--config', action='store', dest="config_file", help='the filename of the YAML configuration file (defaults to \'config.yaml\')', default=DEFAULT_CONFIG_FILE)
-        parser.add_argument('--mode', action='store', dest='mode', type=str, choices=['simple', 'full'], help='the mode to run the archiver in', default='simple')
+        parser.add_argument("--help", "-h", action="store_true", dest="help", help="show a full help message and exit")
+        parser.add_argument("--version", action="version", version=__version__)
+        parser.add_argument(
+            "--config",
+            action="store",
+            dest="config_file",
+            help="the filename of the YAML configuration file (defaults to 'config.yaml')",
+            default=DEFAULT_CONFIG_FILE,
+        )
+        parser.add_argument(
+            "--mode",
+            action="store",
+            dest="mode",
+            type=str,
+            choices=["simple", "full"],
+            help="the mode to run the archiver in",
+            default="simple",
+        )
         # override the default 'help' so we can inject all the configs and show those
-        parser.add_argument('-s', '--store', dest='store', default=False, help='Store the created config in the config file', action=argparse.BooleanOptionalAction)
-        parser.add_argument('--module_paths', dest='module_paths', nargs='+', default=[], help='additional paths to search for modules', action=UniqueAppendAction)
+        parser.add_argument(
+            "-s",
+            "--store",
+            dest="store",
+            default=False,
+            help="Store the created config in the config file",
+            action=argparse.BooleanOptionalAction,
+        )
+        parser.add_argument(
+            "--module_paths",
+            dest="module_paths",
+            nargs="+",
+            default=[],
+            help="additional paths to search for modules",
+            action=UniqueAppendAction,
+        )
 
         self.basic_parser = parser
         return parser
-    
+
     def check_steps(self, config):
         for module_type in MODULE_TYPES:
-            if not config['steps'].get(f"{module_type}s", []):
-                if module_type == 'feeder' or module_type == 'formatter' and config['steps'].get(f"{module_type}"):
-                    raise SetupError(f"It appears you have '{module_type}' set under 'steps' in your configuration file, but as of version 0.13.0 of Auto Archiver, you must use '{module_type}s'. Change this in your configuration file and try again. \
-Here's how that would look: \n\nsteps:\n  {module_type}s:\n  - [your_{module_type}_name_here]\n  {'extractors:...' if module_type == 'feeder' else '...'}\n")
-                if module_type == 'extractor' and config['steps'].get('archivers'):
-                    raise SetupError("As of version 0.13.0 of Auto Archiver, the 'archivers' step name has been changed to 'extractors'. Change this in your configuration file and try again. \
-Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_here]\n  enrichers:...\n")
-                raise SetupError(f"No {module_type}s were configured. Make sure to set at least one {module_type} in your configuration file or on the command line (using --{module_type}s)")
+            if not config["steps"].get(f"{module_type}s", []):
+                if module_type == "feeder" or module_type == "formatter" and config["steps"].get(f"{module_type}"):
+                    raise SetupError(
+                        f"It appears you have '{module_type}' set under 'steps' in your configuration file, but as of version 0.13.0 of Auto Archiver, you must use '{module_type}s'. Change this in your configuration file and try again. \
+Here's how that would look: \n\nsteps:\n  {module_type}s:\n  - [your_{module_type}_name_here]\n  {'extractors:...' if module_type == 'feeder' else '...'}\n"
+                    )
+                if module_type == "extractor" and config["steps"].get("archivers"):
+                    raise SetupError(
+                        "As of version 0.13.0 of Auto Archiver, the 'archivers' step name has been changed to 'extractors'. Change this in your configuration file and try again. \
+Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_here]\n  enrichers:...\n"
+                    )
+                raise SetupError(
+                    f"No {module_type}s were configured. Make sure to set at least one {module_type} in your configuration file or on the command line (using --{module_type}s)"
+                )
 
     def setup_complete_parser(self, basic_config: dict, yaml_config: dict, unused_args: list[str]) -> None:
-
         # modules parser to get the overridden 'steps' values
         modules_parser = argparse.ArgumentParser(
             add_help=False,
@@ -92,7 +134,9 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
         self.add_modules_args(modules_parser)
         cli_modules, unused_args = modules_parser.parse_known_args(unused_args)
         for module_type in MODULE_TYPES:
-            yaml_config['steps'][f"{module_type}s"] = getattr(cli_modules, f"{module_type}s", []) or yaml_config['steps'].get(f"{module_type}s", [])
+            yaml_config["steps"][f"{module_type}s"] = getattr(cli_modules, f"{module_type}s", []) or yaml_config[
+                "steps"
+            ].get(f"{module_type}s", [])
 
         parser = DefaultValidatingParser(
             add_help=False,
@@ -115,30 +159,32 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
             enabled_modules = []
             # first loads the modules from the config file, then from the command line
             for module_type in MODULE_TYPES:
-                enabled_modules.extend(yaml_config['steps'].get(f"{module_type}s", []))
+                enabled_modules.extend(yaml_config["steps"].get(f"{module_type}s", []))
 
             # clear out duplicates, but keep the order
             enabled_modules = list(dict.fromkeys(enabled_modules))
-            avail_modules = self.module_factory.available_modules(limit_to_modules=enabled_modules, suppress_warnings=True)
+            avail_modules = self.module_factory.available_modules(
+                limit_to_modules=enabled_modules, suppress_warnings=True
+            )
             self.add_individual_module_args(avail_modules, parser)
-        elif basic_config.mode == 'simple':
+        elif basic_config.mode == "simple":
             simple_modules = [module for module in self.module_factory.available_modules() if not module.requires_setup]
             self.add_individual_module_args(simple_modules, parser)
 
             # add them to the config
             for module in simple_modules:
                 for module_type in module.type:
-                    yaml_config['steps'].setdefault(f"{module_type}s", []).append(module.name)
+                    yaml_config["steps"].setdefault(f"{module_type}s", []).append(module.name)
         else:
             # load all modules, they're not using the 'simple' mode
             all_modules = self.module_factory.available_modules()
             # add all the modules to the steps
             for module in all_modules:
                 for module_type in module.type:
-                    yaml_config['steps'].setdefault(f"{module_type}s", []).append(module.name)
+                    yaml_config["steps"].setdefault(f"{module_type}s", []).append(module.name)
 
             self.add_individual_module_args(all_modules, parser)
-        
+
         parser.set_defaults(**to_dot_notation(yaml_config))
 
         # reload the parser with the new arguments, now that we have them
@@ -164,43 +210,76 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
             store_yaml(config, basic_config.config_file)
 
         return config
-    
+
     def add_modules_args(self, parser: argparse.ArgumentParser = None):
         if not parser:
             parser = self.parser
 
         # Module loading from the command line
         for module_type in MODULE_TYPES:
-            parser.add_argument(f'--{module_type}s', dest=f'{module_type}s', nargs='+', help=f'the {module_type}s to use', default=[], action=UniqueAppendAction)
+            parser.add_argument(
+                f"--{module_type}s",
+                dest=f"{module_type}s",
+                nargs="+",
+                help=f"the {module_type}s to use",
+                default=[],
+                action=UniqueAppendAction,
+            )
 
     def add_additional_args(self, parser: argparse.ArgumentParser = None):
         if not parser:
             parser = self.parser
 
-        parser.add_argument('--authentication', dest='authentication', help='A dictionary of sites and their authentication methods \
+        parser.add_argument(
+            "--authentication",
+            dest="authentication",
+            help="A dictionary of sites and their authentication methods \
                                                                             (token, username etc.) that extractors can use to log into \
                                                                             a website. If passing this on the command line, use a JSON string. \
-                                                                            You may also pass a path to a valid JSON/YAML file which will be parsed.',
-                                                                            default={},
-                                                                            nargs="?",
-                                                                            action=AuthenticationJsonParseAction)
+                                                                            You may also pass a path to a valid JSON/YAML file which will be parsed.",
+            default={},
+            nargs="?",
+            action=AuthenticationJsonParseAction,
+        )
 
         # logging arguments
-        parser.add_argument('--logging.level', action='store', dest='logging.level', choices=['INFO', 'DEBUG', 'ERROR', 'WARNING'], help='the logging level to use', default='INFO', type=str.upper)
-        parser.add_argument('--logging.file', action='store', dest='logging.file', help='the logging file to write to', default=None)
-        parser.add_argument('--logging.rotation', action='store', dest='logging.rotation', help='the logging rotation to use', default=None)
-
-    def add_individual_module_args(self, modules: list[LazyBaseModule] = None, parser: argparse.ArgumentParser = None) -> None:
+        parser.add_argument(
+            "--logging.level",
+            action="store",
+            dest="logging.level",
+            choices=["INFO", "DEBUG", "ERROR", "WARNING"],
+            help="the logging level to use",
+            default="INFO",
+            type=str.upper,
+        )
+        parser.add_argument(
+            "--logging.file", action="store", dest="logging.file", help="the logging file to write to", default=None
+        )
+        parser.add_argument(
+            "--logging.rotation",
+            action="store",
+            dest="logging.rotation",
+            help="the logging rotation to use",
+            default=None,
+        )
 
+    def add_individual_module_args(
+        self, modules: list[LazyBaseModule] = None, parser: argparse.ArgumentParser = None
+    ) -> None:
         if not modules:
             modules = self.module_factory.available_modules()
-        
+
         for module in modules:
-            if module.name == 'cli_feeder':
+            if module.name == "cli_feeder":
                 # special case. For the CLI feeder, allow passing URLs directly on the command line without setting --cli_feeder.urls=
-                parser.add_argument('urls', nargs='*', default=[], help='URL(s) to archive, either a single URL or a list of urls, should not come from config.yaml')
+                parser.add_argument(
+                    "urls",
+                    nargs="*",
+                    default=[],
+                    help="URL(s) to archive, either a single URL or a list of urls, should not come from config.yaml",
+                )
                 continue
-                
+
             if not module.configs:
                 # this module has no configs, don't show anything in the help
                 # (TODO: do we want to show something about this module though, like a description?)
@@ -209,21 +288,21 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
             group = parser.add_argument_group(module.display_name or module.name, f"{module.description[:100]}...")
 
             for name, kwargs in module.configs.items():
-                if not kwargs.get('metavar', None):
+                if not kwargs.get("metavar", None):
                     # make a nicer metavar, metavar is what's used in the help, e.g. --cli_feeder.urls [METAVAR]
-                    kwargs['metavar'] = name.upper()
+                    kwargs["metavar"] = name.upper()
 
-                if kwargs.get('required', False):
+                if kwargs.get("required", False):
                     # required args shouldn't have a 'default' value, remove it
-                    kwargs.pop('default', None)
+                    kwargs.pop("default", None)
 
-                kwargs.pop('cli_set', None)
-                should_store = kwargs.pop('should_store', False)
-                kwargs['dest'] = f"{module.name}.{kwargs.pop('dest', name)}"
+                kwargs.pop("cli_set", None)
+                should_store = kwargs.pop("should_store", False)
+                kwargs["dest"] = f"{module.name}.{kwargs.pop('dest', name)}"
                 try:
-                    kwargs['type'] = getattr(validators, kwargs.get('type', '__invalid__'))
+                    kwargs["type"] = getattr(validators, kwargs.get("type", "__invalid__"))
                 except AttributeError:
-                    kwargs['type'] = __builtins__.get(kwargs.get('type'), str)
+                    kwargs["type"] = __builtins__.get(kwargs.get("type"), str)
                 arg = group.add_argument(f"--{module.name}.{name}", **kwargs)
                 arg.should_store = should_store
 
@@ -238,12 +317,11 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
         self.basic_parser.exit()
 
     def setup_logging(self, config):
+        logging_config = config["logging"]
 
-        logging_config = config['logging']
-
-        if logging_config.get('enabled', True) is False:
+        if logging_config.get("enabled", True) is False:
             # disabled logging settings, they're set on a higher level
-            logger.disable('auto_archiver')
+            logger.disable("auto_archiver")
             return
 
         # setup loguru logging
@@ -253,38 +331,45 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
             pass
 
         # add other logging info
-        if self.logger_id is None: # note - need direct comparison to None since need to consider falsy value 0
-            self.logger_id = logger.add(sys.stderr, level=logging_config['level'])
-            if log_file := logging_config['file']:
-                logger.add(log_file) if not logging_config['rotation'] else logger.add(log_file, rotation=logging_config['rotation'])
+        if self.logger_id is None:  # note - need direct comparison to None since need to consider falsy value 0
+            self.logger_id = logger.add(sys.stderr, level=logging_config["level"])
+            if log_file := logging_config["file"]:
+                logger.add(log_file) if not logging_config["rotation"] else logger.add(
+                    log_file, rotation=logging_config["rotation"]
+                )
 
     def install_modules(self, modules_by_type):
         """
-        Traverses all modules in 'steps' and loads them into the orchestrator, storing them in the 
+        Traverses all modules in 'steps' and loads them into the orchestrator, storing them in the
         orchestrator's attributes (self.feeders, self.extractors etc.). If no modules of a certain type
         are loaded, the program will exit with an error message.
         """
 
         invalid_modules = []
         for module_type in MODULE_TYPES:
-
             step_items = []
             modules_to_load = modules_by_type[f"{module_type}s"]
             if not modules_to_load:
-                raise SetupError(f"No {module_type}s were configured. Make sure to set at least one {module_type} in your configuration file or on the command line (using --{module_type}s)")
+                raise SetupError(
+                    f"No {module_type}s were configured. Make sure to set at least one {module_type} in your configuration file or on the command line (using --{module_type}s)"
+                )
 
             def check_steps_ok():
                 if not len(step_items):
                     if len(modules_to_load):
-                        logger.error(f"Unable to load any {module_type}s. Tried the following, but none were available: {modules_to_load}")
-                    raise SetupError(f"NO {module_type.upper()}S LOADED. Please check your configuration and try again.")
-                
+                        logger.error(
+                            f"Unable to load any {module_type}s. Tried the following, but none were available: {modules_to_load}"
+                        )
+                    raise SetupError(
+                        f"NO {module_type.upper()}S LOADED. Please check your configuration and try again."
+                    )
 
-                if (module_type == 'feeder' or module_type == 'formatter') and len(step_items) > 1:
-                    raise SetupError(f"Only one {module_type} is allowed, found {len(step_items)} {module_type}s. Please remove one of the following from your configuration file: {modules_to_load}")
+                if (module_type == "feeder" or module_type == "formatter") and len(step_items) > 1:
+                    raise SetupError(
+                        f"Only one {module_type} is allowed, found {len(step_items)} {module_type}s. Please remove one of the following from your configuration file: {modules_to_load}"
+                    )
 
             for module in modules_to_load:
-
                 if module in invalid_modules:
                     continue
 
@@ -293,7 +378,7 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
                     loaded_module: BaseModule = self.module_factory.get_module(module, self.config)
                 except (KeyboardInterrupt, Exception) as e:
                     logger.error(f"Error during setup of modules: {e}\n{traceback.format_exc()}")
-                    if loaded_module and module_type == 'extractor':
+                    if loaded_module and module_type == "extractor":
                         loaded_module.cleanup()
                     raise e
 
@@ -308,11 +393,13 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
 
     def load_config(self, config_file: str) -> dict:
         if not os.path.exists(config_file) and config_file != DEFAULT_CONFIG_FILE:
-            logger.error(f"The configuration file {config_file} was  not found. Make sure the file exists and try again, or run without the --config file to use the default settings.")
+            logger.error(
+                f"The configuration file {config_file} was  not found. Make sure the file exists and try again, or run without the --config file to use the default settings."
+            )
             raise FileNotFoundError(f"Configuration file {config_file} not found")
 
         return read_yaml(config_file)
-    
+
     def setup_config(self, args: list) -> dict:
         """
         Sets up the configuration file, merging the default config with the user's config
@@ -335,13 +422,13 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
         yaml_config = self.load_config(basic_config.config_file)
 
         return self.setup_complete_parser(basic_config, yaml_config, unused_args)
-    
+
     def check_for_updates(self):
         response = requests.get("https://pypi.org/pypi/auto-archiver/json").json()
-        latest_version = response['info']['version']
+        latest_version = response["info"]["version"]
         # check version compared to current version
         if latest_version != __version__:
-            if os.environ.get('RUNNING_IN_DOCKER'):
+            if os.environ.get("RUNNING_IN_DOCKER"):
                 update_cmd = "`docker pull bellingcat/auto-archiver:latest`"
             else:
                 update_cmd = "`pip install --upgrade auto-archiver`"
@@ -351,33 +438,36 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
             logger.warning(f"Make sure to update to the latest version using: {update_cmd}")
             logger.warning("")
 
-        
     def setup(self, args: list):
         """
         Function to configure all setup of the orchestrator: setup configs and load modules.
-        
+
         This method should only ever be called once
         """
 
         self.check_for_updates()
 
         if self.setup_finished:
-            logger.warning("The `setup_config()` function should only ever be run once. \
+            logger.warning(
+                "The `setup_config()` function should only ever be run once. \
                            If you need to re-run the setup, please re-instantiate a new instance of the orchestrator. \
                            For code implementatations, you should call .setup_config() once then you may call .feed() \
-                           multiple times to archive multiple URLs.")
+                           multiple times to archive multiple URLs."
+            )
             return
 
         self.setup_basic_parser()
         self.config = self.setup_config(args)
 
         logger.info(f"======== Welcome to the AUTO ARCHIVER ({__version__}) ==========")
-        self.install_modules(self.config['steps'])
+        self.install_modules(self.config["steps"])
 
         # log out the modules that were loaded
         for module_type in MODULE_TYPES:
-            logger.info(f"{module_type.upper()}S: " + ", ".join(m.display_name for m in getattr(self, f"{module_type}s")))
-        
+            logger.info(
+                f"{module_type.upper()}S: " + ", ".join(m.display_name for m in getattr(self, f"{module_type}s"))
+            )
+
         self.setup_finished = True
 
     def _command_line_run(self, args: list) -> Generator[Metadata]:
@@ -385,9 +475,9 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
         This is the main entry point for the orchestrator, when run from the command line.
 
         :param args: list of arguments to pass to the orchestrator - these are the command line args
-        
+
         You should not call this method from code implementations.
-          
+
         This method sets up the configuration, loads the modules, and runs the feed.
         If you wish to make code invocations yourself, you should use the 'setup' and 'feed' methods separately.
         To test configurations, without loading any modules you can also first call 'setup_configs'
@@ -396,7 +486,7 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
             self.setup(args)
             return self.feed()
         except Exception as e:
-            logger.error(e)
+            logger.error(e, exc_info=True)
             exit(1)
 
     def cleanup(self) -> None:
@@ -405,7 +495,6 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
             e.cleanup()
 
     def feed(self) -> Generator[Metadata]:
-        
         url_count = 0
         for feeder in self.feeders:
             for item in feeder:
@@ -436,7 +525,7 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
             self.cleanup()
             exit()
         except Exception as e:
-            logger.error(f'Got unexpected error on item {item}: {e}\n{traceback.format_exc()}')
+            logger.error(f"Got unexpected error on item {item}: {e}\n{traceback.format_exc()}")
             for d in self.databases:
                 if isinstance(e, AssertionError):
                     d.failed(item, str(e))
@@ -451,13 +540,13 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
 
     def archive(self, result: Metadata) -> Union[Metadata, None]:
         """
-            Runs the archiving process for a single URL
-            1. Each archiver can sanitize its own URLs
-            2. Check for cached results in Databases, and signal start to the databases
-            3. Call Archivers until one succeeds
-            4. Call Enrichers
-            5. Store all downloaded/generated media
-            6. Call selected Formatter and store formatted if needed
+        Runs the archiving process for a single URL
+        1. Each archiver can sanitize its own URLs
+        2. Check for cached results in Databases, and signal start to the databases
+        3. Call Archivers until one succeeds
+        4. Call Enrichers
+        5. Store all downloaded/generated media
+        6. Call selected Formatter and store formatted if needed
         """
 
         original_url = result.get_url().strip()
@@ -528,7 +617,6 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
                 logger.error(f"ERROR database {d.name}: {e}: {traceback.format_exc()}")
 
         return result
-    
 
     def setup_authentication(self, config: dict) -> dict:
         """
@@ -537,7 +625,7 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
         Split up strings into multiple sites if they are comma separated
         """
 
-        authentication = config.get('authentication', {})
+        authentication = config.get("authentication", {})
 
         # extract out concatenated sites
         for key, val in copy(authentication).items():
@@ -546,8 +634,8 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
                     site = site.strip()
                     authentication[site] = val
                 del authentication[key]
-        
-        config['authentication'] = authentication
+
+        config["authentication"] = authentication
         return config
 
     # Helper Properties
diff --git a/src/auto_archiver/core/storage.py b/src/auto_archiver/core/storage.py
index c73e29c..3205f5a 100644
--- a/src/auto_archiver/core/storage.py
+++ b/src/auto_archiver/core/storage.py
@@ -32,16 +32,16 @@ from auto_archiver.utils.misc import random_str
 from auto_archiver.core import Media, BaseModule, Metadata
 from auto_archiver.modules.hash_enricher.hash_enricher import HashEnricher
 
+
 class Storage(BaseModule):
-    
     """
     Base class for implementing storage modules in the media archiving framework.
 
     Subclasses must implement the `get_cdn_url` and `uploadf` methods to define their behavior.
     """
 
-    def store(self, media: Media, url: str, metadata: Metadata=None) -> None:
-        if media.is_stored(in_storage=self): 
+    def store(self, media: Media, url: str, metadata: Metadata = None) -> None:
+        if media.is_stored(in_storage=self):
             logger.debug(f"{media.key} already stored, skipping")
             return
 
@@ -73,18 +73,18 @@ class Storage(BaseModule):
         This method should not be called directly, but instead be called through the 'store' method,
         which sets up the media for storage.
         """
-        logger.debug(f'[{self.__class__.__name__}] storing file {media.filename} with key {media.key}')
-        with open(media.filename, 'rb') as f:
+        logger.debug(f"[{self.__class__.__name__}] storing file {media.filename} with key {media.key}")
+        with open(media.filename, "rb") as f:
             return self.uploadf(f, media, **kwargs)
 
     def set_key(self, media: Media, url: str, metadata: Metadata) -> None:
         """takes the media and optionally item info and generates a key"""
-        
+
         if media.key is not None and len(media.key) > 0:
             # media key is already set
             return
 
-        folder = metadata.get_context('folder', '')
+        folder = metadata.get_context("folder", "")
         filename, ext = os.path.splitext(media.filename)
 
         # Handle path_generator logic
@@ -104,12 +104,11 @@ class Storage(BaseModule):
             filename = random_str(24)
         elif filename_generator == "static":
             # load the hash_enricher module
-            he = self.module_factory.get_module("hash_enricher", self.config)
+            he: HashEnricher = self.module_factory.get_module("hash_enricher", self.config)
             hd = he.calculate_hash(media.filename)
             filename = hd[:24]
         else:
             raise ValueError(f"Invalid filename_generator: {filename_generator}")
-        
-        key = os.path.join(folder, path, f"{filename}{ext}")
 
-        media._key = key
\ No newline at end of file
+        key = os.path.join(folder, path, f"{filename}{ext}")
+        media._key = key
diff --git a/src/auto_archiver/modules/gsheet_feeder_db/__manifest__.py b/src/auto_archiver/modules/gsheet_feeder_db/__manifest__.py
index 6547233..5143218 100644
--- a/src/auto_archiver/modules/gsheet_feeder_db/__manifest__.py
+++ b/src/auto_archiver/modules/gsheet_feeder_db/__manifest__.py
@@ -12,9 +12,7 @@
             "default": None,
             "help": "the id of the sheet to archive (alternative to 'sheet' config)",
         },
-        "header": {"default": 1,
-                   "help": "index of the header row (starts at 1)",
-                   "type": "int"},
+        "header": {"default": 1, "help": "index of the header row (starts at 1)", "type": "int"},
         "service_account": {
             "default": "secrets/service_account.json",
             "help": "service account JSON file path. Learn how to create one: https://gspread.readthedocs.io/en/latest/oauth2.html",
diff --git a/src/auto_archiver/modules/local_storage/local_storage.py b/src/auto_archiver/modules/local_storage/local_storage.py
index 54f4a0e..fdc6978 100644
--- a/src/auto_archiver/modules/local_storage/local_storage.py
+++ b/src/auto_archiver/modules/local_storage/local_storage.py
@@ -1,4 +1,3 @@
-
 import shutil
 from typing import IO
 import os
@@ -8,12 +7,13 @@ from auto_archiver.core import Media
 from auto_archiver.core import Storage
 from auto_archiver.core.consts import SetupError
 
+
 class LocalStorage(Storage):
-
-
     def setup(self) -> None:
         if len(self.save_to) > 200:
-            raise SetupError("Your save_to path is too long, this will cause issues saving files on your computer. Please use a shorter path.")
+            raise SetupError(
+                "Your save_to path is too long, this will cause issues saving files on your computer. Please use a shorter path."
+            )
 
     def get_cdn_url(self, media: Media) -> str:
         dest = media.key
@@ -25,18 +25,18 @@ class LocalStorage(Storage):
     def set_key(self, media, url, metadata):
         # clarify we want to save the file to the save_to folder
 
-        old_folder = metadata.get('folder', '')
-        metadata.set_context('folder', os.path.join(self.save_to, metadata.get('folder', '')))
+        old_folder = metadata.get("folder", "")
+        metadata.set_context("folder", os.path.join(self.save_to, metadata.get("folder", "")))
         super().set_key(media, url, metadata)
         # don't impact other storages that might want a different 'folder' set
-        metadata.set_context('folder', old_folder)
+        metadata.set_context("folder", old_folder)
 
     def upload(self, media: Media, **kwargs) -> bool:
         # override parent so that we can use shutil.copy2 and keep metadata
         dest = media.key
 
         os.makedirs(os.path.dirname(dest), exist_ok=True)
-        logger.debug(f'[{self.__class__.__name__}] storing file {media.filename} with key {media.key} to {dest}')
+        logger.debug(f"[{self.__class__.__name__}] storing file {media.filename} with key {media.key} to {dest}")
 
         res = shutil.copy2(media.filename, dest)
         logger.info(res)
@@ -44,4 +44,4 @@ class LocalStorage(Storage):
 
     # must be implemented even if unused
     def uploadf(self, file: IO[bytes], key: str, **kwargs: dict) -> bool:
-        pass
\ No newline at end of file
+        pass
diff --git a/src/auto_archiver/modules/s3_storage/s3_storage.py b/src/auto_archiver/modules/s3_storage/s3_storage.py
index bb87812..abac4f7 100644
--- a/src/auto_archiver/modules/s3_storage/s3_storage.py
+++ b/src/auto_archiver/modules/s3_storage/s3_storage.py
@@ -1,4 +1,3 @@
-
 from typing import IO
 
 import boto3
@@ -11,18 +10,20 @@ from auto_archiver.utils.misc import calculate_file_hash, random_str
 
 NO_DUPLICATES_FOLDER = "no-dups/"
 
-class S3Storage(Storage):
 
+class S3Storage(Storage):
     def setup(self) -> None:
         self.s3 = boto3.client(
-            's3',
+            "s3",
             region_name=self.region,
             endpoint_url=self.endpoint_url.format(region=self.region),
             aws_access_key_id=self.key,
-            aws_secret_access_key=self.secret
+            aws_secret_access_key=self.secret,
         )
         if self.random_no_duplicate:
-            logger.warning("random_no_duplicate is set to True, this will override `path_generator`, `filename_generator` and `folder`.")
+            logger.warning(
+                "random_no_duplicate is set to True, this will override `path_generator`, `filename_generator` and `folder`."
+            )
 
     def get_cdn_url(self, media: Media) -> str:
         return self.cdn_url.format(bucket=self.bucket, region=self.region, key=media.key)
@@ -32,13 +33,13 @@ class S3Storage(Storage):
             return True
 
         extra_args = kwargs.get("extra_args", {})
-        if not self.private and 'ACL' not in extra_args:
-            extra_args['ACL'] = 'public-read'
+        if not self.private and "ACL" not in extra_args:
+            extra_args["ACL"] = "public-read"
 
-        if 'ContentType' not in extra_args:
+        if "ContentType" not in extra_args:
             try:
                 if media.mimetype:
-                    extra_args['ContentType'] = media.mimetype
+                    extra_args["ContentType"] = media.mimetype
             except Exception as e:
                 logger.warning(f"Unable to get mimetype for {media.key=}, error: {e}")
         self.s3.upload_fileobj(file, Bucket=self.bucket, Key=media.key, ExtraArgs=extra_args)
@@ -50,21 +51,21 @@ class S3Storage(Storage):
             hd = calculate_file_hash(media.filename)
             path = os.path.join(NO_DUPLICATES_FOLDER, hd[:24])
 
-            if existing_key:=self.file_in_folder(path):
+            if existing_key := self.file_in_folder(path):
                 media._key = existing_key
                 media.set("previously archived", True)
                 logger.debug(f"skipping upload of {media.filename} because it already exists in {media.key}")
                 return False
-            
+
             _, ext = os.path.splitext(media.key)
             media._key = os.path.join(path, f"{random_str(24)}{ext}")
         return True
 
-    def file_in_folder(self, path:str) -> str:
+    def file_in_folder(self, path: str) -> str:
         # checks if path exists and is not an empty folder
-        if not path.endswith('/'):
-            path = path + '/' 
-        resp = self.s3.list_objects(Bucket=self.bucket, Prefix=path, Delimiter='/', MaxKeys=1)
-        if 'Contents' in resp:
-            return resp['Contents'][0]['Key']
-        return False
\ No newline at end of file
+        if not path.endswith("/"):
+            path = path + "/"
+        resp = self.s3.list_objects(Bucket=self.bucket, Prefix=path, Delimiter="/", MaxKeys=1)
+        if "Contents" in resp:
+            return resp["Contents"][0]["Key"]
+        return False
diff --git a/src/auto_archiver/modules/tiktok_tikwm_extractor/__init__.py b/src/auto_archiver/modules/tiktok_tikwm_extractor/__init__.py
index 25a20f5..e1008ad 100644
--- a/src/auto_archiver/modules/tiktok_tikwm_extractor/__init__.py
+++ b/src/auto_archiver/modules/tiktok_tikwm_extractor/__init__.py
@@ -1 +1 @@
-from .tiktok_tikwm_extractor import TiktokTikwmExtractor
\ No newline at end of file
+from .tiktok_tikwm_extractor import TiktokTikwmExtractor
diff --git a/src/auto_archiver/modules/tiktok_tikwm_extractor/__manifest__.py b/src/auto_archiver/modules/tiktok_tikwm_extractor/__manifest__.py
index 56d8e3e..7c46a87 100644
--- a/src/auto_archiver/modules/tiktok_tikwm_extractor/__manifest__.py
+++ b/src/auto_archiver/modules/tiktok_tikwm_extractor/__manifest__.py
@@ -2,10 +2,7 @@
     "name": "Tiktok Tikwm Extractor",
     "type": ["extractor"],
     "requires_setup": False,
-    "dependencies": {
-        "python": ["loguru", "requests"],
-        "bin": []
-    },
+    "dependencies": {"python": ["loguru", "requests"], "bin": []},
     "description": """
     Uses an unofficial TikTok video download platform's API to download videos: https://tikwm.com/
 	
@@ -19,5 +16,5 @@
     - If tikwm.com is down, this extractor will not work.
 	- If tikwm.com changes their API, this extractor may break.
 	- If no video is found, this extractor will consider the extraction failed.
-    """
+    """,
 }
diff --git a/src/auto_archiver/modules/tiktok_tikwm_extractor/tiktok_tikwm_extractor.py b/src/auto_archiver/modules/tiktok_tikwm_extractor/tiktok_tikwm_extractor.py
index e7ed91a..3264199 100644
--- a/src/auto_archiver/modules/tiktok_tikwm_extractor/tiktok_tikwm_extractor.py
+++ b/src/auto_archiver/modules/tiktok_tikwm_extractor/tiktok_tikwm_extractor.py
@@ -12,11 +12,12 @@ class TiktokTikwmExtractor(Extractor):
     """
     Extractor for TikTok that uses an unofficial API and can capture content that requires a login, like sensitive content.
     """
+
     TIKWM_ENDPOINT = "https://www.tikwm.com/api/?url={url}"
 
     def download(self, item: Metadata) -> bool | Metadata:
         url = item.get_url()
-        
+
         if not re.match(TikTokIE._VALID_URL, url):
             return False
 
@@ -33,7 +34,7 @@ class TiktokTikwmExtractor(Extractor):
             logger.error(f"failed to parse JSON response from tikwm.com for {url=}")
             return False
 
-        if not json_response.get('msg') == 'success' or not (api_data := json_response.get('data', {})):
+        if not json_response.get("msg") == "success" or not (api_data := json_response.get("data", {})):
             logger.error(f"failed to get a valid response from tikwm.com for {url=}: {json_response}")
             return False
 
@@ -67,7 +68,7 @@ class TiktokTikwmExtractor(Extractor):
         if created_at := api_data.pop("create_time", None):
             result.set_timestamp(datetime.fromtimestamp(created_at, tz=timezone.utc))
 
-        if (author := api_data.pop("author", None)):
+        if author := api_data.pop("author", None):
             result.set("author", author)
 
         result.set("api_data", api_data)
diff --git a/src/auto_archiver/modules/wacz_extractor_enricher/__manifest__.py b/src/auto_archiver/modules/wacz_extractor_enricher/__manifest__.py
index 7916049..97e3bf6 100644
--- a/src/auto_archiver/modules/wacz_extractor_enricher/__manifest__.py
+++ b/src/auto_archiver/modules/wacz_extractor_enricher/__manifest__.py
@@ -14,9 +14,7 @@
             "help": "browsertrix-profile (for profile generation see https://github.com/webrecorder/browsertrix-crawler#creating-and-using-browser-profiles).",
         },
         "docker_commands": {"default": None, "help": "if a custom docker invocation is needed"},
-        "timeout": {"default": 120,
-                    "help": "timeout for WACZ generation in seconds",
-                    "type": "int"},
+        "timeout": {"default": 120, "help": "timeout for WACZ generation in seconds", "type": "int"},
         "extract_media": {
             "default": False,
             "type": "bool",
diff --git a/tests/conftest.py b/tests/conftest.py
index ba1f652..9754b91 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -22,7 +22,9 @@ TESTS_TO_RUN_LAST = ["test_twitter_api_archiver"]
 
 @pytest.fixture
 def setup_module(request):
-    def _setup_module(module_name, config={}):
+    def _setup_module(module_name, config=None):
+        if config is None:
+            config = {}
         module_factory = ModuleFactory()
 
         if isinstance(module_name, type):
diff --git a/tests/extractors/test_tiktok_tikwm_extractor.py b/tests/extractors/test_tiktok_tikwm_extractor.py
index f675ac0..690d448 100644
--- a/tests/extractors/test_tiktok_tikwm_extractor.py
+++ b/tests/extractors/test_tiktok_tikwm_extractor.py
@@ -24,17 +24,20 @@ class TestTiktokTikwmExtractor(TestExtractorBase):
         mock_logger = mocker.patch("auto_archiver.modules.tiktok_tikwm_extractor.tiktok_tikwm_extractor.logger")
         return mock_get, mock_logger
 
-    @pytest.mark.parametrize("url,valid_url", [
-        ("https://bellingcat.com", False),
-        ("https://youtube.com", False),
-        ("https://tiktok.co/", False),
-        ("https://tiktok.com/", False),
-        ("https://www.tiktok.com/", False),
-        ("https://api.cool.tiktok.com/", False),
-        (VALID_EXAMPLE_URL, True),
-        ("https://www.tiktok.com/@bbcnews/video/7478038212070411542", True),
-        ("https://www.tiktok.com/@ggs68taiwan.official/video/7441821351142362375", True),
-    ])
+    @pytest.mark.parametrize(
+        "url,valid_url",
+        [
+            ("https://bellingcat.com", False),
+            ("https://youtube.com", False),
+            ("https://tiktok.co/", False),
+            ("https://tiktok.com/", False),
+            ("https://www.tiktok.com/", False),
+            ("https://api.cool.tiktok.com/", False),
+            (VALID_EXAMPLE_URL, True),
+            ("https://www.tiktok.com/@bbcnews/video/7478038212070411542", True),
+            ("https://www.tiktok.com/@ggs68taiwan.official/video/7441821351142362375", True),
+        ],
+    )
     def test_valid_urls(self, mocker, make_item, url, valid_url):
         mock_get, mock_logger = self.get_mockers(mocker)
         if valid_url:
@@ -53,17 +56,20 @@ class TestTiktokTikwmExtractor(TestExtractorBase):
         mock_logger.error.assert_called_once()
         assert mock_logger.error.call_args[0][0].startswith("failed to parse JSON response")
 
-        mock_get.return_value.json.side_effect = Exception
-        with pytest.raises(Exception):
+        mock_get.return_value.json.side_effect = ValueError
+        with pytest.raises(ValueError):
             self.extractor.download(make_item(self.VALID_EXAMPLE_URL))
         mock_get.assert_called()
         assert mock_get.call_count == 2
         assert mock_get.return_value.json.call_count == 2
 
-    @pytest.mark.parametrize("response", [
-        ({"msg": "failure"}),
-        ({"msg": "success"}),
-    ])
+    @pytest.mark.parametrize(
+        "response",
+        [
+            ({"msg": "failure"}),
+            ({"msg": "success"}),
+        ],
+    )
     def test_unsuccessful_responses(self, mocker, make_item, response):
         mock_get, mock_logger = self.get_mockers(mocker)
         mock_get.return_value.status_code = 200
@@ -74,11 +80,14 @@ class TestTiktokTikwmExtractor(TestExtractorBase):
         mock_logger.error.assert_called_once()
         assert mock_logger.error.call_args[0][0].startswith("failed to get a valid response")
 
-    @pytest.mark.parametrize("response,has_vid", [
-        ({"data": {"id": 123}}, False),
-        ({"data": {"wmplay": "url"}}, True),
-        ({"data": {"play": "url"}}, True),
-    ])
+    @pytest.mark.parametrize(
+        "response,has_vid",
+        [
+            ({"data": {"id": 123}}, False),
+            ({"data": {"wmplay": "url"}}, True),
+            ({"data": {"play": "url"}}, True),
+        ],
+    )
     def test_correct_extraction(self, mocker, make_item, response, has_vid):
         mock_get, mock_logger = self.get_mockers(mocker)
         mock_get.return_value.status_code = 200
@@ -102,16 +111,19 @@ class TestTiktokTikwmExtractor(TestExtractorBase):
     def test_correct_data_extracted(self, mocker, make_item):
         mock_get, _ = self.get_mockers(mocker)
         mock_get.return_value.status_code = 200
-        mock_get.return_value.json.return_value = {"msg": "success", "data": {
-            "wmplay": "url",
-            "origin_cover": "cover.jpg",
-            "title": "Title",
-            "id": 123,
-            "duration": 60,
-            "create_time": 1736301699,
-            "author": "Author",
-            "other": "data"
-        }}
+        mock_get.return_value.json.return_value = {
+            "msg": "success",
+            "data": {
+                "wmplay": "url",
+                "origin_cover": "cover.jpg",
+                "title": "Title",
+                "id": 123,
+                "duration": 60,
+                "create_time": 1736301699,
+                "author": "Author",
+                "other": "data",
+            },
+        }
 
         result = self.extractor.download(make_item(self.VALID_EXAMPLE_URL))
         assert result.is_success()
@@ -129,9 +141,12 @@ class TestTiktokTikwmExtractor(TestExtractorBase):
         result = self.extractor.download(make_item(url))
         assert result.is_success()
         assert len(result.media) == 2
-        assert result.get_title() == "The A23a iceberg is one of the world's oldest and it's so big you can see it from space. #Iceberg  #A23a  #Antarctica  #Ice  #ClimateChange  #DavidAttenborough  #Ocean  #Sea  #SouthGeorgia  #BBCNews "
+        assert (
+            result.get_title()
+            == "The A23a iceberg is one of the world's oldest and it's so big you can see it from space. #Iceberg  #A23a  #Antarctica  #Ice  #ClimateChange  #DavidAttenborough  #Ocean  #Sea  #SouthGeorgia  #BBCNews "
+        )
         assert result.get("author").get("unique_id") == "bbcnews"
-        assert result.get("api_data").get("id") == '7478038212070411542'
+        assert result.get("api_data").get("id") == "7478038212070411542"
         assert result.media[1].get("duration") == 59
         assert result.get("timestamp") == datetime.fromtimestamp(1741122000, tz=timezone.utc)
 
@@ -149,6 +164,6 @@ class TestTiktokTikwmExtractor(TestExtractorBase):
         assert len(result.media) == 2
         assert result.get_title() == "Căng nhất lúc này #ggs68 #ggs68taiwan #taiwan #dailoan #tiktoknews"
         assert result.get("author").get("id") == "7197400619475649562"
-        assert result.get("api_data").get("id") == '7441821351142362375'
+        assert result.get("api_data").get("id") == "7441821351142362375"
         assert result.media[1].get("duration") == 34
         assert result.get("timestamp") == datetime.fromtimestamp(1732684060, tz=timezone.utc)
diff --git a/tests/storages/test_S3_storage.py b/tests/storages/test_S3_storage.py
index 9e27b3f..87da776 100644
--- a/tests/storages/test_S3_storage.py
+++ b/tests/storages/test_S3_storage.py
@@ -8,6 +8,7 @@ class TestS3Storage:
     """
     Test suite for S3Storage.
     """
+
     module_name: str = "s3_storage"
     storage: Type[S3Storage]
     config: dict = {
@@ -32,10 +33,10 @@ class TestS3Storage:
         """Test that S3 client is initialized with correct parameters"""
 
         assert self.storage.s3 is not None
-        assert self.storage.s3.meta.region_name == 'test-region'
+        assert self.storage.s3.meta.region_name == "test-region"
 
     def test_get_cdn_url_generation(self):
-        """Test CDN URL formatting """
+        """Test CDN URL formatting"""
         media = Media("test.txt")
         media._key = "path/to/file.txt"
         url = self.storage.get_cdn_url(media)
@@ -46,14 +47,14 @@ class TestS3Storage:
     def test_uploadf_sets_acl_public(self, mocker):
         media = Media("test.txt")
         mock_file = mocker.MagicMock()
-        mock_s3_upload = mocker.patch.object(self.storage.s3, 'upload_fileobj')
-        mocker.patch.object(self.storage, 'is_upload_needed', return_value=True)
+        mock_s3_upload = mocker.patch.object(self.storage.s3, "upload_fileobj")
+        mocker.patch.object(self.storage, "is_upload_needed", return_value=True)
         self.storage.uploadf(mock_file, media)
         mock_s3_upload.assert_called_once_with(
             mock_file,
-            Bucket='test-bucket',
+            Bucket="test-bucket",
             Key=media.key,
-            ExtraArgs={'ACL': 'public-read', 'ContentType': 'text/plain'}
+            ExtraArgs={"ACL": "public-read", "ContentType": "text/plain"},
         )
 
     def test_upload_decision_logic(self, mocker):
@@ -61,23 +62,29 @@ class TestS3Storage:
         media = Media("test.txt")
         assert self.storage.is_upload_needed(media) is True
         self.storage.random_no_duplicate = True
-        mocker.patch('auto_archiver.modules.s3_storage.s3_storage.calculate_file_hash', return_value='beepboop123beepboop123beepboop123')
-        mock_file_in_folder = mocker.patch.object(self.storage, 'file_in_folder', return_value='existing_key.txt')
+        mocker.patch(
+            "auto_archiver.modules.s3_storage.s3_storage.calculate_file_hash",
+            return_value="beepboop123beepboop123beepboop123",
+        )
+        mock_file_in_folder = mocker.patch.object(self.storage, "file_in_folder", return_value="existing_key.txt")
         assert self.storage.is_upload_needed(media) is False
-        assert media.key == 'existing_key.txt'
-        mock_file_in_folder.assert_called_with('no-dups/beepboop123beepboop123be')
+        assert media.key == "existing_key.txt"
+        mock_file_in_folder.assert_called_with("no-dups/beepboop123beepboop123be")
 
     def test_skips_upload_when_duplicate_exists(self, mocker):
         """Test that upload skips when file_in_folder finds existing object"""
         self.storage.random_no_duplicate = True
-        mocker.patch.object(S3Storage, 'file_in_folder', return_value="existing_folder/existing_file.txt")
+        mocker.patch.object(S3Storage, "file_in_folder", return_value="existing_folder/existing_file.txt")
         media = Media("test.txt")
         media._key = "original_path.txt"
-        mocker.patch('auto_archiver.modules.s3_storage.s3_storage.calculate_file_hash', return_value="beepboop123beepboop123beepboop123")
+        mocker.patch(
+            "auto_archiver.modules.s3_storage.s3_storage.calculate_file_hash",
+            return_value="beepboop123beepboop123beepboop123",
+        )
         assert self.storage.is_upload_needed(media) is False
         assert media.key == "existing_folder/existing_file.txt"
         assert media.get("previously archived") is True
-        mock_upload = mocker.patch.object(self.storage.s3, 'upload_fileobj')
+        mock_upload = mocker.patch.object(self.storage.s3, "upload_fileobj")
         result = self.storage.uploadf(None, media)
         mock_upload.assert_not_called()
         assert result is True
@@ -85,21 +92,18 @@ class TestS3Storage:
     def test_uploads_with_correct_parameters(self, mocker):
         media = Media("test.txt")
         media._key = "original_key.txt"
-        mocker.patch.object(S3Storage, 'is_upload_needed', return_value=True)
-        media.mimetype = 'image/png'
+        mocker.patch.object(S3Storage, "is_upload_needed", return_value=True)
+        media.mimetype = "image/png"
         mock_file = mocker.MagicMock()
-        mock_upload = mocker.patch.object(self.storage.s3, 'upload_fileobj')
+        mock_upload = mocker.patch.object(self.storage.s3, "upload_fileobj")
         self.storage.uploadf(mock_file, media)
         mock_upload.assert_called_once_with(
             mock_file,
-            Bucket='test-bucket',
-            Key='original_key.txt',
-            ExtraArgs={
-                'ACL': 'public-read',
-                'ContentType': 'image/png'
-            }
+            Bucket="test-bucket",
+            Key="original_key.txt",
+            ExtraArgs={"ACL": "public-read", "ContentType": "image/png"},
         )
 
     def test_file_in_folder_exists(self, mocker):
-        mocker.patch.object(self.storage.s3, 'list_objects', return_value={'Contents': [{'Key': 'path/to/file.txt'}]})
-        assert self.storage.file_in_folder('path/to/') == 'path/to/file.txt'
+        mocker.patch.object(self.storage.s3, "list_objects", return_value={"Contents": [{"Key": "path/to/file.txt"}]})
+        assert self.storage.file_in_folder("path/to/") == "path/to/file.txt"
diff --git a/tests/storages/test_local_storage.py b/tests/storages/test_local_storage.py
index c3581df..1230e3d 100644
--- a/tests/storages/test_local_storage.py
+++ b/tests/storages/test_local_storage.py
@@ -1,4 +1,3 @@
-
 import os
 from pathlib import Path
 
@@ -8,6 +7,7 @@ from auto_archiver.core import Media, Metadata
 from auto_archiver.modules.local_storage import LocalStorage
 from auto_archiver.core.consts import SetupError
 
+
 @pytest.fixture
 def local_storage(setup_module, tmp_path) -> LocalStorage:
     save_to = tmp_path / "local_archive"
@@ -20,6 +20,7 @@ def local_storage(setup_module, tmp_path) -> LocalStorage:
     }
     return setup_module("local_storage", configs)
 
+
 @pytest.fixture
 def sample_media(tmp_path) -> Media:
     """Fixture creating a Media object with temporary source file"""
@@ -27,9 +28,11 @@ def sample_media(tmp_path) -> Media:
     src_file.write_text("test content")
     return Media(filename=str(src_file))
 
+
 def test_too_long_save_path(setup_module):
     with pytest.raises(SetupError):
-        setup_module("local_storage", {"save_to": "long"*100})
+        setup_module("local_storage", {"save_to": "long" * 100})
+
 
 def test_get_cdn_url_relative(local_storage):
     local_storage.filename_generator = "random"
@@ -38,6 +41,7 @@ def test_get_cdn_url_relative(local_storage):
     expected = os.path.join(local_storage.save_to, media.key)
     assert local_storage.get_cdn_url(media) == expected
 
+
 def test_get_cdn_url_absolute(local_storage):
     local_storage.filename_generator = "random"
 
@@ -47,14 +51,14 @@ def test_get_cdn_url_absolute(local_storage):
     expected = os.path.abspath(os.path.join(local_storage.save_to, media.key))
     assert local_storage.get_cdn_url(media) == expected
 
+
 def test_upload_file_contents_and_metadata(local_storage, sample_media):
     local_storage.store(sample_media, "https://example.com", Metadata())
     dest = os.path.join(local_storage.save_to, sample_media.key)
     assert Path(sample_media.filename).read_text() == Path(dest).read_text()
 
+
 def test_upload_nonexistent_source(local_storage):
     media = Media(_key="missing.txt", filename="nonexistent.txt")
     with pytest.raises(FileNotFoundError):
         local_storage.upload(media)
-
-
diff --git a/tests/storages/test_storage_base.py b/tests/storages/test_storage_base.py
index 62f2ddc..730304e 100644
--- a/tests/storages/test_storage_base.py
+++ b/tests/storages/test_storage_base.py
@@ -6,32 +6,28 @@ from auto_archiver.core.metadata import Metadata, Media
 from auto_archiver.core.storage import Storage
 from auto_archiver.core.module import ModuleFactory
 
-class TestStorageBase(object):
 
+class TestStorageBase(object):
     module_name: str = None
     config: dict = None
 
     @pytest.fixture(autouse=True)
     def setup_storage(self, setup_module):
-        assert (
-            self.module_name is not None
-        ), "self.module_name must be set on the subclass"
+        assert self.module_name is not None, "self.module_name must be set on the subclass"
         assert self.config is not None, "self.config must be a dict set on the subclass"
-        self.storage: Type[Storage] = setup_module(
-            self.module_name, self.config
-        )
+        self.storage: Type[Storage] = setup_module(self.module_name, self.config)
 
 
 class TestBaseStorage(Storage):
-
     name = "test_storage"
 
     def get_cdn_url(self, media):
         return "cdn_url"
-    
+
     def uploadf(self, file, key, **kwargs):
         return True
 
+
 @pytest.fixture
 def dummy_file(tmp_path):
     # create dummy.txt file
@@ -39,16 +35,18 @@ def dummy_file(tmp_path):
     dummy_file.write_text("test content")
     return str(dummy_file)
 
+
 @pytest.fixture
 def storage_base():
     def _storage_base(config):
         storage_base = TestBaseStorage()
-        storage_base.config_setup({TestBaseStorage.name : config})
+        storage_base.config_setup({TestBaseStorage.name: config})
         storage_base.module_factory = ModuleFactory()
         return storage_base
-    
+
     return _storage_base
 
+
 @pytest.mark.parametrize(
     "path_generator, filename_generator, url, expected_key",
     [
@@ -58,11 +56,11 @@ def storage_base():
         ("url", "random", "https://example.com/file/", "folder/https-example-com-file/pretend-random.txt"),
         ("random", "static", "https://example.com/file/", "folder/pretend-random/6ae8a75555209fd6c44157c0.txt"),
         ("random", "random", "https://example.com/file/", "folder/pretend-random/pretend-random.txt"),
-
     ],
 )
-def test_storage_name_generation(storage_base, path_generator, filename_generator, url, 
-                                 expected_key, mocker, tmp_path, dummy_file):
+def test_storage_name_generation(
+    storage_base, path_generator, filename_generator, url, expected_key, mocker, tmp_path, dummy_file
+):
     mock_random = mocker.patch("auto_archiver.core.storage.random_str")
     mock_random.return_value = "pretend-random"
 
@@ -89,10 +87,10 @@ def test_really_long_name(storage_base, dummy_file):
     }
     storage: Storage = storage_base(config)
 
-    url = f"https://example.com/{'file'*100}"
+    url = f"https://example.com/{'file' * 100}"
     media = Media(filename=dummy_file)
     storage.set_key(media, url, Metadata())
-    assert media.key == f"https-example-com-{'file'*13}/6ae8a75555209fd6c44157c0.txt"
+    assert media.key == f"https-example-com-{'file' * 13}/6ae8a75555209fd6c44157c0.txt"
 
 
 def test_storage_loads_hash_enricher(storage_base, dummy_file):

From 8673bc5979cc61174c339e2880db090a5e3c058e Mon Sep 17 00:00:00 2001
From: erinhmclark <erinhannahmary.clark@gmail.com>
Date: Thu, 13 Mar 2025 13:55:31 +0000
Subject: [PATCH 32/55] Fix unused imports and include rule.

---
 pyproject.toml                                 | 7 +++----
 src/auto_archiver/core/orchestrator.py         | 2 +-
 tests/databases/test_api_db.py                 | 1 -
 tests/databases/test_gsheet_db.py              | 2 +-
 tests/extractors/test_twitter_api_extractor.py | 3 +--
 tests/storages/test_gdrive_storage.py          | 2 --
 tests/test_implementation.py                   | 1 -
 tests/test_modules.py                          | 1 -
 tests/test_orchestrator.py                     | 1 -
 9 files changed, 6 insertions(+), 14 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index 2defdb2..225540e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -98,6 +98,8 @@ markers = [
 line-length = 120
 # Remove this for a more detailed lint report
 output-format = "concise"
+# TODO: temp ignore rule for timestamping_enricher to allow for open PR
+exclude = ["src/auto_archiver/modules/timestamping_enricher/*"]
 
 
 [tool.ruff.lint]
@@ -105,16 +107,13 @@ output-format = "concise"
 # See documentation for more details: https://docs.astral.sh/ruff/rules/
 #extend-select = ["B"]
 
-# Ignore unused imports as some are currently required for lazy loading
-# This can be removed for a `ruff check` run which is manually reviewed
-#ignore = ["F401"]
-
 [tool.ruff.lint.per-file-ignores]
 # Ignore import violations in __init__.py files
 "__init__.py" = ["F401", "F403"]
 # Ignore 'useless expression' in manifest files.
 "__manifest__.py" = ["B018"]
 
+
 [tool.ruff.format]
 docstring-code-format = false
 
diff --git a/src/auto_archiver/core/orchestrator.py b/src/auto_archiver/core/orchestrator.py
index 8c7d112..672994a 100644
--- a/src/auto_archiver/core/orchestrator.py
+++ b/src/auto_archiver/core/orchestrator.py
@@ -486,7 +486,7 @@ Here's how that would look: \n\nsteps:\n  extractors:\n  - [your_extractor_name_
             self.setup(args)
             return self.feed()
         except Exception as e:
-            logger.error(e, exc_info=True)
+            logger.error(e)
             exit(1)
 
     def cleanup(self) -> None:
diff --git a/tests/databases/test_api_db.py b/tests/databases/test_api_db.py
index 4627425..2e87a87 100644
--- a/tests/databases/test_api_db.py
+++ b/tests/databases/test_api_db.py
@@ -1,6 +1,5 @@
 import pytest
 
-from auto_archiver.core import Metadata
 from auto_archiver.modules.api_db import AAApiDb
 
 
diff --git a/tests/databases/test_gsheet_db.py b/tests/databases/test_gsheet_db.py
index 6c9e585..0760c79 100644
--- a/tests/databases/test_gsheet_db.py
+++ b/tests/databases/test_gsheet_db.py
@@ -54,7 +54,7 @@ def mock_media(mocker):
 
 
 @pytest.fixture
-def gsheets_db(mock_gworksheet, setup_module, mocker):
+def gsheets_db(mock_gworksheet, setup_module, mocker) -> GsheetsFeederDB:
     mocker.patch("gspread.service_account")
     config: dict = {
         "sheet": "testsheet",
diff --git a/tests/extractors/test_twitter_api_extractor.py b/tests/extractors/test_twitter_api_extractor.py
index 0664f49..8b8e0d9 100644
--- a/tests/extractors/test_twitter_api_extractor.py
+++ b/tests/extractors/test_twitter_api_extractor.py
@@ -1,6 +1,5 @@
 import os
 import datetime
-import hashlib
 import pytest
 
 from pytwitter.models.media import MediaVariant
@@ -10,7 +9,7 @@ from auto_archiver.modules.twitter_api_extractor import TwitterApiExtractor
 
 @pytest.mark.incremental
 class TestTwitterApiExtractor(TestExtractorBase):
-    extractor_module = "twitter_api_extractor"
+    extractor_module: TwitterApiExtractor = "twitter_api_extractor"
 
     config = {
         "bearer_tokens": [],
diff --git a/tests/storages/test_gdrive_storage.py b/tests/storages/test_gdrive_storage.py
index 08c516f..501bd58 100644
--- a/tests/storages/test_gdrive_storage.py
+++ b/tests/storages/test_gdrive_storage.py
@@ -1,10 +1,8 @@
 from typing import Type
 import pytest
-from oauth2client import service_account
 
 from auto_archiver.core import Media
 from auto_archiver.modules.gdrive_storage import GDriveStorage
-from auto_archiver.core.metadata import Metadata
 from tests.storages.test_storage_base import TestStorageBase
 
 
diff --git a/tests/test_implementation.py b/tests/test_implementation.py
index 973d53d..e52a8d8 100644
--- a/tests/test_implementation.py
+++ b/tests/test_implementation.py
@@ -46,7 +46,6 @@ def autoarchiver(tmp_path, monkeypatch, request):
 def test_run_auto_archiver_no_args(caplog, autoarchiver):
     with pytest.raises(SystemExit):
         autoarchiver()
-
     assert "provide at least one URL via the command line, or set up an alternative feeder" in caplog.text
 
 
diff --git a/tests/test_modules.py b/tests/test_modules.py
index b6018da..f672ca6 100644
--- a/tests/test_modules.py
+++ b/tests/test_modules.py
@@ -1,4 +1,3 @@
-import sys
 import pytest
 from auto_archiver.core.module import ModuleFactory, LazyBaseModule
 from auto_archiver.core.base_module import BaseModule
diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py
index a79aa70..326b93d 100644
--- a/tests/test_orchestrator.py
+++ b/tests/test_orchestrator.py
@@ -1,5 +1,4 @@
 import pytest
-import sys
 from argparse import ArgumentParser, ArgumentTypeError
 from auto_archiver.core.orchestrator import ArchivingOrchestrator
 from auto_archiver.version import __version__

From 16012df30b54b72b50cb11a7a25054f708af3594 Mon Sep 17 00:00:00 2001
From: erinhmclark <erinhannahmary.clark@gmail.com>
Date: Thu, 13 Mar 2025 13:57:04 +0000
Subject: [PATCH 33/55] Revert exception check in test.

---
 tests/extractors/test_tiktok_tikwm_extractor.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/extractors/test_tiktok_tikwm_extractor.py b/tests/extractors/test_tiktok_tikwm_extractor.py
index 690d448..cc50240 100644
--- a/tests/extractors/test_tiktok_tikwm_extractor.py
+++ b/tests/extractors/test_tiktok_tikwm_extractor.py
@@ -56,8 +56,8 @@ class TestTiktokTikwmExtractor(TestExtractorBase):
         mock_logger.error.assert_called_once()
         assert mock_logger.error.call_args[0][0].startswith("failed to parse JSON response")
 
-        mock_get.return_value.json.side_effect = ValueError
-        with pytest.raises(ValueError):
+        mock_get.return_value.json.side_effect = Exception
+        with pytest.raises(Exception):
             self.extractor.download(make_item(self.VALID_EXAMPLE_URL))
         mock_get.assert_called()
         assert mock_get.call_count == 2

From e7489ac4c41fb28270e8738cc13bcc129234a5b7 Mon Sep 17 00:00:00 2001
From: Patrick Robertson <robertson.patrick@gmail.com>
Date: Thu, 13 Mar 2025 14:30:33 +0000
Subject: [PATCH 34/55] Tidy up opentimestamps

* Simplify
* Don't add fake (pending) attestations if the calendar urls all have issues
* Remove unnecessary configs
* Improve docs on upgrading + verifying
---
 .../opentimestamps_enricher/__manifest__.py   | 79 +++++++++++-----
 .../opentimestamps_enricher.py                | 92 +++++++++----------
 .../enrichers/test_opentimestamps_enricher.py | 43 ++-------
 3 files changed, 106 insertions(+), 108 deletions(-)

diff --git a/src/auto_archiver/modules/opentimestamps_enricher/__manifest__.py b/src/auto_archiver/modules/opentimestamps_enricher/__manifest__.py
index ff038e1..733ff1a 100644
--- a/src/auto_archiver/modules/opentimestamps_enricher/__manifest__.py
+++ b/src/auto_archiver/modules/opentimestamps_enricher/__manifest__.py
@@ -9,11 +9,6 @@
         ],
     },
     "configs": {
-        "use_calendars": {
-            "default": True,
-            "help": "Whether to connect to OpenTimestamps calendar servers to create timestamps. If false, creates local timestamp proofs only.",
-            "type": "bool"
-        },
         "calendar_urls": {
             "default": [
                 "https://alice.btc.calendar.opentimestamps.org",
@@ -30,34 +25,76 @@ https://opentimestamps.org/#calendars",
             "help": "Optional whitelist of calendar servers. Override this if you are using your own calendar servers. e.g. ['https://mycalendar.com']",
             "type": "list"
         },
-        "verify_timestamps": {
-            "default": True,
-            "help": "Whether to verify timestamps after creating them.",
-            "type": "bool"
-        }
     },
     "description": """
     Creates OpenTimestamps proofs for archived files, providing blockchain-backed evidence of file existence at a specific time.
 
     Uses OpenTimestamps – a service that timestamps data using the Bitcoin blockchain, providing a decentralized 
-    and secure way to prove that data existed at a certain point in time.
+    and secure way to prove that data existed at a certain point in time. A SHA256 hash of the file to be timestamped is used as the token
+    and sent to each of the 'timestamp calendars' for inclusion in the blockchain. The proof is then saved alongside the original file in a file with
+    the '.ots' extension.
 
     ### Features
-    - Creates cryptographic timestamp proofs that link files to the Bitcoin blockchain
-    - Verifies existing timestamp proofs to confirm the time a file existed
-    - Uses multiple calendar servers to ensure reliability and redundancy
+    - Creates cryptographic timestamp proofs that link files to the Bitcoin or Litecoin blockchain
+    - Verifies timestamp proofs have been submitted to the blockchain (note: does not confirm they have been *added*)
+    - Can use multiple calendar servers to ensure reliability and redundancy
     - Stores timestamp proofs alongside original files for future verification
 
-    ### Notes
-    - Can work offline to create timestamp proofs that can be upgraded later
-    - Verification checks if timestamps have been confirmed in the Bitcoin blockchain
-    - Should run after files have been archived and hashed
+    ### Timestamp status
+    An opentimestamp, when submitted to a timestmap server will have a 'pending' status (Pending Attestation) as it waits to be added
+    to the blockchain. Once it has been added to the blockchain, it will have a 'confirmed' status (Bitcoin Block Timestamp).
+    This process typically takes several hours, depending on the calendar server and the current state of the Bitcoin network. As such,
+    the status of all timestamps added will be 'pending' until they are subsequently confirmed (see 'Upgrading Timestamps' below).
 
-    ### Verifying Timestamps Later
-    If you wish to verify a timestamp (ots) file later, you can install the opentimestamps-client command line tool and use the `ots verify` command.
+    There are two possible statuses for a timestamp:
+    - `Pending`: The timestamp has been submitted to the calendar server but has not yet been confirmed in the Bitcoin blockchain.
+    - `Confirmed`: The timestamp has been confirmed in the Bitcoin or Litecoin blockchain.
+
+    ### Upgrading Timestamps
+    To upgrade a timestamp from 'pending' to 'confirmed', you can use the `ots upgrade` command from the opentimestamps-client package
+    (install it with `pip install opentimesptamps-client`).
+    Example: `ots upgrade my_file.ots`
+
+    Here is a useful script that could be used to upgrade all timestamps in a directory, which could be run on a cron job:
+```{code}  bash
+find . -name "*.ots" -type f | while read file; do
+    echo "Upgrading OTS $file"
+    ots upgrade $file
+done
+# The result might look like:
+# Upgrading OTS ./my_file.ots
+# Got 1 attestation(s) from https://alice.btc.calendar.opentimestamps.org
+# Success! Timestamp complete
+```
+
+```{note} Note: this will only upgrade the .ots files, and will not change the status text in any output .html files or any databases where the
+metadata is stored (e.g. Google Sheets, CSV database, API database etc.).
+```
+
+    ### Verifying Timestamps
+    The easiest way to verify a timestamp (ots) file is to install the opentimestamps-client command line tool and use the `ots verify` command.
     Example: `ots verify my_file.ots`
 
-    Note: if you're using local storage with a filename_generator set to 'static' (a hash) or random, the files will be renamed when they are saved to the
+    ```{code}  bash
+$ ots verify my_file.ots
+Calendar https://bob.btc.calendar.opentimestamps.org: Pending confirmation in Bitcoin blockchain
+Calendar https://finney.calendar.eternitywall.com: Pending confirmation in Bitcoin blockchain
+Calendar https://alice.btc.calendar.opentimestamps.org: Timestamped by transaction 12345; waiting for 6 confirmations
+```
+
+    Note: if you're using a storage with `filename_generator` set to `static` or `random`, the files will be renamed when they are saved to the
     final location meaning you will need to specify the original filename when verifying the timestamp with `ots verify -f original_filename my_file.ots`.
+
+    ### Choosing Calendar Servers
+
+    By default, the OpenTimestamps enricher uses a set of public calendar servers provided by the 'opentimestamps' project.
+    You can customize the list of calendar servers by providing URLs in the `calendar_urls` configuration option.
+
+    ### Calendar WhiteList
+
+    By default, the opentimestamps package only allows their own calendars to be used (see `DEFAULT_CALENDAR_WHITELIST` in `opentimestamps.calendar`),
+    if you want to use your own calendars, then you can override this setting in the `calendar_whitelist` configuration option.
+
+   
     """
 }
\ No newline at end of file
diff --git a/src/auto_archiver/modules/opentimestamps_enricher/opentimestamps_enricher.py b/src/auto_archiver/modules/opentimestamps_enricher/opentimestamps_enricher.py
index cf110a2..d6e8add 100644
--- a/src/auto_archiver/modules/opentimestamps_enricher/opentimestamps_enricher.py
+++ b/src/auto_archiver/modules/opentimestamps_enricher/opentimestamps_enricher.py
@@ -5,7 +5,7 @@ from loguru import logger
 import opentimestamps
 from opentimestamps.calendar import RemoteCalendar, DEFAULT_CALENDAR_WHITELIST
 from opentimestamps.core.timestamp import Timestamp, DetachedTimestampFile
-from opentimestamps.core.notary import PendingAttestation, BitcoinBlockHeaderAttestation
+from opentimestamps.core.notary import PendingAttestation, BitcoinBlockHeaderAttestation, LitecoinBlockHeaderAttestation
 from opentimestamps.core.op import OpSHA256
 from opentimestamps.core import serialize
 from auto_archiver.core import Enricher
@@ -53,44 +53,36 @@ class OpentimestampsEnricher(Enricher):
                 
                 # Submit to calendar servers
                 submitted_to_calendar = False
-                if self.use_calendars:
-                    logger.debug(f"Submitting timestamp to calendar servers for {file_path}")
-                    calendars = []
-                    whitelist = DEFAULT_CALENDAR_WHITELIST
-                    
-                    if self.calendar_whitelist:
-                        whitelist = set(self.calendar_whitelist)
-                    
-                    # Create calendar instances
-                    calendar_urls = []
-                    for url in self.calendar_urls:
-                        if url in whitelist:
-                            calendars.append(RemoteCalendar(url))
-                            calendar_urls.append(url)
-                    
-                    # Submit the hash to each calendar
-                    for calendar in calendars:
-                        try:
-                            calendar_timestamp = calendar.submit(file_hash)
-                            timestamp.merge(calendar_timestamp)
-                            logger.debug(f"Successfully submitted to calendar: {calendar.url}")
-                            submitted_to_calendar = True
-                        except Exception as e:
-                            logger.warning(f"Failed to submit to calendar {calendar.url}: {e}")
-                    
-                    # If all calendar submissions failed, add pending attestations
-                    if not submitted_to_calendar and not timestamp.attestations:
-                        logger.info("All calendar submissions failed, creating pending attestations")
-                        for url in calendar_urls:
-                            pending = PendingAttestation(url)
-                            timestamp.attestations.add(pending)
-                else:
-                    logger.info("Skipping calendar submission as per configuration")
-                    
-                    # Add dummy pending attestation for testing when calendars are disabled
-                    for url in self.calendar_urls:
-                        pending = PendingAttestation(url)
-                        timestamp.attestations.add(pending)
+
+                logger.debug(f"Submitting timestamp to calendar servers for {file_path}")
+                calendars = []
+                whitelist = DEFAULT_CALENDAR_WHITELIST
+                
+                if self.calendar_whitelist:
+                    whitelist = set(self.calendar_whitelist)
+                
+                # Create calendar instances
+                calendar_urls = []
+                for url in self.calendar_urls:
+                    if url in whitelist:
+                        calendars.append(RemoteCalendar(url))
+                        calendar_urls.append(url)
+                
+                # Submit the hash to each calendar
+                for calendar in calendars:
+                    try:
+                        calendar_timestamp = calendar.submit(file_hash)
+                        timestamp.merge(calendar_timestamp)
+                        logger.debug(f"Successfully submitted to calendar: {calendar.url}")
+                        submitted_to_calendar = True
+                    except Exception as e:
+                        logger.warning(f"Failed to submit to calendar {calendar.url}: {e}")
+                
+                # If all calendar submissions failed, add pending attestations
+                if not submitted_to_calendar and not timestamp.attestations:
+                    logger.error(f"Failed to submit to any calendar for {file_path}. **This file will not be timestamped.**")
+                    media.set("opentimestamps", False)
+                    continue
                 
                 # Save the timestamp proof to a file
                 timestamp_path = os.path.join(self.tmp_dir, f"{os.path.basename(file_path)}.ots")
@@ -110,13 +102,9 @@ class OpentimestampsEnricher(Enricher):
                 timestamp_media.mimetype = "application/vnd.opentimestamps"
                 timestamp_media.set("opentimestamps_version", opentimestamps.__version__)
                 
-                # Verify the timestamp if needed
-                if self.verify_timestamps:
-                    verification_info = self.verify_timestamp(detached_timestamp)
-                    for key, value in verification_info.items():
-                        timestamp_media.set(key, value)
-                else:
-                    logger.warning(f"Not verifying the timestamp for media file {file_path}")
+                verification_info = self.verify_timestamp(detached_timestamp)
+                for key, value in verification_info.items():
+                    timestamp_media.set(key, value)
                 
                 media.set("opentimestamp_files", [timestamp_media])
                 timestamp_files.append(timestamp_media.filename)
@@ -132,6 +120,7 @@ class OpentimestampsEnricher(Enricher):
             to_enrich.set("opentimestamps_count", len(timestamp_files))
             logger.success(f"{len(timestamp_files)} OpenTimestamps proofs created for {url=}")
         else:
+            to_enrich.set("opentimestamped", False)
             logger.warning(f"No successful timestamps created for {url=}")
     
     def verify_timestamp(self, detached_timestamp):
@@ -157,11 +146,14 @@ class OpentimestampsEnricher(Enricher):
                 
                 # Process different types of attestations
                 if isinstance(attestation, PendingAttestation):
-                    info["type"] = f"pending"
+                    info["status"] = "pending"
                     info["uri"] = attestation.uri
                 
                 elif isinstance(attestation, BitcoinBlockHeaderAttestation):
-                    info["type"] = "bitcoin"
+                    info["status"] = "confirmed - bitcoin"
+                    info["block_height"] = attestation.height
+                elif isinstance(attestation, LitecoinBlockHeaderAttestation):
+                    info["status"] = "confirmed - litecoin"
                     info["block_height"] = attestation.height
 
                 info["last_check"] = datetime.datetime.now().isoformat()[:-7]
@@ -171,14 +163,12 @@ class OpentimestampsEnricher(Enricher):
             result["attestations"] = attestation_info
             
             # For at least one confirmed attestation
-            if any(a.get("type") == "bitcoin" for a in attestation_info):
+            if any("confirmed" in a.get("status") for a in attestation_info):
                 result["verified"] = True
             else:
                 result["verified"] = False
-                result["pending"] = True
         else:
             result["verified"] = False
-            result["pending"] = False
         result["last_updated"] = datetime.datetime.now().isoformat()[:-7]
         
         return result
\ No newline at end of file
diff --git a/tests/enrichers/test_opentimestamps_enricher.py b/tests/enrichers/test_opentimestamps_enricher.py
index 391fb06..2cdefdf 100644
--- a/tests/enrichers/test_opentimestamps_enricher.py
+++ b/tests/enrichers/test_opentimestamps_enricher.py
@@ -109,12 +109,12 @@ def test_verify_timestamp(setup_module, detached_timestamp_file):
     assert len(verification_info["attestations"]) == 2
     
     # Check attestation types
-    assertion_types = [a["type"] for a in verification_info["attestations"]]
+    assertion_types = [a["status"] for a in verification_info["attestations"]]
     assert "pending" in assertion_types
-    assert "bitcoin" in assertion_types
+    assert "confirmed - bitcoin" in assertion_types
     
     # Check Bitcoin attestation details
-    bitcoin_attestation = next(a for a in verification_info["attestations"] if a["type"] == "bitcoin")
+    bitcoin_attestation = next(a for a in verification_info["attestations"] if a["status"] == "confirmed - bitcoin")
     assert bitcoin_attestation["block_height"] == 783000
 
 def test_verify_pending_only(setup_module, pending_timestamp_file):
@@ -125,10 +125,9 @@ def test_verify_pending_only(setup_module, pending_timestamp_file):
     
     assert verification_info["attestation_count"] == 2
     assert verification_info["verified"] == False
-    assert verification_info["pending"] == True
     
     # All attestations should be of type "pending"
-    assert all(a["type"] == "pending" for a in verification_info["attestations"])
+    assert all(a["status"] == "pending" for a in verification_info["attestations"])
     
     # Check URIs of pending attestations
     uris = [a["uri"] for a in verification_info["attestations"]]
@@ -148,7 +147,7 @@ def test_verify_bitcoin_completed(setup_module, verified_timestamp_file):
     
     # Check that the attestation is a Bitcoin attestation
     attestation = verification_info["attestations"][0]
-    assert attestation["type"] == "bitcoin"
+    assert attestation["status"] == "confirmed - bitcoin"
     assert attestation["block_height"] == 783000
 
 def test_full_enriching(setup_module, sample_file_path, sample_media, mocker):
@@ -199,28 +198,6 @@ def test_full_enriching(setup_module, sample_file_path, sample_media, mocker):
     assert timestamp_media.get("verified") == True
     assert timestamp_media.get("attestation_count") == 1
 
-def test_full_enriching_no_calendars(setup_module, sample_file_path, sample_media, mocker):
-    ots = setup_module("opentimestamps_enricher", {"use_calendars": False})
-    
-    # Create test metadata with sample file
-    metadata = Metadata().set_url("https://example.com")
-    sample_media.filename = sample_file_path
-    metadata.add_media(sample_media)
-    
-    # Run enrichment
-    ots.enrich(metadata)
-    
-    # Verify results
-    assert metadata.get("opentimestamped") == True
-    assert metadata.get("opentimestamps_count") == 1
-
-    timestamp_media = metadata.media[0].get("opentimestamp_files")[0]
-    
-    # Verify status should be false since we didn't use calendars
-    assert timestamp_media.get("verified") == False
-    # We expect 3 pending attestations (one for each calendar URL)
-    assert timestamp_media.get("attestation_count") == 3
-
 def test_full_enriching_calendar_error(setup_module, sample_file_path, sample_media, mocker):
     """Test enrichment when calendar servers return errors"""
     # Mock the calendar submission to raise an exception
@@ -239,14 +216,8 @@ def test_full_enriching_calendar_error(setup_module, sample_file_path, sample_me
     ots.enrich(metadata)
     
     # Verify results
-    assert metadata.get("opentimestamped") == True
-    assert metadata.get("opentimestamps_count") == 1
-    
-    # Verify status should be false since calendar submissions failed
-    timestamp_media = metadata.media[0].get("opentimestamp_files")[0]
-    assert timestamp_media.get("verified") == False
-    # We expect 3 pending attestations (one for each calendar URL that's enabled by default in __manifest__)
-    assert timestamp_media.get("attestation_count") == 3
+    assert metadata.get("opentimestamped") == False
+    assert metadata.get("opentimestamps_count") is None
 
 def test_no_files_to_stamp(setup_module):
     """Test enrichment with no files to timestamp"""

From 15222199d92c99d80a227318634e3346ef4f1552 Mon Sep 17 00:00:00 2001
From: Patrick Robertson <robertson.patrick@gmail.com>
Date: Thu, 13 Mar 2025 14:45:38 +0000
Subject: [PATCH 35/55] Add unit test for if one calendar fails

---
 .../enrichers/test_opentimestamps_enricher.py | 27 ++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)

diff --git a/tests/enrichers/test_opentimestamps_enricher.py b/tests/enrichers/test_opentimestamps_enricher.py
index 2cdefdf..e91f97f 100644
--- a/tests/enrichers/test_opentimestamps_enricher.py
+++ b/tests/enrichers/test_opentimestamps_enricher.py
@@ -165,7 +165,6 @@ def test_full_enriching(setup_module, sample_file_path, sample_media, mocker):
         return test_timestamp
     
     mock_calendar.side_effect = side_effect
-    
 
     ots = setup_module("opentimestamps_enricher")
     
@@ -198,6 +197,32 @@ def test_full_enriching(setup_module, sample_file_path, sample_media, mocker):
     assert timestamp_media.get("verified") == True
     assert timestamp_media.get("attestation_count") == 1
 
+def test_full_enriching_one_calendar_error(setup_module, sample_file_path, sample_media, mocker, pending_timestamp_file):
+    """Test enrichment when one calendar server returns an error"""
+    # Mock the calendar submission to raise an exception
+    mock_calendar = mocker.patch.object(RemoteCalendar, 'submit')
+    
+    test_timestamp = Timestamp(bytes.fromhex("583988e03646c26fa290c5c2408540a2f4e2aa9be087aa4546aefb531385b935"))
+        # Add a bitcoin attestation to the test timestamp
+    bitcoin = BitcoinBlockHeaderAttestation(783000)
+    test_timestamp.attestations.add(bitcoin)
+
+    mock_calendar.side_effect = [test_timestamp, Exception("Calendar server error")]
+
+    ots = setup_module("opentimestamps_enricher", {"calendar_urls": ["https://alice.btc.calendar.opentimestamps.org", "https://bob.btc.calendar.opentimestamps.org"]})
+    
+    # Create test metadata with sample file
+    metadata = Metadata().set_url("https://example.com")
+    sample_media.filename = sample_file_path
+    metadata.add_media(sample_media)
+    
+    # Run enrichment (should complete despite calendar errors)
+    ots.enrich(metadata)
+    
+    # Verify results
+    assert metadata.get("opentimestamped") == True
+    assert metadata.get("opentimestamps_count") == 1 # only alice worked, not bob
+
 def test_full_enriching_calendar_error(setup_module, sample_file_path, sample_media, mocker):
     """Test enrichment when calendar servers return errors"""
     # Mock the calendar submission to raise an exception

From 0bef78b0b4a296f5226072b6577e1d5c2f85db7a Mon Sep 17 00:00:00 2001
From: erinhmclark <erinhannahmary.clark@gmail.com>
Date: Thu, 13 Mar 2025 15:23:35 +0000
Subject: [PATCH 36/55] Remove autouse property of mock_sleep.

---
 tests/conftest.py                                 |  4 ++--
 tests/enrichers/test_wayback_enricher.py          |  6 ++++++
 tests/enrichers/test_whisper_enricher.py          |  6 ++++++
 tests/extractors/test_instagram_tbot_extractor.py |  6 ++++++
 tests/extractors/test_tiktok_tikwm_extractor.py   | 10 +++-------
 tests/storages/test_gdrive_storage.py             |  6 ++++++
 6 files changed, 29 insertions(+), 9 deletions(-)

diff --git a/tests/conftest.py b/tests/conftest.py
index 9754b91..379bfc2 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -145,9 +145,9 @@ def sample_datetime():
     return datetime(2023, 1, 1, 12, 0, tzinfo=timezone.utc)
 
 
-@pytest.fixture(autouse=True)
+@pytest.fixture
 def mock_sleep(mocker):
-    """Globally mock time.sleep to avoid delays."""
+    """Mock time.sleep to avoid delays."""
     return mocker.patch("time.sleep")
 
 
diff --git a/tests/enrichers/test_wayback_enricher.py b/tests/enrichers/test_wayback_enricher.py
index 796c805..113458b 100644
--- a/tests/enrichers/test_wayback_enricher.py
+++ b/tests/enrichers/test_wayback_enricher.py
@@ -5,6 +5,12 @@ from auto_archiver.modules.wayback_extractor_enricher import WaybackExtractorEnr
 from auto_archiver.core import Metadata
 
 
+@pytest.fixture(autouse=True)
+def mock_sleep(mocker):
+    """Mock time.sleep to avoid delays."""
+    return mocker.patch("time.sleep")
+
+
 @pytest.fixture
 def mock_is_auth_wall(mocker):
     """Fixture to mock is_auth_wall behavior."""
diff --git a/tests/enrichers/test_whisper_enricher.py b/tests/enrichers/test_whisper_enricher.py
index 0278e91..f669631 100644
--- a/tests/enrichers/test_whisper_enricher.py
+++ b/tests/enrichers/test_whisper_enricher.py
@@ -7,6 +7,12 @@ from auto_archiver.modules.whisper_enricher import WhisperEnricher
 TEST_S3_URL = "http://cdn.example.com/test.mp4"
 
 
+@pytest.fixture(autouse=True)
+def mock_sleep(mocker):
+    """Mock time.sleep to avoid delays."""
+    return mocker.patch("time.sleep")
+
+
 @pytest.fixture
 def enricher(mocker):
     """Fixture with mocked S3 and API dependencies"""
diff --git a/tests/extractors/test_instagram_tbot_extractor.py b/tests/extractors/test_instagram_tbot_extractor.py
index 1b31d07..47a4bec 100644
--- a/tests/extractors/test_instagram_tbot_extractor.py
+++ b/tests/extractors/test_instagram_tbot_extractor.py
@@ -7,6 +7,12 @@ from auto_archiver.modules.instagram_tbot_extractor import InstagramTbotExtracto
 from tests.extractors.test_extractor_base import TestExtractorBase
 
 
+@pytest.fixture(autouse=True)
+def mock_sleep(mocker):
+    """Mock time.sleep to avoid delays."""
+    return mocker.patch("time.sleep")
+
+
 @pytest.fixture
 def patch_extractor_methods(request, setup_module, mocker):
     mocker.patch.object(InstagramTbotExtractor, "_prepare_session_file", return_value=None)
diff --git a/tests/extractors/test_tiktok_tikwm_extractor.py b/tests/extractors/test_tiktok_tikwm_extractor.py
index cc50240..a21a17a 100644
--- a/tests/extractors/test_tiktok_tikwm_extractor.py
+++ b/tests/extractors/test_tiktok_tikwm_extractor.py
@@ -151,14 +151,10 @@ class TestTiktokTikwmExtractor(TestExtractorBase):
         assert result.get("timestamp") == datetime.fromtimestamp(1741122000, tz=timezone.utc)
 
     @pytest.mark.download
-    def test_download_sensitive_video(self, make_item, mock_sleep):
-        # sleep is needed because of the rate limit
-        mock_sleep.stop()
-        time.sleep(1.1)
-        mock_sleep.start()
-
+    def test_download_sensitive_video(self, make_item):
         url = "https://www.tiktok.com/@ggs68taiwan.official/video/7441821351142362375"
-
+        # Required for rate limiting
+        time.sleep(1.1)
         result = self.extractor.download(make_item(url))
         assert result.is_success()
         assert len(result.media) == 2
diff --git a/tests/storages/test_gdrive_storage.py b/tests/storages/test_gdrive_storage.py
index 501bd58..99df536 100644
--- a/tests/storages/test_gdrive_storage.py
+++ b/tests/storages/test_gdrive_storage.py
@@ -6,6 +6,12 @@ from auto_archiver.modules.gdrive_storage import GDriveStorage
 from tests.storages.test_storage_base import TestStorageBase
 
 
+@pytest.fixture(autouse=True)
+def mock_sleep(mocker):
+    """Mock time.sleep to avoid delays."""
+    return mocker.patch("time.sleep")
+
+
 @pytest.fixture
 def gdrive_storage(setup_module, mocker) -> GDriveStorage:
     module_name: str = "gdrive_storage"

From 10ceb7aa152831a622ffa01f45f51def59ca1d93 Mon Sep 17 00:00:00 2001
From: Patrick Robertson <robertson.patrick@gmail.com>
Date: Thu, 13 Mar 2025 15:59:42 +0000
Subject: [PATCH 37/55] Move tikwm extractor into a droping for the generic
 extractor

---
 .../generic_extractor/generic_extractor.py    | 15 +++-
 .../modules/generic_extractor/tiktok.py       | 74 +++++++++++++++
 .../tiktok_tikwm_extractor/__init__.py        |  1 -
 .../tiktok_tikwm_extractor/__manifest__.py    | 23 -----
 .../tiktok_tikwm_extractor.py                 | 75 ----------------
 .../extractors/test_tiktok_tikwm_extractor.py | 89 ++++++++-----------
 6 files changed, 123 insertions(+), 154 deletions(-)
 create mode 100644 src/auto_archiver/modules/generic_extractor/tiktok.py
 delete mode 100644 src/auto_archiver/modules/tiktok_tikwm_extractor/__init__.py
 delete mode 100644 src/auto_archiver/modules/tiktok_tikwm_extractor/__manifest__.py
 delete mode 100644 src/auto_archiver/modules/tiktok_tikwm_extractor/tiktok_tikwm_extractor.py

diff --git a/src/auto_archiver/modules/generic_extractor/generic_extractor.py b/src/auto_archiver/modules/generic_extractor/generic_extractor.py
index 5acce46..5d8cfc4 100644
--- a/src/auto_archiver/modules/generic_extractor/generic_extractor.py
+++ b/src/auto_archiver/modules/generic_extractor/generic_extractor.py
@@ -12,6 +12,8 @@ from loguru import logger
 from auto_archiver.core.extractor import Extractor
 from auto_archiver.core import Metadata, Media
 
+class SkipYtdlp(Exception):
+    pass
 class GenericExtractor(Extractor):
     _dropins = {}
 
@@ -268,7 +270,8 @@ class GenericExtractor(Extractor):
 
         try:
             if dropin_submodule and dropin_submodule.skip_ytdlp_download(info_extractor, url):
-                raise Exception(f"Skipping using ytdlp to download files for {info_extractor.ie_key()}")
+                logger.debug(f"Skipping using ytdlp to download files for {info_extractor.ie_key()}")
+                raise SkipYtdlp()
 
             # don't download since it can be a live stream
             data = ydl.extract_info(url, ie_key=info_extractor.ie_key(), download=False)
@@ -282,15 +285,19 @@ class GenericExtractor(Extractor):
             if info_extractor.ie_key() == "generic":
                 # don't clutter the logs with issues about the 'generic' extractor not having a dropin
                 return False
+            
+            if not isinstance(e, SkipYtdlp):
+                logger.debug(f'Issue using "{info_extractor.IE_NAME}" extractor to download video (error: {repr(e)}), attempting to use extractor to get post data instead')
 
-            logger.debug(f'Issue using "{info_extractor.IE_NAME}" extractor to download video (error: {repr(e)}), attempting to use extractor to get post data instead')
             try:
                 result = self.get_metadata_for_post(info_extractor, url, ydl)
             except (yt_dlp.utils.DownloadError, yt_dlp.utils.ExtractorError) as post_e:
-                logger.error(f'Error downloading metadata for post: {post_e}')
+                logger.error("Error downloading metadata for post: {error}", error=str(post_e))
                 return False
             except Exception as generic_e:
-                logger.debug(f'Attempt to extract using ytdlp extractor "{info_extractor.IE_NAME}" failed:  \n  {repr(generic_e)}', exc_info=True)
+                logger.debug('Attempt to extract using ytdlp extractor "{name}" failed:  \n  {error}',
+                             name=info_extractor.IE_NAME, error=str(generic_e),
+                            exc_info=True)
                 return False
         
         if result:
diff --git a/src/auto_archiver/modules/generic_extractor/tiktok.py b/src/auto_archiver/modules/generic_extractor/tiktok.py
new file mode 100644
index 0000000..8914f0c
--- /dev/null
+++ b/src/auto_archiver/modules/generic_extractor/tiktok.py
@@ -0,0 +1,74 @@
+import requests
+from loguru import logger
+from auto_archiver.core import Metadata, Media
+from datetime import datetime, timezone
+from .dropin import GenericDropin
+
+class Tiktok(GenericDropin):
+    """
+    TikTok droping for the Generic Extractor that uses an unofficial API if/when ytdlp fails.
+    It's useful for capturing content that requires a login, like sensitive content.
+    """
+
+    TIKWM_ENDPOINT = "https://www.tikwm.com/api/?url={url}"
+
+    def extract_post(self, url: str, ie_instance):
+
+        logger.debug("Using Tikwm API to attempt to download tiktok video from {url=}")
+
+        endpoint = self.TIKWM_ENDPOINT.format(url=url)
+
+        r = requests.get(endpoint)
+        if r.status_code != 200:
+            raise ValueError(f"unexpected status code '{r.status_code}' from tikwm.com for {url=}:")
+
+        try:
+            json_response = r.json()
+        except ValueError:
+            raise ValueError(f"failed to parse JSON response from tikwm.com for {url=}")
+
+        if not json_response.get('msg') == 'success' or not (api_data := json_response.get('data', {})):
+            raise ValueError(f"failed to get a valid response from tikwm.com for {url=}: {repr(json_response)}")
+
+        # tries to get the non-watermarked version first
+        video_url = api_data.pop("play", api_data.pop("wmplay", None))
+        if not video_url:
+            raise ValueError(f"no valid video URL found in response from tikwm.com for {url=}")
+        
+        api_data['video_url'] = video_url
+        return api_data
+
+    
+    def create_metadata(self, post: dict, ie_instance, archiver, url):
+
+                # prepare result, start by downloading video
+        result = Metadata()
+        video_url = post.pop("video_url")
+
+        # get the cover if possible
+        cover_url = post.pop("origin_cover", post.pop("cover", post.pop("ai_dynamic_cover", None)))
+        if cover_url and (cover_downloaded := archiver.download_from_url(cover_url)):
+            result.add_media(Media(cover_downloaded))
+
+        # get the video or fail
+        video_downloaded = archiver.download_from_url(video_url, f"vid_{post.get('id', '')}")
+        if not video_downloaded:
+            logger.error(f"failed to download video from {video_url}")
+            return False
+        video_media = Media(video_downloaded)
+        if duration := post.pop("duration", None):
+            video_media.set("duration", duration)
+        result.add_media(video_media)
+
+        # add remaining metadata
+        result.set_title(post.pop("title", ""))
+
+        if created_at := post.pop("create_time", None):
+            result.set_timestamp(datetime.fromtimestamp(created_at, tz=timezone.utc))
+
+        if (author := post.pop("author", None)):
+            result.set("author", author)
+
+        result.set("api_data", post)
+
+        return result
\ No newline at end of file
diff --git a/src/auto_archiver/modules/tiktok_tikwm_extractor/__init__.py b/src/auto_archiver/modules/tiktok_tikwm_extractor/__init__.py
deleted file mode 100644
index 25a20f5..0000000
--- a/src/auto_archiver/modules/tiktok_tikwm_extractor/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-from .tiktok_tikwm_extractor import TiktokTikwmExtractor
\ No newline at end of file
diff --git a/src/auto_archiver/modules/tiktok_tikwm_extractor/__manifest__.py b/src/auto_archiver/modules/tiktok_tikwm_extractor/__manifest__.py
deleted file mode 100644
index 56d8e3e..0000000
--- a/src/auto_archiver/modules/tiktok_tikwm_extractor/__manifest__.py
+++ /dev/null
@@ -1,23 +0,0 @@
-{
-    "name": "Tiktok Tikwm Extractor",
-    "type": ["extractor"],
-    "requires_setup": False,
-    "dependencies": {
-        "python": ["loguru", "requests"],
-        "bin": []
-    },
-    "description": """
-    Uses an unofficial TikTok video download platform's API to download videos: https://tikwm.com/
-	
-	This extractor complements the generic_extractor which can already get TikTok videos, but this one can extract special videos like those marked as sensitive.
-
-    ### Features
-    - Downloads the video and, if possible, also the video cover.
-	- Stores extra metadata about the post like author information, and more as returned by tikwm.com. 
-
-    ### Notes
-    - If tikwm.com is down, this extractor will not work.
-	- If tikwm.com changes their API, this extractor may break.
-	- If no video is found, this extractor will consider the extraction failed.
-    """
-}
diff --git a/src/auto_archiver/modules/tiktok_tikwm_extractor/tiktok_tikwm_extractor.py b/src/auto_archiver/modules/tiktok_tikwm_extractor/tiktok_tikwm_extractor.py
deleted file mode 100644
index 8b07775..0000000
--- a/src/auto_archiver/modules/tiktok_tikwm_extractor/tiktok_tikwm_extractor.py
+++ /dev/null
@@ -1,75 +0,0 @@
-import re
-import requests
-from loguru import logger
-from datetime import datetime, timezone
-from yt_dlp.extractor.tiktok import TikTokIE
-
-from auto_archiver.core import Extractor
-from auto_archiver.core import Metadata, Media
-
-
-class TiktokTikwmExtractor(Extractor):
-    """
-    Extractor for TikTok that uses an unofficial API and can capture content that requires a login, like sensitive content.
-    """
-    TIKWM_ENDPOINT = "https://www.tikwm.com/api/?url={url}"
-
-    def download(self, item: Metadata) -> Metadata:
-        url = item.get_url()
-        
-        if not re.match(TikTokIE._VALID_URL, url):
-            return False
-
-        endpoint = TiktokTikwmExtractor.TIKWM_ENDPOINT.format(url=url)
-
-        r = requests.get(endpoint)
-        if r.status_code != 200:
-            logger.error(f"unexpected status code '{r.status_code}' from tikwm.com for {url=}:")
-            return False
-
-        try:
-            json_response = r.json()
-        except ValueError:
-            logger.error(f"failed to parse JSON response from tikwm.com for {url=}")
-            return False
-
-        if not json_response.get('msg') == 'success' or not (api_data := json_response.get('data', {})):
-            logger.error(f"failed to get a valid response from tikwm.com for {url=}: {json_response}")
-            return False
-
-        # tries to get the non-watermarked version first
-        video_url = api_data.pop("play", api_data.pop("wmplay", None))
-        if not video_url:
-            logger.error(f"no valid video URL found in response from tikwm.com for {url=}")
-            return False
-
-        # prepare result, start by downloading video
-        result = Metadata()
-
-        # get the cover if possible
-        cover_url = api_data.pop("origin_cover", api_data.pop("cover", api_data.pop("ai_dynamic_cover", None)))
-        if cover_url and (cover_downloaded := self.download_from_url(cover_url)):
-            result.add_media(Media(cover_downloaded))
-
-        # get the video or fail
-        video_downloaded = self.download_from_url(video_url, f"vid_{api_data.get('id', '')}")
-        if not video_downloaded:
-            logger.error(f"failed to download video from {video_url}")
-            return False
-        video_media = Media(video_downloaded)
-        if duration := api_data.pop("duration", None):
-            video_media.set("duration", duration)
-        result.add_media(video_media)
-
-        # add remaining metadata
-        result.set_title(api_data.pop("title", ""))
-
-        if created_at := api_data.pop("create_time", None):
-            result.set_timestamp(datetime.fromtimestamp(created_at, tz=timezone.utc))
-
-        if (author := api_data.pop("author", None)):
-            result.set("author", author)
-
-        result.set("api_data", api_data)
-
-        return result.success("tikwm")
diff --git a/tests/extractors/test_tiktok_tikwm_extractor.py b/tests/extractors/test_tiktok_tikwm_extractor.py
index e8ad8df..51bb57a 100644
--- a/tests/extractors/test_tiktok_tikwm_extractor.py
+++ b/tests/extractors/test_tiktok_tikwm_extractor.py
@@ -1,86 +1,74 @@
 from datetime import datetime, timezone
 import time
 import pytest
+import yt_dlp
 
-from auto_archiver.modules.tiktok_tikwm_extractor.tiktok_tikwm_extractor import TiktokTikwmExtractor
+from auto_archiver.modules.generic_extractor.generic_extractor import GenericExtractor
 from .test_extractor_base import TestExtractorBase
 
 
+@pytest.fixture(autouse=True)
+def skip_ytdlp_own_methods(mocker):
+    # mock this method, so that we skip the ytdlp download in these tests
+    mocker.patch("auto_archiver.modules.generic_extractor.tiktok.Tiktok.skip_ytdlp_download", return_value=True)
+    mocker.patch("auto_archiver.modules.generic_extractor.generic_extractor.GenericExtractor.suitable_extractors",
+                 return_value=[e for e in yt_dlp.YoutubeDL()._ies.values() if e.IE_NAME == 'TikTok'])
+
+@pytest.fixture()
+def mock_get(mocker):
+    return mocker.patch("auto_archiver.modules.generic_extractor.tiktok.requests.get")
+
 class TestTiktokTikwmExtractor(TestExtractorBase):
     """
     Test suite for TestTiktokTikwmExtractor.
     """
 
-    extractor_module = "tiktok_tikwm_extractor"
-    extractor: TiktokTikwmExtractor
+    extractor_module = "generic_extractor"
+    extractor: GenericExtractor
 
     config = {}
 
     VALID_EXAMPLE_URL = "https://www.tiktok.com/@example/video/1234"
 
-    @staticmethod
-    def get_mockers(mocker):
-        mock_get = mocker.patch("auto_archiver.modules.tiktok_tikwm_extractor.tiktok_tikwm_extractor.requests.get")
-        mock_logger = mocker.patch("auto_archiver.modules.tiktok_tikwm_extractor.tiktok_tikwm_extractor.logger")
-        return mock_get, mock_logger
-
-    @pytest.mark.parametrize("url,valid_url", [
-        ("https://bellingcat.com", False),
-        ("https://youtube.com", False),
-        ("https://tiktok.co/", False),
-        ("https://tiktok.com/", False),
-        ("https://www.tiktok.com/", False),
-        ("https://api.cool.tiktok.com/", False),
-        (VALID_EXAMPLE_URL, True),
-        ("https://www.tiktok.com/@bbcnews/video/7478038212070411542", True),
-        ("https://www.tiktok.com/@ggs68taiwan.official/video/7441821351142362375", True),
-    ])
-    def test_valid_urls(self, mocker, make_item, url, valid_url):
-        mock_get, mock_logger = self.get_mockers(mocker)
-        if valid_url:
-            mock_get.return_value.status_code = 404
-        assert self.extractor.download(make_item(url)) == False
-        assert mock_get.call_count == int(valid_url)
-        assert mock_logger.error.call_count == int(valid_url)
-
-    def test_invalid_json_responses(self, mocker, make_item):
-        mock_get, mock_logger = self.get_mockers(mocker)
+    def test_invalid_json_responses(self, mock_get, make_item, caplog):
         mock_get.return_value.status_code = 200
         mock_get.return_value.json.side_effect = ValueError
-        assert self.extractor.download(make_item(self.VALID_EXAMPLE_URL)) == False
-        mock_get.assert_called_once()
-        mock_get.return_value.json.assert_called_once()
-        mock_logger.error.assert_called_once()
-        assert mock_logger.error.call_args[0][0].startswith("failed to parse JSON response")
+        with caplog.at_level('DEBUG'):
+            assert self.extractor.download(make_item(self.VALID_EXAMPLE_URL)) == False
+            mock_get.assert_called_once()
+            mock_get.return_value.json.assert_called_once()
+            assert len(caplog.records) == 2
+            # first message is just the 'Skipping using ytdlp to download files for TikTok' message
+            assert "failed to parse JSON response from tikwm.com for url='https://www.tiktok.com/@example/video/1234'" in caplog.records[1].message
 
         mock_get.return_value.json.side_effect = Exception
-        with pytest.raises(Exception):
-            self.extractor.download(make_item(self.VALID_EXAMPLE_URL))
-        mock_get.assert_called()
-        assert mock_get.call_count == 2
-        assert mock_get.return_value.json.call_count == 2
+        with caplog.at_level('ERROR'):
+            assert self.extractor.download(make_item(self.VALID_EXAMPLE_URL)) == False
+            mock_get.assert_called()
+            assert mock_get.call_count == 2
+            assert mock_get.return_value.json.call_count == 2
+            assert len(caplog.records) == 2
+            assert "failed to parse JSON response from tikwm.com for url='https://www.tiktok.com/@example/video/1234'" in caplog.records[1].message
 
     @pytest.mark.parametrize("response", [
         ({"msg": "failure"}),
         ({"msg": "success"}),
     ])
-    def test_unsuccessful_responses(self, mocker, make_item, response):
-        mock_get, mock_logger = self.get_mockers(mocker)
+    def test_unsuccessful_responses(self, mock_get, make_item, response, caplog):
         mock_get.return_value.status_code = 200
         mock_get.return_value.json.return_value = response
-        assert self.extractor.download(make_item(self.VALID_EXAMPLE_URL)) == False
-        mock_get.assert_called_once()
-        mock_get.return_value.json.assert_called_once()
-        mock_logger.error.assert_called_once()
-        assert mock_logger.error.call_args[0][0].startswith("failed to get a valid response")
+        with caplog.at_level('DEBUG'):
+            assert self.extractor.download(make_item(self.VALID_EXAMPLE_URL)) == False
+            mock_get.assert_called_once()
+            mock_get.return_value.json.assert_called_once()
+            assert "failed to get a valid response from tikwm.com" in caplog.records[1].message
 
     @pytest.mark.parametrize("response,has_vid", [
         ({"data": {"id": 123}}, False),
         ({"data": {"wmplay": "url"}}, True),
         ({"data": {"play": "url"}}, True),
     ])
-    def test_correct_extraction(self, mocker, make_item, response, has_vid):
-        mock_get, mock_logger = self.get_mockers(mocker)
+    def test_correct_extraction(self, mock_get, make_item, response, has_vid):
         mock_get.return_value.status_code = 200
         mock_get.return_value.json.return_value = {"msg": "success", **response}
 
@@ -99,8 +87,7 @@ class TestTiktokTikwmExtractor(TestExtractorBase):
         else:
             mock_logger.error.assert_not_called()
 
-    def test_correct_extraction(self, mocker, make_item):
-        mock_get, _ = self.get_mockers(mocker)
+    def test_correct_extraction(self, mock_get, make_item):
         mock_get.return_value.status_code = 200
         mock_get.return_value.json.return_value = {"msg": "success", "data": {
             "wmplay": "url",

From 2e25e59fa6dac4c742caa02eebfac650044d6fc1 Mon Sep 17 00:00:00 2001
From: Patrick Robertson <robertson.patrick@gmail.com>
Date: Thu, 13 Mar 2025 16:07:49 +0000
Subject: [PATCH 38/55] Fix unit tests - make caplog checks more robust, having
 added a new logger/debug call

---
 tests/extractors/test_tiktok_tikwm_extractor.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/tests/extractors/test_tiktok_tikwm_extractor.py b/tests/extractors/test_tiktok_tikwm_extractor.py
index 51bb57a..3d0c926 100644
--- a/tests/extractors/test_tiktok_tikwm_extractor.py
+++ b/tests/extractors/test_tiktok_tikwm_extractor.py
@@ -37,9 +37,8 @@ class TestTiktokTikwmExtractor(TestExtractorBase):
             assert self.extractor.download(make_item(self.VALID_EXAMPLE_URL)) == False
             mock_get.assert_called_once()
             mock_get.return_value.json.assert_called_once()
-            assert len(caplog.records) == 2
             # first message is just the 'Skipping using ytdlp to download files for TikTok' message
-            assert "failed to parse JSON response from tikwm.com for url='https://www.tiktok.com/@example/video/1234'" in caplog.records[1].message
+            assert "failed to parse JSON response from tikwm.com for url='https://www.tiktok.com/@example/video/1234'" in caplog.text
 
         mock_get.return_value.json.side_effect = Exception
         with caplog.at_level('ERROR'):
@@ -47,8 +46,7 @@ class TestTiktokTikwmExtractor(TestExtractorBase):
             mock_get.assert_called()
             assert mock_get.call_count == 2
             assert mock_get.return_value.json.call_count == 2
-            assert len(caplog.records) == 2
-            assert "failed to parse JSON response from tikwm.com for url='https://www.tiktok.com/@example/video/1234'" in caplog.records[1].message
+            assert "failed to parse JSON response from tikwm.com for url='https://www.tiktok.com/@example/video/1234'" in caplog.text
 
     @pytest.mark.parametrize("response", [
         ({"msg": "failure"}),
@@ -61,7 +59,7 @@ class TestTiktokTikwmExtractor(TestExtractorBase):
             assert self.extractor.download(make_item(self.VALID_EXAMPLE_URL)) == False
             mock_get.assert_called_once()
             mock_get.return_value.json.assert_called_once()
-            assert "failed to get a valid response from tikwm.com" in caplog.records[1].message
+            assert "failed to get a valid response from tikwm.com" in caplog.text
 
     @pytest.mark.parametrize("response,has_vid", [
         ({"data": {"id": 123}}, False),

From b908655cc8d91129940fd85bdb596633edb8c8dd Mon Sep 17 00:00:00 2001
From: Patrick Robertson <robertson.patrick@gmail.com>
Date: Thu, 13 Mar 2025 17:40:00 +0000
Subject: [PATCH 39/55] Remove references to litecoin + several tidy-ups

---
 .../opentimestamps_enricher/__manifest__.py    |  4 ++--
 .../opentimestamps_enricher.py                 | 18 +++++++-----------
 .../enrichers/test_opentimestamps_enricher.py  |  6 +++---
 3 files changed, 12 insertions(+), 16 deletions(-)

diff --git a/src/auto_archiver/modules/opentimestamps_enricher/__manifest__.py b/src/auto_archiver/modules/opentimestamps_enricher/__manifest__.py
index 733ff1a..b489d66 100644
--- a/src/auto_archiver/modules/opentimestamps_enricher/__manifest__.py
+++ b/src/auto_archiver/modules/opentimestamps_enricher/__manifest__.py
@@ -35,7 +35,7 @@ https://opentimestamps.org/#calendars",
     the '.ots' extension.
 
     ### Features
-    - Creates cryptographic timestamp proofs that link files to the Bitcoin or Litecoin blockchain
+    - Creates cryptographic timestamp proofs that link files to the Bitcoin
     - Verifies timestamp proofs have been submitted to the blockchain (note: does not confirm they have been *added*)
     - Can use multiple calendar servers to ensure reliability and redundancy
     - Stores timestamp proofs alongside original files for future verification
@@ -48,7 +48,7 @@ https://opentimestamps.org/#calendars",
 
     There are two possible statuses for a timestamp:
     - `Pending`: The timestamp has been submitted to the calendar server but has not yet been confirmed in the Bitcoin blockchain.
-    - `Confirmed`: The timestamp has been confirmed in the Bitcoin or Litecoin blockchain.
+    - `Confirmed`: The timestamp has been confirmed in the Bitcoin blockchain.
 
     ### Upgrading Timestamps
     To upgrade a timestamp from 'pending' to 'confirmed', you can use the `ots upgrade` command from the opentimestamps-client package
diff --git a/src/auto_archiver/modules/opentimestamps_enricher/opentimestamps_enricher.py b/src/auto_archiver/modules/opentimestamps_enricher/opentimestamps_enricher.py
index d6e8add..4785dd2 100644
--- a/src/auto_archiver/modules/opentimestamps_enricher/opentimestamps_enricher.py
+++ b/src/auto_archiver/modules/opentimestamps_enricher/opentimestamps_enricher.py
@@ -1,16 +1,15 @@
 import os
-import datetime
 
 from loguru import logger
 import opentimestamps
 from opentimestamps.calendar import RemoteCalendar, DEFAULT_CALENDAR_WHITELIST
 from opentimestamps.core.timestamp import Timestamp, DetachedTimestampFile
-from opentimestamps.core.notary import PendingAttestation, BitcoinBlockHeaderAttestation, LitecoinBlockHeaderAttestation
+from opentimestamps.core.notary import PendingAttestation, BitcoinBlockHeaderAttestation
 from opentimestamps.core.op import OpSHA256
 from opentimestamps.core import serialize
 from auto_archiver.core import Enricher
 from auto_archiver.core import Metadata, Media
-from auto_archiver.utils.misc import calculate_file_hash
+from auto_archiver.utils.misc import get_current_timestamp
 
 class OpentimestampsEnricher(Enricher):
 
@@ -34,8 +33,8 @@ class OpentimestampsEnricher(Enricher):
                     continue
                 
                 # Create timestamp for the file - hash is SHA256
-                # Note: ONLY SHA256 is used/supported here. Opentimestamps supports other hashes, but not SHA3-512
-                # see opentimestamps.core.op
+                # Note: hash is hard-coded to SHA256 and does not use hash_enricher to set it.
+                # SHA256 is the recommended hash, ref: https://github.com/bellingcat/auto-archiver/pull/247#discussion_r1992433181
                 logger.debug(f"Creating timestamp for {file_path}")
                 file_hash = None
                 with open(file_path, 'rb') as f:
@@ -150,13 +149,10 @@ class OpentimestampsEnricher(Enricher):
                     info["uri"] = attestation.uri
                 
                 elif isinstance(attestation, BitcoinBlockHeaderAttestation):
-                    info["status"] = "confirmed - bitcoin"
-                    info["block_height"] = attestation.height
-                elif isinstance(attestation, LitecoinBlockHeaderAttestation):
-                    info["status"] = "confirmed - litecoin"
+                    info["status"] = "confirmed"
                     info["block_height"] = attestation.height
 
-                info["last_check"] = datetime.datetime.now().isoformat()[:-7]
+                info["last_check"] = get_current_timestamp()
                 
                 attestation_info.append(info)
             
@@ -169,6 +165,6 @@ class OpentimestampsEnricher(Enricher):
                 result["verified"] = False
         else:
             result["verified"] = False
-        result["last_updated"] = datetime.datetime.now().isoformat()[:-7]
+        result["last_updated"] = get_current_timestamp()
         
         return result
\ No newline at end of file
diff --git a/tests/enrichers/test_opentimestamps_enricher.py b/tests/enrichers/test_opentimestamps_enricher.py
index e91f97f..5b6a079 100644
--- a/tests/enrichers/test_opentimestamps_enricher.py
+++ b/tests/enrichers/test_opentimestamps_enricher.py
@@ -111,10 +111,10 @@ def test_verify_timestamp(setup_module, detached_timestamp_file):
     # Check attestation types
     assertion_types = [a["status"] for a in verification_info["attestations"]]
     assert "pending" in assertion_types
-    assert "confirmed - bitcoin" in assertion_types
+    assert "confirmed" in assertion_types
     
     # Check Bitcoin attestation details
-    bitcoin_attestation = next(a for a in verification_info["attestations"] if a["status"] == "confirmed - bitcoin")
+    bitcoin_attestation = next(a for a in verification_info["attestations"] if a["status"] == "confirmed")
     assert bitcoin_attestation["block_height"] == 783000
 
 def test_verify_pending_only(setup_module, pending_timestamp_file):
@@ -147,7 +147,7 @@ def test_verify_bitcoin_completed(setup_module, verified_timestamp_file):
     
     # Check that the attestation is a Bitcoin attestation
     attestation = verification_info["attestations"][0]
-    assert attestation["status"] == "confirmed - bitcoin"
+    assert attestation["status"] == "confirmed"
     assert attestation["block_height"] == 783000
 
 def test_full_enriching(setup_module, sample_file_path, sample_media, mocker):

From 0efeaaabb181f34e0a692443c4651b2f171a2eb5 Mon Sep 17 00:00:00 2001
From: Patrick Robertson <robertson.patrick@gmail.com>
Date: Tue, 11 Mar 2025 12:24:52 +0000
Subject: [PATCH 40/55] Revert to using time.sleep and .click() - since we only
 want to be waiting the first time (for the page to load)

---
 src/auto_archiver/utils/webdriver.py | 38 ++++++++++------------------
 1 file changed, 14 insertions(+), 24 deletions(-)

diff --git a/src/auto_archiver/utils/webdriver.py b/src/auto_archiver/utils/webdriver.py
index af3b7dd..ccfead5 100644
--- a/src/auto_archiver/utils/webdriver.py
+++ b/src/auto_archiver/utils/webdriver.py
@@ -64,41 +64,31 @@ class CookieSettingDriver(webdriver.Firefox):
                         except Exception as e:
                             logger.warning(f"Failed to add cookie ({cookie.domain}) to webdriver for url {domain}: {e}")
         
-        if self.facebook_accept_cookies:
-            try:
-                logger.debug(f'Trying fb click accept cookie popup.')
-                super(CookieSettingDriver, self).get("http://www.facebook.com")
-                essential_only = self.find_element(By.XPATH, "//span[contains(text(), 'Decline optional cookies')]")
-                essential_only.click()
-                logger.debug(f'fb click worked')
-                # linux server needs a sleep otherwise facebook cookie won't have worked and we'll get a popup on next page
-                time.sleep(2)
-            except Exception as e:
-                logger.warning(f'Failed on fb accept cookies.', e)
+
         
+        super(CookieSettingDriver, self).get(url)
+        time.sleep(2)
+
+        # Try and use some common button text to reject/accept cookies
+        for text in ["Refuse non-essential cookies", "Decline optional cookies", "Reject additional cookies", "Reject all", "Accept all cookies"]:
+            try:
+                xpath = f"//*[contains(translate(text(), 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz'), '{text.lower()}')]"
+                self.find_element(By.XPATH, xpath).click()
+                time.sleep(2)
+            except selenium_exceptions.NoSuchElementException:
+                pass
 
         # now get the actual URL
-        super(CookieSettingDriver, self).get(url)
         if self.facebook_accept_cookies:
             # try and click the 'close' button on the 'login' window to close it
             try:
                 xpath = "//div[@role='dialog']//div[@aria-label='Close']"
-                WebDriverWait(self, 2).until(EC.element_to_be_clickable((By.XPATH, xpath))).click()
+                self.find_element(By.XPATH, xpath).click()
+                time.sleep(2)
             except selenium_exceptions.NoSuchElementException:
                 logger.warning("Unable to find the 'close' button on the facebook login window")
                 pass
 
-        else:
-
-            # for all other sites, try and use some common button text to reject/accept cookies
-            for text in ["Refuse non-essential cookies", "Decline optional cookies", "Reject additional cookies", "Reject all", "Accept all cookies"]:
-                try:
-                    xpath = f"//*[contains(translate(text(), 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz'), '{text.lower()}')]"
-                    WebDriverWait(self, 5).until(EC.element_to_be_clickable((By.XPATH, xpath))).click()
-                    break
-                except selenium_exceptions.WebDriverException:
-                    pass
-
     
 class Webdriver:
     def __init__(self, width: int, height: int, timeout_seconds: int,

From 589c834047b21fe804b81997208e01f666142ee7 Mon Sep 17 00:00:00 2001
From: Patrick Robertson <robertson.patrick@gmail.com>
Date: Tue, 11 Mar 2025 12:25:19 +0000
Subject: [PATCH 41/55] Fix parsing ytdlp args - we should first run them
 through the parse_options method

---
 .../modules/generic_extractor/__manifest__.py |  7 ++++
 .../generic_extractor/generic_extractor.py    | 38 ++++++++++++-------
 2 files changed, 31 insertions(+), 14 deletions(-)

diff --git a/src/auto_archiver/modules/generic_extractor/__manifest__.py b/src/auto_archiver/modules/generic_extractor/__manifest__.py
index 1d3b365..274a4ba 100644
--- a/src/auto_archiver/modules/generic_extractor/__manifest__.py
+++ b/src/auto_archiver/modules/generic_extractor/__manifest__.py
@@ -76,5 +76,12 @@ If you are having issues with the extractor, you can review the version of `yt-d
             "help": "How often to check for yt-dlp updates (days). If positive, will check and update yt-dlp every [num] days. Set it to -1 to disable, or 0 to always update on every run.",
             "type": "int",
         },
+        "ytdlp_args": {
+            "default": "",
+            "help": "Additional arguments to pass to yt-dlp, e.g. --no-check-certificate or --plugin-dirs.\
+See yt-dlp documentation here for more information: https://github.com/yt-dlp/yt-dlp?tab=readme-ov-file#general-options\
+Note: this is not to be confused with 'extractor_args' which are specific to the extractor itself.",
+            "type": "str",
+        },
     },
 }
diff --git a/src/auto_archiver/modules/generic_extractor/generic_extractor.py b/src/auto_archiver/modules/generic_extractor/generic_extractor.py
index 5acce46..56164ff 100644
--- a/src/auto_archiver/modules/generic_extractor/generic_extractor.py
+++ b/src/auto_archiver/modules/generic_extractor/generic_extractor.py
@@ -1,6 +1,7 @@
 import datetime, os
 import importlib
 import subprocess
+
 from typing import Generator, Type
 
 import yt_dlp
@@ -166,7 +167,7 @@ class GenericExtractor(Extractor):
 
         if not dropin:
             # TODO: add a proper link to 'how to create your own dropin'
-            logger.debug(f"""Could not find valid dropin for {info_extractor.IE_NAME}.
+            logger.debug(f"""Could not find valid dropin for {info_extractor.ie_key()}.
                      Why not try creating your own, and make sure it has a valid function called 'create_metadata'. Learn more: https://auto-archiver.readthedocs.io/en/latest/user_guidelines.html#""")
             return False
         
@@ -279,18 +280,18 @@ class GenericExtractor(Extractor):
             result = self.get_metadata_for_video(data, info_extractor, url, ydl)
 
         except Exception as e:
-            if info_extractor.ie_key() == "generic":
+            if info_extractor.IE_NAME == "generic":
                 # don't clutter the logs with issues about the 'generic' extractor not having a dropin
                 return False
 
-            logger.debug(f'Issue using "{info_extractor.IE_NAME}" extractor to download video (error: {repr(e)}), attempting to use extractor to get post data instead')
+            logger.debug(f'Issue using "{info_extractor.IE_NAME}" extractor to download video (error: {repr(e)}), attempting to use dropin to get post data instead')
             try:
                 result = self.get_metadata_for_post(info_extractor, url, ydl)
             except (yt_dlp.utils.DownloadError, yt_dlp.utils.ExtractorError) as post_e:
                 logger.error(f'Error downloading metadata for post: {post_e}')
                 return False
             except Exception as generic_e:
-                logger.debug(f'Attempt to extract using ytdlp extractor "{info_extractor.IE_NAME}" failed:  \n  {repr(generic_e)}', exc_info=True)
+                logger.debug(f'Attempt to extract using ytdlp dropin "{info_extractor.IE_NAME}" failed:  \n  {repr(generic_e)}', exc_info=True)
                 return False
         
         if result:
@@ -314,11 +315,16 @@ class GenericExtractor(Extractor):
             item.set("replaced_url", url)
 
 
-        ydl_options = {'outtmpl': os.path.join(self.tmp_dir, f'%(id)s.%(ext)s'), 
-                       'quiet': False, 'noplaylist': not self.allow_playlist ,
-                       'writesubtitles': self.subtitles,'writeautomaticsub': self.subtitles,
-                       "live_from_start": self.live_from_start, "proxy": self.proxy,
-                       "max_downloads": self.max_downloads, "playlistend": self.max_downloads}
+        ydl_options = ["-o", os.path.join(self.tmp_dir, f'%(id)s.%(ext)s'),
+                       "--quiet",
+                       "--no-playlist" if not self.allow_playlist else "--yes-playlist",
+                       "--write-subs" if self.subtitles else "--no-write-subs",
+                       "--write-auto-subs" if self.subtitles else "--no-write-auto-subs",
+                       "--live-from-start" if self.live_from_start else "--no-live-from-start",
+                       "--proxy", self.proxy if self.proxy else '',
+                       f"--max-downloads {self.max_downloads}" if self.max_downloads != "inf" else '',
+                       f"--playlist-end {self.max_downloads}" if self.max_downloads != "inf" else ''
+                       ]
         
         # set up auth
         auth = self.auth_for_site(url, extract_cookies=False)
@@ -327,19 +333,23 @@ class GenericExtractor(Extractor):
         if auth:
             if 'username' in auth and 'password' in auth:
                 logger.debug(f'Using provided auth username and password for {url}')
-                ydl_options['username'] = auth['username']
-                ydl_options['password'] = auth['password']
+                ydl_options.extend(('--username', auth['username']))
+                ydl_options.extend(('--password', auth['password']))
             elif 'cookie' in auth:
                 logger.debug(f'Using provided auth cookie for {url}')
                 yt_dlp.utils.std_headers['cookie'] = auth['cookie']
             elif 'cookies_from_browser' in auth:
                 logger.debug(f'Using extracted cookies from browser {auth["cookies_from_browser"]} for {url}')
-                ydl_options['cookiesfrombrowser'] = auth['cookies_from_browser']
+                ydl_options.extend(('--cookies-from-browser', auth['cookies_from_browser']))
             elif 'cookies_file' in auth:
                 logger.debug(f'Using cookies from file {auth["cookies_file"]} for {url}')
-                ydl_options['cookiefile'] = auth['cookies_file']
+                ydl_options.extend(('--cookies', auth['cookies_file']))
 
-        ydl = yt_dlp.YoutubeDL(ydl_options) # allsubtitles and subtitleslangs not working as expected, so default lang is always "en"
+        if self.ytdlp_args:
+            ydl_options += self.ytdlp_args.split(" ")
+
+        _, _, _, validated_options = yt_dlp.parse_options(ydl_options)
+        ydl = yt_dlp.YoutubeDL(validated_options) # allsubtitles and subtitleslangs not working as expected, so default lang is always "en"
 
         for info_extractor in self.suitable_extractors(url):
             result = self.download_for_extractor(info_extractor, url, ydl)

From f6b13327f0329b771709a0550895810b91f6cf39 Mon Sep 17 00:00:00 2001
From: Patrick Robertson <robertson.patrick@gmail.com>
Date: Thu, 13 Mar 2025 16:03:52 +0000
Subject: [PATCH 42/55] Tweaks and additional debug logging

---
 .../modules/generic_extractor/generic_extractor.py             | 3 ++-
 src/auto_archiver/utils/webdriver.py                           | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/auto_archiver/modules/generic_extractor/generic_extractor.py b/src/auto_archiver/modules/generic_extractor/generic_extractor.py
index 56164ff..a75e874 100644
--- a/src/auto_archiver/modules/generic_extractor/generic_extractor.py
+++ b/src/auto_archiver/modules/generic_extractor/generic_extractor.py
@@ -346,9 +346,10 @@ class GenericExtractor(Extractor):
                 ydl_options.extend(('--cookies', auth['cookies_file']))
 
         if self.ytdlp_args:
+            logger.debug("Adding additional ytdlp arguments: {self.ytdlp_args}")
             ydl_options += self.ytdlp_args.split(" ")
 
-        _, _, _, validated_options = yt_dlp.parse_options(ydl_options)
+        *_, validated_options = yt_dlp.parse_options(ydl_options)
         ydl = yt_dlp.YoutubeDL(validated_options) # allsubtitles and subtitleslangs not working as expected, so default lang is always "en"
 
         for info_extractor in self.suitable_extractors(url):
diff --git a/src/auto_archiver/utils/webdriver.py b/src/auto_archiver/utils/webdriver.py
index ccfead5..57f2cf1 100644
--- a/src/auto_archiver/utils/webdriver.py
+++ b/src/auto_archiver/utils/webdriver.py
@@ -48,7 +48,7 @@ class CookieSettingDriver(webdriver.Firefox):
                     for name, value in cookie.split("="):
                         self.driver.add_cookie({'name': name, 'value': value})
             elif self.cookiejar:
-                domain = urlparse(url).netloc.lstrip("www.")
+                domain = urlparse(url).netloc
                 regex = re.compile(f"(www)?\.?{domain}$")
                 for cookie in self.cookiejar:
                     if regex.match(cookie.domain):

From 4d67dce4c8eaa5746f7ab952bbc3ea4e11eda3e1 Mon Sep 17 00:00:00 2001
From: msramalho <19508417+msramalho@users.noreply.github.com>
Date: Thu, 13 Mar 2025 19:24:05 +0000
Subject: [PATCH 43/55] minor log fix

---
 src/auto_archiver/modules/generic_extractor/tiktok.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/auto_archiver/modules/generic_extractor/tiktok.py b/src/auto_archiver/modules/generic_extractor/tiktok.py
index 8914f0c..af3751b 100644
--- a/src/auto_archiver/modules/generic_extractor/tiktok.py
+++ b/src/auto_archiver/modules/generic_extractor/tiktok.py
@@ -14,7 +14,7 @@ class Tiktok(GenericDropin):
 
     def extract_post(self, url: str, ie_instance):
 
-        logger.debug("Using Tikwm API to attempt to download tiktok video from {url=}")
+        logger.debug(f"Using Tikwm API to attempt to download tiktok video from {url=}")
 
         endpoint = self.TIKWM_ENDPOINT.format(url=url)
 

From c7c24fbaf2700f22c71b7bde0cfee65a2065ec0d Mon Sep 17 00:00:00 2001
From: erinhmclark <erinhannahmary.clark@gmail.com>
Date: Thu, 13 Mar 2025 20:26:29 +0000
Subject: [PATCH 44/55] Update style_guide.md to clarify pre-commit setup, add
 Docker commands to Makefile and merge ruff actions.

---
 .github/workflows/ruff.yaml            |  7 +---
 Makefile                               | 22 ++++++++----
 docs/source/development/style_guide.md | 49 ++++++++++++++++++++------
 3 files changed, 56 insertions(+), 22 deletions(-)

diff --git a/.github/workflows/ruff.yaml b/.github/workflows/ruff.yaml
index 50407e5..40a6642 100644
--- a/.github/workflows/ruff.yaml
+++ b/.github/workflows/ruff.yaml
@@ -14,9 +14,4 @@ jobs:
       - name: Run Ruff (Lint & Format Check)
         uses: astral-sh/ruff-action@v1
         with:
-          args: "check . --output-format=concise"
-
-      - name: Run Ruff Format Check
-        uses: astral-sh/ruff-action@v1
-        with:
-          args: "format --check ."
+          args: "check . --output-format=concise && ruff format --check ."
diff --git a/Makefile b/Makefile
index c59f272..7877543 100644
--- a/Makefile
+++ b/Makefile
@@ -13,7 +13,9 @@ help:
 	@echo "  make ruff-clean   - Auto-fix Ruff linting and formatting issues"
 	@echo "  make docs         - Generate documentation (same as 'make html')"
 	@echo "  make clean-docs   - Remove generated docs"
-	@echo "  make docker-run   - Run the Docker container"
+	@echo "  make docker-build - Build the Auto Archiver Docker image"
+	@echo "  make docker-compose - Run Auto Archiver with Docker Compose"
+	@echo "  make docker-compose-rebuild - Rebuild and run Auto Archiver with Docker Compose"
 	@echo "  make show-docs    - Build and open the documentation in a browser"
 
 
@@ -56,12 +58,20 @@ show-docs:
 	@echo "Opening documentation in browser..."
 	@open "$(BUILDDIR)/html/index.html"
 
+.PHONY: docker-build
+docker-build:
+	@echo "Building local Auto Archiver Docker image..."
+	@docker compose build  # Uses the same build context as docker-compose.yml
 
-# Run Docker with default settings
-.PHONY: docker-run
-docker-run:
-	@echo "Running Auto Archiver Docker container..."
-	@docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver
+.PHONY: docker-compose
+docker-compose:
+	@echo "Running Auto Archiver with Docker Compose..."
+	@docker compose up
+
+.PHONY: docker-compose-rebuild
+docker-compose-rebuild:
+	@echo "Rebuilding and running Auto Archiver with Docker Compose..."
+	@docker compose up --build
 
 # Catch-all for Sphinx commands
 .PHONY: Makefile
diff --git a/docs/source/development/style_guide.md b/docs/source/development/style_guide.md
index a73a6fc..e7fbded 100644
--- a/docs/source/development/style_guide.md
+++ b/docs/source/development/style_guide.md
@@ -1,21 +1,42 @@
-### Style Guide
+# Style Guide
 
+
+## Ruff
 The project uses [ruff](https://docs.astral.sh/ruff/) for linting and  formatting.
-Our style configurations are set in the `pyproject.toml` file.
+Our style configurations are set in the `pyproject.toml` file, and can be modified from there.
 
-We have a pre-commit hook to run the formatter before you commit, but Ruff can also be [integrated with most editors](https://docs.astral.sh/ruff/editors/setup/) to run automatically.
+### Formatting
+
+We have a pre-commit hook to run the formatter before you commit.
+This requires you to set it up once locally, then it will run automatically when you commit changes.
+
+```shell
+poetry run pre-commit install
+```
+
+Ruff can also be [integrated with most editors](https://docs.astral.sh/ruff/editors/setup/) to run automatically.
+
+### Linting
 
 We recommend you also run the linter before pushing code. 
 
-# Running the linter
-
-We have Makefile commands to run common tasks (Note if you're on Windows you might need to install `make` first, or you can use ruff directly):
+We have Makefile commands to run common tasks (Note if you're on Windows you might need to install `make` first, or alternatively you can use ruff commands directly):
 
 This outputs a report of any issues found:
 ```shell
 make ruff-check
 ```
 
+To see a more detailed linting report, you can remove the following line from the `pyproject.toml` file:
+```toml
+[tool.ruff]
+
+# Remove this for a more detailed lint report
+output-format = "concise"
+```
+
+**Lint Fix**
+
 This command will attempt to fix any issues it can:
 
 ⚠️ Warning: This can cause breaking changes. ⚠️
@@ -31,9 +52,17 @@ This is included with [Git for Windows](https://gitforwindows.org/) or you can i
 choco install make
 ```
 
-**Running directly with ruff**
+**Changing the configs**
 
-Alternatively, you can run the commands directly with ruff.
+Our rules are quite lenient for general usage, but if you want to explore more rigorous checks you can check out the [ruff documentation](https://docs.astral.sh/ruff/configuration/).
+You can then run checks with additional rules to see more nuanced errors which you can review manually.
+One example is to extend the selected rules for linting the `pyproject.toml` file:
 
-Our rules are quite lenient for general usage, but if you want to explore more rigorous checks you can explore the [ruff documentation](https://docs.astral.sh/ruff/configuration/).
-You can then run checks to see more nuanced errors which you can review manually.
\ No newline at end of file
+```toml 
+[tool.ruff.lint]
+# Extend the rules to check for by adding them to this option:
+# See documentation for more details: https://docs.astral.sh/ruff/rules/
+extend-select = ["B"]
+```
+
+Then re-run the `make ruff-check` command to see the new rules in action.
\ No newline at end of file

From ad2784c5de818737647fed0b68af0b0c721b576f Mon Sep 17 00:00:00 2001
From: erinhmclark <erinhannahmary.clark@gmail.com>
Date: Thu, 13 Mar 2025 20:52:21 +0000
Subject: [PATCH 45/55] Update style_guide.md

---
 docs/source/development/style_guide.md | 39 +++++++++++++-------------
 1 file changed, 19 insertions(+), 20 deletions(-)

diff --git a/docs/source/development/style_guide.md b/docs/source/development/style_guide.md
index e7fbded..c3764c1 100644
--- a/docs/source/development/style_guide.md
+++ b/docs/source/development/style_guide.md
@@ -1,11 +1,11 @@
 # Style Guide
 
 
-## Ruff
-The project uses [ruff](https://docs.astral.sh/ruff/) for linting and  formatting.
-Our style configurations are set in the `pyproject.toml` file, and can be modified from there.
+The project uses [Ruff](https://docs.astral.sh/ruff/) for linting and  formatting.
+Our style configurations are set in the `pyproject.toml` file. If needed, you can modify them there.
 
-### Formatting
+
+### **Formatting (Auto-Run Before Commit) 🛠️**  
 
 We have a pre-commit hook to run the formatter before you commit.
 This requires you to set it up once locally, then it will run automatically when you commit changes.
@@ -14,20 +14,24 @@ This requires you to set it up once locally, then it will run automatically when
 poetry run pre-commit install
 ```
 
-Ruff can also be [integrated with most editors](https://docs.astral.sh/ruff/editors/setup/) to run automatically.
+Ruff can also be to run automatically.
+Alternative: Ruff can also be [integrated with most editors](https://docs.astral.sh/ruff/editors/setup/)  for real-time formatting.
 
-### Linting
+### **Linting (Check Before Pushing) 🔍**
 
 We recommend you also run the linter before pushing code. 
 
-We have Makefile commands to run common tasks (Note if you're on Windows you might need to install `make` first, or alternatively you can use ruff commands directly):
+We have [Makefile](../../../Makefile) commands to run common tasks. 
 
-This outputs a report of any issues found:
+Tip: if you're on Windows you might need to install `make` first, or alternatively you can use ruff commands directly.
+
+
+**Lint Check:** This outputs a report of any issues found, without attempting to fix them: 
 ```shell
 make ruff-check
 ```
 
-To see a more detailed linting report, you can remove the following line from the `pyproject.toml` file:
+Tip: To see a more detailed linting report, you can remove the following line from the `pyproject.toml` file:
 ```toml
 [tool.ruff]
 
@@ -35,27 +39,22 @@ To see a more detailed linting report, you can remove the following line from th
 output-format = "concise"
 ```
 
-**Lint Fix**
+**Lint Fix:** This command will attempt to fix some of the issues it picked up with the lint check.
 
-This command will attempt to fix any issues it can:
+Note not all warnings can be fixed automatically.
 
 ⚠️ Warning: This can cause breaking changes. ⚠️
 
-Ensure you check any modifications by this before committing them.
+Most fixes are safe, but some non-standard practices such as dynamic loading are not picked up by linters. Ensure you check any modifications by this before committing them.
 ```shell
 make ruff-fix
 ```
 
-**Note:** If you're on Windows you might not have `make` installed by default.
-This is included with [Git for Windows](https://gitforwindows.org/) or you can install make via [Chocolatey](https://chocolatey.org/):
-```shell
-choco install make
-```
+**Changing Configurations ⚙️**
 
-**Changing the configs**
 
-Our rules are quite lenient for general usage, but if you want to explore more rigorous checks you can check out the [ruff documentation](https://docs.astral.sh/ruff/configuration/).
-You can then run checks with additional rules to see more nuanced errors which you can review manually.
+Our rules are quite lenient for general usage, but if you want to run more rigorous checks you can then run checks with additional rules to see more nuanced errors which you can review manually.
+Check out the [ruff documentation](https://docs.astral.sh/ruff/configuration/) for the full list of rules.
 One example is to extend the selected rules for linting the `pyproject.toml` file:
 
 ```toml 

From 4af3cd7b2a24d839c74f5a1e893b830a40d2f931 Mon Sep 17 00:00:00 2001
From: erinhmclark <erinhannahmary.clark@gmail.com>
Date: Thu, 13 Mar 2025 21:47:09 +0000
Subject: [PATCH 46/55] Revert ruff to separate commands.

---
 .github/workflows/ruff.yaml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ruff.yaml b/.github/workflows/ruff.yaml
index 40a6642..50407e5 100644
--- a/.github/workflows/ruff.yaml
+++ b/.github/workflows/ruff.yaml
@@ -14,4 +14,9 @@ jobs:
       - name: Run Ruff (Lint & Format Check)
         uses: astral-sh/ruff-action@v1
         with:
-          args: "check . --output-format=concise && ruff format --check ."
+          args: "check . --output-format=concise"
+
+      - name: Run Ruff Format Check
+        uses: astral-sh/ruff-action@v1
+        with:
+          args: "format --check ."

From 72f48f01474583aa842abbc688f2c9e177708e4a Mon Sep 17 00:00:00 2001
From: erinhmclark <erinhannahmary.clark@gmail.com>
Date: Fri, 14 Mar 2025 12:11:24 +0000
Subject: [PATCH 47/55] Fix merge conflicts.

---
 .github/workflows/ruff.yaml                   | 30 ++++++++++---------
 .../generic_extractor/generic_extractor.py    |  2 +-
 .../tiktok_tikwm_extractor/__init__.py        |  0
 .../tiktok_tikwm_extractor/__manifest__.py    |  0
 .../tiktok_tikwm_extractor.py                 |  0
 .../extractors/test_tiktok_tikwm_extractor.py | 16 ++++------
 6 files changed, 22 insertions(+), 26 deletions(-)
 delete mode 100644 src/auto_archiver/modules/tiktok_tikwm_extractor/__init__.py
 delete mode 100644 src/auto_archiver/modules/tiktok_tikwm_extractor/__manifest__.py
 delete mode 100644 src/auto_archiver/modules/tiktok_tikwm_extractor/tiktok_tikwm_extractor.py

diff --git a/.github/workflows/ruff.yaml b/.github/workflows/ruff.yaml
index 50407e5..5ccbb1c 100644
--- a/.github/workflows/ruff.yaml
+++ b/.github/workflows/ruff.yaml
@@ -1,22 +1,24 @@
 name: Ruff Formatting & Linting
 
-on: [push, pull_request]
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
 
 jobs:
-  ruff:
-    name: Run Ruff Checks
+  build:
     runs-on: ubuntu-latest
-
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      - name: Run Ruff (Lint & Format Check)
-        uses: astral-sh/ruff-action@v1
+      - uses: actions/checkout@v4
+      - name: Install Python
+        uses: actions/setup-python@v5
         with:
-          args: "check . --output-format=concise"
+          python-version: "3.11"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install ruff
 
-      - name: Run Ruff Format Check
-        uses: astral-sh/ruff-action@v1
-        with:
-          args: "format --check ."
+      - name: Run Ruff
+        run: ruff check --output-format=github . && ruff format --check
\ No newline at end of file
diff --git a/src/auto_archiver/modules/generic_extractor/generic_extractor.py b/src/auto_archiver/modules/generic_extractor/generic_extractor.py
index 4e408df..c06e622 100644
--- a/src/auto_archiver/modules/generic_extractor/generic_extractor.py
+++ b/src/auto_archiver/modules/generic_extractor/generic_extractor.py
@@ -396,7 +396,7 @@ class GenericExtractor(Extractor):
             item.set("replaced_url", url)
 
         ydl_options = {
-            "outtmpl": os.path.join(self.tmp_dir, f"%(id)s.%(ext)s"),
+            "outtmpl": os.path.join(self.tmp_dir, "%(id)s.%(ext)s"),
             "quiet": False,
             "noplaylist": not self.allow_playlist,
             "writesubtitles": self.subtitles,
diff --git a/src/auto_archiver/modules/tiktok_tikwm_extractor/__init__.py b/src/auto_archiver/modules/tiktok_tikwm_extractor/__init__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/src/auto_archiver/modules/tiktok_tikwm_extractor/__manifest__.py b/src/auto_archiver/modules/tiktok_tikwm_extractor/__manifest__.py
deleted file mode 100644
index e69de29..0000000
diff --git a/src/auto_archiver/modules/tiktok_tikwm_extractor/tiktok_tikwm_extractor.py b/src/auto_archiver/modules/tiktok_tikwm_extractor/tiktok_tikwm_extractor.py
deleted file mode 100644
index e69de29..0000000
diff --git a/tests/extractors/test_tiktok_tikwm_extractor.py b/tests/extractors/test_tiktok_tikwm_extractor.py
index b4b10ba..d04d7e4 100644
--- a/tests/extractors/test_tiktok_tikwm_extractor.py
+++ b/tests/extractors/test_tiktok_tikwm_extractor.py
@@ -38,7 +38,7 @@ class TestTiktokTikwmExtractor(TestExtractorBase):
         mock_get.return_value.status_code = 200
         mock_get.return_value.json.side_effect = ValueError
         with caplog.at_level("DEBUG"):
-            assert self.extractor.download(make_item(self.VALID_EXAMPLE_URL)) == False
+            assert self.extractor.download(make_item(self.VALID_EXAMPLE_URL)) is False
             mock_get.assert_called_once()
             mock_get.return_value.json.assert_called_once()
             # first message is just the 'Skipping using ytdlp to download files for TikTok' message
@@ -49,7 +49,7 @@ class TestTiktokTikwmExtractor(TestExtractorBase):
 
         mock_get.return_value.json.side_effect = Exception
         with caplog.at_level("ERROR"):
-            assert self.extractor.download(make_item(self.VALID_EXAMPLE_URL)) == False
+            assert self.extractor.download(make_item(self.VALID_EXAMPLE_URL)) is False
             mock_get.assert_called()
             assert mock_get.call_count == 2
             assert mock_get.return_value.json.call_count == 2
@@ -69,7 +69,7 @@ class TestTiktokTikwmExtractor(TestExtractorBase):
         mock_get.return_value.status_code = 200
         mock_get.return_value.json.return_value = response
         with caplog.at_level("DEBUG"):
-            assert self.extractor.download(make_item(self.VALID_EXAMPLE_URL)) == False
+            assert self.extractor.download(make_item(self.VALID_EXAMPLE_URL)) is False
             mock_get.assert_called_once()
             mock_get.return_value.json.assert_called_once()
             assert "failed to get a valid response from tikwm.com" in caplog.text
@@ -82,10 +82,9 @@ class TestTiktokTikwmExtractor(TestExtractorBase):
             ({"data": {"play": "url"}}, True),
         ],
     )
-    def test_correct_extraction(self, mock_get, make_item, response, has_vid):
+    def test_correct_extraction(self, mock_get, make_item, response, has_vid, mocker):
         mock_get.return_value.status_code = 200
         mock_get.return_value.json.return_value = {"msg": "success", **response}
-
         result = self.extractor.download(make_item(self.VALID_EXAMPLE_URL))
         if not has_vid:
             assert result is False
@@ -95,13 +94,8 @@ class TestTiktokTikwmExtractor(TestExtractorBase):
         mock_get.assert_called()
         assert mock_get.call_count == 1 + int(has_vid)
         mock_get.return_value.json.assert_called_once()
-        if not has_vid:
-            mock_logger.error.assert_called_once()
-            assert mock_logger.error.call_args[0][0].startswith("no valid video URL found")
-        else:
-            mock_logger.error.assert_not_called()
 
-    def test_correct_extraction(self, mock_get, make_item):
+    def test_correct_data_extracted(self, mock_get, make_item):
         mock_get.return_value.status_code = 200
         mock_get.return_value.json.return_value = {
             "msg": "success",

From 17ae75fb95a0b6f753197e615cee07e8d6a5df0a Mon Sep 17 00:00:00 2001
From: Patrick Robertson <robertson.patrick@gmail.com>
Date: Fri, 14 Mar 2025 12:38:12 +0000
Subject: [PATCH 48/55] Ruff fixes

---
 .../opentimestamps_enricher/__manifest__.py   |   8 +-
 .../opentimestamps_enricher.py                |  62 ++++-----
 .../enrichers/test_opentimestamps_enricher.py | 127 ++++++++++--------
 3 files changed, 109 insertions(+), 88 deletions(-)

diff --git a/src/auto_archiver/modules/opentimestamps_enricher/__manifest__.py b/src/auto_archiver/modules/opentimestamps_enricher/__manifest__.py
index b489d66..283d114 100644
--- a/src/auto_archiver/modules/opentimestamps_enricher/__manifest__.py
+++ b/src/auto_archiver/modules/opentimestamps_enricher/__manifest__.py
@@ -18,12 +18,12 @@
             ],
             "help": "List of OpenTimestamps calendar servers to use for timestamping. See here for a list of calendars maintained by opentimestamps:\
 https://opentimestamps.org/#calendars",
-            "type": "list"
+            "type": "list",
         },
         "calendar_whitelist": {
             "default": [],
             "help": "Optional whitelist of calendar servers. Override this if you are using your own calendar servers. e.g. ['https://mycalendar.com']",
-            "type": "list"
+            "type": "list",
         },
     },
     "description": """
@@ -96,5 +96,5 @@ Calendar https://alice.btc.calendar.opentimestamps.org: Timestamped by transacti
     if you want to use your own calendars, then you can override this setting in the `calendar_whitelist` configuration option.
 
    
-    """
-}
\ No newline at end of file
+    """,
+}
diff --git a/src/auto_archiver/modules/opentimestamps_enricher/opentimestamps_enricher.py b/src/auto_archiver/modules/opentimestamps_enricher/opentimestamps_enricher.py
index 4785dd2..d909d8e 100644
--- a/src/auto_archiver/modules/opentimestamps_enricher/opentimestamps_enricher.py
+++ b/src/auto_archiver/modules/opentimestamps_enricher/opentimestamps_enricher.py
@@ -11,8 +11,8 @@ from auto_archiver.core import Enricher
 from auto_archiver.core import Metadata, Media
 from auto_archiver.utils.misc import get_current_timestamp
 
-class OpentimestampsEnricher(Enricher):
 
+class OpentimestampsEnricher(Enricher):
     def enrich(self, to_enrich: Metadata) -> None:
         url = to_enrich.get_url()
         logger.debug(f"OpenTimestamps timestamping files for {url=}")
@@ -31,42 +31,42 @@ class OpentimestampsEnricher(Enricher):
                 if not os.path.exists(file_path):
                     logger.warning(f"File not found: {file_path}")
                     continue
-                
+
                 # Create timestamp for the file - hash is SHA256
                 # Note: hash is hard-coded to SHA256 and does not use hash_enricher to set it.
                 # SHA256 is the recommended hash, ref: https://github.com/bellingcat/auto-archiver/pull/247#discussion_r1992433181
                 logger.debug(f"Creating timestamp for {file_path}")
                 file_hash = None
-                with open(file_path, 'rb') as f:
+                with open(file_path, "rb") as f:
                     file_hash = OpSHA256().hash_fd(f)
 
                 if not file_hash:
                     logger.warning(f"Failed to hash file for timestamping, skipping: {file_path}")
                     continue
-                
+
                 # Create a timestamp with the file hash
                 timestamp = Timestamp(file_hash)
-                
+
                 # Create a detached timestamp file with the hash operation and timestamp
                 detached_timestamp = DetachedTimestampFile(OpSHA256(), timestamp)
-                
+
                 # Submit to calendar servers
                 submitted_to_calendar = False
 
                 logger.debug(f"Submitting timestamp to calendar servers for {file_path}")
                 calendars = []
                 whitelist = DEFAULT_CALENDAR_WHITELIST
-                
+
                 if self.calendar_whitelist:
                     whitelist = set(self.calendar_whitelist)
-                
+
                 # Create calendar instances
                 calendar_urls = []
                 for url in self.calendar_urls:
                     if url in whitelist:
                         calendars.append(RemoteCalendar(url))
                         calendar_urls.append(url)
-                
+
                 # Submit the hash to each calendar
                 for calendar in calendars:
                     try:
@@ -76,17 +76,19 @@ class OpentimestampsEnricher(Enricher):
                         submitted_to_calendar = True
                     except Exception as e:
                         logger.warning(f"Failed to submit to calendar {calendar.url}: {e}")
-                
+
                 # If all calendar submissions failed, add pending attestations
                 if not submitted_to_calendar and not timestamp.attestations:
-                    logger.error(f"Failed to submit to any calendar for {file_path}. **This file will not be timestamped.**")
+                    logger.error(
+                        f"Failed to submit to any calendar for {file_path}. **This file will not be timestamped.**"
+                    )
                     media.set("opentimestamps", False)
                     continue
-                
+
                 # Save the timestamp proof to a file
                 timestamp_path = os.path.join(self.tmp_dir, f"{os.path.basename(file_path)}.ots")
                 try:
-                    with open(timestamp_path, 'wb') as f:
+                    with open(timestamp_path, "wb") as f:
                         # Create a serialization context and write to the file
                         ctx = serialize.BytesSerializationContext()
                         detached_timestamp.serialize(ctx)
@@ -94,25 +96,25 @@ class OpentimestampsEnricher(Enricher):
                 except Exception as e:
                     logger.warning(f"Failed to serialize timestamp file: {e}")
                     continue
-                
+
                 # Create media for the timestamp file
                 timestamp_media = Media(filename=timestamp_path)
                 # explicitly set the mimetype, normally .ots files are 'application/vnd.oasis.opendocument.spreadsheet-template'
                 timestamp_media.mimetype = "application/vnd.opentimestamps"
                 timestamp_media.set("opentimestamps_version", opentimestamps.__version__)
-                
+
                 verification_info = self.verify_timestamp(detached_timestamp)
                 for key, value in verification_info.items():
                     timestamp_media.set(key, value)
-                
+
                 media.set("opentimestamp_files", [timestamp_media])
                 timestamp_files.append(timestamp_media.filename)
                 # Update the original media to indicate it's been timestamped
                 media.set("opentimestamps", True)
-                
+
             except Exception as e:
                 logger.warning(f"Error while timestamping {media.filename}: {e}")
-        
+
         # Add timestamp files to the metadata
         if timestamp_files:
             to_enrich.set("opentimestamped", True)
@@ -121,43 +123,43 @@ class OpentimestampsEnricher(Enricher):
         else:
             to_enrich.set("opentimestamped", False)
             logger.warning(f"No successful timestamps created for {url=}")
-    
+
     def verify_timestamp(self, detached_timestamp):
         """
         Verify a timestamp and extract verification information.
-        
+
         Args:
             detached_timestamp: The detached timestamp to verify.
-            
+
         Returns:
             dict: Information about the verification result.
         """
         result = {}
-        
+
         # Check if we have attestations
         attestations = list(detached_timestamp.timestamp.all_attestations())
         result["attestation_count"] = len(attestations)
-        
+
         if attestations:
             attestation_info = []
             for msg, attestation in attestations:
                 info = {}
-                
+
                 # Process different types of attestations
                 if isinstance(attestation, PendingAttestation):
                     info["status"] = "pending"
                     info["uri"] = attestation.uri
-                
+
                 elif isinstance(attestation, BitcoinBlockHeaderAttestation):
                     info["status"] = "confirmed"
                     info["block_height"] = attestation.height
 
                 info["last_check"] = get_current_timestamp()
-                
+
                 attestation_info.append(info)
-            
+
             result["attestations"] = attestation_info
-            
+
             # For at least one confirmed attestation
             if any("confirmed" in a.get("status") for a in attestation_info):
                 result["verified"] = True
@@ -166,5 +168,5 @@ class OpentimestampsEnricher(Enricher):
         else:
             result["verified"] = False
         result["last_updated"] = get_current_timestamp()
-        
-        return result
\ No newline at end of file
+
+        return result
diff --git a/tests/enrichers/test_opentimestamps_enricher.py b/tests/enrichers/test_opentimestamps_enricher.py
index 5b6a079..8d535d0 100644
--- a/tests/enrichers/test_opentimestamps_enricher.py
+++ b/tests/enrichers/test_opentimestamps_enricher.py
@@ -1,7 +1,4 @@
-from pathlib import Path
 import pytest
-import os
-import tempfile
 import hashlib
 
 from opentimestamps.core.timestamp import Timestamp, DetachedTimestampFile
@@ -26,136 +23,146 @@ def sample_file_path(tmp_path):
     tmp_file.write_text("This is a test file content for OpenTimestamps")
     return str(tmp_file)
 
+
 @pytest.fixture
 def detached_timestamp_file():
     """Create a simple detached timestamp file for testing"""
     file_hash = hashlib.sha256(b"Test content").digest()
     from opentimestamps.core.op import OpSHA256
+
     file_hash_op = OpSHA256()
     timestamp = Timestamp(file_hash)
-    
+
     # Add a pending attestation
     pending = PendingAttestation("https://example.calendar.com")
     timestamp.attestations.add(pending)
-    
+
     # Add a bitcoin attestation
     bitcoin = BitcoinBlockHeaderAttestation(783000)  # Some block height
     timestamp.attestations.add(bitcoin)
-    
+
     return DetachedTimestampFile(file_hash_op, timestamp)
 
+
 @pytest.fixture
 def verified_timestamp_file():
     """Create a timestamp file with a Bitcoin attestation"""
     file_hash = hashlib.sha256(b"Verified content").digest()
     from opentimestamps.core.op import OpSHA256
+
     file_hash_op = OpSHA256()
     timestamp = Timestamp(file_hash)
-    
+
     # Add only a Bitcoin attestation
     bitcoin = BitcoinBlockHeaderAttestation(783000)  # Some block height
     timestamp.attestations.add(bitcoin)
-    
+
     return DetachedTimestampFile(file_hash_op, timestamp)
 
+
 @pytest.fixture
 def pending_timestamp_file():
     """Create a timestamp file with only pending attestations"""
     file_hash = hashlib.sha256(b"Pending content").digest()
     from opentimestamps.core.op import OpSHA256
+
     file_hash_op = OpSHA256()
     timestamp = Timestamp(file_hash)
-    
+
     # Add only pending attestations
     pending1 = PendingAttestation("https://example1.calendar.com")
     pending2 = PendingAttestation("https://example2.calendar.com")
     timestamp.attestations.add(pending1)
     timestamp.attestations.add(pending2)
-    
+
     return DetachedTimestampFile(file_hash_op, timestamp)
 
+
 @pytest.mark.download
 def test_download_tsr(setup_module, mocker):
     """Test submitting a hash to calendar servers"""
     # Mock the RemoteCalendar submit method
-    mock_submit = mocker.patch.object(RemoteCalendar, 'submit')
+    mock_submit = mocker.patch.object(RemoteCalendar, "submit")
     test_timestamp = Timestamp(hashlib.sha256(b"test").digest())
     mock_submit.return_value = test_timestamp
-    
 
     ots = setup_module("opentimestamps_enricher")
-    
+
     # Create a calendar
     calendar = RemoteCalendar("https://alice.btc.calendar.opentimestamps.org")
-    
+
     # Test submission
     file_hash = hashlib.sha256(b"Test file content").digest()
     result = calendar.submit(file_hash)
-    
+
     assert mock_submit.called
     assert isinstance(result, Timestamp)
     assert result == test_timestamp
 
+
 def test_verify_timestamp(setup_module, detached_timestamp_file):
     """Test the verification of timestamp attestations"""
     ots = setup_module("opentimestamps_enricher")
-    
+
     # Test verification
     verification_info = ots.verify_timestamp(detached_timestamp_file)
-    
+
     # Check verification results
     assert verification_info["attestation_count"] == 2
     assert verification_info["verified"] == True
     assert len(verification_info["attestations"]) == 2
-    
+
     # Check attestation types
     assertion_types = [a["status"] for a in verification_info["attestations"]]
     assert "pending" in assertion_types
     assert "confirmed" in assertion_types
-    
+
     # Check Bitcoin attestation details
     bitcoin_attestation = next(a for a in verification_info["attestations"] if a["status"] == "confirmed")
     assert bitcoin_attestation["block_height"] == 783000
 
+
 def test_verify_pending_only(setup_module, pending_timestamp_file):
     """Test verification of timestamps with only pending attestations"""
     ots = setup_module("opentimestamps_enricher")
-    
+
     verification_info = ots.verify_timestamp(pending_timestamp_file)
-    
+
     assert verification_info["attestation_count"] == 2
     assert verification_info["verified"] == False
-    
+
     # All attestations should be of type "pending"
     assert all(a["status"] == "pending" for a in verification_info["attestations"])
-    
+
     # Check URIs of pending attestations
     uris = [a["uri"] for a in verification_info["attestations"]]
     assert "https://example1.calendar.com" in uris
     assert "https://example2.calendar.com" in uris
 
+
 def test_verify_bitcoin_completed(setup_module, verified_timestamp_file):
     """Test verification of timestamps with completed Bitcoin attestations"""
 
     ots = setup_module("opentimestamps_enricher")
-    
+
     verification_info = ots.verify_timestamp(verified_timestamp_file)
-    
+
     assert verification_info["attestation_count"] == 1
     assert verification_info["verified"] == True
     assert "pending" not in verification_info
-    
+
     # Check that the attestation is a Bitcoin attestation
     attestation = verification_info["attestations"][0]
     assert attestation["status"] == "confirmed"
     assert attestation["block_height"] == 783000
 
+
 def test_full_enriching(setup_module, sample_file_path, sample_media, mocker):
     """Test the complete enrichment process"""
 
     # Mock the calendar submission to avoid network requests
-    mock_calendar = mocker.patch.object(RemoteCalendar, 'submit')
-    
+    mock_calendar = mocker.patch.object(RemoteCalendar, "submit")
+
     # Create a function that returns a new timestamp for each call
     def side_effect(digest):
         test_timestamp = Timestamp(digest)
@@ -163,97 +170,109 @@ def test_full_enriching(setup_module, sample_file_path, sample_media, mocker):
         bitcoin = BitcoinBlockHeaderAttestation(783000)
         test_timestamp.attestations.add(bitcoin)
         return test_timestamp
-    
+
     mock_calendar.side_effect = side_effect
 
     ots = setup_module("opentimestamps_enricher")
-    
+
     # Create test metadata with sample file
     metadata = Metadata().set_url("https://example.com")
     sample_media.filename = sample_file_path
     metadata.add_media(sample_media)
-    
+
     # Run enrichment
     ots.enrich(metadata)
-    
+
     # Verify results
     assert metadata.get("opentimestamped") == True
     assert metadata.get("opentimestamps_count") == 1
-    
+
     # Check that we have one parent media item: the original
     assert len(metadata.media) == 1
-    
+
     # Check that the original media was updated
     assert metadata.media[0].get("opentimestamps") == True
-    
+
     # Check the timestamp file media is a child of the original
     assert len(metadata.media[0].get("opentimestamp_files")) == 1
 
     timestamp_media = metadata.media[0].get("opentimestamp_files")[0]
 
     assert timestamp_media.get("opentimestamps_version") is not None
-    
+
     # Check verification results on the timestamp media
     assert timestamp_media.get("verified") == True
     assert timestamp_media.get("attestation_count") == 1
 
-def test_full_enriching_one_calendar_error(setup_module, sample_file_path, sample_media, mocker, pending_timestamp_file):
+
+def test_full_enriching_one_calendar_error(
+    setup_module, sample_file_path, sample_media, mocker, pending_timestamp_file
+):
     """Test enrichment when one calendar server returns an error"""
     # Mock the calendar submission to raise an exception
-    mock_calendar = mocker.patch.object(RemoteCalendar, 'submit')
-    
+    mock_calendar = mocker.patch.object(RemoteCalendar, "submit")
+
     test_timestamp = Timestamp(bytes.fromhex("583988e03646c26fa290c5c2408540a2f4e2aa9be087aa4546aefb531385b935"))
-        # Add a bitcoin attestation to the test timestamp
+    # Add a bitcoin attestation to the test timestamp
     bitcoin = BitcoinBlockHeaderAttestation(783000)
     test_timestamp.attestations.add(bitcoin)
 
     mock_calendar.side_effect = [test_timestamp, Exception("Calendar server error")]
 
-    ots = setup_module("opentimestamps_enricher", {"calendar_urls": ["https://alice.btc.calendar.opentimestamps.org", "https://bob.btc.calendar.opentimestamps.org"]})
-    
+    ots = setup_module(
+        "opentimestamps_enricher",
+        {
+            "calendar_urls": [
+                "https://alice.btc.calendar.opentimestamps.org",
+                "https://bob.btc.calendar.opentimestamps.org",
+            ]
+        },
+    )
+
     # Create test metadata with sample file
     metadata = Metadata().set_url("https://example.com")
     sample_media.filename = sample_file_path
     metadata.add_media(sample_media)
-    
+
     # Run enrichment (should complete despite calendar errors)
     ots.enrich(metadata)
-    
+
     # Verify results
     assert metadata.get("opentimestamped") == True
-    assert metadata.get("opentimestamps_count") == 1 # only alice worked, not bob
+    assert metadata.get("opentimestamps_count") == 1  # only alice worked, not bob
+
 
 def test_full_enriching_calendar_error(setup_module, sample_file_path, sample_media, mocker):
     """Test enrichment when calendar servers return errors"""
     # Mock the calendar submission to raise an exception
-    mock_calendar = mocker.patch.object(RemoteCalendar, 'submit')
+    mock_calendar = mocker.patch.object(RemoteCalendar, "submit")
     mock_calendar.side_effect = Exception("Calendar server error")
-    
 
     ots = setup_module("opentimestamps_enricher")
-    
+
     # Create test metadata with sample file
     metadata = Metadata().set_url("https://example.com")
     sample_media.filename = sample_file_path
     metadata.add_media(sample_media)
-    
+
     # Run enrichment (should complete despite calendar errors)
     ots.enrich(metadata)
-    
+
     # Verify results
     assert metadata.get("opentimestamped") == False
     assert metadata.get("opentimestamps_count") is None
 
+
 def test_no_files_to_stamp(setup_module):
     """Test enrichment with no files to timestamp"""
     ots = setup_module("opentimestamps_enricher")
-    
+
     # Create empty metadata
     metadata = Metadata().set_url("https://example.com")
-    
+
     # Run enrichment
     ots.enrich(metadata)
-    
+
     # Verify no timestamping occurred
     assert metadata.get("opentimestamped") is None
-    assert len(metadata.media) == 0
\ No newline at end of file
+    assert len(metadata.media) == 0

From abaeec0cc6342ee7e843b05b6cc2d029d2103465 Mon Sep 17 00:00:00 2001
From: Patrick Robertson <robertson.patrick@gmail.com>
Date: Fri, 14 Mar 2025 12:48:06 +0000
Subject: [PATCH 49/55] Add ruff check

---
 .pre-commit-config.yaml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 78421d7..833a540 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -3,7 +3,10 @@ repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
     rev: v0.9.10
     hooks:
-      - id: ruff-format
+        - id: ruff
+          args: [ --fix ]
+        - id: ruff-format
+
 
       # Runs Ruff linting - just checks without fixing, but blocks commit if errors are found.
 #      - id: ruff

From a8e5585e6c40c5dad8fad32e591fb90d7c52217e Mon Sep 17 00:00:00 2001
From: Patrick Robertson <robertson.patrick@gmail.com>
Date: Fri, 14 Mar 2025 12:51:47 +0000
Subject: [PATCH 50/55] github format

---
 .pre-commit-config.yaml                                         | 2 +-
 .../modules/generic_extractor/generic_extractor.py              | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 833a540..0ec35a5 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -4,7 +4,7 @@ repos:
     rev: v0.9.10
     hooks:
         - id: ruff
-          args: [ --fix ]
+          args: [ --fix, --output-format=github]
         - id: ruff-format
 
 
diff --git a/src/auto_archiver/modules/generic_extractor/generic_extractor.py b/src/auto_archiver/modules/generic_extractor/generic_extractor.py
index 72b526d..534fb71 100644
--- a/src/auto_archiver/modules/generic_extractor/generic_extractor.py
+++ b/src/auto_archiver/modules/generic_extractor/generic_extractor.py
@@ -398,7 +398,7 @@ class GenericExtractor(Extractor):
 
         ydl_options = [
             "-o",
-            os.path.join(self.tmp_dir, f"%(id)s.%(ext)s"),
+            os.path.join(self.tmp_dir, "%(id)s.%(ext)s"),
             "--quiet",
             "--no-playlist" if not self.allow_playlist else "--yes-playlist",
             "--write-subs" if self.subtitles else "--no-write-subs",

From b21467c922f538a4093ba7c0382b5c78def28728 Mon Sep 17 00:00:00 2001
From: Patrick Robertson <robertson.patrick@gmail.com>
Date: Fri, 14 Mar 2025 12:59:37 +0000
Subject: [PATCH 51/55] Fix ruff checks

---
 .../enrichers/test_opentimestamps_enricher.py  | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/tests/enrichers/test_opentimestamps_enricher.py b/tests/enrichers/test_opentimestamps_enricher.py
index 8d535d0..99ddd66 100644
--- a/tests/enrichers/test_opentimestamps_enricher.py
+++ b/tests/enrichers/test_opentimestamps_enricher.py
@@ -86,8 +86,6 @@ def test_download_tsr(setup_module, mocker):
     test_timestamp = Timestamp(hashlib.sha256(b"test").digest())
     mock_submit.return_value = test_timestamp
 
-    ots = setup_module("opentimestamps_enricher")
-
     # Create a calendar
     calendar = RemoteCalendar("https://alice.btc.calendar.opentimestamps.org")
 
@@ -109,7 +107,7 @@ def test_verify_timestamp(setup_module, detached_timestamp_file):
 
     # Check verification results
     assert verification_info["attestation_count"] == 2
-    assert verification_info["verified"] == True
+    assert verification_info["verified"] is True
     assert len(verification_info["attestations"]) == 2
 
     # Check attestation types
@@ -129,7 +127,7 @@ def test_verify_pending_only(setup_module, pending_timestamp_file):
     verification_info = ots.verify_timestamp(pending_timestamp_file)
 
     assert verification_info["attestation_count"] == 2
-    assert verification_info["verified"] == False
+    assert verification_info["verified"] is False
 
     # All attestations should be of type "pending"
     assert all(a["status"] == "pending" for a in verification_info["attestations"])
@@ -148,7 +146,7 @@ def test_verify_bitcoin_completed(setup_module, verified_timestamp_file):
     verification_info = ots.verify_timestamp(verified_timestamp_file)
 
     assert verification_info["attestation_count"] == 1
-    assert verification_info["verified"] == True
+    assert verification_info["verified"] is True
     assert "pending" not in verification_info
 
     # Check that the attestation is a Bitcoin attestation
@@ -184,14 +182,14 @@ def test_full_enriching(setup_module, sample_file_path, sample_media, mocker):
     ots.enrich(metadata)
 
     # Verify results
-    assert metadata.get("opentimestamped") == True
+    assert metadata.get("opentimestamped") is True
     assert metadata.get("opentimestamps_count") == 1
 
     # Check that we have one parent media item: the original
     assert len(metadata.media) == 1
 
     # Check that the original media was updated
-    assert metadata.media[0].get("opentimestamps") == True
+    assert metadata.media[0].get("opentimestamps") is True
 
     # Check the timestamp file media is a child of the original
     assert len(metadata.media[0].get("opentimestamp_files")) == 1
@@ -201,7 +199,7 @@ def test_full_enriching(setup_module, sample_file_path, sample_media, mocker):
     assert timestamp_media.get("opentimestamps_version") is not None
 
     # Check verification results on the timestamp media
-    assert timestamp_media.get("verified") == True
+    assert timestamp_media.get("verified") is True
     assert timestamp_media.get("attestation_count") == 1
 
 
@@ -238,7 +236,7 @@ def test_full_enriching_one_calendar_error(
     ots.enrich(metadata)
 
     # Verify results
-    assert metadata.get("opentimestamped") == True
+    assert metadata.get("opentimestamped") is True
     assert metadata.get("opentimestamps_count") == 1  # only alice worked, not bob
 
 
@@ -259,7 +257,7 @@ def test_full_enriching_calendar_error(setup_module, sample_file_path, sample_me
     ots.enrich(metadata)
 
     # Verify results
-    assert metadata.get("opentimestamped") == False
+    assert metadata.get("opentimestamped") is False
     assert metadata.get("opentimestamps_count") is None
 
 

From 562d06916ecd5bf8d1d17c17497e902f4e475a39 Mon Sep 17 00:00:00 2001
From: erinhmclark <erinhannahmary.clark@gmail.com>
Date: Fri, 14 Mar 2025 13:08:57 +0000
Subject: [PATCH 52/55] Revert pre commit

---
 .pre-commit-config.yaml | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 0ec35a5..78421d7 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -3,10 +3,7 @@ repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
     rev: v0.9.10
     hooks:
-        - id: ruff
-          args: [ --fix, --output-format=github]
-        - id: ruff-format
-
+      - id: ruff-format
 
       # Runs Ruff linting - just checks without fixing, but blocks commit if errors are found.
 #      - id: ruff

From 29cc1d317f145e3c89a90b42810fdad4c7698459 Mon Sep 17 00:00:00 2001
From: Patrick Robertson <robertson.patrick@gmail.com>
Date: Fri, 14 Mar 2025 13:40:57 +0000
Subject: [PATCH 53/55] Fix pre-commit for ruff check

---
 .pre-commit-config.yaml | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 78421d7..aca3a0d 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -3,8 +3,5 @@ repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
     rev: v0.9.10
     hooks:
-      - id: ruff-format
-
-      # Runs Ruff linting - just checks without fixing, but blocks commit if errors are found.
-#      - id: ruff
-#        args: ["--output-format=concise"]
\ No newline at end of file
+      - id: ruff
+      - id: ruff-format
\ No newline at end of file

From 6920585f6daa7bf96400cbd247ef26759e9c50b2 Mon Sep 17 00:00:00 2001
From: Patrick Robertson <robertson.patrick@gmail.com>
Date: Fri, 14 Mar 2025 13:42:40 +0000
Subject: [PATCH 54/55] Version bump to 0.13.6

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index 225540e..cc53553 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [project]
 name = "auto-archiver"
-version = "0.13.5"
+version = "0.13.6"
 description = "Automatically archive links to videos, images, and social media content from Google Sheets (and more)."
 
 requires-python = ">=3.10,<3.13"

From 282380d8cc080528f9398e44035f9bedf505ff32 Mon Sep 17 00:00:00 2001
From: Patrick Robertson <robertson.patrick@gmail.com>
Date: Fri, 14 Mar 2025 14:20:32 +0000
Subject: [PATCH 55/55] Add note on skipping pre-commit hook

---
 docs/source/development/style_guide.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/source/development/style_guide.md b/docs/source/development/style_guide.md
index c3764c1..390f11a 100644
--- a/docs/source/development/style_guide.md
+++ b/docs/source/development/style_guide.md
@@ -17,6 +17,9 @@ poetry run pre-commit install
 Ruff can also be to run automatically.
 Alternative: Ruff can also be [integrated with most editors](https://docs.astral.sh/ruff/editors/setup/)  for real-time formatting.
 
+If you wish to disable the pre-commit hook (for example, if you want to commit some WIP code) you can use the `--no-verify` flag when you commit.
+For example: `git commit -m "WIP Code" --no-verify`
+
 ### **Linting (Check Before Pushing) 🔍**
 
 We recommend you also run the linter before pushing code.