From 3432d35af6077b46de34bc32af2d15561084ff2d Mon Sep 17 00:00:00 2001
From: msramalho <19508417+msramalho@users.noreply.github.com>
Date: Thu, 17 Oct 2024 11:27:35 +0100
Subject: [PATCH 01/33] isolating api methods, introducing first tests

---
 .gitignore                        |    3 +-
 README.md                         |    2 +-
 docker-compose.dev.yml            |    2 -
 docker-compose.yml                |    3 +-
 src/Dockerfile                    |    4 +-
 src/Pipfile                       |    3 +-
 src/Pipfile.lock                  | 2920 +++++++++++++++++++----------
 src/core/__init__.py              |    0
 src/core/config.py                |   27 +
 src/core/events.py                |   44 +
 src/core/logging.py               |   15 +
 src/db/crud.py                    |    6 +-
 src/db/database.py                |   12 +-
 src/db/schemas.py                 |   14 +-
 src/endpoints/__init__.py         |    5 +
 src/endpoints/default.py          |   36 +
 src/endpoints/interoperability.py |   26 +
 src/endpoints/sheet.py            |   23 +
 src/endpoints/task.py             |   47 +
 src/endpoints/url.py              |   57 +
 src/main.py                       |  225 +--
 src/security.py                   |    8 +-
 src/static/favicon.ico            |  Bin 0 -> 95499 bytes
 src/utils/__init__.py             |    0
 src/utils/metrics.py              |   66 +
 src/worker.py                     |   15 +-
 26 files changed, 2340 insertions(+), 1223 deletions(-)
 create mode 100644 src/core/__init__.py
 create mode 100644 src/core/config.py
 create mode 100644 src/core/events.py
 create mode 100644 src/core/logging.py
 create mode 100644 src/endpoints/__init__.py
 create mode 100644 src/endpoints/default.py
 create mode 100644 src/endpoints/interoperability.py
 create mode 100644 src/endpoints/sheet.py
 create mode 100644 src/endpoints/task.py
 create mode 100644 src/endpoints/url.py
 create mode 100644 src/static/favicon.ico
 create mode 100644 src/utils/__init__.py
 create mode 100644 src/utils/metrics.py

diff --git a/.gitignore b/.gitignore
index cf49566..4311303 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,4 +13,5 @@ redis/data/*
 #temp
 tests
 src/user-groups.yaml
-wit*
\ No newline at end of file
+wit*
+src/crawls
\ No newline at end of file
diff --git a/README.md b/README.md
index 50b5f56..013f455 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
 # Auto Archiver API
 
-An api that uses celery workers to process URL archive requests via [bellingcat/auto-archiver](https://github.com/bellingcat/auto-archiver), it allows authentication via Google OAuth Apps an d enables CORS, everything runs on docker but development can be done without docker (except for redis).
+An api that uses celery workers to process URL archive requests via [bellingcat/auto-archiver](https://github.com/bellingcat/auto-archiver), it allows authentication via Google OAuth Apps and enables CORS, everything runs on docker but development can be done without docker (except for redis).
 
 
 ## Development
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index ba01011..aa15f48 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -1,5 +1,3 @@
-version: '3.8'
-
 services:
   web:
     restart: "no"
diff --git a/docker-compose.yml b/docker-compose.yml
index ce2413e..6549b28 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -9,8 +9,6 @@ x-base-setup: &base-setup
     CELERY_BROKER_URL: *broker-url
     CELERY_RESULT_BACKEND: *broker-url
 
-version: '3.8'
-
 volumes:
   crawls:
 
@@ -34,6 +32,7 @@ services:
       - /var/run/docker.sock:/var/run/docker.sock
       - crawls:/crawls # BROWSERTRIX_HOME_HOST:BROWSERTRIX_HOME_CONTAINER, do not change /crawls
     environment:
+      # celery broker-url needs to be duplicated here, do not remove
       CELERY_BROKER_URL: *broker-url
       CELERY_RESULT_BACKEND: *broker-url
       WACZ_ENABLE_DOCKER: 1 # Enable calling docker from this container
diff --git a/src/Dockerfile b/src/Dockerfile
index bd44d70..5bab89e 100644
--- a/src/Dockerfile
+++ b/src/Dockerfile
@@ -7,8 +7,8 @@ WORKDIR /app
 RUN curl -fsSL https://get.docker.com -o get-docker.sh && \
     sh get-docker.sh
 # set environment variables
-ENV PYTHONUNBUFFERED 1
-ENV PYTHONDONTWRITEBYTECODE 1
+ENV PYTHONUNBUFFERED=1
+ENV PYTHONDONTWRITEBYTECODE=1
 
 # install dependencies
 RUN pip install --upgrade pip && \
diff --git a/src/Pipfile b/src/Pipfile
index 9af8ba1..32e26d4 100644
--- a/src/Pipfile
+++ b/src/Pipfile
@@ -23,7 +23,8 @@ auto-archiver = "*"
 
 [dev-packages]
 watchdog = "*"
-pytest = "==6.2.4"
+pytest = "*"
+httpx = "*"
 
 [requires]
 python_version = "3.10"
diff --git a/src/Pipfile.lock b/src/Pipfile.lock
index ab5bd79..1d2a301 100644
--- a/src/Pipfile.lock
+++ b/src/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "40a080ecb6d794f1a9964a4a3a53271bbb462feaca2fa1206484d7167da314ae"
+            "sha256": "d601bfa04bab584dcde6b8b7b619538bd03b7d2e13de53f802eacc6427703996"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -26,85 +26,85 @@
         },
         "aiohttp": {
             "hashes": [
-                "sha256:0605cc2c0088fcaae79f01c913a38611ad09ba68ff482402d3410bf59039bfb8",
-                "sha256:0a158704edf0abcac8ac371fbb54044f3270bdbc93e254a82b6c82be1ef08f3c",
-                "sha256:0cbf56238f4bbf49dab8c2dc2e6b1b68502b1e88d335bea59b3f5b9f4c001475",
-                "sha256:1732102949ff6087589408d76cd6dea656b93c896b011ecafff418c9661dc4ed",
-                "sha256:18f634d540dd099c262e9f887c8bbacc959847cfe5da7a0e2e1cf3f14dbf2daf",
-                "sha256:239f975589a944eeb1bad26b8b140a59a3a320067fb3cd10b75c3092405a1372",
-                "sha256:2faa61a904b83142747fc6a6d7ad8fccff898c849123030f8e75d5d967fd4a81",
-                "sha256:320e8618eda64e19d11bdb3bd04ccc0a816c17eaecb7e4945d01deee2a22f95f",
-                "sha256:38d80498e2e169bc61418ff36170e0aad0cd268da8b38a17c4cf29d254a8b3f1",
-                "sha256:3916c8692dbd9d55c523374a3b8213e628424d19116ac4308e434dbf6d95bbdd",
-                "sha256:393c7aba2b55559ef7ab791c94b44f7482a07bf7640d17b341b79081f5e5cd1a",
-                "sha256:3b7b30258348082826d274504fbc7c849959f1989d86c29bc355107accec6cfb",
-                "sha256:3fcb4046d2904378e3aeea1df51f697b0467f2aac55d232c87ba162709478c46",
-                "sha256:4109adee842b90671f1b689901b948f347325045c15f46b39797ae1bf17019de",
-                "sha256:4558e5012ee03d2638c681e156461d37b7a113fe13970d438d95d10173d25f78",
-                "sha256:45731330e754f5811c314901cebdf19dd776a44b31927fa4b4dbecab9e457b0c",
-                "sha256:4715a9b778f4293b9f8ae7a0a7cef9829f02ff8d6277a39d7f40565c737d3771",
-                "sha256:471f0ef53ccedec9995287f02caf0c068732f026455f07db3f01a46e49d76bbb",
-                "sha256:4d3ebb9e1316ec74277d19c5f482f98cc65a73ccd5430540d6d11682cd857430",
-                "sha256:4ff550491f5492ab5ed3533e76b8567f4b37bd2995e780a1f46bca2024223233",
-                "sha256:52c27110f3862a1afbcb2af4281fc9fdc40327fa286c4625dfee247c3ba90156",
-                "sha256:55b39c8684a46e56ef8c8d24faf02de4a2b2ac60d26cee93bc595651ff545de9",
-                "sha256:5a7ee16aab26e76add4afc45e8f8206c95d1d75540f1039b84a03c3b3800dd59",
-                "sha256:5ca51eadbd67045396bc92a4345d1790b7301c14d1848feaac1d6a6c9289e888",
-                "sha256:5d6b3f1fabe465e819aed2c421a6743d8debbde79b6a8600739300630a01bf2c",
-                "sha256:60cdbd56f4cad9f69c35eaac0fbbdf1f77b0ff9456cebd4902f3dd1cf096464c",
-                "sha256:6380c039ec52866c06d69b5c7aad5478b24ed11696f0e72f6b807cfb261453da",
-                "sha256:639d0042b7670222f33b0028de6b4e2fad6451462ce7df2af8aee37dcac55424",
-                "sha256:66331d00fb28dc90aa606d9a54304af76b335ae204d1836f65797d6fe27f1ca2",
-                "sha256:67c3119f5ddc7261d47163ed86d760ddf0e625cd6246b4ed852e82159617b5fb",
-                "sha256:694d828b5c41255e54bc2dddb51a9f5150b4eefa9886e38b52605a05d96566e8",
-                "sha256:6ae79c1bc12c34082d92bf9422764f799aee4746fd7a392db46b7fd357d4a17a",
-                "sha256:702e2c7c187c1a498a4e2b03155d52658fdd6fda882d3d7fbb891a5cf108bb10",
-                "sha256:714d4e5231fed4ba2762ed489b4aec07b2b9953cf4ee31e9871caac895a839c0",
-                "sha256:7b179eea70833c8dee51ec42f3b4097bd6370892fa93f510f76762105568cf09",
-                "sha256:7f64cbd44443e80094309875d4f9c71d0401e966d191c3d469cde4642bc2e031",
-                "sha256:82a6a97d9771cb48ae16979c3a3a9a18b600a8505b1115cfe354dfb2054468b4",
-                "sha256:84dabd95154f43a2ea80deffec9cb44d2e301e38a0c9d331cc4aa0166fe28ae3",
-                "sha256:8676e8fd73141ded15ea586de0b7cda1542960a7b9ad89b2b06428e97125d4fa",
-                "sha256:88e311d98cc0bf45b62fc46c66753a83445f5ab20038bcc1b8a1cc05666f428a",
-                "sha256:8b4f72fbb66279624bfe83fd5eb6aea0022dad8eec62b71e7bf63ee1caadeafe",
-                "sha256:8c64a6dc3fe5db7b1b4d2b5cb84c4f677768bdc340611eca673afb7cf416ef5a",
-                "sha256:8cf142aa6c1a751fcb364158fd710b8a9be874b81889c2bd13aa8893197455e2",
-                "sha256:8d1964eb7617907c792ca00b341b5ec3e01ae8c280825deadbbd678447b127e1",
-                "sha256:93e22add827447d2e26d67c9ac0161756007f152fdc5210277d00a85f6c92323",
-                "sha256:9c69e77370cce2d6df5d12b4e12bdcca60c47ba13d1cbbc8645dd005a20b738b",
-                "sha256:9dbc053ac75ccc63dc3a3cc547b98c7258ec35a215a92bd9f983e0aac95d3d5b",
-                "sha256:9e3a1ae66e3d0c17cf65c08968a5ee3180c5a95920ec2731f53343fac9bad106",
-                "sha256:a6ea1a5b409a85477fd8e5ee6ad8f0e40bf2844c270955e09360418cfd09abac",
-                "sha256:a81b1143d42b66ffc40a441379387076243ef7b51019204fd3ec36b9f69e77d6",
-                "sha256:ad7f2919d7dac062f24d6f5fe95d401597fbb015a25771f85e692d043c9d7832",
-                "sha256:afc52b8d969eff14e069a710057d15ab9ac17cd4b6753042c407dcea0e40bf75",
-                "sha256:b3df71da99c98534be076196791adca8819761f0bf6e08e07fd7da25127150d6",
-                "sha256:c088c4d70d21f8ca5c0b8b5403fe84a7bc8e024161febdd4ef04575ef35d474d",
-                "sha256:c26959ca7b75ff768e2776d8055bf9582a6267e24556bb7f7bd29e677932be72",
-                "sha256:c413016880e03e69d166efb5a1a95d40f83d5a3a648d16486592c49ffb76d0db",
-                "sha256:c6021d296318cb6f9414b48e6a439a7f5d1f665464da507e8ff640848ee2a58a",
-                "sha256:c671dc117c2c21a1ca10c116cfcd6e3e44da7fcde37bf83b2be485ab377b25da",
-                "sha256:c7a4b7a6cf5b6eb11e109a9755fd4fda7d57395f8c575e166d363b9fc3ec4678",
-                "sha256:c8a02fbeca6f63cb1f0475c799679057fc9268b77075ab7cf3f1c600e81dd46b",
-                "sha256:cd2adf5c87ff6d8b277814a28a535b59e20bfea40a101db6b3bdca7e9926bc24",
-                "sha256:d1469f228cd9ffddd396d9948b8c9cd8022b6d1bf1e40c6f25b0fb90b4f893ed",
-                "sha256:d153f652a687a8e95ad367a86a61e8d53d528b0530ef382ec5aaf533140ed00f",
-                "sha256:d5ab8e1f6bee051a4bf6195e38a5c13e5e161cb7bad83d8854524798bd9fcd6e",
-                "sha256:da00da442a0e31f1c69d26d224e1efd3a1ca5bcbf210978a2ca7426dfcae9f58",
-                "sha256:da22dab31d7180f8c3ac7c7635f3bcd53808f374f6aa333fe0b0b9e14b01f91a",
-                "sha256:e0ae53e33ee7476dd3d1132f932eeb39bf6125083820049d06edcdca4381f342",
-                "sha256:e7a6a8354f1b62e15d48e04350f13e726fa08b62c3d7b8401c0a1314f02e3558",
-                "sha256:e9a3d838441bebcf5cf442700e3963f58b5c33f015341f9ea86dcd7d503c07e2",
-                "sha256:edea7d15772ceeb29db4aff55e482d4bcfb6ae160ce144f2682de02f6d693551",
-                "sha256:f22eb3a6c1080d862befa0a89c380b4dafce29dc6cd56083f630073d102eb595",
-                "sha256:f26383adb94da5e7fb388d441bf09c61e5e35f455a3217bfd790c6b6bc64b2ee",
-                "sha256:f3c2890ca8c59ee683fd09adf32321a40fe1cf164e3387799efb2acebf090c11",
-                "sha256:f64fd07515dad67f24b6ea4a66ae2876c01031de91c93075b8093f07c0a2d93d",
-                "sha256:fcde4c397f673fdec23e6b05ebf8d4751314fa7c24f93334bf1f1364c1c69ac7",
-                "sha256:ff84aeb864e0fac81f676be9f4685f0527b660f1efdc40dcede3c251ef1e867f"
+                "sha256:02ab6006ec3c3463b528374c4cdce86434e7b89ad355e7bf29e2f16b46c7dd6f",
+                "sha256:04fa38875e53eb7e354ece1607b1d2fdee2d175ea4e4d745f6ec9f751fe20c7c",
+                "sha256:0b0a6a36ed7e164c6df1e18ee47afbd1990ce47cb428739d6c99aaabfaf1b3af",
+                "sha256:0d406b01a9f5a7e232d1b0d161b40c05275ffbcbd772dc18c1d5a570961a1ca4",
+                "sha256:0e49b08eafa4f5707ecfb321ab9592717a319e37938e301d462f79b4e860c32a",
+                "sha256:0e7ba7ff228c0d9a2cd66194e90f2bca6e0abca810b786901a569c0de082f489",
+                "sha256:11cb254e397a82efb1805d12561e80124928e04e9c4483587ce7390b3866d213",
+                "sha256:11ff168d752cb41e8492817e10fb4f85828f6a0142b9726a30c27c35a1835f01",
+                "sha256:176df045597e674fa950bf5ae536be85699e04cea68fa3a616cf75e413737eb5",
+                "sha256:219a16763dc0294842188ac8a12262b5671817042b35d45e44fd0a697d8c8361",
+                "sha256:22698f01ff5653fe66d16ffb7658f582a0ac084d7da1323e39fd9eab326a1f26",
+                "sha256:237533179d9747080bcaad4d02083ce295c0d2eab3e9e8ce103411a4312991a0",
+                "sha256:289ba9ae8e88d0ba16062ecf02dd730b34186ea3b1e7489046fc338bdc3361c4",
+                "sha256:2c59e0076ea31c08553e868cec02d22191c086f00b44610f8ab7363a11a5d9d8",
+                "sha256:2c9376e2b09895c8ca8b95362283365eb5c03bdc8428ade80a864160605715f1",
+                "sha256:3135713c5562731ee18f58d3ad1bf41e1d8883eb68b363f2ffde5b2ea4b84cc7",
+                "sha256:3b9c7426923bb7bd66d409da46c41e3fb40f5caf679da624439b9eba92043fa6",
+                "sha256:3c0266cd6f005e99f3f51e583012de2778e65af6b73860038b968a0a8888487a",
+                "sha256:41473de252e1797c2d2293804e389a6d6986ef37cbb4a25208de537ae32141dd",
+                "sha256:4831df72b053b1eed31eb00a2e1aff6896fb4485301d4ccb208cac264b648db4",
+                "sha256:49f0c1b3c2842556e5de35f122fc0f0b721334ceb6e78c3719693364d4af8499",
+                "sha256:4b4c452d0190c5a820d3f5c0f3cd8a28ace48c54053e24da9d6041bf81113183",
+                "sha256:4ee8caa925aebc1e64e98432d78ea8de67b2272252b0a931d2ac3bd876ad5544",
+                "sha256:500f1c59906cd142d452074f3811614be04819a38ae2b3239a48b82649c08821",
+                "sha256:5216b6082c624b55cfe79af5d538e499cd5f5b976820eac31951fb4325974501",
+                "sha256:54311eb54f3a0c45efb9ed0d0a8f43d1bc6060d773f6973efd90037a51cd0a3f",
+                "sha256:54631fb69a6e44b2ba522f7c22a6fb2667a02fd97d636048478db2fd8c4e98fe",
+                "sha256:565760d6812b8d78d416c3c7cfdf5362fbe0d0d25b82fed75d0d29e18d7fc30f",
+                "sha256:598db66eaf2e04aa0c8900a63b0101fdc5e6b8a7ddd805c56d86efb54eb66672",
+                "sha256:5c4fa235d534b3547184831c624c0b7c1e262cd1de847d95085ec94c16fddcd5",
+                "sha256:69985d50a2b6f709412d944ffb2e97d0be154ea90600b7a921f95a87d6f108a2",
+                "sha256:69da0f3ed3496808e8cbc5123a866c41c12c15baaaead96d256477edf168eb57",
+                "sha256:6c93b7c2e52061f0925c3382d5cb8980e40f91c989563d3d32ca280069fd6a87",
+                "sha256:70907533db712f7aa791effb38efa96f044ce3d4e850e2d7691abd759f4f0ae0",
+                "sha256:81b77f868814346662c96ab36b875d7814ebf82340d3284a31681085c051320f",
+                "sha256:82eefaf1a996060602f3cc1112d93ba8b201dbf5d8fd9611227de2003dddb3b7",
+                "sha256:85c3e3c9cb1d480e0b9a64c658cd66b3cfb8e721636ab8b0e746e2d79a7a9eed",
+                "sha256:8a22a34bc594d9d24621091d1b91511001a7eea91d6652ea495ce06e27381f70",
+                "sha256:8cef8710fb849d97c533f259103f09bac167a008d7131d7b2b0e3a33269185c0",
+                "sha256:8d44e7bf06b0c0a70a20f9100af9fcfd7f6d9d3913e37754c12d424179b4e48f",
+                "sha256:8d7f98fde213f74561be1d6d3fa353656197f75d4edfbb3d94c9eb9b0fc47f5d",
+                "sha256:8d8e4450e7fe24d86e86b23cc209e0023177b6d59502e33807b732d2deb6975f",
+                "sha256:8fc49a87ac269d4529da45871e2ffb6874e87779c3d0e2ccd813c0899221239d",
+                "sha256:90ec72d231169b4b8d6085be13023ece8fa9b1bb495e4398d847e25218e0f431",
+                "sha256:91c742ca59045dce7ba76cab6e223e41d2c70d79e82c284a96411f8645e2afff",
+                "sha256:9b05d33ff8e6b269e30a7957bd3244ffbce2a7a35a81b81c382629b80af1a8bf",
+                "sha256:9b05d5cbe9dafcdc733262c3a99ccf63d2f7ce02543620d2bd8db4d4f7a22f83",
+                "sha256:9c5857612c9813796960c00767645cb5da815af16dafb32d70c72a8390bbf690",
+                "sha256:a34086c5cc285be878622e0a6ab897a986a6e8bf5b67ecb377015f06ed316587",
+                "sha256:ab221850108a4a063c5b8a70f00dd7a1975e5a1713f87f4ab26a46e5feac5a0e",
+                "sha256:b796b44111f0cab6bbf66214186e44734b5baab949cb5fb56154142a92989aeb",
+                "sha256:b8c3a67eb87394386847d188996920f33b01b32155f0a94f36ca0e0c635bf3e3",
+                "sha256:bcb6532b9814ea7c5a6a3299747c49de30e84472fa72821b07f5a9818bce0f66",
+                "sha256:bcc0ea8d5b74a41b621ad4a13d96c36079c81628ccc0b30cfb1603e3dfa3a014",
+                "sha256:bea94403a21eb94c93386d559bce297381609153e418a3ffc7d6bf772f59cc35",
+                "sha256:bff7e2811814fa2271be95ab6e84c9436d027a0e59665de60edf44e529a42c1f",
+                "sha256:c72444d17777865734aa1a4d167794c34b63e5883abb90356a0364a28904e6c0",
+                "sha256:c7b5d5d64e2a14e35a9240b33b89389e0035e6de8dbb7ffa50d10d8b65c57449",
+                "sha256:c7e939f1ae428a86e4abbb9a7c4732bf4706048818dfd979e5e2839ce0159f23",
+                "sha256:c88a15f272a0ad3d7773cf3a37cc7b7d077cbfc8e331675cf1346e849d97a4e5",
+                "sha256:c9110c06eaaac7e1f5562caf481f18ccf8f6fdf4c3323feab28a93d34cc646bd",
+                "sha256:ca7ca5abfbfe8d39e653870fbe8d7710be7a857f8a8386fc9de1aae2e02ce7e4",
+                "sha256:cae4c0c2ca800c793cae07ef3d40794625471040a87e1ba392039639ad61ab5b",
+                "sha256:cdefe289681507187e375a5064c7599f52c40343a8701761c802c1853a504558",
+                "sha256:cf2a0ac0615842b849f40c4d7f304986a242f1e68286dbf3bd7a835e4f83acfd",
+                "sha256:cfeadf42840c1e870dc2042a232a8748e75a36b52d78968cda6736de55582766",
+                "sha256:d737e69d193dac7296365a6dcb73bbbf53bb760ab25a3727716bbd42022e8d7a",
+                "sha256:d7481f581251bb5558ba9f635db70908819caa221fc79ee52a7f58392778c636",
+                "sha256:df9cf74b9bc03d586fc53ba470828d7b77ce51b0582d1d0b5b2fb673c0baa32d",
+                "sha256:e1f80197f8b0b846a8d5cf7b7ec6084493950d0882cc5537fb7b96a69e3c8590",
+                "sha256:ecca113f19d5e74048c001934045a2b9368d77b0b17691d905af18bd1c21275e",
+                "sha256:ee2527134f95e106cc1653e9ac78846f3a2ec1004cf20ef4e02038035a74544d",
+                "sha256:f27fdaadce22f2ef950fc10dcdf8048407c3b42b73779e48a4e76b3c35bca26c",
+                "sha256:f694dc8a6a3112059258a725a4ebe9acac5fe62f11c77ac4dcf896edfa78ca28",
+                "sha256:f800164276eec54e0af5c99feb9494c295118fc10a11b997bbb1348ba1a52065",
+                "sha256:ffcd828e37dc219a72c9012ec44ad2e7e3066bec6ff3aaa19e7d435dbf4032ca"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==3.9.5"
+            "version": "==3.9.1"
         },
         "aiosignal": {
             "hashes": [
@@ -123,14 +123,22 @@
             "markers": "python_version >= '3.8'",
             "version": "==0.20.0"
         },
+        "alabaster": {
+            "hashes": [
+                "sha256:75a8b99c28a5dad50dd7f8ccdd447a121ddb3892da9e53d1ca5cca3106d58d65",
+                "sha256:b46733c07dce03ae4e150330b975c75737fa60f0a7c591b6c8bf4928a28e2c92"
+            ],
+            "markers": "python_version >= '3.9'",
+            "version": "==0.7.16"
+        },
         "alembic": {
             "hashes": [
-                "sha256:2edcc97bed0bd3272611ce3a98d98279e9c209e7186e43e75bbb1b2bdfdbcc43",
-                "sha256:4932c8558bf68f2ee92b9bbcb8218671c627064d5b08939437af6d77dc05e595"
+                "sha256:203503117415561e203aa14541740643a611f641517f0209fcae63e9fa09f1a2",
+                "sha256:908e905976d15235fae59c9ac42c4c5b75cfcefe3d27c0fbf7ae15a37715d80e"
             ],
             "index": "pypi",
             "markers": "python_version >= '3.8'",
-            "version": "==1.13.1"
+            "version": "==1.13.3"
         },
         "amqp": {
             "hashes": [
@@ -140,13 +148,21 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
             "version": "==2.6.1"
         },
-        "anyio": {
+        "annotated-types": {
             "hashes": [
-                "sha256:048e05d0f6caeed70d731f3db756d35dcc1f35747c8c403364a8332c630441b8",
-                "sha256:f75253795a87df48568485fd18cdd2a3fa5c4f7c5be8e5e36637733fce06fed6"
+                "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53",
+                "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==4.3.0"
+            "version": "==0.7.0"
+        },
+        "anyio": {
+            "hashes": [
+                "sha256:5aadc6a1bbb7cdb0bede386cac5e2940f5e2ff3aa20277e991cf028e0585ce94",
+                "sha256:c1b2d8f46a8a812513012e1107cb0e68c17159a7a594208005a57dc776e1bdc7"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==4.4.0"
         },
         "argparse": {
             "hashes": [
@@ -167,7 +183,7 @@
                 "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f",
                 "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"
             ],
-            "markers": "python_version < '3.11'",
+            "markers": "python_version >= '3.7'",
             "version": "==4.0.3"
         },
         "attrs": {
@@ -180,28 +196,44 @@
         },
         "authlib": {
             "hashes": [
-                "sha256:959ea62a5b7b5123c5059758296122b57cd2585ae2ed1c0622c21b371ffdae06",
-                "sha256:9637e4de1fb498310a56900b3e2043a206b03cb11c05422014b0302cbc814be3"
+                "sha256:4b16130117f9eb82aa6eec97f6dd4673c3f960ac0283ccdae2897ee4bc030ba2",
+                "sha256:ede026a95e9f5cdc2d4364a52103f5405e75aa156357e831ef2bfd0bc5094dfc"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==1.3.0"
+            "version": "==1.3.2"
         },
         "auto-archiver": {
             "hashes": [
-                "sha256:046f31ca1614a22fc8871c87dc2db3c069b6513e59277d1eb3fc72e1bf531cf2",
-                "sha256:e931e8020eaf48b740e910183305e2e8a1b85ad96b7dba0487a314fccda340fd"
+                "sha256:3cee45b9a17feba214503eb1be4e8552e40cadbba128964585e0f53a45966fc8",
+                "sha256:b9f1fb490fc268462325ec3f3c97c425a9c62dd0a2b4e58c771b64e8d29f0a87"
             ],
             "index": "pypi",
+            "markers": "python_version >= '3.10'",
+            "version": "==0.12.0"
+        },
+        "babel": {
+            "hashes": [
+                "sha256:08706bdad8d0a3413266ab61bd6c34d0c28d6e1e7badf40a2cebe67644e2e1fb",
+                "sha256:8daf0e265d05768bc6c7a314cf1321e9a123afc328cc635c18622a2f30a04413"
+            ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.11.5"
+            "version": "==2.15.0"
+        },
+        "backports.tarfile": {
+            "hashes": [
+                "sha256:77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34",
+                "sha256:d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==1.2.0"
         },
         "beautifulsoup4": {
             "hashes": [
-                "sha256:74e3d1928edc070d21748185c46e3fb33490f22f52a3addee9aee0f4f7781051",
-                "sha256:b80878c9f40111313e55da8ba20bdba06d8fa3969fc68304167741bbf9e082ed"
+                "sha256:7e05ad0b6c26108d9990e2235e8a9b4e2c03ead6f391ceb60347f8ebea6b80ba",
+                "sha256:c684ddec071aa120819889aa9e8940f85c3f3cdaa08e23b9fa26510387897bd5"
             ],
             "markers": "python_full_version >= '3.6.0'",
-            "version": "==4.12.3"
+            "version": "==4.13.0b2"
         },
         "billiard": {
             "hashes": [
@@ -210,6 +242,42 @@
             ],
             "version": "==3.6.4.0"
         },
+        "black": {
+            "hashes": [
+                "sha256:257d724c2c9b1660f353b36c802ccece186a30accc7742c176d29c146df6e474",
+                "sha256:37aae07b029fa0174d39daf02748b379399b909652a806e5708199bd93899da1",
+                "sha256:415e686e87dbbe6f4cd5ef0fbf764af7b89f9057b97c908742b6008cc554b9c0",
+                "sha256:48a85f2cb5e6799a9ef05347b476cce6c182d6c71ee36925a6c194d074336ef8",
+                "sha256:7768a0dbf16a39aa5e9a3ded568bb545c8c2727396d063bbaf847df05b08cd96",
+                "sha256:7e122b1c4fb252fd85df3ca93578732b4749d9be076593076ef4d07a0233c3e1",
+                "sha256:88c57dc656038f1ab9f92b3eb5335ee9b021412feaa46330d5eba4e51fe49b04",
+                "sha256:8e537d281831ad0e71007dcdcbe50a71470b978c453fa41ce77186bbe0ed6021",
+                "sha256:98e123f1d5cfd42f886624d84464f7756f60ff6eab89ae845210631714f6db94",
+                "sha256:accf49e151c8ed2c0cdc528691838afd217c50412534e876a19270fea1e28e2d",
+                "sha256:b1530ae42e9d6d5b670a34db49a94115a64596bc77710b1d05e9801e62ca0a7c",
+                "sha256:b9176b9832e84308818a99a561e90aa479e73c523b3f77afd07913380ae2eab7",
+                "sha256:bdde6f877a18f24844e381d45e9947a49e97933573ac9d4345399be37621e26c",
+                "sha256:be8bef99eb46d5021bf053114442914baeb3649a89dc5f3a555c88737e5e98fc",
+                "sha256:bf10f7310db693bb62692609b397e8d67257c55f949abde4c67f9cc574492cc7",
+                "sha256:c872b53057f000085da66a19c55d68f6f8ddcac2642392ad3a355878406fbd4d",
+                "sha256:d36ed1124bb81b32f8614555b34cc4259c3fbc7eec17870e8ff8ded335b58d8c",
+                "sha256:da33a1a5e49c4122ccdfd56cd021ff1ebc4a1ec4e2d01594fef9b6f267a9e741",
+                "sha256:dd1b5a14e417189db4c7b64a6540f31730713d173f0b63e55fabd52d61d8fdce",
+                "sha256:e151054aa00bad1f4e1f04919542885f89f5f7d086b8a59e5000e6c616896ffb",
+                "sha256:eaea3008c281f1038edb473c1aa8ed8143a5535ff18f978a318f10302b254063",
+                "sha256:ef703f83fc32e131e9bcc0a5094cfe85599e7109f896fe8bc96cc402f3eb4b6e"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==24.4.2"
+        },
+        "bleach": {
+            "hashes": [
+                "sha256:1a1a85c1595e07d8db14c5f09f09e6433502c51c595970edc090551f0db99414",
+                "sha256:33c16e3353dbd13028ab4799a0f89a83f113405c766e9c122df8a06f5b85b3f4"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==6.0.0"
+        },
         "blinker": {
             "hashes": [
                 "sha256:1779309f71bf239144b9399d06ae925637cf6634cf6bd131104184531bf67c01",
@@ -220,19 +288,19 @@
         },
         "boto3": {
             "hashes": [
-                "sha256:b633e8fbf7145bdb995ce68a27d096bb89fd393185b0e773418d81cd78db5a03",
-                "sha256:f2c11635be0de7b7c06eb606ece1add125e02d6ed521592294a0a21af09af135"
+                "sha256:2bf7e7f376aee52155fc4ae4487f29333a6bcdf3a05c3bc4fede10b972d951a6",
+                "sha256:e74bc6d69c04ca611b7f58afe08e2ded6cb6504a4a80557b656abeefee395f88"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==1.34.105"
+            "version": "==1.35.41"
         },
         "botocore": {
             "hashes": [
-                "sha256:727d5d3e800ac8b705fca6e19b6fefa1e728a81d62a712df9bd32ed0117c740b",
-                "sha256:a459d060b541beecb50681e6e8a39313cca981e146a59ba7c5229d62f631a016"
+                "sha256:8a09a32136df8768190a6c92f0240cd59c30deb99c89026563efadbbed41fa00",
+                "sha256:915c4d81e3a0be3b793c1e2efdf19af1d0a9cd4a2d8de08ee18216c14d67764b"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==1.34.105"
+            "version": "==1.35.41"
         },
         "brotli": {
             "hashes": [
@@ -332,11 +400,11 @@
         },
         "cachetools": {
             "hashes": [
-                "sha256:0abad1021d3f8325b2fc1d2e9c8b9c9d57b04c3932657a72465447332c24d945",
-                "sha256:ba29e2dfa0b8b556606f097407ed1aa62080ee108ab0dc5ec9d6a723a007d105"
+                "sha256:02134e8439cdc2ffb62023ce1debca2944c3f289d66bb17ead3ab3dede74b292",
+                "sha256:2cc24fb4cbe39633fb7badd9db9ca6295d766d9c2995f245725a46715d050f2a"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==5.3.3"
+            "version": "==5.5.0"
         },
         "celery": {
             "hashes": [
@@ -349,11 +417,11 @@
         },
         "certifi": {
             "hashes": [
-                "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f",
-                "sha256:dc383c07b76109f368f6106eee2b593b04a011ea4d55f652c6ca24a754d1cdd1"
+                "sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b",
+                "sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==2024.2.2"
+            "version": "==2024.7.4"
         },
         "certvalidator": {
             "hashes": [
@@ -364,61 +432,76 @@
         },
         "cffi": {
             "hashes": [
-                "sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc",
-                "sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a",
-                "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417",
-                "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab",
-                "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520",
-                "sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36",
-                "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743",
-                "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8",
-                "sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed",
-                "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684",
-                "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56",
-                "sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324",
-                "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d",
-                "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235",
-                "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e",
-                "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088",
-                "sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000",
-                "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7",
-                "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e",
-                "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673",
-                "sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c",
-                "sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe",
-                "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2",
-                "sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098",
-                "sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8",
-                "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a",
-                "sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0",
-                "sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b",
-                "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896",
-                "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e",
-                "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9",
-                "sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2",
-                "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b",
-                "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6",
-                "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404",
-                "sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f",
-                "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0",
-                "sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4",
-                "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc",
-                "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936",
-                "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba",
-                "sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872",
-                "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb",
-                "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614",
-                "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1",
-                "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d",
-                "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969",
-                "sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b",
-                "sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4",
-                "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627",
-                "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956",
-                "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357"
+                "sha256:157cfe06e48356a7552e68cb73976a710f2620a5f9eb25a5fe7066cf71601b68",
+                "sha256:1c9f4df436f3780f2dbea2ff066cea0bb2f74425883bc5b098812768da2b34f7",
+                "sha256:1da24a9bf6fd9ab987a915887f0d3577d0a0b3946d582b776b380294dc5fce18",
+                "sha256:1db9f6fcf79e92ee2d193cd989dde4e1419193ff11eef4bcc00cb06293e22f4b",
+                "sha256:1fee79745f50734490d3358f9cb6578f57850bb61287256115dda2a513abe3c6",
+                "sha256:205051765f126c1480d1eaf6268c644262bae3ed610423f0783349f04e7f5a6b",
+                "sha256:22eac8f9c77df0899a6cd373d6a62da40644573a5e27982f7713bd2a9f0b0edf",
+                "sha256:2e5562c744d495f838dc0fbe9cd76cff27ebea0a2e747dd84dd8a7e47bcd3c8f",
+                "sha256:3113951a250b021d2092e870fe86cd4292a633a786f7ece67200663406409659",
+                "sha256:319ec248b55d34a49d7a43c48f2cf922b526e5ad2a3988887cc822a0c226b983",
+                "sha256:35bd512b1a16723b8c50665c3fe83c80789f7e5599c8f0721ef145552b6853e7",
+                "sha256:3745df375d5e66261295840fa219797251ff6a30afedfae650576ab2b10f43db",
+                "sha256:39b9131ed6c28f63132dee75d1fa5653436cb46fc7e6a097af29f32c7f5f8eca",
+                "sha256:3c4b0e03d0d9f3a31110994bf657076f3821ad1a88e2cdb7c3e43b4e4f96e7b0",
+                "sha256:3ea7190f834a5979e30bc4af334c031303a4f16f38216599645034751d683171",
+                "sha256:3f60cc0a65ac412887ba284c946242ed4e07065003b358a4d288334f6c2a54ed",
+                "sha256:475d2832950f9a65740aeb20d5baf6d84cf0d08a7063c8c6c407ec24cac41881",
+                "sha256:494abc4dc78792d210249127a75021049c7832468f9daa6e81ec0dfc1f55d9d0",
+                "sha256:4f17c3cfc4a7a53693bda38ac1631f30ceb2430f4a038550f5515728592ccd6f",
+                "sha256:58463f9a28f4357f4a07a94fbb0dca91486f6948f19a4971e0bedd6292ef0394",
+                "sha256:614afb2f32d5ea64a946643d798f3391d53bba868290e7433f4eaae7d1692e06",
+                "sha256:625eb8d8d377438cfbf64899e09969d20cd139019838a60644f05216f7c7767d",
+                "sha256:6a891c9e564527b4e65d65f87e3e989c3369329d04b39c49f279a91266287b85",
+                "sha256:6aff0256e080afb8964e091f94222c2808cdf7c5f13d58f88e799e2fbde53a9d",
+                "sha256:6bce1aa64c52c3cb0c7326dd81d1dc5a4831946b29721592983eb4ae80beb2ac",
+                "sha256:6df680dccdb5fcd257343532d5354c0059a6e5e4bc27b24a6a310cc51ba35a31",
+                "sha256:7249add87681d15f1a291e096f49350b28c44be958c5ca650d8c4dfbce3a458f",
+                "sha256:730a92dd144eb89f69c7b61ba4e6ac48ee6a01ba92f70c17e277c3e2c49b253d",
+                "sha256:752c6a06036a24b54936f488ad13b0a83b7d1e0f9fefbe3a4fc237676b1091cf",
+                "sha256:7953cd1968a8ea99482d7bfcf5bb9c56d56e91660b97ee940923394c8194d921",
+                "sha256:7e12962a21ba417611c7f9ae3e7f42d5354b68bf3c894af7796c171f6a965acf",
+                "sha256:84269088c987aa34045ee808b8a3c5f44397403f1afeff65429cd7c9e123dc01",
+                "sha256:85b997ce260a93010a72767c0f2f7c405524cada076792a9baad75cef435f293",
+                "sha256:8b77f45d5b938f8fa6d3087892458c57458f55a90410ce15c61585627930838b",
+                "sha256:8e7b261c3ea000b9a7c4fd40dd54ec3749d4592808025261d82e82f6457e8b7f",
+                "sha256:8fe736c2666e20090ae52af3b0297fb9273830f9d31f6041d7a8c7172fb6a566",
+                "sha256:94af5cfe8eb0d2742435458b8c8708aeb88f17fb48372bc4dacb87671e1ba867",
+                "sha256:98c7f31f55c4d0f9dba7da07bab8cd822cff6ac8dbea28ea8385e3a1e7074ac6",
+                "sha256:98e89b4eabb3f98c7882016cb4c498bded7882ad655f80d7a9d23043a1d12d43",
+                "sha256:98eaba1ed99a0a219cabe7d8bb716d9d87aeeb1b6f33792bcf84cc222c1a37b1",
+                "sha256:9b5cb07680e7d3c522733d14fbc0cac0660b597a2e33d8bbd305537b65eb6a51",
+                "sha256:9e39b8008534eedae1bde35d7cd5b71069f8aa7e6c079ae549a0de68299af43c",
+                "sha256:a23431415147e0c711742b4e273b362758e632bd11a1e676c58011f0ed96da42",
+                "sha256:a33648455eefb348b265bd10764833ab7d5f3811912a90dcefc00328d548da0d",
+                "sha256:a4b7e94db6e6bc2582fa540175384070edbd63c61103b182f57be3a958c0b7ad",
+                "sha256:a72748e56cd5edfc808c508da6e4b903225d1ed4c45463c28edf188ffea6d442",
+                "sha256:b3245d8073632f958cf239a198c0c3bed112a59d6ee2202e85367955b92794c6",
+                "sha256:b57fa5d8a1a2cc960613e0e578867d21a018f4405e9bad31c7b0af2b14004f2b",
+                "sha256:b6f35a638639298d4f9dca59db1f7568860ea179ace42318d658698850f2f540",
+                "sha256:b7cb4755dc605ac5f2cf0b00e4063fdc2ca474da7bdc473877f8b5cba133b43e",
+                "sha256:ba993bea9f3195dc2f8dd9e3739f97f41eac5d71f5804d1ef87ee1283a13a280",
+                "sha256:bf62263af2a3fadaf992775e0e555d657546dee30d3ca8a2ed1559c90006d46e",
+                "sha256:c207ccc9f2e459eab7952401dc9237e36d6b630b5020890736e6b18002a750f3",
+                "sha256:c82e1f580f3dd473a9d8b213071dfd8da07f7a433b04ba6be4773ada211d3fdb",
+                "sha256:ca0dd9cfb6a3fd91d6f1de5a2e2ee7a0f4b5b753309ec4edce32d5505dbc9149",
+                "sha256:cfc1d8a64c44544a01b06b1688dca70433dc47e2d46f462c9ee6dc02ab233ba8",
+                "sha256:d1089e9654cbbeb4e3ba84caa5eb0a92371fcac6ba43b14514680d339068abed",
+                "sha256:d50cef1600b59ec5669a28050286a456682443f20be9b0226c0fe5502860216e",
+                "sha256:e27ceb498d5a93f7fe833c5a3a85f8b9f0a4f1a182f1d37936e9ed31dda6926b",
+                "sha256:e3ae055e90ea13480185a1ef5325ebd9ac092e03f5f473be3e93eac62bfd43df",
+                "sha256:e547a347a983bda467ae8d8b607d278cdf8a37bea735399d655c82cba3f5d725",
+                "sha256:e6c686d93378b18a7b26bbb376dab75716a72bd95c04b7f2cff9094ac66a4582",
+                "sha256:ec95c379f5ebd92cd09e3e8183da9afee8c2da2544593fe091421ed2d757f3c1",
+                "sha256:f6e933e0118a97df454139ca84a28473a024429c7c1eb82619a56ef886b07583",
+                "sha256:f9155a5b35097cbe7a2e31611daf681b7119d895090bb101bf94805fb6bc7834",
+                "sha256:fa76f23281fd49c305002f510c773ecf6216118f2e7083b34ffa06983d6db96a",
+                "sha256:ffe885231b8b58f18149e9eaece2d556602aeb233161c069618bda31f3a30d04"
             ],
             "markers": "platform_python_implementation != 'PyPy'",
-            "version": "==1.16.0"
+            "version": "==1.17.0rc1"
         },
         "charset-normalizer": {
             "hashes": [
@@ -531,43 +614,119 @@
             ],
             "version": "==1.2.71"
         },
+        "colorama": {
+            "hashes": [
+                "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44",
+                "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'",
+            "version": "==0.4.6"
+        },
+        "commonmark": {
+            "hashes": [
+                "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60",
+                "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9"
+            ],
+            "version": "==0.9.1"
+        },
+        "coverage": {
+            "extras": [
+                "toml"
+            ],
+            "hashes": [
+                "sha256:0086cd4fc71b7d485ac93ca4239c8f75732c2ae3ba83f6be1c9be59d9e2c6382",
+                "sha256:01c322ef2bbe15057bc4bf132b525b7e3f7206f071799eb8aa6ad1940bcf5fb1",
+                "sha256:03cafe82c1b32b770a29fd6de923625ccac3185a54a5e66606da26d105f37dac",
+                "sha256:044a0985a4f25b335882b0966625270a8d9db3d3409ddc49a4eb00b0ef5e8cee",
+                "sha256:07ed352205574aad067482e53dd606926afebcb5590653121063fbf4e2175166",
+                "sha256:0d1b923fc4a40c5832be4f35a5dab0e5ff89cddf83bb4174499e02ea089daf57",
+                "sha256:0e7b27d04131c46e6894f23a4ae186a6a2207209a05df5b6ad4caee6d54a222c",
+                "sha256:1fad32ee9b27350687035cb5fdf9145bc9cf0a094a9577d43e909948ebcfa27b",
+                "sha256:289cc803fa1dc901f84701ac10c9ee873619320f2f9aff38794db4a4a0268d51",
+                "sha256:3c59105f8d58ce500f348c5b56163a4113a440dad6daa2294b5052a10db866da",
+                "sha256:46c3d091059ad0b9c59d1034de74a7f36dcfa7f6d3bde782c49deb42438f2450",
+                "sha256:482855914928c8175735a2a59c8dc5806cf7d8f032e4820d52e845d1f731dca2",
+                "sha256:49c76cdfa13015c4560702574bad67f0e15ca5a2872c6a125f6327ead2b731dd",
+                "sha256:4b03741e70fb811d1a9a1d75355cf391f274ed85847f4b78e35459899f57af4d",
+                "sha256:4bea27c4269234e06f621f3fac3925f56ff34bc14521484b8f66a580aacc2e7d",
+                "sha256:4d5fae0a22dc86259dee66f2cc6c1d3e490c4a1214d7daa2a93d07491c5c04b6",
+                "sha256:543ef9179bc55edfd895154a51792b01c017c87af0ebaae092720152e19e42ca",
+                "sha256:54dece71673b3187c86226c3ca793c5f891f9fc3d8aa183f2e3653da18566169",
+                "sha256:6379688fb4cfa921ae349c76eb1a9ab26b65f32b03d46bb0eed841fd4cb6afb1",
+                "sha256:65fa405b837060db569a61ec368b74688f429b32fa47a8929a7a2f9b47183713",
+                "sha256:6616d1c9bf1e3faea78711ee42a8b972367d82ceae233ec0ac61cc7fec09fa6b",
+                "sha256:6fe885135c8a479d3e37a7aae61cbd3a0fb2deccb4dda3c25f92a49189f766d6",
+                "sha256:7221f9ac9dad9492cecab6f676b3eaf9185141539d5c9689d13fd6b0d7de840c",
+                "sha256:76d5f82213aa78098b9b964ea89de4617e70e0d43e97900c2778a50856dac605",
+                "sha256:7792f0ab20df8071d669d929c75c97fecfa6bcab82c10ee4adb91c7a54055463",
+                "sha256:831b476d79408ab6ccfadaaf199906c833f02fdb32c9ab907b1d4aa0713cfa3b",
+                "sha256:9146579352d7b5f6412735d0f203bbd8d00113a680b66565e205bc605ef81bc6",
+                "sha256:9cc44bf0315268e253bf563f3560e6c004efe38f76db03a1558274a6e04bf5d5",
+                "sha256:a73d18625f6a8a1cbb11eadc1d03929f9510f4131879288e3f7922097a429f63",
+                "sha256:a8659fd33ee9e6ca03950cfdcdf271d645cf681609153f218826dd9805ab585c",
+                "sha256:a94925102c89247530ae1dab7dc02c690942566f22e189cbd53579b0693c0783",
+                "sha256:ad4567d6c334c46046d1c4c20024de2a1c3abc626817ae21ae3da600f5779b44",
+                "sha256:b2e16f4cd2bc4d88ba30ca2d3bbf2f21f00f382cf4e1ce3b1ddc96c634bc48ca",
+                "sha256:bbdf9a72403110a3bdae77948b8011f644571311c2fb35ee15f0f10a8fc082e8",
+                "sha256:beb08e8508e53a568811016e59f3234d29c2583f6b6e28572f0954a6b4f7e03d",
+                "sha256:c4cbe651f3904e28f3a55d6f371203049034b4ddbce65a54527a3f189ca3b390",
+                "sha256:c7b525ab52ce18c57ae232ba6f7010297a87ced82a2383b1afd238849c1ff933",
+                "sha256:ca5d79cfdae420a1d52bf177de4bc2289c321d6c961ae321503b2ca59c17ae67",
+                "sha256:cdab02a0a941af190df8782aafc591ef3ad08824f97850b015c8c6a8b3877b0b",
+                "sha256:d17c6a415d68cfe1091d3296ba5749d3d8696e42c37fca5d4860c5bf7b729f03",
+                "sha256:d39bd10f0ae453554798b125d2f39884290c480f56e8a02ba7a6ed552005243b",
+                "sha256:d4b3cd1ca7cd73d229487fa5caca9e4bc1f0bca96526b922d61053ea751fe791",
+                "sha256:d50a252b23b9b4dfeefc1f663c568a221092cbaded20a05a11665d0dbec9b8fb",
+                "sha256:da8549d17489cd52f85a9829d0e1d91059359b3c54a26f28bec2c5d369524807",
+                "sha256:dcd070b5b585b50e6617e8972f3fbbee786afca71b1936ac06257f7e178f00f6",
+                "sha256:ddaaa91bfc4477d2871442bbf30a125e8fe6b05da8a0015507bfbf4718228ab2",
+                "sha256:df423f351b162a702c053d5dddc0fc0ef9a9e27ea3f449781ace5f906b664428",
+                "sha256:dff044f661f59dace805eedb4a7404c573b6ff0cdba4a524141bc63d7be5c7fd",
+                "sha256:e7e128f85c0b419907d1f38e616c4f1e9f1d1b37a7949f44df9a73d5da5cd53c",
+                "sha256:ed8d1d1821ba5fc88d4a4f45387b65de52382fa3ef1f0115a4f7a20cdfab0e94",
+                "sha256:f2501d60d7497fd55e391f423f965bbe9e650e9ffc3c627d5f0ac516026000b8",
+                "sha256:f7db0b6ae1f96ae41afe626095149ecd1b212b424626175a6633c2999eaad45b"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==7.6.0"
+        },
         "cryptography": {
             "hashes": [
-                "sha256:02c0eee2d7133bdbbc5e24441258d5d2244beb31da5ed19fbb80315f4bbbff55",
-                "sha256:0d563795db98b4cd57742a78a288cdbdc9daedac29f2239793071fe114f13785",
-                "sha256:16268d46086bb8ad5bf0a2b5544d8a9ed87a0e33f5e77dd3c3301e63d941a83b",
-                "sha256:1a58839984d9cb34c855197043eaae2c187d930ca6d644612843b4fe8513c886",
-                "sha256:2954fccea107026512b15afb4aa664a5640cd0af630e2ee3962f2602693f0c82",
-                "sha256:2e47577f9b18723fa294b0ea9a17d5e53a227867a0a4904a1a076d1646d45ca1",
-                "sha256:31adb7d06fe4383226c3e963471f6837742889b3c4caa55aac20ad951bc8ffda",
-                "sha256:3577d029bc3f4827dd5bf8bf7710cac13527b470bbf1820a3f394adb38ed7d5f",
-                "sha256:36017400817987670037fbb0324d71489b6ead6231c9604f8fc1f7d008087c68",
-                "sha256:362e7197754c231797ec45ee081f3088a27a47c6c01eff2ac83f60f85a50fe60",
-                "sha256:3de9a45d3b2b7d8088c3fbf1ed4395dfeff79d07842217b38df14ef09ce1d8d7",
-                "sha256:4f698edacf9c9e0371112792558d2f705b5645076cc0aaae02f816a0171770fd",
-                "sha256:5482e789294854c28237bba77c4c83be698be740e31a3ae5e879ee5444166582",
-                "sha256:5e44507bf8d14b36b8389b226665d597bc0f18ea035d75b4e53c7b1ea84583cc",
-                "sha256:779245e13b9a6638df14641d029add5dc17edbef6ec915688f3acb9e720a5858",
-                "sha256:789caea816c6704f63f6241a519bfa347f72fbd67ba28d04636b7c6b7da94b0b",
-                "sha256:7f8b25fa616d8b846aef64b15c606bb0828dbc35faf90566eb139aa9cff67af2",
-                "sha256:8cb8ce7c3347fcf9446f201dc30e2d5a3c898d009126010cbd1f443f28b52678",
-                "sha256:93a3209f6bb2b33e725ed08ee0991b92976dfdcf4e8b38646540674fc7508e13",
-                "sha256:a3a5ac8b56fe37f3125e5b72b61dcde43283e5370827f5233893d461b7360cd4",
-                "sha256:a47787a5e3649008a1102d3df55424e86606c9bae6fb77ac59afe06d234605f8",
-                "sha256:a79165431551042cc9d1d90e6145d5d0d3ab0f2d66326c201d9b0e7f5bf43604",
-                "sha256:a987f840718078212fdf4504d0fd4c6effe34a7e4740378e59d47696e8dfb477",
-                "sha256:a9bc127cdc4ecf87a5ea22a2556cab6c7eda2923f84e4f3cc588e8470ce4e42e",
-                "sha256:bd13b5e9b543532453de08bcdc3cc7cebec6f9883e886fd20a92f26940fd3e7a",
-                "sha256:c65f96dad14f8528a447414125e1fc8feb2ad5a272b8f68477abbcc1ea7d94b9",
-                "sha256:d8e3098721b84392ee45af2dd554c947c32cc52f862b6a3ae982dbb90f577f14",
-                "sha256:e6b79d0adb01aae87e8a44c2b64bc3f3fe59515280e00fb6d57a7267a2583cda",
-                "sha256:e6b8f1881dac458c34778d0a424ae5769de30544fc678eac51c1c8bb2183e9da",
-                "sha256:e9b2a6309f14c0497f348d08a065d52f3020656f675819fc405fb63bbcd26562",
-                "sha256:ecbfbc00bf55888edda9868a4cf927205de8499e7fabe6c050322298382953f2",
-                "sha256:efd0bf5205240182e0f13bcaea41be4fdf5c22c5129fc7ced4a0282ac86998c9"
+                "sha256:013629ae70b40af70c9a7a5db40abe5d9054e6f4380e50ce769947b73bf3caad",
+                "sha256:2346b911eb349ab547076f47f2e035fc8ff2c02380a7cbbf8d87114fa0f1c583",
+                "sha256:2f66d9cd9147ee495a8374a45ca445819f8929a3efcd2e3df6428e46c3cbb10b",
+                "sha256:2f88d197e66c65be5e42cd72e5c18afbfae3f741742070e3019ac8f4ac57262c",
+                "sha256:31f721658a29331f895a5a54e7e82075554ccfb8b163a18719d342f5ffe5ecb1",
+                "sha256:343728aac38decfdeecf55ecab3264b015be68fc2816ca800db649607aeee648",
+                "sha256:5226d5d21ab681f432a9c1cf8b658c0cb02533eece706b155e5fbd8a0cdd3949",
+                "sha256:57080dee41209e556a9a4ce60d229244f7a66ef52750f813bfbe18959770cfba",
+                "sha256:5a94eccb2a81a309806027e1670a358b99b8fe8bfe9f8d329f27d72c094dde8c",
+                "sha256:6b7c4f03ce01afd3b76cf69a5455caa9cfa3de8c8f493e0d3ab7d20611c8dae9",
+                "sha256:7016f837e15b0a1c119d27ecd89b3515f01f90a8615ed5e9427e30d9cdbfed3d",
+                "sha256:81884c4d096c272f00aeb1f11cf62ccd39763581645b0812e99a91505fa48e0c",
+                "sha256:81d8a521705787afe7a18d5bfb47ea9d9cc068206270aad0b96a725022e18d2e",
+                "sha256:8d09d05439ce7baa8e9e95b07ec5b6c886f548deb7e0f69ef25f64b3bce842f2",
+                "sha256:961e61cefdcb06e0c6d7e3a1b22ebe8b996eb2bf50614e89384be54c48c6b63d",
+                "sha256:9c0c1716c8447ee7dbf08d6db2e5c41c688544c61074b54fc4564196f55c25a7",
+                "sha256:a0608251135d0e03111152e41f0cc2392d1e74e35703960d4190b2e0f4ca9c70",
+                "sha256:a0c5b2b0585b6af82d7e385f55a8bc568abff8923af147ee3c07bd8b42cda8b2",
+                "sha256:ad803773e9df0b92e0a817d22fd8a3675493f690b96130a5e24f1b8fabbea9c7",
+                "sha256:b297f90c5723d04bcc8265fc2a0f86d4ea2e0f7ab4b6994459548d3a6b992a14",
+                "sha256:ba4f0a211697362e89ad822e667d8d340b4d8d55fae72cdd619389fb5912eefe",
+                "sha256:c4783183f7cb757b73b2ae9aed6599b96338eb957233c58ca8f49a49cc32fd5e",
+                "sha256:c9bb2ae11bfbab395bdd072985abde58ea9860ed84e59dbc0463a5d0159f5b71",
+                "sha256:cafb92b2bc622cd1aa6a1dce4b93307792633f4c5fe1f46c6b97cf67073ec961",
+                "sha256:d45b940883a03e19e944456a558b67a41160e367a719833c53de6911cabba2b7",
+                "sha256:dc0fdf6787f37b1c6b08e6dfc892d9d068b5bdb671198c72072828b80bd5fe4c",
+                "sha256:dea567d1b0e8bc5764b9443858b673b734100c2871dc93163f58c46a97a83d28",
+                "sha256:dec9b018df185f08483f294cae6ccac29e7a6e0678996587363dc352dc65c842",
+                "sha256:e3ec3672626e1b9e55afd0df6d774ff0e953452886e06e0f1eb7eb0c832e8902",
+                "sha256:e599b53fd95357d92304510fb7bda8523ed1f79ca98dce2f43c115950aa78801",
+                "sha256:fa76fbb7596cc5839320000cdd5d0955313696d9511debab7ee7278fc8b5c84a",
+                "sha256:fff12c88a672ab9c9c1cf7b0c80e3ad9e2ebd9d828d955c126be4fd3e5578c9e"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==42.0.7"
+            "version": "==42.0.8"
         },
         "dataclasses-json": {
             "hashes": [
@@ -585,31 +744,39 @@
             "markers": "python_version >= '3.7'",
             "version": "==1.2.0"
         },
+        "docutils": {
+            "hashes": [
+                "sha256:23010f129180089fbcd3bc08cfefccb3b890b0050e1ca00c867036e9d161b98c",
+                "sha256:679987caf361a7539d76e584cbeddc311e3aee937877c87346f31debc63e9d06"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==0.18.1"
+        },
         "exceptiongroup": {
             "hashes": [
-                "sha256:5258b9ed329c5bbdd31a309f53cbfb0b155341807f6ff7606a1e801a891b29ad",
-                "sha256:a4785e48b045528f5bfe627b6ad554ff32def154f42372786903b7abcfe1aa16"
+                "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b",
+                "sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc"
             ],
-            "markers": "python_version < '3.11'",
-            "version": "==1.2.1"
+            "markers": "python_version >= '3.7'",
+            "version": "==1.2.2"
         },
         "fastapi": {
             "hashes": [
-                "sha256:266775f0dcc95af9d3ef39bad55cff525329a931d5fd51930aadd4f428bf7ff3",
-                "sha256:87a1f6fb632a218222c5984be540055346a8f5d8a68e8f6fb647b1dc9934de4b"
+                "sha256:3995739e0b09fa12f984bce8fa9ae197b35d433750d3d312422d846e283697ee",
+                "sha256:61704c71286579cc5a598763905928f24ee98bfcc07aabe84cfefb98812bbc86"
             ],
             "index": "pypi",
             "markers": "python_version >= '3.8'",
-            "version": "==0.110.0"
+            "version": "==0.115.2"
         },
         "fastapi-utils": {
             "hashes": [
-                "sha256:0e6c7fc1870b80e681494957abf65d4f4f42f4c7f70005918e9181b22f1bd759",
-                "sha256:dd0be7dc7f03fa681b25487a206651d99f2330d5a567fb8ab6cb5f8a06a29360"
+                "sha256:074509405b02e2651dfe2d11862dd760bacc1a64508f3d8cc44e52a6dc1ed342",
+                "sha256:4fc4d6a10b5c5c3f2ec564d360fc1188507b911e4b06ee4d4c111906d7ddeef1"
             ],
             "index": "pypi",
-            "markers": "python_version >= '3.6' and python_version < '4.0'",
-            "version": "==0.2.1"
+            "markers": "python_version >= '3.7' and python_version < '4.0'",
+            "version": "==0.7.0"
         },
         "ffmpeg-python": {
             "hashes": [
@@ -620,11 +787,19 @@
         },
         "filelock": {
             "hashes": [
-                "sha256:43339835842f110ca7ae60f1e1c160714c5a6afd15a2873419ab185334975c0f",
-                "sha256:6ea72da3be9b8c82afd3edcf99f2fffbb5076335a5ae4d03248bb5b6c3eae78a"
+                "sha256:2082e5703d51fbf98ea75855d9d5527e33d8ff23099bec374a134febee6946b0",
+                "sha256:c249fbfcd5db47e5e2d6d62198e565475ee65e4831e2561c8e313fa7eb961435"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==3.14.0"
+            "version": "==3.16.1"
+        },
+        "flake8": {
+            "hashes": [
+                "sha256:2e416edcc62471a64cea09353f4e7bdba32aeb079b6e360554c659a122b1bc6a",
+                "sha256:48a07b626b55236e0fb4784ee69a465fbf59d79eec1f5b4785c3d3bc57d17aa5"
+            ],
+            "markers": "python_full_version >= '3.8.1'",
+            "version": "==7.1.0"
         },
         "flask": {
             "hashes": [
@@ -725,6 +900,14 @@
             "markers": "python_version >= '3.8'",
             "version": "==1.4.1"
         },
+        "furo": {
+            "hashes": [
+                "sha256:4ab2be254a2d5e52792d0ca793a12c35582dd09897228a6dd47885dabd5c9521",
+                "sha256:b99e7867a5cc833b2b34d7230631dd6558c7a29f93071fdbb5709634bb33c5a5"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==2023.3.27"
+        },
         "future": {
             "hashes": [
                 "sha256:929292d34f5872e70396626ef385ec22355a1fae8ad29e1a734c3e43f9fbc216",
@@ -735,27 +918,27 @@
         },
         "google-api-core": {
             "hashes": [
-                "sha256:8661eec4078c35428fd3f69a2c7ee29e342896b70f01d1a1cbcb334372dd6251",
-                "sha256:cf1b7c2694047886d2af1128a03ae99e391108a08804f87cfd35970e49c9cd10"
+                "sha256:4a152fd11a9f774ea606388d423b68aa7e6d6a0ffe4c8266f74979613ec09f81",
+                "sha256:6869eacb2a37720380ba5898312af79a4d30b8bca1548fb4093e0697dc4bdf5d"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.19.0"
+            "version": "==2.21.0"
         },
         "google-api-python-client": {
             "hashes": [
-                "sha256:984cc8cc8eb4923468b1926d2b8effc5b459a4dda3c845896eb87c153b28ef84",
-                "sha256:d50f7e2dfdbb7fc2732f6a0cba1c54d7bb676390679526c6bb628c901e43ec86"
+                "sha256:1a5232e9cfed8c201799d9327e4d44dc7ea7daa3c6e1627fca41aa201539c0da",
+                "sha256:b9d68c6b14ec72580d66001bd33c5816b78e2134b93ccc5cf8f624516b561750"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.129.0"
+            "version": "==2.149.0"
         },
         "google-auth": {
             "hashes": [
-                "sha256:672dff332d073227550ffc7457868ac4218d6c500b155fe6cc17d2b13602c360",
-                "sha256:d452ad095688cd52bae0ad6fafe027f6a6d6f560e810fec20914e17a09526415"
+                "sha256:25df55f327ef021de8be50bad0dfd4a916ad0de96da86cd05661c9297723ad3f",
+                "sha256:f4c64ed4e01e8e8b646ef34c018f8bf3338df0c8e37d8b3bba40e7f574a3278a"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.29.0"
+            "version": "==2.35.0"
         },
         "google-auth-httplib2": {
             "hashes": [
@@ -766,91 +949,106 @@
         },
         "google-auth-oauthlib": {
             "hashes": [
-                "sha256:292d2d3783349f2b0734a0a0207b1e1e322ac193c2c09d8f7c613fb7cc501ea8",
-                "sha256:297c1ce4cb13a99b5834c74a1fe03252e1e499716718b190f56bcb9c4abc4faf"
+                "sha256:2d58a27262d55aa1b87678c3ba7142a080098cbc2024f903c62355deb235d91f",
+                "sha256:afd0cad092a2eaa53cd8e8298557d6de1034c6cb4a740500b5357b648af97263"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==1.2.0"
+            "version": "==1.2.1"
         },
         "googleapis-common-protos": {
             "hashes": [
-                "sha256:17ad01b11d5f1d0171c06d3ba5c04c54474e883b66b949722b4938ee2694ef4e",
-                "sha256:ae45f75702f7c08b541f750854a678bd8f534a1a6bace6afe975f1d0a82d6632"
+                "sha256:2972e6c496f435b92590fd54045060867f3fe9be2c82ab148fc8885035479a63",
+                "sha256:334a29d07cddc3aa01dee4988f9afd9b2916ee2ff49d6b757155dc0d197852c0"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==1.63.0"
+            "version": "==1.65.0"
         },
         "greenlet": {
             "hashes": [
-                "sha256:01bc7ea167cf943b4c802068e178bbf70ae2e8c080467070d01bfa02f337ee67",
-                "sha256:0448abc479fab28b00cb472d278828b3ccca164531daab4e970a0458786055d6",
-                "sha256:086152f8fbc5955df88382e8a75984e2bb1c892ad2e3c80a2508954e52295257",
-                "sha256:098d86f528c855ead3479afe84b49242e174ed262456c342d70fc7f972bc13c4",
-                "sha256:149e94a2dd82d19838fe4b2259f1b6b9957d5ba1b25640d2380bea9c5df37676",
-                "sha256:1551a8195c0d4a68fac7a4325efac0d541b48def35feb49d803674ac32582f61",
-                "sha256:15d79dd26056573940fcb8c7413d84118086f2ec1a8acdfa854631084393efcc",
-                "sha256:1996cb9306c8595335bb157d133daf5cf9f693ef413e7673cb07e3e5871379ca",
-                "sha256:1a7191e42732df52cb5f39d3527217e7ab73cae2cb3694d241e18f53d84ea9a7",
-                "sha256:1ea188d4f49089fc6fb283845ab18a2518d279c7cd9da1065d7a84e991748728",
-                "sha256:1f672519db1796ca0d8753f9e78ec02355e862d0998193038c7073045899f305",
-                "sha256:2516a9957eed41dd8f1ec0c604f1cdc86758b587d964668b5b196a9db5bfcde6",
-                "sha256:2797aa5aedac23af156bbb5a6aa2cd3427ada2972c828244eb7d1b9255846379",
-                "sha256:2dd6e660effd852586b6a8478a1d244b8dc90ab5b1321751d2ea15deb49ed414",
-                "sha256:3ddc0f794e6ad661e321caa8d2f0a55ce01213c74722587256fb6566049a8b04",
-                "sha256:3ed7fb269f15dc662787f4119ec300ad0702fa1b19d2135a37c2c4de6fadfd4a",
-                "sha256:419b386f84949bf0e7c73e6032e3457b82a787c1ab4a0e43732898a761cc9dbf",
-                "sha256:43374442353259554ce33599da8b692d5aa96f8976d567d4badf263371fbe491",
-                "sha256:52f59dd9c96ad2fc0d5724107444f76eb20aaccb675bf825df6435acb7703559",
-                "sha256:57e8974f23e47dac22b83436bdcf23080ade568ce77df33159e019d161ce1d1e",
-                "sha256:5b51e85cb5ceda94e79d019ed36b35386e8c37d22f07d6a751cb659b180d5274",
-                "sha256:649dde7de1a5eceb258f9cb00bdf50e978c9db1b996964cd80703614c86495eb",
-                "sha256:64d7675ad83578e3fc149b617a444fab8efdafc9385471f868eb5ff83e446b8b",
-                "sha256:68834da854554926fbedd38c76e60c4a2e3198c6fbed520b106a8986445caaf9",
-                "sha256:6b66c9c1e7ccabad3a7d037b2bcb740122a7b17a53734b7d72a344ce39882a1b",
-                "sha256:70fb482fdf2c707765ab5f0b6655e9cfcf3780d8d87355a063547b41177599be",
-                "sha256:7170375bcc99f1a2fbd9c306f5be8764eaf3ac6b5cb968862cad4c7057756506",
-                "sha256:73a411ef564e0e097dbe7e866bb2dda0f027e072b04da387282b02c308807405",
-                "sha256:77457465d89b8263bca14759d7c1684df840b6811b2499838cc5b040a8b5b113",
-                "sha256:7f362975f2d179f9e26928c5b517524e89dd48530a0202570d55ad6ca5d8a56f",
-                "sha256:81bb9c6d52e8321f09c3d165b2a78c680506d9af285bfccbad9fb7ad5a5da3e5",
-                "sha256:881b7db1ebff4ba09aaaeae6aa491daeb226c8150fc20e836ad00041bcb11230",
-                "sha256:894393ce10ceac937e56ec00bb71c4c2f8209ad516e96033e4b3b1de270e200d",
-                "sha256:99bf650dc5d69546e076f413a87481ee1d2d09aaaaaca058c9251b6d8c14783f",
-                "sha256:9da2bd29ed9e4f15955dd1595ad7bc9320308a3b766ef7f837e23ad4b4aac31a",
-                "sha256:afaff6cf5200befd5cec055b07d1c0a5a06c040fe5ad148abcd11ba6ab9b114e",
-                "sha256:b1b5667cced97081bf57b8fa1d6bfca67814b0afd38208d52538316e9422fc61",
-                "sha256:b37eef18ea55f2ffd8f00ff8fe7c8d3818abd3e25fb73fae2ca3b672e333a7a6",
-                "sha256:b542be2440edc2d48547b5923c408cbe0fc94afb9f18741faa6ae970dbcb9b6d",
-                "sha256:b7dcbe92cc99f08c8dd11f930de4d99ef756c3591a5377d1d9cd7dd5e896da71",
-                "sha256:b7f009caad047246ed379e1c4dbcb8b020f0a390667ea74d2387be2998f58a22",
-                "sha256:bba5387a6975598857d86de9eac14210a49d554a77eb8261cc68b7d082f78ce2",
-                "sha256:c5e1536de2aad7bf62e27baf79225d0d64360d4168cf2e6becb91baf1ed074f3",
-                "sha256:c5ee858cfe08f34712f548c3c363e807e7186f03ad7a5039ebadb29e8c6be067",
-                "sha256:c9db1c18f0eaad2f804728c67d6c610778456e3e1cc4ab4bbd5eeb8e6053c6fc",
-                "sha256:d353cadd6083fdb056bb46ed07e4340b0869c305c8ca54ef9da3421acbdf6881",
-                "sha256:d46677c85c5ba00a9cb6f7a00b2bfa6f812192d2c9f7d9c4f6a55b60216712f3",
-                "sha256:d4d1ac74f5c0c0524e4a24335350edad7e5f03b9532da7ea4d3c54d527784f2e",
-                "sha256:d73a9fe764d77f87f8ec26a0c85144d6a951a6c438dfe50487df5595c6373eac",
-                "sha256:da70d4d51c8b306bb7a031d5cff6cc25ad253affe89b70352af5f1cb68e74b53",
-                "sha256:daf3cb43b7cf2ba96d614252ce1684c1bccee6b2183a01328c98d36fcd7d5cb0",
-                "sha256:dca1e2f3ca00b84a396bc1bce13dd21f680f035314d2379c4160c98153b2059b",
-                "sha256:dd4f49ae60e10adbc94b45c0b5e6a179acc1736cf7a90160b404076ee283cf83",
-                "sha256:e1f145462f1fa6e4a4ae3c0f782e580ce44d57c8f2c7aae1b6fa88c0b2efdb41",
-                "sha256:e3391d1e16e2a5a1507d83e4a8b100f4ee626e8eca43cf2cadb543de69827c4c",
-                "sha256:fcd2469d6a2cf298f198f0487e0a5b1a47a42ca0fa4dfd1b6862c999f018ebbf",
-                "sha256:fd096eb7ffef17c456cfa587523c5f92321ae02427ff955bebe9e3c63bc9f0da",
-                "sha256:fe754d231288e1e64323cfad462fcee8f0288654c10bdf4f603a39ed923bef33"
+                "sha256:0153404a4bb921f0ff1abeb5ce8a5131da56b953eda6e14b88dc6bbc04d2049e",
+                "sha256:03a088b9de532cbfe2ba2034b2b85e82df37874681e8c470d6fb2f8c04d7e4b7",
+                "sha256:04b013dc07c96f83134b1e99888e7a79979f1a247e2a9f59697fa14b5862ed01",
+                "sha256:05175c27cb459dcfc05d026c4232f9de8913ed006d42713cb8a5137bd49375f1",
+                "sha256:09fc016b73c94e98e29af67ab7b9a879c307c6731a2c9da0db5a7d9b7edd1159",
+                "sha256:0bbae94a29c9e5c7e4a2b7f0aae5c17e8e90acbfd3bf6270eeba60c39fce3563",
+                "sha256:0fde093fb93f35ca72a556cf72c92ea3ebfda3d79fc35bb19fbe685853869a83",
+                "sha256:1443279c19fca463fc33e65ef2a935a5b09bb90f978beab37729e1c3c6c25fe9",
+                "sha256:1776fd7f989fc6b8d8c8cb8da1f6b82c5814957264d1f6cf818d475ec2bf6395",
+                "sha256:1d3755bcb2e02de341c55b4fca7a745a24a9e7212ac953f6b3a48d117d7257aa",
+                "sha256:23f20bb60ae298d7d8656c6ec6db134bca379ecefadb0b19ce6f19d1f232a942",
+                "sha256:275f72decf9932639c1c6dd1013a1bc266438eb32710016a1c742df5da6e60a1",
+                "sha256:2846930c65b47d70b9d178e89c7e1a69c95c1f68ea5aa0a58646b7a96df12441",
+                "sha256:3319aa75e0e0639bc15ff54ca327e8dc7a6fe404003496e3c6925cd3142e0e22",
+                "sha256:346bed03fe47414091be4ad44786d1bd8bef0c3fcad6ed3dee074a032ab408a9",
+                "sha256:36b89d13c49216cadb828db8dfa6ce86bbbc476a82d3a6c397f0efae0525bdd0",
+                "sha256:37b9de5a96111fc15418819ab4c4432e4f3c2ede61e660b1e33971eba26ef9ba",
+                "sha256:396979749bd95f018296af156201d6211240e7a23090f50a8d5d18c370084dc3",
+                "sha256:3b2813dc3de8c1ee3f924e4d4227999285fd335d1bcc0d2be6dc3f1f6a318ec1",
+                "sha256:411f015496fec93c1c8cd4e5238da364e1da7a124bcb293f085bf2860c32c6f6",
+                "sha256:47da355d8687fd65240c364c90a31569a133b7b60de111c255ef5b606f2ae291",
+                "sha256:48ca08c771c268a768087b408658e216133aecd835c0ded47ce955381105ba39",
+                "sha256:4afe7ea89de619adc868e087b4d2359282058479d7cfb94970adf4b55284574d",
+                "sha256:4ce3ac6cdb6adf7946475d7ef31777c26d94bccc377e070a7986bd2d5c515467",
+                "sha256:4ead44c85f8ab905852d3de8d86f6f8baf77109f9da589cb4fa142bd3b57b475",
+                "sha256:54558ea205654b50c438029505def3834e80f0869a70fb15b871c29b4575ddef",
+                "sha256:5e06afd14cbaf9e00899fae69b24a32f2196c19de08fcb9f4779dd4f004e5e7c",
+                "sha256:62ee94988d6b4722ce0028644418d93a52429e977d742ca2ccbe1c4f4a792511",
+                "sha256:63e4844797b975b9af3a3fb8f7866ff08775f5426925e1e0bbcfe7932059a12c",
+                "sha256:6510bf84a6b643dabba74d3049ead221257603a253d0a9873f55f6a59a65f822",
+                "sha256:667a9706c970cb552ede35aee17339a18e8f2a87a51fba2ed39ceeeb1004798a",
+                "sha256:6ef9ea3f137e5711f0dbe5f9263e8c009b7069d8a1acea822bd5e9dae0ae49c8",
+                "sha256:7017b2be767b9d43cc31416aba48aab0d2309ee31b4dbf10a1d38fb7972bdf9d",
+                "sha256:7124e16b4c55d417577c2077be379514321916d5790fa287c9ed6f23bd2ffd01",
+                "sha256:73aaad12ac0ff500f62cebed98d8789198ea0e6f233421059fa68a5aa7220145",
+                "sha256:77c386de38a60d1dfb8e55b8c1101d68c79dfdd25c7095d51fec2dd800892b80",
+                "sha256:7876452af029456b3f3549b696bb36a06db7c90747740c5302f74a9e9fa14b13",
+                "sha256:7939aa3ca7d2a1593596e7ac6d59391ff30281ef280d8632fa03d81f7c5f955e",
+                "sha256:8320f64b777d00dd7ccdade271eaf0cad6636343293a25074cc5566160e4de7b",
+                "sha256:85f3ff71e2e60bd4b4932a043fbbe0f499e263c628390b285cb599154a3b03b1",
+                "sha256:8b8b36671f10ba80e159378df9c4f15c14098c4fd73a36b9ad715f057272fbef",
+                "sha256:93147c513fac16385d1036b7e5b102c7fbbdb163d556b791f0f11eada7ba65dc",
+                "sha256:935e943ec47c4afab8965954bf49bfa639c05d4ccf9ef6e924188f762145c0ff",
+                "sha256:94b6150a85e1b33b40b1464a3f9988dcc5251d6ed06842abff82e42632fac120",
+                "sha256:94ebba31df2aa506d7b14866fed00ac141a867e63143fe5bca82a8e503b36437",
+                "sha256:95ffcf719966dd7c453f908e208e14cde192e09fde6c7186c8f1896ef778d8cd",
+                "sha256:98884ecf2ffb7d7fe6bd517e8eb99d31ff7855a840fa6d0d63cd07c037f6a981",
+                "sha256:99cfaa2110534e2cf3ba31a7abcac9d328d1d9f1b95beede58294a60348fba36",
+                "sha256:9e8f8c9cb53cdac7ba9793c276acd90168f416b9ce36799b9b885790f8ad6c0a",
+                "sha256:a0dfc6c143b519113354e780a50381508139b07d2177cb6ad6a08278ec655798",
+                "sha256:b2795058c23988728eec1f36a4e5e4ebad22f8320c85f3587b539b9ac84128d7",
+                "sha256:b42703b1cf69f2aa1df7d1030b9d77d3e584a70755674d60e710f0af570f3761",
+                "sha256:b7cede291382a78f7bb5f04a529cb18e068dd29e0fb27376074b6d0317bf4dd0",
+                "sha256:b8a678974d1f3aa55f6cc34dc480169d58f2e6d8958895d68845fa4ab566509e",
+                "sha256:b8da394b34370874b4572676f36acabac172602abf054cbc4ac910219f3340af",
+                "sha256:c3a701fe5a9695b238503ce5bbe8218e03c3bcccf7e204e455e7462d770268aa",
+                "sha256:c4aab7f6381f38a4b42f269057aee279ab0fc7bf2e929e3d4abfae97b682a12c",
+                "sha256:ca9d0ff5ad43e785350894d97e13633a66e2b50000e8a183a50a88d834752d42",
+                "sha256:d0028e725ee18175c6e422797c407874da24381ce0690d6b9396c204c7f7276e",
+                "sha256:d21e10da6ec19b457b82636209cbe2331ff4306b54d06fa04b7c138ba18c8a81",
+                "sha256:d5e975ca70269d66d17dd995dafc06f1b06e8cb1ec1e9ed54c1d1e4a7c4cf26e",
+                "sha256:da7a9bff22ce038e19bf62c4dd1ec8391062878710ded0a845bcf47cc0200617",
+                "sha256:db32b5348615a04b82240cc67983cb315309e88d444a288934ee6ceaebcad6cc",
+                "sha256:dcc62f31eae24de7f8dce72134c8651c58000d3b1868e01392baea7c32c247de",
+                "sha256:dfc59d69fc48664bc693842bd57acfdd490acafda1ab52c7836e3fc75c90a111",
+                "sha256:e347b3bfcf985a05e8c0b7d462ba6f15b1ee1c909e2dcad795e49e91b152c383",
+                "sha256:e4d333e558953648ca09d64f13e6d8f0523fa705f51cae3f03b5983489958c70",
+                "sha256:ed10eac5830befbdd0c32f83e8aa6288361597550ba669b04c48f0f9a2c843c6",
+                "sha256:efc0f674aa41b92da8c49e0346318c6075d734994c3c4e4430b1c3f853e498e4",
+                "sha256:f1695e76146579f8c06c1509c7ce4dfe0706f49c6831a817ac04eebb2fd02011",
+                "sha256:f1d4aeb8891338e60d1ab6127af1fe45def5259def8094b9c7e34690c8858803",
+                "sha256:f406b22b7c9a9b4f8aa9d2ab13d6ae0ac3e85c9a809bd590ad53fed2bf70dc79",
+                "sha256:f6ff3b14f2df4c41660a7dec01045a045653998784bf8cfcb5a525bdffffbc8f"
             ],
-            "markers": "python_version >= '3' and platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32')))))",
-            "version": "==3.0.3"
+            "markers": "python_version < '3.13' and platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32')))))",
+            "version": "==3.1.1"
         },
         "gspread": {
             "hashes": [
-                "sha256:576b72b628b251d2ee41e02b982d3c714d511d2a5aa3a88e587ed9efc4d6e752",
-                "sha256:67aa3335cfcbb2625e41b53129433ba779f2890244a4c532624362ce5e022342"
+                "sha256:cf03627f44e9e03a0a3de241d1748709db22af4fc8c11a13aa389d0bce6053fd",
+                "sha256:d3b45ea70db9723ac04259bf2650881b0568b943fd04a7f161e88d97ab21bd29"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==6.1.0"
+            "version": "==6.1.3"
         },
         "h11": {
             "hashes": [
@@ -862,11 +1060,11 @@
         },
         "httpcore": {
             "hashes": [
-                "sha256:34a38e2f9291467ee3b44e89dd52615370e152954ba21721378a87b2960f7a61",
-                "sha256:421f18bac248b25d310f3cacd198d55b8e6125c107797b609ff9b7a6ba7991b5"
+                "sha256:27b59625743b85577a8c0e10e55b50b5368a4f2cfe8cc7bcfa9cf00829c2682f",
+                "sha256:73f6dbd6eb8c21bbf7ef8efad555481853f5f6acdeaff1edb0694289269ee17f"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==1.0.5"
+            "version": "==1.0.6"
         },
         "httplib2": {
             "hashes": [
@@ -878,19 +1076,19 @@
         },
         "httpx": {
             "hashes": [
-                "sha256:71d5465162c13681bff01ad59b2cc68dd838ea1f10e51574bac27103f00c91a5",
-                "sha256:a0cb88a46f32dc874e04ee956e4c2764aba2aa228f650b06788ba6bda2962ab5"
+                "sha256:7bb2708e112d8fdd7829cd4243970f0c223274051cb35ee80c03301ee29a3df0",
+                "sha256:f7c2be1d2f3c3c3160d441802406b206c2b76f5947b11115e6df10c6c65e66c2"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.27.0"
+            "version": "==0.27.2"
         },
         "humanize": {
             "hashes": [
-                "sha256:582a265c931c683a7e9b8ed9559089dea7edcf6cc95be39a3cbc2c5d5ac2bcfa",
-                "sha256:ce284a76d5b1377fd8836733b983bfb0b76f1aa1c090de2566fcf008d7f6ab16"
+                "sha256:b53caaec8532bcb2fff70c8826f904c35943f8cecaca29d272d9df38092736c0",
+                "sha256:e66f36020a2d5a974c504bd2555cf770621dbdbb6d82f94a6857c0b1ea2608be"
             ],
-            "markers": "python_version >= '3.8'",
-            "version": "==4.9.0"
+            "markers": "python_version >= '3.9'",
+            "version": "==4.11.0"
         },
         "idna": {
             "hashes": [
@@ -900,12 +1098,44 @@
             "markers": "python_version >= '3.5'",
             "version": "==3.7"
         },
-        "instaloader": {
+        "imagesize": {
             "hashes": [
-                "sha256:7478a1f0ed5c05911832c50cb19747243a461b5d434907f9fdb7d2d750d1b4f5"
+                "sha256:0d8d18d08f840c19d0ee7ca1fd82490fdc3729b7ac93f49870406ddde8ef8d8b",
+                "sha256:69150444affb9cb0d5cc5a92b3676f0b2fb7cd9ae39e947a5e11a36b4497cd4a"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==1.4.1"
+        },
+        "importlib-metadata": {
+            "hashes": [
+                "sha256:15584cf2b1bf449d98ff8a6ff1abef57bf20f3ac6454f431736cd3e660921b2f",
+                "sha256:188bd24e4c346d3f0a933f275c2fec67050326a856b9a359881d7c2a697e8812"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==4.11"
+            "version": "==8.0.0"
+        },
+        "iniconfig": {
+            "hashes": [
+                "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3",
+                "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==2.0.0"
+        },
+        "instaloader": {
+            "hashes": [
+                "sha256:36774ea1076eeb236f8782d221e3737f71ddc023042f0b13761429ef137f1133"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==4.13.1"
+        },
+        "isort": {
+            "hashes": [
+                "sha256:0ec8b74806e80fec33e6e7ba89d35e17b3eb1c4c74316ea44cf877cc26e8b118",
+                "sha256:cde11e804641edbe1b6b95d56582eb541f27eebc77864c6015545944bb0e9c76"
+            ],
+            "markers": "python_full_version >= '3.7.0'",
+            "version": "==6.0.0b2"
         },
         "itsdangerous": {
             "hashes": [
@@ -915,6 +1145,38 @@
             "markers": "python_version >= '3.8'",
             "version": "==2.2.0"
         },
+        "jaraco.classes": {
+            "hashes": [
+                "sha256:47a024b51d0239c0dd8c8540c6c7f484be3b8fcf0b2d85c13825780d3b3f3acd",
+                "sha256:f662826b6bed8cace05e7ff873ce0f9283b5c924470fe664fff1c2f00f581790"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==3.4.0"
+        },
+        "jaraco.context": {
+            "hashes": [
+                "sha256:3e16388f7da43d384a1a7cd3452e72e14732ac9fe459678773a3608a812bf266",
+                "sha256:c2f67165ce1f9be20f32f650f25d8edfc1646a8aeee48ae06fb35f90763576d2"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==5.3.0"
+        },
+        "jaraco.functools": {
+            "hashes": [
+                "sha256:3b24ccb921d6b593bdceb56ce14799204f473976e2a9d4b15b04d0f2c2326664",
+                "sha256:d33fa765374c0611b52f8b3a795f8900869aa88c84769d4d1746cd68fb28c3e8"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==4.0.1"
+        },
+        "jeepney": {
+            "hashes": [
+                "sha256:5efe48d255973902f6badc3ce55e2aa6c5c3b3bc642059ef3a91247bcfcc5806",
+                "sha256:c0a454ad016ca575060802ee4d590dd912e35c122fa04e70306de3d076cce755"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==0.8.0"
+        },
         "jinja2": {
             "hashes": [
                 "sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369",
@@ -940,6 +1202,14 @@
             "markers": "python_version >= '3.8'",
             "version": "==4.0.0"
         },
+        "keyring": {
+            "hashes": [
+                "sha256:2458681cdefc0dbc0b7eb6cf75d0b98e59f9ad9b2d4edd319d18f68bdca95e50",
+                "sha256:daaffd42dbda25ddafb1ad5fec4024e5bbcfe424597ca1ca452b299861e49f1b"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==25.2.1"
+        },
         "kombu": {
             "hashes": [
                 "sha256:be48cdffb54a2194d93ad6533d73f69408486483d189fe9f5990ee24255b0e0a",
@@ -948,6 +1218,13 @@
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
             "version": "==4.6.11"
         },
+        "livereload": {
+            "hashes": [
+                "sha256:776f2f865e59fde56490a56bcc6773b6917366bce0c267c60ee8aaf1a0959869",
+                "sha256:ad4ac6f53b2d62bb6ce1a5e6e96f1f00976a32348afedcb4b6d68df2a1d346e4"
+            ],
+            "version": "==2.6.3"
+        },
         "loguru": {
             "hashes": [
                 "sha256:003d71e3d3ed35f0f8984898359d65b79e5b21943f78af86aa5491210429b8eb",
@@ -959,163 +1236,163 @@
         },
         "lxml": {
             "hashes": [
-                "sha256:02437fb7308386867c8b7b0e5bc4cd4b04548b1c5d089ffb8e7b31009b961dc3",
-                "sha256:02f6a8eb6512fdc2fd4ca10a49c341c4e109aa6e9448cc4859af5b949622715a",
-                "sha256:05f8757b03208c3f50097761be2dea0aba02e94f0dc7023ed73a7bb14ff11eb0",
-                "sha256:06668e39e1f3c065349c51ac27ae430719d7806c026fec462e5693b08b95696b",
-                "sha256:07542787f86112d46d07d4f3c4e7c760282011b354d012dc4141cc12a68cef5f",
-                "sha256:08ea0f606808354eb8f2dfaac095963cb25d9d28e27edcc375d7b30ab01abbf6",
-                "sha256:0969e92af09c5687d769731e3f39ed62427cc72176cebb54b7a9d52cc4fa3b73",
-                "sha256:0a028b61a2e357ace98b1615fc03f76eb517cc028993964fe08ad514b1e8892d",
-                "sha256:0b3f5016e00ae7630a4b83d0868fca1e3d494c78a75b1c7252606a3a1c5fc2ad",
-                "sha256:13e69be35391ce72712184f69000cda04fc89689429179bc4c0ae5f0b7a8c21b",
-                "sha256:16a8326e51fcdffc886294c1e70b11ddccec836516a343f9ed0f82aac043c24a",
-                "sha256:19b4e485cd07b7d83e3fe3b72132e7df70bfac22b14fe4bf7a23822c3a35bff5",
-                "sha256:1a2569a1f15ae6c8c64108a2cd2b4a858fc1e13d25846be0666fc144715e32ab",
-                "sha256:1a7aca7964ac4bb07680d5c9d63b9d7028cace3e2d43175cb50bba8c5ad33316",
-                "sha256:1d8a701774dfc42a2f0b8ccdfe7dbc140500d1049e0632a611985d943fcf12df",
-                "sha256:1e275ea572389e41e8b039ac076a46cb87ee6b8542df3fff26f5baab43713bca",
-                "sha256:2304d3c93f2258ccf2cf7a6ba8c761d76ef84948d87bf9664e14d203da2cd264",
-                "sha256:23441e2b5339bc54dc949e9e675fa35efe858108404ef9aa92f0456929ef6fe8",
-                "sha256:23cfafd56887eaed93d07bc4547abd5e09d837a002b791e9767765492a75883f",
-                "sha256:28bf95177400066596cdbcfc933312493799382879da504633d16cf60bba735b",
-                "sha256:2eb2227ce1ff998faf0cd7fe85bbf086aa41dfc5af3b1d80867ecfe75fb68df3",
-                "sha256:2fb0ba3e8566548d6c8e7dd82a8229ff47bd8fb8c2da237607ac8e5a1b8312e5",
-                "sha256:303f540ad2dddd35b92415b74b900c749ec2010e703ab3bfd6660979d01fd4ed",
-                "sha256:339ee4a4704bc724757cd5dd9dc8cf4d00980f5d3e6e06d5847c1b594ace68ab",
-                "sha256:33ce9e786753743159799fdf8e92a5da351158c4bfb6f2db0bf31e7892a1feb5",
-                "sha256:343ab62e9ca78094f2306aefed67dcfad61c4683f87eee48ff2fd74902447726",
-                "sha256:34e17913c431f5ae01d8658dbf792fdc457073dcdfbb31dc0cc6ab256e664a8d",
-                "sha256:364d03207f3e603922d0d3932ef363d55bbf48e3647395765f9bfcbdf6d23632",
-                "sha256:38b67afb0a06b8575948641c1d6d68e41b83a3abeae2ca9eed2ac59892b36706",
-                "sha256:3a745cc98d504d5bd2c19b10c79c61c7c3df9222629f1b6210c0368177589fb8",
-                "sha256:3b019d4ee84b683342af793b56bb35034bd749e4cbdd3d33f7d1107790f8c472",
-                "sha256:3b6a30a9ab040b3f545b697cb3adbf3696c05a3a68aad172e3fd7ca73ab3c835",
-                "sha256:3d1e35572a56941b32c239774d7e9ad724074d37f90c7a7d499ab98761bd80cf",
-                "sha256:3d98de734abee23e61f6b8c2e08a88453ada7d6486dc7cdc82922a03968928db",
-                "sha256:453d037e09a5176d92ec0fd282e934ed26d806331a8b70ab431a81e2fbabf56d",
-                "sha256:45f9494613160d0405682f9eee781c7e6d1bf45f819654eb249f8f46a2c22545",
-                "sha256:4820c02195d6dfb7b8508ff276752f6b2ff8b64ae5d13ebe02e7667e035000b9",
-                "sha256:49095a38eb333aaf44c06052fd2ec3b8f23e19747ca7ec6f6c954ffea6dbf7be",
-                "sha256:4aefd911793b5d2d7a921233a54c90329bf3d4a6817dc465f12ffdfe4fc7b8fe",
-                "sha256:4bc6cb140a7a0ad1f7bc37e018d0ed690b7b6520ade518285dc3171f7a117905",
-                "sha256:4c30a2f83677876465f44c018830f608fa3c6a8a466eb223535035fbc16f3438",
-                "sha256:50127c186f191b8917ea2fb8b206fbebe87fd414a6084d15568c27d0a21d60db",
-                "sha256:50ccb5d355961c0f12f6cf24b7187dbabd5433f29e15147a67995474f27d1776",
-                "sha256:519895c99c815a1a24a926d5b60627ce5ea48e9f639a5cd328bda0515ea0f10c",
-                "sha256:54401c77a63cc7d6dc4b4e173bb484f28a5607f3df71484709fe037c92d4f0ed",
-                "sha256:546cf886f6242dff9ec206331209db9c8e1643ae642dea5fdbecae2453cb50fd",
-                "sha256:55ce6b6d803890bd3cc89975fca9de1dff39729b43b73cb15ddd933b8bc20484",
-                "sha256:56793b7a1a091a7c286b5f4aa1fe4ae5d1446fe742d00cdf2ffb1077865db10d",
-                "sha256:57f0a0bbc9868e10ebe874e9f129d2917750adf008fe7b9c1598c0fbbfdde6a6",
-                "sha256:5b8c041b6265e08eac8a724b74b655404070b636a8dd6d7a13c3adc07882ef30",
-                "sha256:5e097646944b66207023bc3c634827de858aebc226d5d4d6d16f0b77566ea182",
-                "sha256:60499fe961b21264e17a471ec296dcbf4365fbea611bf9e303ab69db7159ce61",
-                "sha256:610b5c77428a50269f38a534057444c249976433f40f53e3b47e68349cca1425",
-                "sha256:625e3ef310e7fa3a761d48ca7ea1f9d8718a32b1542e727d584d82f4453d5eeb",
-                "sha256:657a972f46bbefdbba2d4f14413c0d079f9ae243bd68193cb5061b9732fa54c1",
-                "sha256:69ab77a1373f1e7563e0fb5a29a8440367dec051da6c7405333699d07444f511",
-                "sha256:6a520b4f9974b0a0a6ed73c2154de57cdfd0c8800f4f15ab2b73238ffed0b36e",
-                "sha256:6d68ce8e7b2075390e8ac1e1d3a99e8b6372c694bbe612632606d1d546794207",
-                "sha256:6dcc3d17eac1df7859ae01202e9bb11ffa8c98949dcbeb1069c8b9a75917e01b",
-                "sha256:6dfdc2bfe69e9adf0df4915949c22a25b39d175d599bf98e7ddf620a13678585",
-                "sha256:739e36ef7412b2bd940f75b278749106e6d025e40027c0b94a17ef7968d55d56",
-                "sha256:7429e7faa1a60cad26ae4227f4dd0459efde239e494c7312624ce228e04f6391",
-                "sha256:74da9f97daec6928567b48c90ea2c82a106b2d500f397eeb8941e47d30b1ca85",
-                "sha256:74e4f025ef3db1c6da4460dd27c118d8cd136d0391da4e387a15e48e5c975147",
-                "sha256:75a9632f1d4f698b2e6e2e1ada40e71f369b15d69baddb8968dcc8e683839b18",
-                "sha256:76acba4c66c47d27c8365e7c10b3d8016a7da83d3191d053a58382311a8bf4e1",
-                "sha256:79d1fb9252e7e2cfe4de6e9a6610c7cbb99b9708e2c3e29057f487de5a9eaefa",
-                "sha256:7ce7ad8abebe737ad6143d9d3bf94b88b93365ea30a5b81f6877ec9c0dee0a48",
-                "sha256:7ed07b3062b055d7a7f9d6557a251cc655eed0b3152b76de619516621c56f5d3",
-                "sha256:8268cbcd48c5375f46e000adb1390572c98879eb4f77910c6053d25cc3ac2c67",
-                "sha256:875a3f90d7eb5c5d77e529080d95140eacb3c6d13ad5b616ee8095447b1d22e7",
-                "sha256:89feb82ca055af0fe797a2323ec9043b26bc371365847dbe83c7fd2e2f181c34",
-                "sha256:8a7e24cb69ee5f32e003f50e016d5fde438010c1022c96738b04fc2423e61706",
-                "sha256:8ab6a358d1286498d80fe67bd3d69fcbc7d1359b45b41e74c4a26964ca99c3f8",
-                "sha256:8b8df03a9e995b6211dafa63b32f9d405881518ff1ddd775db4e7b98fb545e1c",
-                "sha256:8cf85a6e40ff1f37fe0f25719aadf443686b1ac7652593dc53c7ef9b8492b115",
-                "sha256:8e8d351ff44c1638cb6e980623d517abd9f580d2e53bfcd18d8941c052a5a009",
-                "sha256:9164361769b6ca7769079f4d426a41df6164879f7f3568be9086e15baca61466",
-                "sha256:96e85aa09274955bb6bd483eaf5b12abadade01010478154b0ec70284c1b1526",
-                "sha256:981a06a3076997adf7c743dcd0d7a0415582661e2517c7d961493572e909aa1d",
-                "sha256:9cd5323344d8ebb9fb5e96da5de5ad4ebab993bbf51674259dbe9d7a18049525",
-                "sha256:9d6c6ea6a11ca0ff9cd0390b885984ed31157c168565702959c25e2191674a14",
-                "sha256:a02d3c48f9bb1e10c7788d92c0c7db6f2002d024ab6e74d6f45ae33e3d0288a3",
-                "sha256:a233bb68625a85126ac9f1fc66d24337d6e8a0f9207b688eec2e7c880f012ec0",
-                "sha256:a2f6a1bc2460e643785a2cde17293bd7a8f990884b822f7bca47bee0a82fc66b",
-                "sha256:a6d17e0370d2516d5bb9062c7b4cb731cff921fc875644c3d751ad857ba9c5b1",
-                "sha256:ab67ed772c584b7ef2379797bf14b82df9aa5f7438c5b9a09624dd834c1c1aaf",
-                "sha256:ac6540c9fff6e3813d29d0403ee7a81897f1d8ecc09a8ff84d2eea70ede1cdbf",
-                "sha256:ae4073a60ab98529ab8a72ebf429f2a8cc612619a8c04e08bed27450d52103c0",
-                "sha256:ae791f6bd43305aade8c0e22f816b34f3b72b6c820477aab4d18473a37e8090b",
-                "sha256:aef5474d913d3b05e613906ba4090433c515e13ea49c837aca18bde190853dff",
-                "sha256:b0b3f2df149efb242cee2ffdeb6674b7f30d23c9a7af26595099afaf46ef4e88",
-                "sha256:b128092c927eaf485928cec0c28f6b8bead277e28acf56800e972aa2c2abd7a2",
-                "sha256:b16db2770517b8799c79aa80f4053cd6f8b716f21f8aca962725a9565ce3ee40",
-                "sha256:b336b0416828022bfd5a2e3083e7f5ba54b96242159f83c7e3eebaec752f1716",
-                "sha256:b47633251727c8fe279f34025844b3b3a3e40cd1b198356d003aa146258d13a2",
-                "sha256:b537bd04d7ccd7c6350cdaaaad911f6312cbd61e6e6045542f781c7f8b2e99d2",
-                "sha256:b5e4ef22ff25bfd4ede5f8fb30f7b24446345f3e79d9b7455aef2836437bc38a",
-                "sha256:b74b9ea10063efb77a965a8d5f4182806fbf59ed068b3c3fd6f30d2ac7bee734",
-                "sha256:bb2dc4898180bea79863d5487e5f9c7c34297414bad54bcd0f0852aee9cfdb87",
-                "sha256:bbc4b80af581e18568ff07f6395c02114d05f4865c2812a1f02f2eaecf0bfd48",
-                "sha256:bcc98f911f10278d1daf14b87d65325851a1d29153caaf146877ec37031d5f36",
-                "sha256:be49ad33819d7dcc28a309b86d4ed98e1a65f3075c6acd3cd4fe32103235222b",
-                "sha256:bec4bd9133420c5c52d562469c754f27c5c9e36ee06abc169612c959bd7dbb07",
-                "sha256:c689d0d5381f56de7bd6966a4541bff6e08bf8d3871bbd89a0c6ab18aa699573",
-                "sha256:c7079d5eb1c1315a858bbf180000757db8ad904a89476653232db835c3114001",
-                "sha256:cb3942960f0beb9f46e2a71a3aca220d1ca32feb5a398656be934320804c0df9",
-                "sha256:cd9e78285da6c9ba2d5c769628f43ef66d96ac3085e59b10ad4f3707980710d3",
-                "sha256:cf2a978c795b54c539f47964ec05e35c05bd045db5ca1e8366988c7f2fe6b3ce",
-                "sha256:d14a0d029a4e176795cef99c056d58067c06195e0c7e2dbb293bf95c08f772a3",
-                "sha256:d237ba6664b8e60fd90b8549a149a74fcc675272e0e95539a00522e4ca688b04",
-                "sha256:d26a618ae1766279f2660aca0081b2220aca6bd1aa06b2cf73f07383faf48927",
-                "sha256:d28cb356f119a437cc58a13f8135ab8a4c8ece18159eb9194b0d269ec4e28083",
-                "sha256:d4ed0c7cbecde7194cd3228c044e86bf73e30a23505af852857c09c24e77ec5d",
-                "sha256:d83e2d94b69bf31ead2fa45f0acdef0757fa0458a129734f59f67f3d2eb7ef32",
-                "sha256:d8bbcd21769594dbba9c37d3c819e2d5847656ca99c747ddb31ac1701d0c0ed9",
-                "sha256:d9b342c76003c6b9336a80efcc766748a333573abf9350f4094ee46b006ec18f",
-                "sha256:dc911208b18842a3a57266d8e51fc3cfaccee90a5351b92079beed912a7914c2",
-                "sha256:dfa7c241073d8f2b8e8dbc7803c434f57dbb83ae2a3d7892dd068d99e96efe2c",
-                "sha256:e282aedd63c639c07c3857097fc0e236f984ceb4089a8b284da1c526491e3f3d",
-                "sha256:e290d79a4107d7d794634ce3e985b9ae4f920380a813717adf61804904dc4393",
-                "sha256:e3d9d13603410b72787579769469af730c38f2f25505573a5888a94b62b920f8",
-                "sha256:e481bba1e11ba585fb06db666bfc23dbe181dbafc7b25776156120bf12e0d5a6",
-                "sha256:e49b052b768bb74f58c7dda4e0bdf7b79d43a9204ca584ffe1fb48a6f3c84c66",
-                "sha256:eb00b549b13bd6d884c863554566095bf6fa9c3cecb2e7b399c4bc7904cb33b5",
-                "sha256:ec87c44f619380878bd49ca109669c9f221d9ae6883a5bcb3616785fa8f94c97",
-                "sha256:edcfa83e03370032a489430215c1e7783128808fd3e2e0a3225deee278585196",
-                "sha256:f11ae142f3a322d44513de1018b50f474f8f736bc3cd91d969f464b5bfef8836",
-                "sha256:f2a09f6184f17a80897172863a655467da2b11151ec98ba8d7af89f17bf63dae",
-                "sha256:f5b65529bb2f21ac7861a0e94fdbf5dc0daab41497d18223b46ee8515e5ad297",
-                "sha256:f60fdd125d85bf9c279ffb8e94c78c51b3b6a37711464e1f5f31078b45002421",
-                "sha256:f61efaf4bed1cc0860e567d2ecb2363974d414f7f1f124b1df368bbf183453a6",
-                "sha256:f90e552ecbad426eab352e7b2933091f2be77115bb16f09f78404861c8322981",
-                "sha256:f956196ef61369f1685d14dad80611488d8dc1ef00be57c0c5a03064005b0f30",
-                "sha256:fb91819461b1b56d06fa4bcf86617fac795f6a99d12239fb0c68dbeba41a0a30",
-                "sha256:fbc9d316552f9ef7bba39f4edfad4a734d3d6f93341232a9dddadec4f15d425f",
-                "sha256:ff69a9a0b4b17d78170c73abe2ab12084bdf1691550c5629ad1fe7849433f324",
-                "sha256:ffb2be176fed4457e445fe540617f0252a72a8bc56208fd65a690fdb1f57660b"
+                "sha256:01220dca0d066d1349bd6a1726856a78f7929f3878f7e2ee83c296c69495309e",
+                "sha256:02ced472497b8362c8e902ade23e3300479f4f43e45f4105c85ef43b8db85229",
+                "sha256:052d99051e77a4f3e8482c65014cf6372e61b0a6f4fe9edb98503bb5364cfee3",
+                "sha256:07da23d7ee08577760f0a71d67a861019103e4812c87e2fab26b039054594cc5",
+                "sha256:094cb601ba9f55296774c2d57ad68730daa0b13dc260e1f941b4d13678239e70",
+                "sha256:0a7056921edbdd7560746f4221dca89bb7a3fe457d3d74267995253f46343f15",
+                "sha256:0c120f43553ec759f8de1fee2f4794452b0946773299d44c36bfe18e83caf002",
+                "sha256:0d7b36afa46c97875303a94e8f3ad932bf78bace9e18e603f2085b652422edcd",
+                "sha256:0fdf3a3059611f7585a78ee10399a15566356116a4288380921a4b598d807a22",
+                "sha256:109fa6fede314cc50eed29e6e56c540075e63d922455346f11e4d7a036d2b8cf",
+                "sha256:146173654d79eb1fc97498b4280c1d3e1e5d58c398fa530905c9ea50ea849b22",
+                "sha256:1473427aff3d66a3fa2199004c3e601e6c4500ab86696edffdbc84954c72d832",
+                "sha256:1483fd3358963cc5c1c9b122c80606a3a79ee0875bcac0204149fa09d6ff2727",
+                "sha256:168f2dfcfdedf611eb285efac1516c8454c8c99caf271dccda8943576b67552e",
+                "sha256:17e8d968d04a37c50ad9c456a286b525d78c4a1c15dd53aa46c1d8e06bf6fa30",
+                "sha256:18feb4b93302091b1541221196a2155aa296c363fd233814fa11e181adebc52f",
+                "sha256:1afe0a8c353746e610bd9031a630a95bcfb1a720684c3f2b36c4710a0a96528f",
+                "sha256:1d04f064bebdfef9240478f7a779e8c5dc32b8b7b0b2fc6a62e39b928d428e51",
+                "sha256:1fdc9fae8dd4c763e8a31e7630afef517eab9f5d5d31a278df087f307bf601f4",
+                "sha256:1ffc23010330c2ab67fac02781df60998ca8fe759e8efde6f8b756a20599c5de",
+                "sha256:20094fc3f21ea0a8669dc4c61ed7fa8263bd37d97d93b90f28fc613371e7a875",
+                "sha256:213261f168c5e1d9b7535a67e68b1f59f92398dd17a56d934550837143f79c42",
+                "sha256:218c1b2e17a710e363855594230f44060e2025b05c80d1f0661258142b2add2e",
+                "sha256:23e0553b8055600b3bf4a00b255ec5c92e1e4aebf8c2c09334f8368e8bd174d6",
+                "sha256:25f1b69d41656b05885aa185f5fdf822cb01a586d1b32739633679699f220391",
+                "sha256:2b3778cb38212f52fac9fe913017deea2fdf4eb1a4f8e4cfc6b009a13a6d3fcc",
+                "sha256:2bc9fd5ca4729af796f9f59cd8ff160fe06a474da40aca03fcc79655ddee1a8b",
+                "sha256:2c226a06ecb8cdef28845ae976da407917542c5e6e75dcac7cc33eb04aaeb237",
+                "sha256:2c3406b63232fc7e9b8783ab0b765d7c59e7c59ff96759d8ef9632fca27c7ee4",
+                "sha256:2c86bf781b12ba417f64f3422cfc302523ac9cd1d8ae8c0f92a1c66e56ef2e86",
+                "sha256:2d9b8d9177afaef80c53c0a9e30fa252ff3036fb1c6494d427c066a4ce6a282f",
+                "sha256:2dec2d1130a9cda5b904696cec33b2cfb451304ba9081eeda7f90f724097300a",
+                "sha256:2dfab5fa6a28a0b60a20638dc48e6343c02ea9933e3279ccb132f555a62323d8",
+                "sha256:2ecdd78ab768f844c7a1d4a03595038c166b609f6395e25af9b0f3f26ae1230f",
+                "sha256:315f9542011b2c4e1d280e4a20ddcca1761993dda3afc7a73b01235f8641e903",
+                "sha256:36aef61a1678cb778097b4a6eeae96a69875d51d1e8f4d4b491ab3cfb54b5a03",
+                "sha256:384aacddf2e5813a36495233b64cb96b1949da72bef933918ba5c84e06af8f0e",
+                "sha256:3879cc6ce938ff4eb4900d901ed63555c778731a96365e53fadb36437a131a99",
+                "sha256:3c174dc350d3ec52deb77f2faf05c439331d6ed5e702fc247ccb4e6b62d884b7",
+                "sha256:3eb44520c4724c2e1a57c0af33a379eee41792595023f367ba3952a2d96c2aab",
+                "sha256:406246b96d552e0503e17a1006fd27edac678b3fcc9f1be71a2f94b4ff61528d",
+                "sha256:41ce1f1e2c7755abfc7e759dc34d7d05fd221723ff822947132dc934d122fe22",
+                "sha256:423b121f7e6fa514ba0c7918e56955a1d4470ed35faa03e3d9f0e3baa4c7e492",
+                "sha256:44264ecae91b30e5633013fb66f6ddd05c006d3e0e884f75ce0b4755b3e3847b",
+                "sha256:482c2f67761868f0108b1743098640fbb2a28a8e15bf3f47ada9fa59d9fe08c3",
+                "sha256:4b0c7a688944891086ba192e21c5229dea54382f4836a209ff8d0a660fac06be",
+                "sha256:4c1fefd7e3d00921c44dc9ca80a775af49698bbfd92ea84498e56acffd4c5469",
+                "sha256:4e109ca30d1edec1ac60cdbe341905dc3b8f55b16855e03a54aaf59e51ec8c6f",
+                "sha256:501d0d7e26b4d261fca8132854d845e4988097611ba2531408ec91cf3fd9d20a",
+                "sha256:516f491c834eb320d6c843156440fe7fc0d50b33e44387fcec5b02f0bc118a4c",
+                "sha256:51806cfe0279e06ed8500ce19479d757db42a30fd509940b1701be9c86a5ff9a",
+                "sha256:562e7494778a69086f0312ec9689f6b6ac1c6b65670ed7d0267e49f57ffa08c4",
+                "sha256:56b9861a71575f5795bde89256e7467ece3d339c9b43141dbdd54544566b3b94",
+                "sha256:5b8f5db71b28b8c404956ddf79575ea77aa8b1538e8b2ef9ec877945b3f46442",
+                "sha256:5c2fb570d7823c2bbaf8b419ba6e5662137f8166e364a8b2b91051a1fb40ab8b",
+                "sha256:5c54afdcbb0182d06836cc3d1be921e540be3ebdf8b8a51ee3ef987537455f84",
+                "sha256:5d6a6972b93c426ace71e0be9a6f4b2cfae9b1baed2eed2006076a746692288c",
+                "sha256:609251a0ca4770e5a8768ff902aa02bf636339c5a93f9349b48eb1f606f7f3e9",
+                "sha256:62d172f358f33a26d6b41b28c170c63886742f5b6772a42b59b4f0fa10526cb1",
+                "sha256:62f7fdb0d1ed2065451f086519865b4c90aa19aed51081979ecd05a21eb4d1be",
+                "sha256:658f2aa69d31e09699705949b5fc4719cbecbd4a97f9656a232e7d6c7be1a367",
+                "sha256:65ab5685d56914b9a2a34d67dd5488b83213d680b0c5d10b47f81da5a16b0b0e",
+                "sha256:68934b242c51eb02907c5b81d138cb977b2129a0a75a8f8b60b01cb8586c7b21",
+                "sha256:68b87753c784d6acb8a25b05cb526c3406913c9d988d51f80adecc2b0775d6aa",
+                "sha256:69959bd3167b993e6e710b99051265654133a98f20cec1d9b493b931942e9c16",
+                "sha256:6a7095eeec6f89111d03dabfe5883a1fd54da319c94e0fb104ee8f23616b572d",
+                "sha256:6b038cc86b285e4f9fea2ba5ee76e89f21ed1ea898e287dc277a25884f3a7dfe",
+                "sha256:6ba0d3dcac281aad8a0e5b14c7ed6f9fa89c8612b47939fc94f80b16e2e9bc83",
+                "sha256:6e91cf736959057f7aac7adfc83481e03615a8e8dd5758aa1d95ea69e8931dba",
+                "sha256:6ee8c39582d2652dcd516d1b879451500f8db3fe3607ce45d7c5957ab2596040",
+                "sha256:6f651ebd0b21ec65dfca93aa629610a0dbc13dbc13554f19b0113da2e61a4763",
+                "sha256:71a8dd38fbd2f2319136d4ae855a7078c69c9a38ae06e0c17c73fd70fc6caad8",
+                "sha256:74068c601baff6ff021c70f0935b0c7bc528baa8ea210c202e03757c68c5a4ff",
+                "sha256:7437237c6a66b7ca341e868cda48be24b8701862757426852c9b3186de1da8a2",
+                "sha256:747a3d3e98e24597981ca0be0fd922aebd471fa99d0043a3842d00cdcad7ad6a",
+                "sha256:74bcb423462233bc5d6066e4e98b0264e7c1bed7541fff2f4e34fe6b21563c8b",
+                "sha256:78d9b952e07aed35fe2e1a7ad26e929595412db48535921c5013edc8aa4a35ce",
+                "sha256:7b1cd427cb0d5f7393c31b7496419da594fe600e6fdc4b105a54f82405e6626c",
+                "sha256:7d3d1ca42870cdb6d0d29939630dbe48fa511c203724820fc0fd507b2fb46577",
+                "sha256:7e2f58095acc211eb9d8b5771bf04df9ff37d6b87618d1cbf85f92399c98dae8",
+                "sha256:7f41026c1d64043a36fda21d64c5026762d53a77043e73e94b71f0521939cc71",
+                "sha256:81b4e48da4c69313192d8c8d4311e5d818b8be1afe68ee20f6385d0e96fc9512",
+                "sha256:86a6b24b19eaebc448dc56b87c4865527855145d851f9fc3891673ff97950540",
+                "sha256:874a216bf6afaf97c263b56371434e47e2c652d215788396f60477540298218f",
+                "sha256:89e043f1d9d341c52bf2af6d02e6adde62e0a46e6755d5eb60dc6e4f0b8aeca2",
+                "sha256:8c72e9563347c7395910de6a3100a4840a75a6f60e05af5e58566868d5eb2d6a",
+                "sha256:8dc2c0395bea8254d8daebc76dcf8eb3a95ec2a46fa6fae5eaccee366bfe02ce",
+                "sha256:8f0de2d390af441fe8b2c12626d103540b5d850d585b18fcada58d972b74a74e",
+                "sha256:92e67a0be1639c251d21e35fe74df6bcc40cba445c2cda7c4a967656733249e2",
+                "sha256:94d6c3782907b5e40e21cadf94b13b0842ac421192f26b84c45f13f3c9d5dc27",
+                "sha256:97acf1e1fd66ab53dacd2c35b319d7e548380c2e9e8c54525c6e76d21b1ae3b1",
+                "sha256:9ada35dd21dc6c039259596b358caab6b13f4db4d4a7f8665764d616daf9cc1d",
+                "sha256:9c52100e2c2dbb0649b90467935c4b0de5528833c76a35ea1a2691ec9f1ee7a1",
+                "sha256:9e41506fec7a7f9405b14aa2d5c8abbb4dbbd09d88f9496958b6d00cb4d45330",
+                "sha256:9e4b47ac0f5e749cfc618efdf4726269441014ae1d5583e047b452a32e221920",
+                "sha256:9fb81d2824dff4f2e297a276297e9031f46d2682cafc484f49de182aa5e5df99",
+                "sha256:a0eabd0a81625049c5df745209dc7fcef6e2aea7793e5f003ba363610aa0a3ff",
+                "sha256:a3d819eb6f9b8677f57f9664265d0a10dd6551d227afb4af2b9cd7bdc2ccbf18",
+                "sha256:a87de7dd873bf9a792bf1e58b1c3887b9264036629a5bf2d2e6579fe8e73edff",
+                "sha256:aa617107a410245b8660028a7483b68e7914304a6d4882b5ff3d2d3eb5948d8c",
+                "sha256:aac0bbd3e8dd2d9c45ceb82249e8bdd3ac99131a32b4d35c8af3cc9db1657179",
+                "sha256:ab6dd83b970dc97c2d10bc71aa925b84788c7c05de30241b9e96f9b6d9ea3080",
+                "sha256:ace2c2326a319a0bb8a8b0e5b570c764962e95818de9f259ce814ee666603f19",
+                "sha256:ae5fe5c4b525aa82b8076c1a59d642c17b6e8739ecf852522c6321852178119d",
+                "sha256:b11a5d918a6216e521c715b02749240fb07ae5a1fefd4b7bf12f833bc8b4fe70",
+                "sha256:b1c8c20847b9f34e98080da785bb2336ea982e7f913eed5809e5a3c872900f32",
+                "sha256:b369d3db3c22ed14c75ccd5af429086f166a19627e84a8fdade3f8f31426e52a",
+                "sha256:b710bc2b8292966b23a6a0121f7a6c51d45d2347edcc75f016ac123b8054d3f2",
+                "sha256:bd96517ef76c8654446fc3db9242d019a1bb5fe8b751ba414765d59f99210b79",
+                "sha256:c00f323cc00576df6165cc9d21a4c21285fa6b9989c5c39830c3903dc4303ef3",
+                "sha256:c162b216070f280fa7da844531169be0baf9ccb17263cf5a8bf876fcd3117fa5",
+                "sha256:c1a69e58a6bb2de65902051d57fde951febad631a20a64572677a1052690482f",
+                "sha256:c1f794c02903c2824fccce5b20c339a1a14b114e83b306ff11b597c5f71a1c8d",
+                "sha256:c24037349665434f375645fa9d1f5304800cec574d0310f618490c871fd902b3",
+                "sha256:c300306673aa0f3ed5ed9372b21867690a17dba38c68c44b287437c362ce486b",
+                "sha256:c56a1d43b2f9ee4786e4658c7903f05da35b923fb53c11025712562d5cc02753",
+                "sha256:c6379f35350b655fd817cd0d6cbeef7f265f3ae5fedb1caae2eb442bbeae9ab9",
+                "sha256:c802e1c2ed9f0c06a65bc4ed0189d000ada8049312cfeab6ca635e39c9608957",
+                "sha256:cb83f8a875b3d9b458cada4f880fa498646874ba4011dc974e071a0a84a1b033",
+                "sha256:cf120cce539453ae086eacc0130a324e7026113510efa83ab42ef3fcfccac7fb",
+                "sha256:dd36439be765e2dde7660212b5275641edbc813e7b24668831a5c8ac91180656",
+                "sha256:dd5350b55f9fecddc51385463a4f67a5da829bc741e38cf689f38ec9023f54ab",
+                "sha256:df5c7333167b9674aa8ae1d4008fa4bc17a313cc490b2cca27838bbdcc6bb15b",
+                "sha256:e63601ad5cd8f860aa99d109889b5ac34de571c7ee902d6812d5d9ddcc77fa7d",
+                "sha256:e92ce66cd919d18d14b3856906a61d3f6b6a8500e0794142338da644260595cd",
+                "sha256:e99f5507401436fdcc85036a2e7dc2e28d962550afe1cbfc07c40e454256a859",
+                "sha256:ea2e2f6f801696ad7de8aec061044d6c8c0dd4037608c7cab38a9a4d316bfb11",
+                "sha256:eafa2c8658f4e560b098fe9fc54539f86528651f61849b22111a9b107d18910c",
+                "sha256:ecd4ad8453ac17bc7ba3868371bffb46f628161ad0eefbd0a855d2c8c32dd81a",
+                "sha256:ee70d08fd60c9565ba8190f41a46a54096afa0eeb8f76bd66f2c25d3b1b83005",
+                "sha256:eec1bb8cdbba2925bedc887bc0609a80e599c75b12d87ae42ac23fd199445654",
+                "sha256:ef0c1fe22171dd7c7c27147f2e9c3e86f8bdf473fed75f16b0c2e84a5030ce80",
+                "sha256:f2901429da1e645ce548bf9171784c0f74f0718c3f6150ce166be39e4dd66c3e",
+                "sha256:f422a209d2455c56849442ae42f25dbaaba1c6c3f501d58761c619c7836642ec",
+                "sha256:f65e5120863c2b266dbcc927b306c5b78e502c71edf3295dfcb9501ec96e5fc7",
+                "sha256:f7d4a670107d75dfe5ad080bed6c341d18c4442f9378c9f58e5851e86eb79965",
+                "sha256:f914c03e6a31deb632e2daa881fe198461f4d06e57ac3d0e05bbcab8eae01945",
+                "sha256:fb66442c2546446944437df74379e9cf9e9db353e61301d1a0e26482f43f0dd8"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==5.2.2"
+            "version": "==5.3.0"
         },
         "mako": {
             "hashes": [
-                "sha256:2a0c8ad7f6274271b3bb7467dd37cf9cc6dab4bc19cb69a4ef10669402de698e",
-                "sha256:32a99d70754dfce237019d17ffe4a282d2d3351b9c476e90d8a60e63f133b80c"
+                "sha256:260f1dbc3a519453a9c856dedfe4beb4e50bd5a26d96386cb6c80856556bb91a",
+                "sha256:48dbc20568c1d276a2698b36d968fa76161bf127194907ea6fc594fa81f943bc"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==1.3.2"
+            "version": "==1.3.5"
         },
         "markdown-it-py": {
             "hashes": [
-                "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1",
-                "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb"
+                "sha256:5a35f8d1870171d9acc47b99612dc146129b631baf04970128b568f190d0cc30",
+                "sha256:7c9a5e412688bc771c67432cbfebcdd686c93ce6484913dccf06cb5a0bea35a1"
             ],
-            "markers": "python_version >= '3.8'",
-            "version": "==3.0.0"
+            "markers": "python_version >= '3.7'",
+            "version": "==2.2.0"
         },
         "markupsafe": {
             "hashes": [
@@ -1185,11 +1462,27 @@
         },
         "marshmallow": {
             "hashes": [
-                "sha256:70b54a6282f4704d12c0a41599682c5c5450e843b9ec406308653b47c59648a1",
-                "sha256:82408deadd8b33d56338d2182d455db632c6313aa2af61916672146bb32edc56"
+                "sha256:4972f529104a220bb8637d595aa4c9762afbe7f7a77d82dc58c1615d70c5823e",
+                "sha256:71a2dce49ef901c3f97ed296ae5051135fd3febd2bf43afe0ae9a82143a494d9"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==3.21.2"
+            "version": "==3.22.0"
+        },
+        "mccabe": {
+            "hashes": [
+                "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325",
+                "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==0.7.0"
+        },
+        "mdit-py-plugins": {
+            "hashes": [
+                "sha256:ca9a0714ea59a24b2b044a1831f48d817dd0c817e84339f20e7889f392d77c4e",
+                "sha256:eee0adc7195e5827e17e02d2a258a2ba159944a0748f59c5099a4a27f78fcf6a"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==0.3.5"
         },
         "mdurl": {
             "hashes": [
@@ -1230,101 +1523,93 @@
             ],
             "version": "==0.15.0"
         },
+        "more-itertools": {
+            "hashes": [
+                "sha256:e5d93ef411224fbcef366a6e8ddc4c5781bc6359d43412a65dd5964e46111463",
+                "sha256:ea6a02e24a9161e51faad17a8782b92a0df82c12c1c8886fec7f0c3fa1a1b320"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==10.3.0"
+        },
         "multidict": {
             "hashes": [
-                "sha256:01265f5e40f5a17f8241d52656ed27192be03bfa8764d88e8220141d1e4b3556",
-                "sha256:0275e35209c27a3f7951e1ce7aaf93ce0d163b28948444bec61dd7badc6d3f8c",
-                "sha256:04bde7a7b3de05732a4eb39c94574db1ec99abb56162d6c520ad26f83267de29",
-                "sha256:04da1bb8c8dbadf2a18a452639771951c662c5ad03aefe4884775454be322c9b",
-                "sha256:09a892e4a9fb47331da06948690ae38eaa2426de97b4ccbfafbdcbe5c8f37ff8",
-                "sha256:0d63c74e3d7ab26de115c49bffc92cc77ed23395303d496eae515d4204a625e7",
-                "sha256:107c0cdefe028703fb5dafe640a409cb146d44a6ae201e55b35a4af8e95457dd",
-                "sha256:141b43360bfd3bdd75f15ed811850763555a251e38b2405967f8e25fb43f7d40",
-                "sha256:14c2976aa9038c2629efa2c148022ed5eb4cb939e15ec7aace7ca932f48f9ba6",
-                "sha256:19fe01cea168585ba0f678cad6f58133db2aa14eccaf22f88e4a6dccadfad8b3",
-                "sha256:1d147090048129ce3c453f0292e7697d333db95e52616b3793922945804a433c",
-                "sha256:1d9ea7a7e779d7a3561aade7d596649fbecfa5c08a7674b11b423783217933f9",
-                "sha256:215ed703caf15f578dca76ee6f6b21b7603791ae090fbf1ef9d865571039ade5",
-                "sha256:21fd81c4ebdb4f214161be351eb5bcf385426bf023041da2fd9e60681f3cebae",
-                "sha256:220dd781e3f7af2c2c1053da9fa96d9cf3072ca58f057f4c5adaaa1cab8fc442",
-                "sha256:228b644ae063c10e7f324ab1ab6b548bdf6f8b47f3ec234fef1093bc2735e5f9",
-                "sha256:29bfeb0dff5cb5fdab2023a7a9947b3b4af63e9c47cae2a10ad58394b517fddc",
-                "sha256:2f4848aa3baa109e6ab81fe2006c77ed4d3cd1e0ac2c1fbddb7b1277c168788c",
-                "sha256:2faa5ae9376faba05f630d7e5e6be05be22913782b927b19d12b8145968a85ea",
-                "sha256:2ffc42c922dbfddb4a4c3b438eb056828719f07608af27d163191cb3e3aa6cc5",
-                "sha256:37b15024f864916b4951adb95d3a80c9431299080341ab9544ed148091b53f50",
-                "sha256:3cc2ad10255f903656017363cd59436f2111443a76f996584d1077e43ee51182",
-                "sha256:3d25f19500588cbc47dc19081d78131c32637c25804df8414463ec908631e453",
-                "sha256:403c0911cd5d5791605808b942c88a8155c2592e05332d2bf78f18697a5fa15e",
-                "sha256:411bf8515f3be9813d06004cac41ccf7d1cd46dfe233705933dd163b60e37600",
-                "sha256:425bf820055005bfc8aa9a0b99ccb52cc2f4070153e34b701acc98d201693733",
-                "sha256:435a0984199d81ca178b9ae2c26ec3d49692d20ee29bc4c11a2a8d4514c67eda",
-                "sha256:4a6a4f196f08c58c59e0b8ef8ec441d12aee4125a7d4f4fef000ccb22f8d7241",
-                "sha256:4cc0ef8b962ac7a5e62b9e826bd0cd5040e7d401bc45a6835910ed699037a461",
-                "sha256:51d035609b86722963404f711db441cf7134f1889107fb171a970c9701f92e1e",
-                "sha256:53689bb4e102200a4fafa9de9c7c3c212ab40a7ab2c8e474491914d2305f187e",
-                "sha256:55205d03e8a598cfc688c71ca8ea5f66447164efff8869517f175ea632c7cb7b",
-                "sha256:5c0631926c4f58e9a5ccce555ad7747d9a9f8b10619621f22f9635f069f6233e",
-                "sha256:5cb241881eefd96b46f89b1a056187ea8e9ba14ab88ba632e68d7a2ecb7aadf7",
-                "sha256:60d698e8179a42ec85172d12f50b1668254628425a6bd611aba022257cac1386",
-                "sha256:612d1156111ae11d14afaf3a0669ebf6c170dbb735e510a7438ffe2369a847fd",
-                "sha256:6214c5a5571802c33f80e6c84713b2c79e024995b9c5897f794b43e714daeec9",
-                "sha256:6939c95381e003f54cd4c5516740faba40cf5ad3eeff460c3ad1d3e0ea2549bf",
-                "sha256:69db76c09796b313331bb7048229e3bee7928eb62bab5e071e9f7fcc4879caee",
-                "sha256:6bf7a982604375a8d49b6cc1b781c1747f243d91b81035a9b43a2126c04766f5",
-                "sha256:766c8f7511df26d9f11cd3a8be623e59cca73d44643abab3f8c8c07620524e4a",
-                "sha256:76c0de87358b192de7ea9649beb392f107dcad9ad27276324c24c91774ca5271",
-                "sha256:76f067f5121dcecf0d63a67f29080b26c43c71a98b10c701b0677e4a065fbd54",
-                "sha256:7901c05ead4b3fb75113fb1dd33eb1253c6d3ee37ce93305acd9d38e0b5f21a4",
-                "sha256:79660376075cfd4b2c80f295528aa6beb2058fd289f4c9252f986751a4cd0496",
-                "sha256:79a6d2ba910adb2cbafc95dad936f8b9386e77c84c35bc0add315b856d7c3abb",
-                "sha256:7afcdd1fc07befad18ec4523a782cde4e93e0a2bf71239894b8d61ee578c1319",
-                "sha256:7be7047bd08accdb7487737631d25735c9a04327911de89ff1b26b81745bd4e3",
-                "sha256:7c6390cf87ff6234643428991b7359b5f59cc15155695deb4eda5c777d2b880f",
-                "sha256:7df704ca8cf4a073334e0427ae2345323613e4df18cc224f647f251e5e75a527",
-                "sha256:85f67aed7bb647f93e7520633d8f51d3cbc6ab96957c71272b286b2f30dc70ed",
-                "sha256:896ebdcf62683551312c30e20614305f53125750803b614e9e6ce74a96232604",
-                "sha256:92d16a3e275e38293623ebf639c471d3e03bb20b8ebb845237e0d3664914caef",
-                "sha256:99f60d34c048c5c2fabc766108c103612344c46e35d4ed9ae0673d33c8fb26e8",
-                "sha256:9fe7b0653ba3d9d65cbe7698cca585bf0f8c83dbbcc710db9c90f478e175f2d5",
-                "sha256:a3145cb08d8625b2d3fee1b2d596a8766352979c9bffe5d7833e0503d0f0b5e5",
-                "sha256:aeaf541ddbad8311a87dd695ed9642401131ea39ad7bc8cf3ef3967fd093b626",
-                "sha256:b55358304d7a73d7bdf5de62494aaf70bd33015831ffd98bc498b433dfe5b10c",
-                "sha256:b82cc8ace10ab5bd93235dfaab2021c70637005e1ac787031f4d1da63d493c1d",
-                "sha256:c0868d64af83169e4d4152ec612637a543f7a336e4a307b119e98042e852ad9c",
-                "sha256:c1c1496e73051918fcd4f58ff2e0f2f3066d1c76a0c6aeffd9b45d53243702cc",
-                "sha256:c9bf56195c6bbd293340ea82eafd0071cb3d450c703d2c93afb89f93b8386ccc",
-                "sha256:cbebcd5bcaf1eaf302617c114aa67569dd3f090dd0ce8ba9e35e9985b41ac35b",
-                "sha256:cd6c8fca38178e12c00418de737aef1261576bd1b6e8c6134d3e729a4e858b38",
-                "sha256:ceb3b7e6a0135e092de86110c5a74e46bda4bd4fbfeeb3a3bcec79c0f861e450",
-                "sha256:cf590b134eb70629e350691ecca88eac3e3b8b3c86992042fb82e3cb1830d5e1",
-                "sha256:d3eb1ceec286eba8220c26f3b0096cf189aea7057b6e7b7a2e60ed36b373b77f",
-                "sha256:d65f25da8e248202bd47445cec78e0025c0fe7582b23ec69c3b27a640dd7a8e3",
-                "sha256:d6f6d4f185481c9669b9447bf9d9cf3b95a0e9df9d169bbc17e363b7d5487755",
-                "sha256:d84a5c3a5f7ce6db1f999fb9438f686bc2e09d38143f2d93d8406ed2dd6b9226",
-                "sha256:d946b0a9eb8aaa590df1fe082cee553ceab173e6cb5b03239716338629c50c7a",
-                "sha256:dce1c6912ab9ff5f179eaf6efe7365c1f425ed690b03341911bf4939ef2f3046",
-                "sha256:de170c7b4fe6859beb8926e84f7d7d6c693dfe8e27372ce3b76f01c46e489fcf",
-                "sha256:e02021f87a5b6932fa6ce916ca004c4d441509d33bbdbeca70d05dff5e9d2479",
-                "sha256:e030047e85cbcedbfc073f71836d62dd5dadfbe7531cae27789ff66bc551bd5e",
-                "sha256:e0e79d91e71b9867c73323a3444724d496c037e578a0e1755ae159ba14f4f3d1",
-                "sha256:e4428b29611e989719874670fd152b6625500ad6c686d464e99f5aaeeaca175a",
-                "sha256:e4972624066095e52b569e02b5ca97dbd7a7ddd4294bf4e7247d52635630dd83",
-                "sha256:e7be68734bd8c9a513f2b0cfd508802d6609da068f40dc57d4e3494cefc92929",
-                "sha256:e8e94e6912639a02ce173341ff62cc1201232ab86b8a8fcc05572741a5dc7d93",
-                "sha256:ea1456df2a27c73ce51120fa2f519f1bea2f4a03a917f4a43c8707cf4cbbae1a",
-                "sha256:ebd8d160f91a764652d3e51ce0d2956b38efe37c9231cd82cfc0bed2e40b581c",
-                "sha256:eca2e9d0cc5a889850e9bbd68e98314ada174ff6ccd1129500103df7a94a7a44",
-                "sha256:edd08e6f2f1a390bf137080507e44ccc086353c8e98c657e666c017718561b89",
-                "sha256:f285e862d2f153a70586579c15c44656f888806ed0e5b56b64489afe4a2dbfba",
-                "sha256:f2a1dee728b52b33eebff5072817176c172050d44d67befd681609b4746e1c2e",
-                "sha256:f7e301075edaf50500f0b341543c41194d8df3ae5caf4702f2095f3ca73dd8da",
-                "sha256:fb616be3538599e797a2017cccca78e354c767165e8858ab5116813146041a24",
-                "sha256:fce28b3c8a81b6b36dfac9feb1de115bab619b3c13905b419ec71d03a3fc1423",
-                "sha256:fe5d7785250541f7f5019ab9cba2c71169dc7d74d0f45253f8313f436458a4ef"
+                "sha256:01a3a55bd90018c9c080fbb0b9f4891db37d148a0a18722b42f94694f8b6d4c9",
+                "sha256:0b1a97283e0c85772d613878028fec909f003993e1007eafa715b24b377cb9b8",
+                "sha256:0dfad7a5a1e39c53ed00d2dd0c2e36aed4650936dc18fd9a1826a5ae1cad6f03",
+                "sha256:11bdf3f5e1518b24530b8241529d2050014c884cf18b6fc69c0c2b30ca248710",
+                "sha256:1502e24330eb681bdaa3eb70d6358e818e8e8f908a22a1851dfd4e15bc2f8161",
+                "sha256:16ab77bbeb596e14212e7bab8429f24c1579234a3a462105cda4a66904998664",
+                "sha256:16d232d4e5396c2efbbf4f6d4df89bfa905eb0d4dc5b3549d872ab898451f569",
+                "sha256:21a12c4eb6ddc9952c415f24eef97e3e55ba3af61f67c7bc388dcdec1404a067",
+                "sha256:27c523fbfbdfd19c6867af7346332b62b586eed663887392cff78d614f9ec313",
+                "sha256:281af09f488903fde97923c7744bb001a9b23b039a909460d0f14edc7bf59706",
+                "sha256:33029f5734336aa0d4c0384525da0387ef89148dc7191aae00ca5fb23d7aafc2",
+                "sha256:3601a3cece3819534b11d4efc1eb76047488fddd0c85a3948099d5da4d504636",
+                "sha256:3666906492efb76453c0e7b97f2cf459b0682e7402c0489a95484965dbc1da49",
+                "sha256:36c63aaa167f6c6b04ef2c85704e93af16c11d20de1d133e39de6a0e84582a93",
+                "sha256:39ff62e7d0f26c248b15e364517a72932a611a9b75f35b45be078d81bdb86603",
+                "sha256:43644e38f42e3af682690876cff722d301ac585c5b9e1eacc013b7a3f7b696a0",
+                "sha256:4372381634485bec7e46718edc71528024fcdc6f835baefe517b34a33c731d60",
+                "sha256:458f37be2d9e4c95e2d8866a851663cbc76e865b78395090786f6cd9b3bbf4f4",
+                "sha256:45e1ecb0379bfaab5eef059f50115b54571acfbe422a14f668fc8c27ba410e7e",
+                "sha256:4b9d9e4e2b37daddb5c23ea33a3417901fa7c7b3dee2d855f63ee67a0b21e5b1",
+                "sha256:4ceef517eca3e03c1cceb22030a3e39cb399ac86bff4e426d4fc6ae49052cc60",
+                "sha256:4d1a3d7ef5e96b1c9e92f973e43aa5e5b96c659c9bc3124acbbd81b0b9c8a951",
+                "sha256:4dcbb0906e38440fa3e325df2359ac6cb043df8e58c965bb45f4e406ecb162cc",
+                "sha256:509eac6cf09c794aa27bcacfd4d62c885cce62bef7b2c3e8b2e49d365b5003fe",
+                "sha256:52509b5be062d9eafc8170e53026fbc54cf3b32759a23d07fd935fb04fc22d95",
+                "sha256:52f2dffc8acaba9a2f27174c41c9e57f60b907bb9f096b36b1a1f3be71c6284d",
+                "sha256:574b7eae1ab267e5f8285f0fe881f17efe4b98c39a40858247720935b893bba8",
+                "sha256:5979b5632c3e3534e42ca6ff856bb24b2e3071b37861c2c727ce220d80eee9ed",
+                "sha256:59d43b61c59d82f2effb39a93c48b845efe23a3852d201ed2d24ba830d0b4cf2",
+                "sha256:5a4dcf02b908c3b8b17a45fb0f15b695bf117a67b76b7ad18b73cf8e92608775",
+                "sha256:5cad9430ab3e2e4fa4a2ef4450f548768400a2ac635841bc2a56a2052cdbeb87",
+                "sha256:5fc1b16f586f049820c5c5b17bb4ee7583092fa0d1c4e28b5239181ff9532e0c",
+                "sha256:62501642008a8b9871ddfccbf83e4222cf8ac0d5aeedf73da36153ef2ec222d2",
+                "sha256:64bdf1086b6043bf519869678f5f2757f473dee970d7abf6da91ec00acb9cb98",
+                "sha256:64da238a09d6039e3bd39bb3aee9c21a5e34f28bfa5aa22518581f910ff94af3",
+                "sha256:666daae833559deb2d609afa4490b85830ab0dfca811a98b70a205621a6109fe",
+                "sha256:67040058f37a2a51ed8ea8f6b0e6ee5bd78ca67f169ce6122f3e2ec80dfe9b78",
+                "sha256:6748717bb10339c4760c1e63da040f5f29f5ed6e59d76daee30305894069a660",
+                "sha256:6b181d8c23da913d4ff585afd1155a0e1194c0b50c54fcfe286f70cdaf2b7176",
+                "sha256:6ed5f161328b7df384d71b07317f4d8656434e34591f20552c7bcef27b0ab88e",
+                "sha256:7582a1d1030e15422262de9f58711774e02fa80df0d1578995c76214f6954988",
+                "sha256:7d18748f2d30f94f498e852c67d61261c643b349b9d2a581131725595c45ec6c",
+                "sha256:7d6ae9d593ef8641544d6263c7fa6408cc90370c8cb2bbb65f8d43e5b0351d9c",
+                "sha256:81a4f0b34bd92df3da93315c6a59034df95866014ac08535fc819f043bfd51f0",
+                "sha256:8316a77808c501004802f9beebde51c9f857054a0c871bd6da8280e718444449",
+                "sha256:853888594621e6604c978ce2a0444a1e6e70c8d253ab65ba11657659dcc9100f",
+                "sha256:99b76c052e9f1bc0721f7541e5e8c05db3941eb9ebe7b8553c625ef88d6eefde",
+                "sha256:a2e4369eb3d47d2034032a26c7a80fcb21a2cb22e1173d761a162f11e562caa5",
+                "sha256:ab55edc2e84460694295f401215f4a58597f8f7c9466faec545093045476327d",
+                "sha256:af048912e045a2dc732847d33821a9d84ba553f5c5f028adbd364dd4765092ac",
+                "sha256:b1a2eeedcead3a41694130495593a559a668f382eee0727352b9a41e1c45759a",
+                "sha256:b1e8b901e607795ec06c9e42530788c45ac21ef3aaa11dbd0c69de543bfb79a9",
+                "sha256:b41156839806aecb3641f3208c0dafd3ac7775b9c4c422d82ee2a45c34ba81ca",
+                "sha256:b692f419760c0e65d060959df05f2a531945af31fda0c8a3b3195d4efd06de11",
+                "sha256:bc779e9e6f7fda81b3f9aa58e3a6091d49ad528b11ed19f6621408806204ad35",
+                "sha256:bf6774e60d67a9efe02b3616fee22441d86fab4c6d335f9d2051d19d90a40063",
+                "sha256:c048099e4c9e9d615545e2001d3d8a4380bd403e1a0578734e0d31703d1b0c0b",
+                "sha256:c5cb09abb18c1ea940fb99360ea0396f34d46566f157122c92dfa069d3e0e982",
+                "sha256:cc8e1d0c705233c5dd0c5e6460fbad7827d5d36f310a0fadfd45cc3029762258",
+                "sha256:d5e3fc56f88cc98ef8139255cf8cd63eb2c586531e43310ff859d6bb3a6b51f1",
+                "sha256:d6aa0418fcc838522256761b3415822626f866758ee0bc6632c9486b179d0b52",
+                "sha256:d6c254ba6e45d8e72739281ebc46ea5eb5f101234f3ce171f0e9f5cc86991480",
+                "sha256:d6d635d5209b82a3492508cf5b365f3446afb65ae7ebd755e70e18f287b0adf7",
+                "sha256:dcfe792765fab89c365123c81046ad4103fcabbc4f56d1c1997e6715e8015461",
+                "sha256:ddd3915998d93fbcd2566ddf9cf62cdb35c9e093075f862935573d265cf8f65d",
+                "sha256:ddff9c4e225a63a5afab9dd15590432c22e8057e1a9a13d28ed128ecf047bbdc",
+                "sha256:e41b7e2b59679edfa309e8db64fdf22399eec4b0b24694e1b2104fb789207779",
+                "sha256:e69924bfcdda39b722ef4d9aa762b2dd38e4632b3641b1d9a57ca9cd18f2f83a",
+                "sha256:ea20853c6dbbb53ed34cb4d080382169b6f4554d394015f1bef35e881bf83547",
+                "sha256:ee2a1ece51b9b9e7752e742cfb661d2a29e7bcdba2d27e66e28a99f1890e4fa0",
+                "sha256:eeb6dcc05e911516ae3d1f207d4b0520d07f54484c49dfc294d6e7d63b734171",
+                "sha256:f70b98cd94886b49d91170ef23ec5c0e8ebb6f242d734ed7ed677b24d50c82cf",
+                "sha256:fc35cb4676846ef752816d5be2193a1e8367b4c1397b74a565a9d0389c433a1d",
+                "sha256:ff959bee35038c4624250473988b24f846cbeb2c6639de3602c073f10410ceba"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==6.0.5"
+            "version": "==6.0.4"
         },
         "mutagen": {
             "hashes": [
@@ -1334,6 +1619,39 @@
             "markers": "python_version >= '3.7'",
             "version": "==1.47.0"
         },
+        "mypy": {
+            "hashes": [
+                "sha256:0cd62192a4a32b77ceb31272d9e74d23cd88c8060c34d1d3622db3267679a5d9",
+                "sha256:1b3a2ffce52cc4dbaeee4df762f20a2905aa171ef157b82192f2e2f368eec05d",
+                "sha256:1f8f492d7db9e3593ef42d4f115f04e556130f2819ad33ab84551403e97dd4c0",
+                "sha256:2189ff1e39db399f08205e22a797383613ce1cb0cb3b13d8bcf0170e45b96cc3",
+                "sha256:378c03f53f10bbdd55ca94e46ec3ba255279706a6aacaecac52ad248f98205d3",
+                "sha256:37fd87cab83f09842653f08de066ee68f1182b9b5282e4634cdb4b407266bade",
+                "sha256:3c4c2992f6ea46ff7fce0072642cfb62af7a2484efe69017ed8b095f7b39ef31",
+                "sha256:51a46974340baaa4145363b9e051812a2446cf583dfaeba124af966fa44593f7",
+                "sha256:5bb9cd11c01c8606a9d0b83ffa91d0b236a0e91bc4126d9ba9ce62906ada868e",
+                "sha256:5cc3ca0a244eb9a5249c7c583ad9a7e881aa5d7b73c35652296ddcdb33b2b9c7",
+                "sha256:604282c886497645ffb87b8f35a57ec773a4a2721161e709a4422c1636ddde5c",
+                "sha256:6166a88b15f1759f94a46fa474c7b1b05d134b1b61fca627dd7335454cc9aa6b",
+                "sha256:6bacf8f3a3d7d849f40ca6caea5c055122efe70e81480c8328ad29c55c69e93e",
+                "sha256:6be84c06e6abd72f960ba9a71561c14137a583093ffcf9bbfaf5e613d63fa531",
+                "sha256:701b5f71413f1e9855566a34d6e9d12624e9e0a8818a5704d74d6b0402e66c04",
+                "sha256:71d8ac0b906354ebda8ef1673e5fde785936ac1f29ff6987c7483cfbd5a4235a",
+                "sha256:8addf6313777dbb92e9564c5d32ec122bf2c6c39d683ea64de6a1fd98b90fe37",
+                "sha256:901c89c2d67bba57aaaca91ccdb659aa3a312de67f23b9dfb059727cce2e2e0a",
+                "sha256:97a131ee36ac37ce9581f4220311247ab6cba896b4395b9c87af0675a13a755f",
+                "sha256:a1bbb3a6f5ff319d2b9d40b4080d46cd639abe3516d5a62c070cf0114a457d84",
+                "sha256:a2cbc68cb9e943ac0814c13e2452d2046c2f2b23ff0278e26599224cf164e78d",
+                "sha256:b8edd4e9bbbc9d7b79502eb9592cab808585516ae1bcc1446eb9122656c6066f",
+                "sha256:bd6f629b67bb43dc0d9211ee98b96d8dabc97b1ad38b9b25f5e4c4d7569a0c6a",
+                "sha256:c2ae450d60d7d020d67ab440c6e3fae375809988119817214440033f26ddf7bf",
+                "sha256:d8681909f7b44d0b7b86e653ca152d6dff0eb5eb41694e163c6092124f8246d7",
+                "sha256:e36f229acfe250dc660790840916eb49726c928e8ce10fbdf90715090fe4ae02",
+                "sha256:fe85ed6836165d52ae8b88f99527d3d1b2362e0cb90b005409b8bed90e9059b3"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==1.10.1"
+        },
         "mypy-extensions": {
             "hashes": [
                 "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d",
@@ -1342,47 +1660,93 @@
             "markers": "python_version >= '3.5'",
             "version": "==1.0.0"
         },
+        "myst-parser": {
+            "hashes": [
+                "sha256:61b275b85d9f58aa327f370913ae1bec26ebad372cc99f3ab85c8ec3ee8d9fb8",
+                "sha256:79317f4bb2c13053dd6e64f9da1ba1da6cd9c40c8a430c447a7b146a594c246d"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==0.18.1"
+        },
+        "nh3": {
+            "hashes": [
+                "sha256:0411beb0589eacb6734f28d5497ca2ed379eafab8ad8c84b31bb5c34072b7164",
+                "sha256:14c5a72e9fe82aea5fe3072116ad4661af5cf8e8ff8fc5ad3450f123e4925e86",
+                "sha256:19aaba96e0f795bd0a6c56291495ff59364f4300d4a39b29a0abc9cb3774a84b",
+                "sha256:34c03fa78e328c691f982b7c03d4423bdfd7da69cd707fe572f544cf74ac23ad",
+                "sha256:36c95d4b70530b320b365659bb5034341316e6a9b30f0b25fa9c9eff4c27a204",
+                "sha256:3a157ab149e591bb638a55c8c6bcb8cdb559c8b12c13a8affaba6cedfe51713a",
+                "sha256:42c64511469005058cd17cc1537578eac40ae9f7200bedcfd1fc1a05f4f8c200",
+                "sha256:5f36b271dae35c465ef5e9090e1fdaba4a60a56f0bb0ba03e0932a66f28b9189",
+                "sha256:6955369e4d9f48f41e3f238a9e60f9410645db7e07435e62c6a9ea6135a4907f",
+                "sha256:7b7c2a3c9eb1a827d42539aa64091640bd275b81e097cd1d8d82ef91ffa2e811",
+                "sha256:8ce0f819d2f1933953fca255db2471ad58184a60508f03e6285e5114b6254844",
+                "sha256:94a166927e53972a9698af9542ace4e38b9de50c34352b962f4d9a7d4c927af4",
+                "sha256:a7f1b5b2c15866f2db413a3649a8fe4fd7b428ae58be2c0f6bca5eefd53ca2be",
+                "sha256:c8b3a1cebcba9b3669ed1a84cc65bf005728d2f0bc1ed2a6594a992e817f3a50",
+                "sha256:de3ceed6e661954871d6cd78b410213bdcb136f79aafe22aa7182e028b8c7307",
+                "sha256:f0eca9ca8628dbb4e916ae2491d72957fdd35f7a5d326b7032a345f111ac07fe"
+            ],
+            "version": "==0.2.18"
+        },
         "numpy": {
             "hashes": [
-                "sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b",
-                "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818",
-                "sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20",
-                "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0",
-                "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010",
-                "sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a",
-                "sha256:3373d5d70a5fe74a2c1bb6d2cfd9609ecf686d47a2d7b1d37a8f3b6bf6003aea",
-                "sha256:47711010ad8555514b434df65f7d7b076bb8261df1ca9bb78f53d3b2db02e95c",
-                "sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71",
-                "sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110",
-                "sha256:52b8b60467cd7dd1e9ed082188b4e6bb35aa5cdd01777621a1658910745b90be",
-                "sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a",
-                "sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a",
-                "sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5",
-                "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed",
-                "sha256:679b0076f67ecc0138fd2ede3a8fd196dddc2ad3254069bcb9faf9a79b1cebcd",
-                "sha256:7349ab0fa0c429c82442a27a9673fc802ffdb7c7775fad780226cb234965e53c",
-                "sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e",
-                "sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0",
-                "sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c",
-                "sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a",
-                "sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b",
-                "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0",
-                "sha256:a354325ee03388678242a4d7ebcd08b5c727033fcff3b2f536aea978e15ee9e6",
-                "sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2",
-                "sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a",
-                "sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30",
-                "sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218",
-                "sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5",
-                "sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07",
-                "sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2",
-                "sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4",
-                "sha256:d5241e0a80d808d70546c697135da2c613f30e28251ff8307eb72ba696945764",
-                "sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef",
-                "sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3",
-                "sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f"
+                "sha256:05b2d4e667895cc55e3ff2b56077e4c8a5604361fc21a042845ea3ad67465aa8",
+                "sha256:12edb90831ff481f7ef5f6bc6431a9d74dc0e5ff401559a71e5e4611d4f2d466",
+                "sha256:13311c2db4c5f7609b462bc0f43d3c465424d25c626d95040f073e30f7570e35",
+                "sha256:13532a088217fa624c99b843eeb54640de23b3414b14aa66d023805eb731066c",
+                "sha256:13602b3174432a35b16c4cfb5de9a12d229727c3dd47a6ce35111f2ebdf66ff4",
+                "sha256:1600068c262af1ca9580a527d43dc9d959b0b1d8e56f8a05d830eea39b7c8af6",
+                "sha256:1b8cde4f11f0a975d1fd59373b32e2f5a562ade7cde4f85b7137f3de8fbb29a0",
+                "sha256:1c193d0b0238638e6fc5f10f1b074a6993cb13b0b431f64079a509d63d3aa8b7",
+                "sha256:1ebec5fd716c5a5b3d8dfcc439be82a8407b7b24b230d0ad28a81b61c2f4659a",
+                "sha256:242b39d00e4944431a3cd2db2f5377e15b5785920421993770cddb89992c3f3a",
+                "sha256:259ec80d54999cc34cd1eb8ded513cb053c3bf4829152a2e00de2371bd406f5e",
+                "sha256:2abbf905a0b568706391ec6fa15161fad0fb5d8b68d73c461b3c1bab6064dd62",
+                "sha256:2cbba4b30bf31ddbe97f1c7205ef976909a93a66bb1583e983adbd155ba72ac2",
+                "sha256:2ffef621c14ebb0188a8633348504a35c13680d6da93ab5cb86f4e54b7e922b5",
+                "sha256:30d53720b726ec36a7f88dc873f0eec8447fbc93d93a8f079dfac2629598d6ee",
+                "sha256:32e16a03138cabe0cb28e1007ee82264296ac0983714094380b408097a418cfe",
+                "sha256:43cca367bf94a14aca50b89e9bc2061683116cfe864e56740e083392f533ce7a",
+                "sha256:456e3b11cb79ac9946c822a56346ec80275eaf2950314b249b512896c0d2505e",
+                "sha256:4d6ec0d4222e8ffdab1744da2560f07856421b367928026fb540e1945f2eeeaf",
+                "sha256:5006b13a06e0b38d561fab5ccc37581f23c9511879be7693bd33c7cd15ca227c",
+                "sha256:675c741d4739af2dc20cd6c6a5c4b7355c728167845e3c6b0e824e4e5d36a6c3",
+                "sha256:6cdb606a7478f9ad91c6283e238544451e3a95f30fb5467fbf715964341a8a86",
+                "sha256:6d95f286b8244b3649b477ac066c6906fbb2905f8ac19b170e2175d3d799f4df",
+                "sha256:76322dcdb16fccf2ac56f99048af32259dcc488d9b7e25b51e5eca5147a3fb98",
+                "sha256:7c1c60328bd964b53f8b835df69ae8198659e2b9302ff9ebb7de4e5a5994db3d",
+                "sha256:860ec6e63e2c5c2ee5e9121808145c7bf86c96cca9ad396c0bd3e0f2798ccbe2",
+                "sha256:8e00ea6fc82e8a804433d3e9cedaa1051a1422cb6e443011590c14d2dea59146",
+                "sha256:9c6c754df29ce6a89ed23afb25550d1c2d5fdb9901d9c67a16e0b16eaf7e2550",
+                "sha256:a26ae94658d3ba3781d5e103ac07a876b3e9b29db53f68ed7df432fd033358a8",
+                "sha256:a65acfdb9c6ebb8368490dbafe83c03c7e277b37e6857f0caeadbbc56e12f4fb",
+                "sha256:a7d80b2e904faa63068ead63107189164ca443b42dd1930299e0d1cb041cec2e",
+                "sha256:a84498e0d0a1174f2b3ed769b67b656aa5460c92c9554039e11f20a05650f00d",
+                "sha256:ab4754d432e3ac42d33a269c8567413bdb541689b02d93788af4131018cbf366",
+                "sha256:ad369ed238b1959dfbade9018a740fb9392c5ac4f9b5173f420bd4f37ba1f7a0",
+                "sha256:b1d0fcae4f0949f215d4632be684a539859b295e2d0cb14f78ec231915d644db",
+                "sha256:b42a1a511c81cc78cbc4539675713bbcf9d9c3913386243ceff0e9429ca892fe",
+                "sha256:bd33f82e95ba7ad632bc57837ee99dba3d7e006536200c4e9124089e1bf42426",
+                "sha256:bdd407c40483463898b84490770199d5714dcc9dd9b792f6c6caccc523c00952",
+                "sha256:c6eef7a2dbd0abfb0d9eaf78b73017dbfd0b54051102ff4e6a7b2980d5ac1a03",
+                "sha256:c82af4b2ddd2ee72d1fc0c6695048d457e00b3582ccde72d8a1c991b808bb20f",
+                "sha256:d666cb72687559689e9906197e3bec7b736764df6a2e58ee265e360663e9baf7",
+                "sha256:d7bf0a4f9f15b32b5ba53147369e94296f5fffb783db5aacc1be15b4bf72f43b",
+                "sha256:d82075752f40c0ddf57e6e02673a17f6cb0f8eb3f587f63ca1eaab5594da5b17",
+                "sha256:da65fb46d4cbb75cb417cddf6ba5e7582eb7bb0b47db4b99c9fe5787ce5d91f5",
+                "sha256:e2b49c3c0804e8ecb05d59af8386ec2f74877f7ca8fd9c1e00be2672e4d399b1",
+                "sha256:e585c8ae871fd38ac50598f4763d73ec5497b0de9a0ab4ef5b69f01c6a046142",
+                "sha256:e8d3ca0a72dd8846eb6f7dfe8f19088060fcb76931ed592d29128e0219652884",
+                "sha256:ef444c57d664d35cac4e18c298c47d7b504c66b17c2ea91312e979fcfbdfb08a",
+                "sha256:f1eb068ead09f4994dec71c24b2844f1e4e4e013b9629f812f292f04bd1510d9",
+                "sha256:f2ded8d9b6f68cc26f8425eda5d3877b47343e68ca23d0d0846f4d312ecaa445",
+                "sha256:f751ed0a2f250541e19dfca9f1eafa31a392c71c832b6bb9e113b10d050cb0f1",
+                "sha256:faa88bc527d0f097abdc2c663cddf37c05a1c2f113716601555249805cf573f1",
+                "sha256:fc44e3c68ff00fd991b59092a54350e6e4911152682b4782f68070985aa9e648"
             ],
-            "markers": "python_version >= '3.9'",
-            "version": "==1.26.4"
+            "markers": "python_version >= '3.10'",
+            "version": "==2.1.2"
         },
         "oauth2client": {
             "hashes": [
@@ -1416,100 +1780,140 @@
         },
         "packaging": {
             "hashes": [
-                "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5",
-                "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9"
+                "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002",
+                "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==24.0"
+            "markers": "python_version >= '3.8'",
+            "version": "==24.1"
+        },
+        "pathspec": {
+            "hashes": [
+                "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08",
+                "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==0.12.1"
         },
         "pdqhash": {
             "hashes": [
-                "sha256:1b3d4a6251033d8bfbb8d054fcdbcd0ea6b7a93c6a0b7433c9947c4fb4a3f5ef",
-                "sha256:211431c347e737ff247a39fb24924c93c853d793842961cf476d40fa9a586eca",
-                "sha256:230694fab7b7f70fc494ee8bc3a0a5960ed82c4a78421a357e1312d933947854",
-                "sha256:2b0519a07551a753380cf76db24212b4db7c0ea30516b620a5c902d25b467e6a",
-                "sha256:70a905a777826293ab5e4eba0e05e8c44467d873959d336a0558d0f632a90133",
-                "sha256:9f07f4e8482ec9eada19ea834c72c72df1159eba2d2a1761c4a00f70789f591d",
-                "sha256:a2daed23628cf58eb866bcef9731de6c159578e32e939a71939e0d8032b11a86",
-                "sha256:c688f3f6ea65debb6213577f46ba2be9ebb7db7d23f96ad6ed2df05832de93c3",
-                "sha256:e06e531c4fd204d6a9d58eef4a64354eaa62f2e3ca78ed1721ac571644f0c571"
+                "sha256:262ec2881b80877a4005f408000f27e492d08b0d4e84269e1cfcc8a31e96c3bf",
+                "sha256:33bc5d22c458e5245c2649c0e54968d13e7b5940f6ace268f4a05016481b2253",
+                "sha256:68549e2071499a5a19748505eb00f52384f24830fe14a437f982d046a8edb498",
+                "sha256:6eab8a3112853f18adfaf1482b48bff8003d822de927b7397a421c5e7f0f76d7",
+                "sha256:7efd5b4e1ded44ec2a32ea2d32c29fef37d1adca03ce0867975526aea0ffe7fe",
+                "sha256:91652e70d017c8fd60003ea0bfcf4eefeceb3896b93deb2dec43a2ae896da6cd",
+                "sha256:9648abdfdbccb5edfc55fa2a61183766e51d140080fe08213f5daa885c3d5c66",
+                "sha256:a7d943874df8b2ca8c97755f60d1b6ae66e654fe8b2bb6ac8e8be216cae7d130",
+                "sha256:d63886b1edea4134eaa9862987393391e8958f35569918b91804b528f23c5e6c",
+                "sha256:df6375fef513089191cadcbf07c90715d40e882867141e04360a39d8a0861cb5",
+                "sha256:fc6c53cdc395f5421c857e4e30a92862cc918e18f91d7e4452bb5eb746e454f2"
             ],
-            "version": "==0.2.5"
+            "version": "==0.2.7"
         },
         "pillow": {
             "hashes": [
-                "sha256:048ad577748b9fa4a99a0548c64f2cb8d672d5bf2e643a739ac8faff1164238c",
-                "sha256:048eeade4c33fdf7e08da40ef402e748df113fd0b4584e32c4af74fe78baaeb2",
-                "sha256:0ba26351b137ca4e0db0342d5d00d2e355eb29372c05afd544ebf47c0956ffeb",
-                "sha256:0ea2a783a2bdf2a561808fe4a7a12e9aa3799b701ba305de596bc48b8bdfce9d",
-                "sha256:1530e8f3a4b965eb6a7785cf17a426c779333eb62c9a7d1bbcf3ffd5bf77a4aa",
-                "sha256:16563993329b79513f59142a6b02055e10514c1a8e86dca8b48a893e33cf91e3",
-                "sha256:19aeb96d43902f0a783946a0a87dbdad5c84c936025b8419da0a0cd7724356b1",
-                "sha256:1a1d1915db1a4fdb2754b9de292642a39a7fb28f1736699527bb649484fb966a",
-                "sha256:1b87bd9d81d179bd8ab871603bd80d8645729939f90b71e62914e816a76fc6bd",
-                "sha256:1dfc94946bc60ea375cc39cff0b8da6c7e5f8fcdc1d946beb8da5c216156ddd8",
-                "sha256:2034f6759a722da3a3dbd91a81148cf884e91d1b747992ca288ab88c1de15999",
-                "sha256:261ddb7ca91fcf71757979534fb4c128448b5b4c55cb6152d280312062f69599",
-                "sha256:2ed854e716a89b1afcedea551cd85f2eb2a807613752ab997b9974aaa0d56936",
-                "sha256:3102045a10945173d38336f6e71a8dc71bcaeed55c3123ad4af82c52807b9375",
-                "sha256:339894035d0ede518b16073bdc2feef4c991ee991a29774b33e515f1d308e08d",
-                "sha256:412444afb8c4c7a6cc11a47dade32982439925537e483be7c0ae0cf96c4f6a0b",
-                "sha256:4203efca580f0dd6f882ca211f923168548f7ba334c189e9eab1178ab840bf60",
-                "sha256:45ebc7b45406febf07fef35d856f0293a92e7417ae7933207e90bf9090b70572",
-                "sha256:4b5ec25d8b17217d635f8935dbc1b9aa5907962fae29dff220f2659487891cd3",
-                "sha256:4c8e73e99da7db1b4cad7f8d682cf6abad7844da39834c288fbfa394a47bbced",
-                "sha256:4e6f7d1c414191c1199f8996d3f2282b9ebea0945693fb67392c75a3a320941f",
-                "sha256:4eaa22f0d22b1a7e93ff0a596d57fdede2e550aecffb5a1ef1106aaece48e96b",
-                "sha256:50b8eae8f7334ec826d6eeffaeeb00e36b5e24aa0b9df322c247539714c6df19",
-                "sha256:50fd3f6b26e3441ae07b7c979309638b72abc1a25da31a81a7fbd9495713ef4f",
-                "sha256:51243f1ed5161b9945011a7360e997729776f6e5d7005ba0c6879267d4c5139d",
-                "sha256:5d512aafa1d32efa014fa041d38868fda85028e3f930a96f85d49c7d8ddc0383",
-                "sha256:5f77cf66e96ae734717d341c145c5949c63180842a545c47a0ce7ae52ca83795",
-                "sha256:6b02471b72526ab8a18c39cb7967b72d194ec53c1fd0a70b050565a0f366d355",
-                "sha256:6fb1b30043271ec92dc65f6d9f0b7a830c210b8a96423074b15c7bc999975f57",
-                "sha256:7161ec49ef0800947dc5570f86568a7bb36fa97dd09e9827dc02b718c5643f09",
-                "sha256:72d622d262e463dfb7595202d229f5f3ab4b852289a1cd09650362db23b9eb0b",
-                "sha256:74d28c17412d9caa1066f7a31df8403ec23d5268ba46cd0ad2c50fb82ae40462",
-                "sha256:78618cdbccaa74d3f88d0ad6cb8ac3007f1a6fa5c6f19af64b55ca170bfa1edf",
-                "sha256:793b4e24db2e8742ca6423d3fde8396db336698c55cd34b660663ee9e45ed37f",
-                "sha256:798232c92e7665fe82ac085f9d8e8ca98826f8e27859d9a96b41d519ecd2e49a",
-                "sha256:81d09caa7b27ef4e61cb7d8fbf1714f5aec1c6b6c5270ee53504981e6e9121ad",
-                "sha256:8ab74c06ffdab957d7670c2a5a6e1a70181cd10b727cd788c4dd9005b6a8acd9",
-                "sha256:8eb0908e954d093b02a543dc963984d6e99ad2b5e36503d8a0aaf040505f747d",
-                "sha256:90b9e29824800e90c84e4022dd5cc16eb2d9605ee13f05d47641eb183cd73d45",
-                "sha256:9797a6c8fe16f25749b371c02e2ade0efb51155e767a971c61734b1bf6293994",
-                "sha256:9d2455fbf44c914840c793e89aa82d0e1763a14253a000743719ae5946814b2d",
-                "sha256:9d3bea1c75f8c53ee4d505c3e67d8c158ad4df0d83170605b50b64025917f338",
-                "sha256:9e2ec1e921fd07c7cda7962bad283acc2f2a9ccc1b971ee4b216b75fad6f0463",
-                "sha256:9e91179a242bbc99be65e139e30690e081fe6cb91a8e77faf4c409653de39451",
-                "sha256:a0eaa93d054751ee9964afa21c06247779b90440ca41d184aeb5d410f20ff591",
-                "sha256:a2c405445c79c3f5a124573a051062300936b0281fee57637e706453e452746c",
-                "sha256:aa7e402ce11f0885305bfb6afb3434b3cd8f53b563ac065452d9d5654c7b86fd",
-                "sha256:aff76a55a8aa8364d25400a210a65ff59d0168e0b4285ba6bf2bd83cf675ba32",
-                "sha256:b09b86b27a064c9624d0a6c54da01c1beaf5b6cadfa609cf63789b1d08a797b9",
-                "sha256:b14f16f94cbc61215115b9b1236f9c18403c15dd3c52cf629072afa9d54c1cbf",
-                "sha256:b50811d664d392f02f7761621303eba9d1b056fb1868c8cdf4231279645c25f5",
-                "sha256:b7bc2176354defba3edc2b9a777744462da2f8e921fbaf61e52acb95bafa9828",
-                "sha256:c78e1b00a87ce43bb37642c0812315b411e856a905d58d597750eb79802aaaa3",
-                "sha256:c83341b89884e2b2e55886e8fbbf37c3fa5efd6c8907124aeb72f285ae5696e5",
-                "sha256:ca2870d5d10d8726a27396d3ca4cf7976cec0f3cb706debe88e3a5bd4610f7d2",
-                "sha256:ccce24b7ad89adb5a1e34a6ba96ac2530046763912806ad4c247356a8f33a67b",
-                "sha256:cd5e14fbf22a87321b24c88669aad3a51ec052eb145315b3da3b7e3cc105b9a2",
-                "sha256:ce49c67f4ea0609933d01c0731b34b8695a7a748d6c8d186f95e7d085d2fe475",
-                "sha256:d33891be6df59d93df4d846640f0e46f1a807339f09e79a8040bc887bdcd7ed3",
-                "sha256:d3b2348a78bc939b4fed6552abfd2e7988e0f81443ef3911a4b8498ca084f6eb",
-                "sha256:d886f5d353333b4771d21267c7ecc75b710f1a73d72d03ca06df49b09015a9ef",
-                "sha256:d93480005693d247f8346bc8ee28c72a2191bdf1f6b5db469c096c0c867ac015",
-                "sha256:dc1a390a82755a8c26c9964d457d4c9cbec5405896cba94cf51f36ea0d855002",
-                "sha256:dd78700f5788ae180b5ee8902c6aea5a5726bac7c364b202b4b3e3ba2d293170",
-                "sha256:e46f38133e5a060d46bd630faa4d9fa0202377495df1f068a8299fd78c84de84",
-                "sha256:e4b878386c4bf293578b48fc570b84ecfe477d3b77ba39a6e87150af77f40c57",
-                "sha256:f0d0591a0aeaefdaf9a5e545e7485f89910c977087e7de2b6c388aec32011e9f",
-                "sha256:fdcbb4068117dfd9ce0138d068ac512843c52295ed996ae6dd1faf537b6dbc27",
-                "sha256:ff61bfd9253c3915e6d41c651d5f962da23eda633cf02262990094a18a55371a"
+                "sha256:00177a63030d612148e659b55ba99527803288cea7c75fb05766ab7981a8c1b7",
+                "sha256:006bcdd307cc47ba43e924099a038cbf9591062e6c50e570819743f5607404f5",
+                "sha256:084a07ef0821cfe4858fe86652fffac8e187b6ae677e9906e192aafcc1b69903",
+                "sha256:0ae08bd8ffc41aebf578c2af2f9d8749d91f448b3bfd41d7d9ff573d74f2a6b2",
+                "sha256:0e038b0745997c7dcaae350d35859c9715c71e92ffb7e0f4a8e8a16732150f38",
+                "sha256:1187739620f2b365de756ce086fdb3604573337cc28a0d3ac4a01ab6b2d2a6d2",
+                "sha256:16095692a253047fe3ec028e951fa4221a1f3ed3d80c397e83541a3037ff67c9",
+                "sha256:1a61b54f87ab5786b8479f81c4b11f4d61702830354520837f8cc791ebba0f5f",
+                "sha256:1c1d72714f429a521d8d2d018badc42414c3077eb187a59579f28e4270b4b0fc",
+                "sha256:1e2688958a840c822279fda0086fec1fdab2f95bf2b717b66871c4ad9859d7e8",
+                "sha256:20ec184af98a121fb2da42642dea8a29ec80fc3efbaefb86d8fdd2606619045d",
+                "sha256:21a0d3b115009ebb8ac3d2ebec5c2982cc693da935f4ab7bb5c8ebe2f47d36f2",
+                "sha256:224aaa38177597bb179f3ec87eeefcce8e4f85e608025e9cfac60de237ba6316",
+                "sha256:2679d2258b7f1192b378e2893a8a0a0ca472234d4c2c0e6bdd3380e8dfa21b6a",
+                "sha256:27a7860107500d813fcd203b4ea19b04babe79448268403172782754870dac25",
+                "sha256:290f2cc809f9da7d6d622550bbf4c1e57518212da51b6a30fe8e0a270a5b78bd",
+                "sha256:2e46773dc9f35a1dd28bd6981332fd7f27bec001a918a72a79b4133cf5291dba",
+                "sha256:3107c66e43bda25359d5ef446f59c497de2b5ed4c7fdba0894f8d6cf3822dafc",
+                "sha256:375b8dd15a1f5d2feafff536d47e22f69625c1aa92f12b339ec0b2ca40263273",
+                "sha256:45c566eb10b8967d71bf1ab8e4a525e5a93519e29ea071459ce517f6b903d7fa",
+                "sha256:499c3a1b0d6fc8213519e193796eb1a86a1be4b1877d678b30f83fd979811d1a",
+                "sha256:4ad70c4214f67d7466bea6a08061eba35c01b1b89eaa098040a35272a8efb22b",
+                "sha256:4b60c9520f7207aaf2e1d94de026682fc227806c6e1f55bba7606d1c94dd623a",
+                "sha256:5178952973e588b3f1360868847334e9e3bf49d19e169bbbdfaf8398002419ae",
+                "sha256:52a2d8323a465f84faaba5236567d212c3668f2ab53e1c74c15583cf507a0291",
+                "sha256:598b4e238f13276e0008299bd2482003f48158e2b11826862b1eb2ad7c768b97",
+                "sha256:5bd2d3bdb846d757055910f0a59792d33b555800813c3b39ada1829c372ccb06",
+                "sha256:5c39ed17edea3bc69c743a8dd3e9853b7509625c2462532e62baa0732163a904",
+                "sha256:5d203af30149ae339ad1b4f710d9844ed8796e97fda23ffbc4cc472968a47d0b",
+                "sha256:5ddbfd761ee00c12ee1be86c9c0683ecf5bb14c9772ddbd782085779a63dd55b",
+                "sha256:607bbe123c74e272e381a8d1957083a9463401f7bd01287f50521ecb05a313f8",
+                "sha256:61b887f9ddba63ddf62fd02a3ba7add935d053b6dd7d58998c630e6dbade8527",
+                "sha256:6619654954dc4936fcff82db8eb6401d3159ec6be81e33c6000dfd76ae189947",
+                "sha256:674629ff60030d144b7bca2b8330225a9b11c482ed408813924619c6f302fdbb",
+                "sha256:6ec0d5af64f2e3d64a165f490d96368bb5dea8b8f9ad04487f9ab60dc4bb6003",
+                "sha256:6f4dba50cfa56f910241eb7f883c20f1e7b1d8f7d91c750cd0b318bad443f4d5",
+                "sha256:70fbbdacd1d271b77b7721fe3cdd2d537bbbd75d29e6300c672ec6bb38d9672f",
+                "sha256:72bacbaf24ac003fea9bff9837d1eedb6088758d41e100c1552930151f677739",
+                "sha256:7326a1787e3c7b0429659e0a944725e1b03eeaa10edd945a86dead1913383944",
+                "sha256:73853108f56df97baf2bb8b522f3578221e56f646ba345a372c78326710d3830",
+                "sha256:73e3a0200cdda995c7e43dd47436c1548f87a30bb27fb871f352a22ab8dcf45f",
+                "sha256:75acbbeb05b86bc53cbe7b7e6fe00fbcf82ad7c684b3ad82e3d711da9ba287d3",
+                "sha256:8069c5179902dcdce0be9bfc8235347fdbac249d23bd90514b7a47a72d9fecf4",
+                "sha256:846e193e103b41e984ac921b335df59195356ce3f71dcfd155aa79c603873b84",
+                "sha256:8594f42df584e5b4bb9281799698403f7af489fba84c34d53d1c4bfb71b7c4e7",
+                "sha256:86510e3f5eca0ab87429dd77fafc04693195eec7fd6a137c389c3eeb4cfb77c6",
+                "sha256:8853a3bf12afddfdf15f57c4b02d7ded92c7a75a5d7331d19f4f9572a89c17e6",
+                "sha256:88a58d8ac0cc0e7f3a014509f0455248a76629ca9b604eca7dc5927cc593c5e9",
+                "sha256:8ba470552b48e5835f1d23ecb936bb7f71d206f9dfeee64245f30c3270b994de",
+                "sha256:8c676b587da5673d3c75bd67dd2a8cdfeb282ca38a30f37950511766b26858c4",
+                "sha256:8ec4a89295cd6cd4d1058a5e6aec6bf51e0eaaf9714774e1bfac7cfc9051db47",
+                "sha256:94f3e1780abb45062287b4614a5bc0874519c86a777d4a7ad34978e86428b8dd",
+                "sha256:9a0f748eaa434a41fccf8e1ee7a3eed68af1b690e75328fd7a60af123c193b50",
+                "sha256:a5629742881bcbc1f42e840af185fd4d83a5edeb96475a575f4da50d6ede337c",
+                "sha256:a65149d8ada1055029fcb665452b2814fe7d7082fcb0c5bed6db851cb69b2086",
+                "sha256:b3c5ac4bed7519088103d9450a1107f76308ecf91d6dabc8a33a2fcfb18d0fba",
+                "sha256:b4fd7bd29610a83a8c9b564d457cf5bd92b4e11e79a4ee4716a63c959699b306",
+                "sha256:bcd1fb5bb7b07f64c15618c89efcc2cfa3e95f0e3bcdbaf4642509de1942a699",
+                "sha256:c12b5ae868897c7338519c03049a806af85b9b8c237b7d675b8c5e089e4a618e",
+                "sha256:c26845094b1af3c91852745ae78e3ea47abf3dbcd1cf962f16b9a5fbe3ee8488",
+                "sha256:c6a660307ca9d4867caa8d9ca2c2658ab685de83792d1876274991adec7b93fa",
+                "sha256:c809a70e43c7977c4a42aefd62f0131823ebf7dd73556fa5d5950f5b354087e2",
+                "sha256:c8b2351c85d855293a299038e1f89db92a2f35e8d2f783489c6f0b2b5f3fe8a3",
+                "sha256:cb929ca942d0ec4fac404cbf520ee6cac37bf35be479b970c4ffadf2b6a1cad9",
+                "sha256:d2c0a187a92a1cb5ef2c8ed5412dd8d4334272617f532d4ad4de31e0495bd923",
+                "sha256:d69bfd8ec3219ae71bcde1f942b728903cad25fafe3100ba2258b973bd2bc1b2",
+                "sha256:daffdf51ee5db69a82dd127eabecce20729e21f7a3680cf7cbb23f0829189790",
+                "sha256:e58876c91f97b0952eb766123bfef372792ab3f4e3e1f1a2267834c2ab131734",
+                "sha256:eda2616eb2313cbb3eebbe51f19362eb434b18e3bb599466a1ffa76a033fb916",
+                "sha256:ee217c198f2e41f184f3869f3e485557296d505b5195c513b2bfe0062dc537f1",
+                "sha256:f02541ef64077f22bf4924f225c0fd1248c168f86e4b7abdedd87d6ebaceab0f",
+                "sha256:f1b82c27e89fffc6da125d5eb0ca6e68017faf5efc078128cfaa42cf5cb38798",
+                "sha256:fba162b8872d30fea8c52b258a542c5dfd7b235fb5cb352240c8d63b414013eb",
+                "sha256:fbbcb7b57dc9c794843e3d1258c0fbf0f48656d46ffe9e09b63bbd6e8cd5d0a2",
+                "sha256:fcb4621042ac4b7865c179bb972ed0da0218a076dc1820ffc48b1d74c1e37fe9"
+            ],
+            "markers": "python_version >= '3.9'",
+            "version": "==11.0.0"
+        },
+        "pkginfo": {
+            "hashes": [
+                "sha256:5df73835398d10db79f8eecd5cd86b1f6d29317589ea70796994d49399af6297",
+                "sha256:889a6da2ed7ffc58ab5b900d888ddce90bce912f2d2de1dc1c26f4cb9fe65097"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==1.10.0"
+        },
+        "platformdirs": {
+            "hashes": [
+                "sha256:2d7a1657e36a80ea911db832a8a6ece5ee53d8de21edd5cc5879af6530b1bfee",
+                "sha256:38b7b51f512eed9e84a22788b4bce1de17c0adb134d6becb09836e37d8654cd3"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==10.3.0"
+            "version": "==4.2.2"
+        },
+        "pluggy": {
+            "hashes": [
+                "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1",
+                "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==1.5.0"
         },
         "prometheus-client": {
             "hashes": [
@@ -1520,37 +1924,67 @@
         },
         "prometheus-fastapi-instrumentator": {
             "hashes": [
-                "sha256:1820d7a90389ce100f7d1285495ead388818ae0882e761c1f3e6e62a410bdf13",
-                "sha256:2279ac1cf5b9566a4c3a07f78c9c5ee19648ed90976ab87d73d672abc1bfa017"
+                "sha256:5ba67c9212719f244ad7942d75ded80693b26331ee5dfc1e7571e4794a9ccbed",
+                "sha256:96030c43c776ee938a3dae58485ec24caed7e05bfc60fe067161e0d5b5757052"
             ],
             "index": "pypi",
-            "markers": "python_full_version >= '3.7.0' and python_full_version < '4.0.0'",
-            "version": "==6.1.0"
+            "markers": "python_full_version >= '3.8.1' and python_full_version < '4.0.0'",
+            "version": "==7.0.0"
         },
         "proto-plus": {
             "hashes": [
-                "sha256:89075171ef11988b3fa157f5dbd8b9cf09d65fffee97e29ce403cd8defba19d2",
-                "sha256:a829c79e619e1cf632de091013a4173deed13a55f326ef84f05af6f50ff4c82c"
+                "sha256:30b72a5ecafe4406b0d339db35b56c4059064e69227b8c3bda7462397f966445",
+                "sha256:402576830425e5f6ce4c2a6702400ac79897dab0b4343821aa5188b0fab81a12"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==1.23.0"
+            "markers": "python_version >= '3.7'",
+            "version": "==1.24.0"
         },
         "protobuf": {
             "hashes": [
-                "sha256:19b270aeaa0099f16d3ca02628546b8baefe2955bbe23224aaf856134eccf1e4",
-                "sha256:209ba4cc916bab46f64e56b85b090607a676f66b473e6b762e6f1d9d591eb2e8",
-                "sha256:25b5d0b42fd000320bd7830b349e3b696435f3b329810427a6bcce6a5492cc5c",
-                "sha256:7c8daa26095f82482307bc717364e7c13f4f1c99659be82890dcfc215194554d",
-                "sha256:c053062984e61144385022e53678fbded7aea14ebb3e0305ae3592fb219ccfa4",
-                "sha256:d4198877797a83cbfe9bffa3803602bbe1625dc30d8a097365dbc762e5790faa",
-                "sha256:e3c97a1555fd6388f857770ff8b9703083de6bf1f9274a002a332d65fbb56c8c",
-                "sha256:e7cb0ae90dd83727f0c0718634ed56837bfeeee29a5f82a7514c03ee1364c019",
-                "sha256:f0700d54bcf45424477e46a9f0944155b46fb0639d69728739c0e47bab83f2b9",
-                "sha256:f1279ab38ecbfae7e456a108c5c0681e4956d5b1090027c1de0f934dfdb4b35c",
-                "sha256:f4f118245c4a087776e0a8408be33cf09f6c547442c00395fbfb116fac2f8ac2"
+                "sha256:2c69461a7fcc8e24be697624c09a839976d82ae75062b11a0972e41fd2cd9132",
+                "sha256:35cfcb15f213449af7ff6198d6eb5f739c37d7e4f1c09b5d0641babf2cc0c68f",
+                "sha256:52235802093bd8a2811abbe8bf0ab9c5f54cca0a751fdd3f6ac2a21438bffece",
+                "sha256:59379674ff119717404f7454647913787034f03fe7049cbef1d74a97bb4593f0",
+                "sha256:5e8a95246d581eef20471b5d5ba010d55f66740942b95ba9b872d918c459452f",
+                "sha256:87317e9bcda04a32f2ee82089a204d3a2f0d3c8aeed16568c7daf4756e4f1fe0",
+                "sha256:8ddc60bf374785fb7cb12510b267f59067fa10087325b8e1855b898a0d81d276",
+                "sha256:a8b9403fc70764b08d2f593ce44f1d2920c5077bf7d311fefec999f8c40f78b7",
+                "sha256:c0ea0123dac3399a2eeb1a1443d82b7afc9ff40241433296769f7da42d142ec3",
+                "sha256:ca53faf29896c526863366a52a8f4d88e69cd04ec9571ed6082fa117fac3ab36",
+                "sha256:eeea10f3dc0ac7e6b4933d32db20662902b4ab81bf28df12218aa389e9c2102d"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==4.25.3"
+            "version": "==5.28.2"
+        },
+        "psutil": {
+            "hashes": [
+                "sha256:02615ed8c5ea222323408ceba16c60e99c3f91639b07da6373fb7e6539abc56d",
+                "sha256:05806de88103b25903dff19bb6692bd2e714ccf9e668d050d144012055cbca73",
+                "sha256:26bd09967ae00920df88e0352a91cff1a78f8d69b3ecabbfe733610c0af486c8",
+                "sha256:27cc40c3493bb10de1be4b3f07cae4c010ce715290a5be22b98493509c6299e2",
+                "sha256:36f435891adb138ed3c9e58c6af3e2e6ca9ac2f365efe1f9cfef2794e6c93b4e",
+                "sha256:50187900d73c1381ba1454cf40308c2bf6f34268518b3f36a9b663ca87e65e36",
+                "sha256:611052c4bc70432ec770d5d54f64206aa7203a101ec273a0cd82418c86503bb7",
+                "sha256:6be126e3225486dff286a8fb9a06246a5253f4c7c53b475ea5f5ac934e64194c",
+                "sha256:7d79560ad97af658a0f6adfef8b834b53f64746d45b403f225b85c5c2c140eee",
+                "sha256:8cb6403ce6d8e047495a701dc7c5bd788add903f8986d523e3e20b98b733e421",
+                "sha256:8db4c1b57507eef143a15a6884ca10f7c73876cdf5d51e713151c1236a0e68cf",
+                "sha256:aee678c8720623dc456fa20659af736241f575d79429a0e5e9cf88ae0605cc81",
+                "sha256:bc56c2a1b0d15aa3eaa5a60c9f3f8e3e565303b465dbf57a1b730e7a2b9844e0",
+                "sha256:bd1184ceb3f87651a67b2708d4c3338e9b10c5df903f2e3776b62303b26cb631",
+                "sha256:d06016f7f8625a1825ba3732081d77c94589dca78b7a3fc072194851e88461a4",
+                "sha256:d16bbddf0693323b8c6123dd804100241da461e41d6e332fb0ba6058f630f8c8"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
+            "version": "==5.9.8"
+        },
+        "py": {
+            "hashes": [
+                "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719",
+                "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==1.11.0"
         },
         "pyaes": {
             "hashes": [
@@ -1560,19 +1994,27 @@
         },
         "pyasn1": {
             "hashes": [
-                "sha256:3a35ab2c4b5ef98e17dfdec8ab074046fbda76e281c5a706ccd82328cfc8f64c",
-                "sha256:cca4bb0f2df5504f02f6f8a775b6e416ff9b0b3b16f7ee80b5a3153d9b804473"
+                "sha256:0d632f46f2ba09143da3a8afe9e33fb6f92fa2320ab7e886e2d0f7672af84629",
+                "sha256:6f580d2bdd84365380830acf45550f2511469f673cb4a5ae3857a3170128b034"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.6.0"
+            "version": "==0.6.1"
         },
         "pyasn1-modules": {
             "hashes": [
-                "sha256:831dbcea1b177b28c9baddf4c6d1013c24c3accd14a1873fffaa6a2e905f17b6",
-                "sha256:be04f15b66c206eed667e0bb5ab27e2b1855ea54a842e5037738099e8ca4ae0b"
+                "sha256:49bfa96b45a292b711e986f222502c1c9a5e1f4e568fc30e2574a6c7d07838fd",
+                "sha256:c28e2dbf9c06ad61c71a075c7e0f9fd0f1b0bb2d2ad4377f240d33ac2ab60a7c"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.4.0"
+            "version": "==0.4.1"
+        },
+        "pycodestyle": {
+            "hashes": [
+                "sha256:442f950141b4f43df752dd303511ffded3a04c2b6fb7f65980574f0c31e6e79c",
+                "sha256:949a39f6b86c3e1515ba1787c2022131d165a8ad271b11370a8819aa070269e4"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==2.12.0"
         },
         "pycparser": {
             "hashes": [
@@ -1622,45 +2064,114 @@
         },
         "pydantic": {
             "hashes": [
-                "sha256:08b6ec0917c30861e3fe71a93be1648a2aa4f62f866142ba21670b24444d7fd8",
-                "sha256:0fbb503bbbbab0c588ed3cd21975a1d0d4163b87e360fec17a792f7d8c4ff29f",
-                "sha256:1245f4f61f467cb3dfeced2b119afef3db386aec3d24a22a1de08c65038b255f",
-                "sha256:13e86a19dca96373dcf3190fcb8797d40a6f12f154a244a8d1e8e03b8f280593",
-                "sha256:21efacc678a11114c765eb52ec0db62edffa89e9a562a94cbf8fa10b5db5c046",
-                "sha256:23d47a4b57a38e8652bcab15a658fdb13c785b9ce217cc3a729504ab4e1d6bc9",
-                "sha256:24a7679fab2e0eeedb5a8924fc4a694b3bcaac7d305aeeac72dd7d4e05ecbebf",
-                "sha256:282613a5969c47c83a8710cc8bfd1e70c9223feb76566f74683af889faadc0ea",
-                "sha256:336709883c15c050b9c55a63d6c7ff09be883dbc17805d2b063395dd9d9d0022",
-                "sha256:412ab4a3f6dbd2bf18aefa9f79c7cca23744846b31f1d6555c2ee2b05a2e14ca",
-                "sha256:466669501d08ad8eb3c4fecd991c5e793c4e0bbd62299d05111d4f827cded64f",
-                "sha256:46f17b832fe27de7850896f3afee50ea682220dd218f7e9c88d436788419dca6",
-                "sha256:49a46a0994dd551ec051986806122767cf144b9702e31d47f6d493c336462597",
-                "sha256:4ae57b4d8e3312d486e2498d42aed3ece7b51848336964e43abbf9671584e67f",
-                "sha256:53e3819bd20a42470d6dd0fe7fc1c121c92247bca104ce608e609b59bc7a77ee",
-                "sha256:596f12a1085e38dbda5cbb874d0973303e34227b400b6414782bf205cc14940c",
-                "sha256:646b2b12df4295b4c3148850c85bff29ef6d0d9621a8d091e98094871a62e5c7",
-                "sha256:798a3d05ee3b71967844a1164fd5bdb8c22c6d674f26274e78b9f29d81770c4e",
-                "sha256:7f4fcec873f90537c382840f330b90f4715eebc2bc9925f04cb92de593eae054",
-                "sha256:82d886bd3c3fbeaa963692ef6b643159ccb4b4cefaf7ff1617720cbead04fd1d",
-                "sha256:8e3a76f571970fcd3c43ad982daf936ae39b3e90b8a2e96c04113a369869dc87",
-                "sha256:8ee853cd12ac2ddbf0ecbac1c289f95882b2d4482258048079d13be700aa114c",
-                "sha256:9d578ac4bf7fdf10ce14caba6f734c178379bd35c486c6deb6f49006e1ba78a7",
-                "sha256:a42c7d17706911199798d4c464b352e640cab4351efe69c2267823d619a937e5",
-                "sha256:aad4e10efa5474ed1a611b6d7f0d130f4aafadceb73c11d9e72823e8f508e663",
-                "sha256:ad8c2bc677ae5f6dbd3cf92f2c7dc613507eafe8f71719727cbc0a7dec9a8c01",
-                "sha256:bc3ed06ab13660b565eed80887fcfbc0070f0aa0691fbb351657041d3e874efe",
-                "sha256:bfb113860e9288d0886e3b9e49d9cf4a9d48b441f52ded7d96db7819028514cc",
-                "sha256:c37c28449752bb1f47975d22ef2882d70513c546f8f37201e0fec3a97b816eee",
-                "sha256:c66609e138c31cba607d8e2a7b6a5dc38979a06c900815495b2d90ce6ded35b4",
-                "sha256:d604be0f0b44d473e54fdcb12302495fe0467c56509a2f80483476f3ba92b33c",
-                "sha256:d986e115e0b39604b9eee3507987368ff8148222da213cd38c359f6f57b3b347",
-                "sha256:dba49d52500c35cfec0b28aa8b3ea5c37c9df183ffc7210b10ff2a415c125c4a",
-                "sha256:e897c9f35281f7889873a3e6d6b69aa1447ceb024e8495a5f0d02ecd17742a7f",
-                "sha256:f9f674b5c3bebc2eba401de64f29948ae1e646ba2735f884d1594c5f675d6f2a",
-                "sha256:fa7790e94c60f809c95602a26d906eba01a0abee9cc24150e4ce2189352deb1b"
+                "sha256:d155cef71265d1e9807ed1c32b4c8deec042a44a50a4188b25ac67ecd81a9c0f",
+                "sha256:f048cec7b26778210e28a0459867920654d48e5e62db0958433636cde4254f12"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==1.10.14"
+            "markers": "python_version >= '3.8'",
+            "version": "==2.9.2"
+        },
+        "pydantic-core": {
+            "hashes": [
+                "sha256:0a7df63886be5e270da67e0966cf4afbae86069501d35c8c1b3b6c168f42cb36",
+                "sha256:0cb3da3fd1b6a5d0279a01877713dbda118a2a4fc6f0d821a57da2e464793f05",
+                "sha256:0dbd8dbed2085ed23b5c04afa29d8fd2771674223135dc9bc937f3c09284d071",
+                "sha256:0dff76e0602ca7d4cdaacc1ac4c005e0ce0dcfe095d5b5259163a80d3a10d327",
+                "sha256:1278e0d324f6908e872730c9102b0112477a7f7cf88b308e4fc36ce1bdb6d58c",
+                "sha256:128585782e5bfa515c590ccee4b727fb76925dd04a98864182b22e89a4e6ed36",
+                "sha256:1498bec4c05c9c787bde9125cfdcc63a41004ff167f495063191b863399b1a29",
+                "sha256:19442362866a753485ba5e4be408964644dd6a09123d9416c54cd49171f50744",
+                "sha256:1b84d168f6c48fabd1f2027a3d1bdfe62f92cade1fb273a5d68e621da0e44e6d",
+                "sha256:1e90d2e3bd2c3863d48525d297cd143fe541be8bbf6f579504b9712cb6b643ec",
+                "sha256:20152074317d9bed6b7a95ade3b7d6054845d70584216160860425f4fbd5ee9e",
+                "sha256:216f9b2d7713eb98cb83c80b9c794de1f6b7e3145eef40400c62e86cee5f4e1e",
+                "sha256:233710f069d251feb12a56da21e14cca67994eab08362207785cf8c598e74577",
+                "sha256:255a8ef062cbf6674450e668482456abac99a5583bbafb73f9ad469540a3a232",
+                "sha256:2584f7cf844ac4d970fba483a717dbe10c1c1c96a969bf65d61ffe94df1b2863",
+                "sha256:2971bb5ffe72cc0f555c13e19b23c85b654dd2a8f7ab493c262071377bfce9f6",
+                "sha256:29d2c342c4bc01b88402d60189f3df065fb0dda3654744d5a165a5288a657368",
+                "sha256:2e203fdf807ac7e12ab59ca2bfcabb38c7cf0b33c41efeb00f8e5da1d86af480",
+                "sha256:33e3d65a85a2a4a0dc3b092b938a4062b1a05f3a9abde65ea93b233bca0e03f2",
+                "sha256:374a5e5049eda9e0a44c696c7ade3ff355f06b1fe0bb945ea3cac2bc336478a2",
+                "sha256:37b0fe330e4a58d3c58b24d91d1eb102aeec675a3db4c292ec3928ecd892a9a6",
+                "sha256:3d5639516376dce1940ea36edf408c554475369f5da2abd45d44621cb616f769",
+                "sha256:42c6dcb030aefb668a2b7009c85b27f90e51e6a3b4d5c9bc4c57631292015b0d",
+                "sha256:4a7cd62e831afe623fbb7aabbb4fe583212115b3ef38a9f6b71869ba644624a2",
+                "sha256:4ba762ed58e8d68657fc1281e9bb72e1c3e79cc5d464be146e260c541ec12d84",
+                "sha256:4fc714bdbfb534f94034efaa6eadd74e5b93c8fa6315565a222f7b6f42ca1166",
+                "sha256:4ffa2ebd4c8530079140dd2d7f794a9d9a73cbb8e9d59ffe24c63436efa8f271",
+                "sha256:5a1504ad17ba4210df3a045132a7baeeba5a200e930f57512ee02909fc5c4cb5",
+                "sha256:5c364564d17da23db1106787675fc7af45f2f7b58b4173bfdd105564e132e6fb",
+                "sha256:5e11661ce0fd30a6790e8bcdf263b9ec5988e95e63cf901972107efc49218b13",
+                "sha256:5f54b118ce5de9ac21c363d9b3caa6c800341e8c47a508787e5868c6b79c9323",
+                "sha256:5f5ff8d839f4566a474a969508fe1c5e59c31c80d9e140566f9a37bba7b8d556",
+                "sha256:61817945f2fe7d166e75fbfb28004034b48e44878177fc54d81688e7b85a3665",
+                "sha256:624e278a7d29b6445e4e813af92af37820fafb6dcc55c012c834f9e26f9aaaef",
+                "sha256:63e46b3169866bd62849936de036f901a9356e36376079b05efa83caeaa02ceb",
+                "sha256:6531b7ca5f951d663c339002e91aaebda765ec7d61b7d1e3991051906ddde119",
+                "sha256:68665f4c17edcceecc112dfed5dbe6f92261fb9d6054b47d01bf6371a6196126",
+                "sha256:696dd8d674d6ce621ab9d45b205df149399e4bb9aa34102c970b721554828510",
+                "sha256:6f783e0ec4803c787bcea93e13e9932edab72068f68ecffdf86a99fd5918878b",
+                "sha256:723314c1d51722ab28bfcd5240d858512ffd3116449c557a1336cbe3919beb87",
+                "sha256:74b9127ffea03643e998e0c5ad9bd3811d3dac8c676e47db17b0ee7c3c3bf35f",
+                "sha256:7530e201d10d7d14abce4fb54cfe5b94a0aefc87da539d0346a484ead376c3cc",
+                "sha256:77733e3892bb0a7fa797826361ce8a9184d25c8dffaec60b7ffe928153680ba8",
+                "sha256:78ddaaa81421a29574a682b3179d4cf9e6d405a09b99d93ddcf7e5239c742e21",
+                "sha256:7c9129eb40958b3d4500fa2467e6a83356b3b61bfff1b414c7361d9220f9ae8f",
+                "sha256:7d32706badfe136888bdea71c0def994644e09fff0bfe47441deaed8e96fdbc6",
+                "sha256:81965a16b675b35e1d09dd14df53f190f9129c0202356ed44ab2728b1c905658",
+                "sha256:8394d940e5d400d04cad4f75c0598665cbb81aecefaca82ca85bd28264af7f9b",
+                "sha256:86d2f57d3e1379a9525c5ab067b27dbb8a0642fb5d454e17a9ac434f9ce523e3",
+                "sha256:883a91b5dd7d26492ff2f04f40fbb652de40fcc0afe07e8129e8ae779c2110eb",
+                "sha256:88ad334a15b32a791ea935af224b9de1bf99bcd62fabf745d5f3442199d86d59",
+                "sha256:9261d3ce84fa1d38ed649c3638feefeae23d32ba9182963e465d58d62203bd24",
+                "sha256:97df63000f4fea395b2824da80e169731088656d1818a11b95f3b173747b6cd9",
+                "sha256:98d134c954828488b153d88ba1f34e14259284f256180ce659e8d83e9c05eaa3",
+                "sha256:996a38a83508c54c78a5f41456b0103c30508fed9abcad0a59b876d7398f25fd",
+                "sha256:9a5bce9d23aac8f0cf0836ecfc033896aa8443b501c58d0602dbfd5bd5b37753",
+                "sha256:9a6b5099eeec78827553827f4c6b8615978bb4b6a88e5d9b93eddf8bb6790f55",
+                "sha256:9d18368b137c6295db49ce7218b1a9ba15c5bc254c96d7c9f9e924a9bc7825ad",
+                "sha256:a4fa4fc04dff799089689f4fd502ce7d59de529fc2f40a2c8836886c03e0175a",
+                "sha256:a5c7ba8ffb6d6f8f2ab08743be203654bb1aaa8c9dcb09f82ddd34eadb695605",
+                "sha256:aea443fffa9fbe3af1a9ba721a87f926fe548d32cab71d188a6ede77d0ff244e",
+                "sha256:b10bd51f823d891193d4717448fab065733958bdb6a6b351967bd349d48d5c9b",
+                "sha256:ba1a0996f6c2773bd83e63f18914c1de3c9dd26d55f4ac302a7efe93fb8e7433",
+                "sha256:bb2802e667b7051a1bebbfe93684841cc9351004e2badbd6411bf357ab8d5ac8",
+                "sha256:cfdd16ab5e59fc31b5e906d1a3f666571abc367598e3e02c83403acabc092e07",
+                "sha256:d06b0c8da4f16d1d1e352134427cb194a0a6e19ad5db9161bf32b2113409e728",
+                "sha256:d0776dea117cf5272382634bd2a5c1b6eb16767c223c6a5317cd3e2a757c61a0",
+                "sha256:d18ca8148bebe1b0a382a27a8ee60350091a6ddaf475fa05ef50dc35b5df6327",
+                "sha256:d4488a93b071c04dc20f5cecc3631fc78b9789dd72483ba15d423b5b3689b555",
+                "sha256:d5f7a395a8cf1621939692dba2a6b6a830efa6b3cee787d82c7de1ad2930de64",
+                "sha256:d7a80d21d613eec45e3d41eb22f8f94ddc758a6c4720842dc74c0581f54993d6",
+                "sha256:d97683ddee4723ae8c95d1eddac7c192e8c552da0c73a925a89fa8649bf13eea",
+                "sha256:dcedcd19a557e182628afa1d553c3895a9f825b936415d0dbd3cd0bbcfd29b4b",
+                "sha256:de6d1d1b9e5101508cb37ab0d972357cac5235f5c6533d1071964c47139257df",
+                "sha256:df49e7a0861a8c36d089c1ed57d308623d60416dab2647a4a17fe050ba85de0e",
+                "sha256:df933278128ea1cd77772673c73954e53a1c95a4fdf41eef97c2b779271bd0bd",
+                "sha256:e08277a400de01bc72436a0ccd02bdf596631411f592ad985dcee21445bd0068",
+                "sha256:e38e63e6f3d1cec5a27e0afe90a085af8b6806ee208b33030e65b6516353f1a3",
+                "sha256:e55541f756f9b3ee346b840103f32779c695a19826a4c442b7954550a0972040",
+                "sha256:ec4e55f79b1c4ffb2eecd8a0cfba9955a2588497d96851f4c8f99aa4a1d39b12",
+                "sha256:ed1a53de42fbe34853ba90513cea21673481cd81ed1be739f7f2efb931b24916",
+                "sha256:ed541d70698978a20eb63d8c5d72f2cc6d7079d9d90f6b50bad07826f1320f5f",
+                "sha256:f09e2ff1f17c2b51f2bc76d1cc33da96298f0a036a137f5440ab3ec5360b624f",
+                "sha256:f220b0eea5965dec25480b6333c788fb72ce5f9129e8759ef876a1d805d00801",
+                "sha256:f3e0da4ebaef65158d4dfd7d3678aad692f7666877df0002b8a522cdf088f231",
+                "sha256:f455ee30a9d61d3e1a15abd5068827773d6e4dc513e795f380cdd59932c782d5",
+                "sha256:f5ef8f42bec47f21d07668a043f077d507e5bf4e668d5c6dfe6aaba89de1a5b8",
+                "sha256:f69a8e0b033b747bb3e36a44e7732f0c99f7edd5cea723d45bc0d6e95377ffee",
+                "sha256:ff02b6d461a6de369f07ec15e465a88895f3223eb75073ffea56b84d9331f607"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==2.23.4"
+        },
+        "pyflakes": {
+            "hashes": [
+                "sha256:1c61603ff154621fb2a9172037d84dca3500def8c8b630657d1701f026f8af3f",
+                "sha256:84b5be138a2dfbb40689ca07e2152deb896a65c3a3e24c251c5c62489568074a"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==3.2.0"
         },
         "pygments": {
             "hashes": [
@@ -1672,19 +2183,19 @@
         },
         "pyopenssl": {
             "hashes": [
-                "sha256:17ed5be5936449c5418d1cd269a1a9e9081bc54c17aed272b45856a3d3dc86ad",
-                "sha256:cabed4bfaa5df9f1a16c0ef64a0cb65318b5cd077a7eda7d6970131ca2f41a6f"
+                "sha256:4247f0dbe3748d560dcbb2ff3ea01af0f9a1a001ef5f7c4c647956ed8cbf0e95",
+                "sha256:967d5719b12b243588573f39b0c677637145c7a1ffedcd495a487e58177fbb8d"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==24.1.0"
+            "version": "==24.2.1"
         },
         "pyparsing": {
             "hashes": [
-                "sha256:a1bac0ce561155ecc3ed78ca94d3c9378656ad4c94c1270de543f621420f94ad",
-                "sha256:f9db75911801ed778fe61bb643079ff86601aca99fcae6345aa67292038fb742"
+                "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb",
+                "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"
             ],
-            "markers": "python_version >= '3.1'",
-            "version": "==3.1.2"
+            "markers": "python_full_version >= '3.6.8'",
+            "version": "==3.0.9"
         },
         "pysocks": {
             "hashes": [
@@ -1696,11 +2207,35 @@
         },
         "pysubs2": {
             "hashes": [
-                "sha256:0261611e71735ff7763972c519c72593c8063efcb9039c54af65f31b81cec116",
-                "sha256:1f96d9dfb5f859a54a00e04621beb20ff21ea1d788821b2f4935c5c0ef8dc68e"
+                "sha256:b0130f373390736754531be4e68a0fa521e825fa15cc8ff506e4f8ca2c17459a",
+                "sha256:de438c868d2c656781c4a78f220ec3a6fd6d52be49266c81fe912d2527002d44"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==1.6.1"
+            "markers": "python_version >= '3.8'",
+            "version": "==1.7.3"
+        },
+        "pytest": {
+            "hashes": [
+                "sha256:c434598117762e2bd304e526244f67bf66bbd7b5d6cf22138be51ff661980343",
+                "sha256:de4bb8104e201939ccdc688b27a89a7be2079b22e2bd2b07f806b6ba71117977"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==8.2.2"
+        },
+        "pytest-cov": {
+            "hashes": [
+                "sha256:4f0764a1219df53214206bf1feea4633c3b558a2925c8b59f144f682861ce652",
+                "sha256:5837b58e9f6ebd335b0f8060eecce69b662415b16dc503883a02f45dfeb14857"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==5.0.0"
+        },
+        "pytest-sphinx": {
+            "hashes": [
+                "sha256:3b63c8181b9de6a5e5c9826d1b4dc0c827245bec8e64c9f16f269be08be5ecd5",
+                "sha256:856e760e64dfbfc89e362e187d641140a267b97881d3ef8aeefb72cc8438ac40"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==0.6.3"
         },
         "python-dateutil": {
             "hashes": [
@@ -1737,67 +2272,77 @@
         },
         "pytz": {
             "hashes": [
-                "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812",
-                "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319"
+                "sha256:1e760e2fe6a8163bc0b3d9a19c4f84342afa0a2affebfaa84b01b978a02ecaa7",
+                "sha256:e68985985296d9a66a881eb3193b0906246245294a881e7c8afe623866ac6a5c"
             ],
-            "version": "==2024.1"
+            "version": "==2022.1"
         },
         "pyyaml": {
             "hashes": [
-                "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5",
-                "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc",
-                "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df",
-                "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741",
-                "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206",
-                "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27",
-                "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595",
-                "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62",
-                "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98",
-                "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696",
-                "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290",
-                "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9",
-                "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d",
-                "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6",
-                "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867",
-                "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47",
-                "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486",
-                "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6",
-                "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3",
-                "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007",
-                "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938",
-                "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0",
-                "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c",
-                "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735",
-                "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d",
-                "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28",
-                "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4",
-                "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba",
-                "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8",
-                "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef",
-                "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5",
-                "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd",
-                "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3",
-                "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0",
-                "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515",
-                "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c",
-                "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c",
-                "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924",
-                "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34",
-                "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43",
-                "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859",
-                "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673",
-                "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54",
-                "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a",
-                "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b",
-                "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab",
-                "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa",
-                "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c",
-                "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585",
-                "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d",
-                "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"
+                "sha256:0101357af42f5c9fc7e9acc5c5ab8c3049f50db7425de175b6c7a5959cb6023d",
+                "sha256:0ae563b7e3ed5e918cd0184060e28b48b7e672b975bf7c6f4a892cee9d886ada",
+                "sha256:0fe2c1c5401a3a98f06337fed48f57340cf652a685484834b44f5ceeadb772ba",
+                "sha256:1eb00dd3344da80264261ab126c95481824669ed9e5ecc82fb2d88b1fce668ee",
+                "sha256:2086b30215c433c1e480c08c1db8b43c1edd36c59cf43d36b424e6f35fcaf1ad",
+                "sha256:29b4a67915232f79506211e69943e3102e211c616181ceff0adf34e21b469357",
+                "sha256:2e9bc8a34797f0621f56160b961d47a088644370f79d34bedc934fb89e3f47dd",
+                "sha256:30ec6b9afc17353a9abcff109880edf6e8d5b924eb1eeed7fe9376febc1f9800",
+                "sha256:31573d7e161d2f905311f036b12e65c058389b474dbd35740f4880b91e2ca2be",
+                "sha256:36d7bf63558843ea2a81de9d0c3e9c56c353b1df8e6c1faaec86df5adedf2e02",
+                "sha256:3af6b36bc195d741cd5b511810246cad143b99c953b4591e679e194a820d7b7c",
+                "sha256:414629800a1ddccd7303471650843fc801801cc579a195d2fe617b5b455409e3",
+                "sha256:459113f2b9cd68881201a3bd1a858ece3281dc0e92ece6e917d23b128f0fcb31",
+                "sha256:46e4fae38d00b40a62d32d60f1baa1b9ef33aff28c2aafd96b05d5cc770f1583",
+                "sha256:4bf821ccd51e8d5bc1a4021b8bd85a92b498832ac1cd1a53b399f0eb7c1c4258",
+                "sha256:50bd6560a6df3de59336b9a9086cbdea5aa9eee5361661448ee45c21eeb0da68",
+                "sha256:53056b51f111223e603bed1db5367f54596d44cacfa50f07e082a11929612957",
+                "sha256:53c5f0749a93e3296078262c9acf632de246241ff2f22bbedfe49d4b55e9bbdd",
+                "sha256:54c754cee6937bb9b72d6a16163160dec80b93a43020ac6fc9f13729c030c30b",
+                "sha256:58cc18ccbade0c48fb55102aa971a5b4e571e2b22187d083dda33f8708fa4ee7",
+                "sha256:5921fd128fbf27ab7c7ad1a566d2cd9557b84ade130743a7c110a55e7dec3b3c",
+                "sha256:5c758cc29713c9166750a30156ca3d90ac2515d5dea3c874377ae8829cf03087",
+                "sha256:60bf91e73354c96754220a9c04a9502c2ad063231cd754b59f8e4511157e32e2",
+                "sha256:6f0f728a88c6eb58a3b762726b965bb6acf12d97f8ea2cb4fecf856a727f9bdc",
+                "sha256:6f31c5935310da69ea0efe996a962d488f080312f0eb43beff1717acb5fe9bed",
+                "sha256:728b447d0cedec409ea1a3f0ad1a6cc3cec0a8d086611b45f038a9230a2242f3",
+                "sha256:72ffbc5c0cc71877104387548a450f2b7b7c4926b40dc9443e7598fe92aa13d9",
+                "sha256:73d8b233309ecd45c33c51cd55aa1be1dcab1799a9e54f6c753d8cab054b8c34",
+                "sha256:765029d1cf96e9e761329ee1c20f1ca2de8644e7350a151b198260698b96e30f",
+                "sha256:7ee3d180d886a3bc50f753b76340f1c314f9e8c507f5b107212112214c3a66fd",
+                "sha256:826fb4d5ac2c48b9d6e71423def2669d4646c93b6c13612a71b3ac7bb345304b",
+                "sha256:84c39ceec517cd8f01cb144efb08904a32050be51c55b7a59bc7958c8091568d",
+                "sha256:88bfe675bb19ae12a9c77c52322a28a8e2a8d3d213fbcfcded5c3f5ca3ead352",
+                "sha256:8e0a1ebd5c5842595365bf90db3ef7e9a8d6a79c9aedb1d05b675c81c7267fd3",
+                "sha256:9426067a10b369474396bf57fdf895b899045a25d1848798844693780b147436",
+                "sha256:9c5c0de7ec50d4df88b62f4b019ab7b3bb2883c826a1044268e9afb344c57b17",
+                "sha256:ad0c172fe15beffc32e3a8260f18e6708eb0e15ae82c9b3f80fbe04de0ef5729",
+                "sha256:ad206c7f5f08d393b872d3399f597246fdc6ebebff09c5ae5268ac45aebf4f8d",
+                "sha256:b0a163f4f84d1e0fe6a07ccad3b02e9b243790b8370ff0408ae5932c50c4d96d",
+                "sha256:b0dd9c7497d60126445e79e542ff01351c6b6dc121299d89787f5685b382c626",
+                "sha256:b1de10c488d6f02e498eb6956b89081bea31abf3133223c17749e7137734da75",
+                "sha256:b408f36eeb4e2be6f802f1be82daf1b578f3de5a51917c6e467aedb46187d827",
+                "sha256:bae077a01367e4bf5fddf00fd6c8b743e676385911c7c615e29e1c45ace8813b",
+                "sha256:bc3c3600fec6c2a719106381d6282061d8c108369cdec58b6f280610eba41e09",
+                "sha256:c16522bf91daa4ea9dedc1243b56b5a226357ab98b3133089ca627ef99baae6f",
+                "sha256:ca5136a77e2d64b4cf5106fb940376650ae232c74c09a8ff29dbb1e262495b31",
+                "sha256:d6e0f7ee5f8d851b1d91149a3e5074dbf5aacbb63e4b771fcce16508339a856f",
+                "sha256:e7930a0612e74fcca37019ca851b50d73b5f0c3dab7f3085a7c15d2026118315",
+                "sha256:e8e6dd230a158a836cda3cc521fcbedea16f22b16b8cfa8054d0c6cea5d0a531",
+                "sha256:eee36bf4bc11e39e3f17c171f25cdedff3d7c73b148aedc8820257ce2aa56d3b",
+                "sha256:f07adc282d51aaa528f3141ac1922d16d32fe89413ee59bfb8a73ed689ad3d23",
+                "sha256:f09816c047fdb588dddba53d321f1cb8081e38ad2a40ea6a7560a88b7a2f0ea8",
+                "sha256:fea4c4310061cd70ef73b39801231b9dc3dc638bb8858e38364b144fbd335a1a"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==6.0.1"
+            "version": "==6.0.2rc1"
+        },
+        "readme-renderer": {
+            "hashes": [
+                "sha256:1818dd28140813509eeed8d62687f7cd4f7bad90d4db586001c5dc09d4fde311",
+                "sha256:19db308d86ecd60e5affa3b2a98f017af384678c63c88e5d4556a380e674f3f9"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==43.0"
         },
         "redis": {
             "hashes": [
@@ -1810,100 +2355,115 @@
         },
         "regex": {
             "hashes": [
-                "sha256:031219782d97550c2098d9a68ce9e9eaefe67d2d81d8ff84c8354f9c009e720c",
-                "sha256:0709ba544cf50bd5cb843df4b8bb6701bae2b70a8e88da9add8386cbca5c1385",
-                "sha256:0a9f89d7db5ef6bdf53e5cc8e6199a493d0f1374b3171796b464a74ebe8e508a",
-                "sha256:0bc94873ba11e34837bffd7e5006703abeffc4514e2f482022f46ce05bd25e67",
-                "sha256:0ce56a923f4c01d7568811bfdffe156268c0a7aae8a94c902b92fe34c4bde785",
-                "sha256:0faecb6d5779753a6066a3c7a0471a8d29fe25d9981ca9e552d6d1b8f8b6a594",
-                "sha256:1118ba9def608250250f4b3e3f48c62f4562ba16ca58ede491b6e7554bfa09ff",
-                "sha256:12446827f43c7881decf2c126762e11425de5eb93b3b0d8b581344c16db7047a",
-                "sha256:14905ed75c7a6edf423eb46c213ed3f4507c38115f1ed3c00f4ec9eafba50e58",
-                "sha256:15e593386ec6331e0ab4ac0795b7593f02ab2f4b30a698beb89fbdc34f92386a",
-                "sha256:160ba087232c5c6e2a1e7ad08bd3a3f49b58c815be0504d8c8aacfb064491cd8",
-                "sha256:161a206c8f3511e2f5fafc9142a2cc25d7fe9a1ec5ad9b4ad2496a7c33e1c5d2",
-                "sha256:169fd0acd7a259f58f417e492e93d0e15fc87592cd1e971c8c533ad5703b5830",
-                "sha256:193b7c6834a06f722f0ce1ba685efe80881de7c3de31415513862f601097648c",
-                "sha256:1a3903128f9e17a500618e80c68165c78c741ebb17dd1a0b44575f92c3c68b02",
-                "sha256:1d5bd666466c8f00a06886ce1397ba8b12371c1f1c6d1bef11013e9e0a1464a8",
-                "sha256:224a9269f133564109ce668213ef3cb32bc72ccf040b0b51c72a50e569e9dc9e",
-                "sha256:236cace6c1903effd647ed46ce6dd5d76d54985fc36dafc5256032886736c85d",
-                "sha256:249fbcee0a277c32a3ce36d8e36d50c27c968fdf969e0fbe342658d4e010fbc8",
-                "sha256:29d839829209f3c53f004e1de8c3113efce6d98029f044fa5cfee666253ee7e6",
-                "sha256:2c8982ee19ccecabbaeac1ba687bfef085a6352a8c64f821ce2f43e6d76a9298",
-                "sha256:2f30a5ab8902f93930dc6f627c4dd5da2703333287081c85cace0fc6e21c25af",
-                "sha256:304e7e2418146ae4d0ef0e9ffa28f881f7874b45b4994cc2279b21b6e7ae50c8",
-                "sha256:32e5f3b8e32918bfbdd12eca62e49ab3031125c454b507127ad6ecbd86e62fca",
-                "sha256:334b79ce9c08f26b4659a53f42892793948a613c46f1b583e985fd5a6bf1c149",
-                "sha256:33d19f0cde6838c81acffff25c7708e4adc7dd02896c9ec25c3939b1500a1778",
-                "sha256:3799e36d60a35162bb35b2246d8bb012192b7437dff807ef79c14e7352706306",
-                "sha256:42be5de7cc8c1edac55db92d82b68dc8e683b204d6f5414c5a51997a323d7081",
-                "sha256:44b3267cea873684af022822195298501568ed44d542f9a2d9bebc0212e99069",
-                "sha256:458d68d34fb74b906709735c927c029e62f7d06437a98af1b5b6258025223210",
-                "sha256:45cc13d398b6359a7708986386f72bd156ae781c3e83a68a6d4cee5af04b1ce9",
-                "sha256:4e7eaf9df15423d07b6050fb91f86c66307171b95ea53e2d87a7993b6d02c7f7",
-                "sha256:4fad420b14ae1970a1f322e8ae84a1d9d89375eb71e1b504060ab2d1bfe68f3c",
-                "sha256:504b5116e2bd1821efd815941edff7535e93372a098e156bb9dffde30264e798",
-                "sha256:50e7e96a527488334379e05755b210b7da4a60fc5d6481938c1fa053e0c92184",
-                "sha256:51d27844763c273a122e08a3e86e7aefa54ee09fb672d96a645ece0454d8425e",
-                "sha256:5253dcb0bfda7214523de58b002eb0090cb530d7c55993ce5f6d17faf953ece7",
-                "sha256:534efd2653ebc4f26fc0e47234e53bf0cb4715bb61f98c64d2774a278b58c846",
-                "sha256:560278c9975694e1f0bc50da187abf2cdc1e4890739ea33df2bc4a85eeef143e",
-                "sha256:571452362d552de508c37191b6abbbb660028b8b418e2d68c20779e0bc8eaaa8",
-                "sha256:62b5f7910b639f3c1d122d408421317c351e213ca39c964ad4121f27916631c6",
-                "sha256:696639a73ca78a380acfaa0a1f6dd8220616a99074c05bba9ba8bb916914b224",
-                "sha256:6ccdeef4584450b6f0bddd5135354908dacad95425fcb629fe36d13e48b60f32",
-                "sha256:70364a097437dd0a90b31cd77f09f7387ad9ac60ef57590971f43b7fca3082a5",
-                "sha256:7117cb7d6ac7f2e985f3d18aa8a1728864097da1a677ffa69e970ca215baebf1",
-                "sha256:7467ad8b0eac0b28e52679e972b9b234b3de0ea5cee12eb50091d2b68145fe36",
-                "sha256:7d35d4cc9270944e95f9c88af757b0c9fc43f396917e143a5756608462c5223b",
-                "sha256:7dda3091838206969c2b286f9832dff41e2da545b99d1cfaea9ebd8584d02708",
-                "sha256:853cc36e756ff673bf984e9044ccc8fad60b95a748915dddeab9488aea974c73",
-                "sha256:8722f72068b3e1156a4b2e1afde6810f1fc67155a9fa30a4b9d5b4bc46f18fb0",
-                "sha256:8c6c71cf92b09e5faa72ea2c68aa1f61c9ce11cb66fdc5069d712f4392ddfd00",
-                "sha256:903350bf44d7e4116b4d5898b30b15755d61dcd3161e3413a49c7db76f0bee5a",
-                "sha256:91b53dea84415e8115506cc62e441a2b54537359c63d856d73cb1abe05af4c9a",
-                "sha256:951be1eae7b47660412dc4938777a975ebc41936d64e28081bf2e584b47ec246",
-                "sha256:972b49f2fe1047b9249c958ec4fa1bdd2cf8ce305dc19d27546d5a38e57732d8",
-                "sha256:9a8625849387b9d558d528e263ecc9c0fbde86cfa5c2f0eef43fff480ae24d71",
-                "sha256:9cdbb1998da94607d5eec02566b9586f0e70d6438abf1b690261aac0edda7ab6",
-                "sha256:9e6d4d6ae1827b2f8c7200aaf7501c37cf3f3896c86a6aaf2566448397c823dd",
-                "sha256:aab65121229c2ecdf4a31b793d99a6a0501225bd39b616e653c87b219ed34a49",
-                "sha256:ab98016541543692a37905871a5ffca59b16e08aacc3d7d10a27297b443f572d",
-                "sha256:ad45f3bccfcb00868f2871dce02a755529838d2b86163ab8a246115e80cfb7d6",
-                "sha256:b43b78f9386d3d932a6ce5af4b45f393d2e93693ee18dc4800d30a8909df700e",
-                "sha256:b66421f8878a0c82fc0c272a43e2121c8d4c67cb37429b764f0d5ad70b82993b",
-                "sha256:ba034c8db4b264ef1601eb33cd23d87c5013b8fb48b8161debe2e5d3bd9156b0",
-                "sha256:bbdc5db2c98ac2bf1971ffa1410c87ca7a15800415f788971e8ba8520fc0fda9",
-                "sha256:bc0db93ad039fc2fe32ccd3dd0e0e70c4f3d6e37ae83f0a487e1aba939bd2fbd",
-                "sha256:bf7c8ee4861d9ef5b1120abb75846828c811f932d63311596ad25fa168053e00",
-                "sha256:bf9596cba92ce7b1fd32c7b07c6e3212c7eed0edc271757e48bfcd2b54646452",
-                "sha256:c43395a3b7cc9862801a65c6994678484f186ce13c929abab44fb8a9e473a55a",
-                "sha256:c46a76a599fcbf95f98755275c5527304cc4f1bb69919434c1e15544d7052910",
-                "sha256:ca23b41355ba95929e9505ee04e55495726aa2282003ed9b012d86f857d3e49b",
-                "sha256:cd832bd9b6120d6074f39bdfbb3c80e416848b07ac72910f1c7f03131a6debc3",
-                "sha256:cfa6d61a76c77610ba9274c1a90a453062bdf6887858afbe214d18ad41cf6bde",
-                "sha256:d8a0f0ab5453e409586b11ebe91c672040bc804ca98d03a656825f7890cbdf88",
-                "sha256:e91b1976358e17197157b405cab408a5f4e33310cda211c49fc6da7cffd0b2f0",
-                "sha256:ea057306ab469130167014b662643cfaed84651c792948891d003cf0039223a5",
-                "sha256:eda3dd46df535da787ffb9036b5140f941ecb91701717df91c9daf64cabef953",
-                "sha256:f03b1dbd4d9596dd84955bb40f7d885204d6aac0d56a919bb1e0ff2fb7e1735a",
-                "sha256:fa9335674d7c819674467c7b46154196c51efbaf5f5715187fd366814ba3fa39"
+                "sha256:01c2acb51f8a7d6494c8c5eafe3d8e06d76563d8a8a4643b37e9b2dd8a2ff623",
+                "sha256:02087ea0a03b4af1ed6ebab2c54d7118127fee8d71b26398e8e4b05b78963199",
+                "sha256:040562757795eeea356394a7fb13076ad4f99d3c62ab0f8bdfb21f99a1f85664",
+                "sha256:042c55879cfeb21a8adacc84ea347721d3d83a159da6acdf1116859e2427c43f",
+                "sha256:079400a8269544b955ffa9e31f186f01d96829110a3bf79dc338e9910f794fca",
+                "sha256:07f45f287469039ffc2c53caf6803cd506eb5f5f637f1d4acb37a738f71dd066",
+                "sha256:09d77559e80dcc9d24570da3745ab859a9cf91953062e4ab126ba9d5993688ca",
+                "sha256:0cbff728659ce4bbf4c30b2a1be040faafaa9eca6ecde40aaff86f7889f4ab39",
+                "sha256:0e12c481ad92d129c78f13a2a3662317e46ee7ef96c94fd332e1c29131875b7d",
+                "sha256:0ea51dcc0835eea2ea31d66456210a4e01a076d820e9039b04ae8d17ac11dee6",
+                "sha256:0ffbcf9221e04502fc35e54d1ce9567541979c3fdfb93d2c554f0ca583a19b35",
+                "sha256:1494fa8725c285a81d01dc8c06b55287a1ee5e0e382d8413adc0a9197aac6408",
+                "sha256:16e13a7929791ac1216afde26f712802e3df7bf0360b32e4914dca3ab8baeea5",
+                "sha256:18406efb2f5a0e57e3a5881cd9354c1512d3bb4f5c45d96d110a66114d84d23a",
+                "sha256:18e707ce6c92d7282dfce370cd205098384b8ee21544e7cb29b8aab955b66fa9",
+                "sha256:220e92a30b426daf23bb67a7962900ed4613589bab80382be09b48896d211e92",
+                "sha256:23b30c62d0f16827f2ae9f2bb87619bc4fba2044911e2e6c2eb1af0161cdb766",
+                "sha256:23f9985c8784e544d53fc2930fc1ac1a7319f5d5332d228437acc9f418f2f168",
+                "sha256:297f54910247508e6e5cae669f2bc308985c60540a4edd1c77203ef19bfa63ca",
+                "sha256:2b08fce89fbd45664d3df6ad93e554b6c16933ffa9d55cb7e01182baaf971508",
+                "sha256:2cce2449e5927a0bf084d346da6cd5eb016b2beca10d0013ab50e3c226ffc0df",
+                "sha256:313ea15e5ff2a8cbbad96ccef6be638393041b0a7863183c2d31e0c6116688cf",
+                "sha256:323c1f04be6b2968944d730e5c2091c8c89767903ecaa135203eec4565ed2b2b",
+                "sha256:35f4a6f96aa6cb3f2f7247027b07b15a374f0d5b912c0001418d1d55024d5cb4",
+                "sha256:3b37fa423beefa44919e009745ccbf353d8c981516e807995b2bd11c2c77d268",
+                "sha256:3ce4f1185db3fbde8ed8aa223fc9620f276c58de8b0d4f8cc86fd1360829edb6",
+                "sha256:46989629904bad940bbec2106528140a218b4a36bb3042d8406980be1941429c",
+                "sha256:4838e24ee015101d9f901988001038f7f0d90dc0c3b115541a1365fb439add62",
+                "sha256:49b0e06786ea663f933f3710a51e9385ce0cba0ea56b67107fd841a55d56a231",
+                "sha256:4db21ece84dfeefc5d8a3863f101995de646c6cb0536952c321a2650aa202c36",
+                "sha256:54c4a097b8bc5bb0dfc83ae498061d53ad7b5762e00f4adaa23bee22b012e6ba",
+                "sha256:54d9ff35d4515debf14bc27f1e3b38bfc453eff3220f5bce159642fa762fe5d4",
+                "sha256:55b96e7ce3a69a8449a66984c268062fbaa0d8ae437b285428e12797baefce7e",
+                "sha256:57fdd2e0b2694ce6fc2e5ccf189789c3e2962916fb38779d3e3521ff8fe7a822",
+                "sha256:587d4af3979376652010e400accc30404e6c16b7df574048ab1f581af82065e4",
+                "sha256:5b513b6997a0b2f10e4fd3a1313568e373926e8c252bd76c960f96fd039cd28d",
+                "sha256:5ddcd9a179c0a6fa8add279a4444015acddcd7f232a49071ae57fa6e278f1f71",
+                "sha256:6113c008a7780792efc80f9dfe10ba0cd043cbf8dc9a76ef757850f51b4edc50",
+                "sha256:635a1d96665f84b292e401c3d62775851aedc31d4f8784117b3c68c4fcd4118d",
+                "sha256:64ce2799bd75039b480cc0360907c4fb2f50022f030bf9e7a8705b636e408fad",
+                "sha256:69dee6a020693d12a3cf892aba4808fe168d2a4cef368eb9bf74f5398bfd4ee8",
+                "sha256:6a2644a93da36c784e546de579ec1806bfd2763ef47babc1b03d765fe560c9f8",
+                "sha256:6b41e1adc61fa347662b09398e31ad446afadff932a24807d3ceb955ed865cc8",
+                "sha256:6c188c307e8433bcb63dc1915022deb553b4203a70722fc542c363bf120a01fd",
+                "sha256:6edd623bae6a737f10ce853ea076f56f507fd7726bee96a41ee3d68d347e4d16",
+                "sha256:73d6d2f64f4d894c96626a75578b0bf7d9e56dcda8c3d037a2118fdfe9b1c664",
+                "sha256:7a22ccefd4db3f12b526eccb129390942fe874a3a9fdbdd24cf55773a1faab1a",
+                "sha256:7fb89ee5d106e4a7a51bce305ac4efb981536301895f7bdcf93ec92ae0d91c7f",
+                "sha256:846bc79ee753acf93aef4184c040d709940c9d001029ceb7b7a52747b80ed2dd",
+                "sha256:85ab7824093d8f10d44330fe1e6493f756f252d145323dd17ab6b48733ff6c0a",
+                "sha256:8dee5b4810a89447151999428fe096977346cf2f29f4d5e29609d2e19e0199c9",
+                "sha256:8e5fb5f77c8745a60105403a774fe2c1759b71d3e7b4ca237a5e67ad066c7199",
+                "sha256:98eeee2f2e63edae2181c886d7911ce502e1292794f4c5ee71e60e23e8d26b5d",
+                "sha256:9d4a76b96f398697fe01117093613166e6aa8195d63f1b4ec3f21ab637632963",
+                "sha256:9e8719792ca63c6b8340380352c24dcb8cd7ec49dae36e963742a275dfae6009",
+                "sha256:a0b2b80321c2ed3fcf0385ec9e51a12253c50f146fddb2abbb10f033fe3d049a",
+                "sha256:a4cc92bb6db56ab0c1cbd17294e14f5e9224f0cc6521167ef388332604e92679",
+                "sha256:a738b937d512b30bf75995c0159c0ddf9eec0775c9d72ac0202076c72f24aa96",
+                "sha256:a8f877c89719d759e52783f7fe6e1c67121076b87b40542966c02de5503ace42",
+                "sha256:a906ed5e47a0ce5f04b2c981af1c9acf9e8696066900bf03b9d7879a6f679fc8",
+                "sha256:ae2941333154baff9838e88aa71c1d84f4438189ecc6021a12c7573728b5838e",
+                "sha256:b0d0a6c64fcc4ef9c69bd5b3b3626cc3776520a1637d8abaa62b9edc147a58f7",
+                "sha256:b5b029322e6e7b94fff16cd120ab35a253236a5f99a79fb04fda7ae71ca20ae8",
+                "sha256:b7aaa315101c6567a9a45d2839322c51c8d6e81f67683d529512f5bcfb99c802",
+                "sha256:be1c8ed48c4c4065ecb19d882a0ce1afe0745dfad8ce48c49586b90a55f02366",
+                "sha256:c0256beda696edcf7d97ef16b2a33a8e5a875affd6fa6567b54f7c577b30a137",
+                "sha256:c157bb447303070f256e084668b702073db99bbb61d44f85d811025fcf38f784",
+                "sha256:c57d08ad67aba97af57a7263c2d9006d5c404d721c5f7542f077f109ec2a4a29",
+                "sha256:c69ada171c2d0e97a4b5aa78fbb835e0ffbb6b13fc5da968c09811346564f0d3",
+                "sha256:c94bb0a9f1db10a1d16c00880bdebd5f9faf267273b8f5bd1878126e0fbde771",
+                "sha256:cb130fccd1a37ed894824b8c046321540263013da72745d755f2d35114b81a60",
+                "sha256:ced479f601cd2f8ca1fd7b23925a7e0ad512a56d6e9476f79b8f381d9d37090a",
+                "sha256:d05ac6fa06959c4172eccd99a222e1fbf17b5670c4d596cb1e5cde99600674c4",
+                "sha256:d552c78411f60b1fdaafd117a1fca2f02e562e309223b9d44b7de8be451ec5e0",
+                "sha256:dd4490a33eb909ef5078ab20f5f000087afa2a4daa27b4c072ccb3cb3050ad84",
+                "sha256:df5cbb1fbc74a8305b6065d4ade43b993be03dbe0f8b30032cced0d7740994bd",
+                "sha256:e28f9faeb14b6f23ac55bfbbfd3643f5c7c18ede093977f1df249f73fd22c7b1",
+                "sha256:e464b467f1588e2c42d26814231edecbcfe77f5ac414d92cbf4e7b55b2c2a776",
+                "sha256:e4c22e1ac1f1ec1e09f72e6c44d8f2244173db7eb9629cc3a346a8d7ccc31142",
+                "sha256:e53b5fbab5d675aec9f0c501274c467c0f9a5d23696cfc94247e1fb56501ed89",
+                "sha256:e93f1c331ca8e86fe877a48ad64e77882c0c4da0097f2212873a69bbfea95d0c",
+                "sha256:e997fd30430c57138adc06bba4c7c2968fb13d101e57dd5bb9355bf8ce3fa7e8",
+                "sha256:e9a091b0550b3b0207784a7d6d0f1a00d1d1c8a11699c1a4d93db3fbefc3ad35",
+                "sha256:eab4bb380f15e189d1313195b062a6aa908f5bd687a0ceccd47c8211e9cf0d4a",
+                "sha256:eb1ae19e64c14c7ec1995f40bd932448713d3c73509e82d8cd7744dc00e29e86",
+                "sha256:ecea58b43a67b1b79805f1a0255730edaf5191ecef84dbc4cc85eb30bc8b63b9",
+                "sha256:ee439691d8c23e76f9802c42a95cfeebf9d47cf4ffd06f18489122dbb0a7ad64",
+                "sha256:eee9130eaad130649fd73e5cd92f60e55708952260ede70da64de420cdcad554",
+                "sha256:f47cd43a5bfa48f86925fe26fbdd0a488ff15b62468abb5d2a1e092a4fb10e85",
+                "sha256:f6fff13ef6b5f29221d6904aa816c34701462956aa72a77f1f151a8ec4f56aeb",
+                "sha256:f745ec09bc1b0bd15cfc73df6fa4f726dcc26bb16c23a03f9e3367d357eeedd0",
+                "sha256:f8404bf61298bb6f8224bb9176c1424548ee1181130818fcd2cbffddc768bed8",
+                "sha256:f9268774428ec173654985ce55fc6caf4c6d11ade0f6f914d48ef4719eb05ebb",
+                "sha256:faa3c142464efec496967359ca99696c896c591c56c53506bac1ad465f66e919"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2024.5.10"
+            "version": "==2024.9.11"
         },
         "requests": {
             "extras": [
                 "socks"
             ],
             "hashes": [
-                "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f",
-                "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"
+                "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760",
+                "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"
             ],
             "index": "pypi",
-            "markers": "python_version >= '3.7'",
-            "version": "==2.31.0"
+            "markers": "python_version >= '3.8'",
+            "version": "==2.32.3"
         },
         "requests-oauthlib": {
             "hashes": [
@@ -1928,6 +2488,14 @@
             ],
             "version": "==1.3.4"
         },
+        "rfc3986": {
+            "hashes": [
+                "sha256:50b1502b60e289cb37883f3dfd34532b8873c7de9f49bb546641ce9cbd256ebd",
+                "sha256:97aacf9dbd4bfd829baad6e6309fa6573aaf1be3f6fa735c8ab05e46cecb261c"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==2.0.0"
+        },
         "rich": {
             "hashes": [
                 "sha256:4edbae314f59eb482f54e9e30bf00d33350aaa94f4bfcd4e9e3110e64d0d7222",
@@ -1946,19 +2514,27 @@
         },
         "s3transfer": {
             "hashes": [
-                "sha256:5683916b4c724f799e600f41dd9e10a9ff19871bf87623cc8f491cb4f5fa0a19",
-                "sha256:ceb252b11bcf87080fb7850a224fb6e05c8a776bab8f2b64b7f25b969464839d"
+                "sha256:263ed587a5803c6c708d3ce44dc4dfedaab4c1a32e8329bab818933d79ddcf5d",
+                "sha256:4f50ed74ab84d474ce614475e0b8d5047ff080810aac5d01ea25231cfc944b0c"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.10.1"
+            "version": "==0.10.3"
+        },
+        "secretstorage": {
+            "hashes": [
+                "sha256:2403533ef369eca6d2ba81718576c5e0f564d5cca1b58f73a8b23e7d4eeebd77",
+                "sha256:f356e6628222568e3af06f2eba8df495efa13b3b63081dafd4f7d9a7b7bc9f99"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==3.3.3"
         },
         "selenium": {
             "hashes": [
-                "sha256:0bd564ee166980d419a8aaf4ac00289bc152afcf2eadca5efe8c8e36711853fd",
-                "sha256:b1d0c33b38ca27d0499183e48e1dd09ff26973481f5d3ef2983073813ae6588d"
+                "sha256:3798d2d12b4a570bc5790163ba57fef10b2afee958bf1d80f2a3cf07c4141f33",
+                "sha256:95d08d3b82fb353f3c474895154516604c7f0e6a9a565ae6498ef36c9bac6921"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==4.20.0"
+            "version": "==4.25.0"
         },
         "six": {
             "hashes": [
@@ -1976,6 +2552,13 @@
             "markers": "python_version >= '3.7'",
             "version": "==1.3.1"
         },
+        "snowballstemmer": {
+            "hashes": [
+                "sha256:09b16deb8547d3412ad7b590689584cd0fe25ec8db3be37788be3810cbf19cb1",
+                "sha256:c8e1716e83cc398ae16824e5572ae04e0d9fc2c6b985fb0f900f5f0c96ecba1a"
+            ],
+            "version": "==2.2.0"
+        },
         "snscrape": {
             "hashes": [
                 "sha256:6eedb85c7e79f35361dde1949e1e7e2dee44e9f8469668438c9f8e72980f482f",
@@ -1999,80 +2582,172 @@
             "markers": "python_version >= '3.8'",
             "version": "==2.5"
         },
+        "sphinx": {
+            "hashes": [
+                "sha256:b18e978ea7565720f26019c702cd85c84376e948370f1cd43d60265010e1c7b0",
+                "sha256:d3e57663eed1d7c5c50895d191fdeda0b54ded6f44d5621b50709466c338d1e8"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==5.0.2"
+        },
+        "sphinx-autobuild": {
+            "hashes": [
+                "sha256:1c0ed37a1970eed197f9c5a66d65759e7c4e4cba7b5a5d77940752bf1a59f2c7",
+                "sha256:f2522779d30fcbf0253e09714f274ce8c608cb6ebcd67922b1c54de59faba702"
+            ],
+            "markers": "python_version >= '3.9'",
+            "version": "==2024.4.16"
+        },
+        "sphinx-autodoc-typehints": {
+            "hashes": [
+                "sha256:6c841db55e0e9be0483ff3962a2152b60e79306f4288d8c4e7e86ac84486a5ea",
+                "sha256:9be46aeeb1b315eb5df1f3a7cb262149895d16c7d7dcd77b92513c3c3a1e85e6"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==1.19.1"
+        },
+        "sphinx-basic-ng": {
+            "hashes": [
+                "sha256:9ec55a47c90c8c002b5960c57492ec3021f5193cb26cebc2dc4ea226848651c9",
+                "sha256:eb09aedbabfb650607e9b4b68c9d240b90b1e1be221d6ad71d61c52e29f7932b"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==1.0.0b2"
+        },
+        "sphinx-copybutton": {
+            "hashes": [
+                "sha256:4cf17c82fb9646d1bc9ca92ac280813a3b605d8c421225fd9913154103ee1fbd",
+                "sha256:fb543fd386d917746c9a2c50360c7905b605726b9355cd26e9974857afeae06e"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==0.5.2"
+        },
+        "sphinxcontrib-applehelp": {
+            "hashes": [
+                "sha256:c40a4f96f3776c4393d933412053962fac2b84f4c99a7982ba42e09576a70619",
+                "sha256:cb61eb0ec1b61f349e5cc36b2028e9e7ca765be05e49641c97241274753067b4"
+            ],
+            "markers": "python_version >= '3.9'",
+            "version": "==1.0.8"
+        },
+        "sphinxcontrib-devhelp": {
+            "hashes": [
+                "sha256:6485d09629944511c893fa11355bda18b742b83a2b181f9a009f7e500595c90f",
+                "sha256:9893fd3f90506bc4b97bdb977ceb8fbd823989f4316b28c3841ec128544372d3"
+            ],
+            "markers": "python_version >= '3.9'",
+            "version": "==1.0.6"
+        },
+        "sphinxcontrib-htmlhelp": {
+            "hashes": [
+                "sha256:0dc87637d5de53dd5eec3a6a01753b1ccf99494bd756aafecd74b4fa9e729015",
+                "sha256:393f04f112b4d2f53d93448d4bce35842f62b307ccdc549ec1585e950bc35e04"
+            ],
+            "markers": "python_version >= '3.9'",
+            "version": "==2.0.5"
+        },
+        "sphinxcontrib-jsmath": {
+            "hashes": [
+                "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178",
+                "sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8"
+            ],
+            "markers": "python_version >= '3.5'",
+            "version": "==1.0.1"
+        },
+        "sphinxcontrib-qthelp": {
+            "hashes": [
+                "sha256:053dedc38823a80a7209a80860b16b722e9e0209e32fea98c90e4e6624588ed6",
+                "sha256:e2ae3b5c492d58fcbd73281fbd27e34b8393ec34a073c792642cd8e529288182"
+            ],
+            "markers": "python_version >= '3.9'",
+            "version": "==1.0.7"
+        },
+        "sphinxcontrib-serializinghtml": {
+            "hashes": [
+                "sha256:326369b8df80a7d2d8d7f99aa5ac577f51ea51556ed974e7716cfd4fca3f6cb7",
+                "sha256:93f3f5dc458b91b192fe10c397e324f262cf163d79f3282c158e8436a2c4511f"
+            ],
+            "markers": "python_version >= '3.9'",
+            "version": "==1.1.10"
+        },
         "sqlalchemy": {
             "hashes": [
-                "sha256:0525c4905b4b52d8ccc3c203c9d7ab2a80329ffa077d4bacf31aefda7604dc65",
-                "sha256:0535d5b57d014d06ceeaeffd816bb3a6e2dddeb670222570b8c4953e2d2ea678",
-                "sha256:0892e7ac8bc76da499ad3ee8de8da4d7905a3110b952e2a35a940dab1ffa550e",
-                "sha256:0d661cff58c91726c601cc0ee626bf167b20cc4d7941c93c5f3ac28dc34ddbea",
-                "sha256:1980e6eb6c9be49ea8f89889989127daafc43f0b1b6843d71efab1514973cca0",
-                "sha256:1a09d5bd1a40d76ad90e5570530e082ddc000e1d92de495746f6257dc08f166b",
-                "sha256:245c67c88e63f1523e9216cad6ba3107dea2d3ee19adc359597a628afcabfbcb",
-                "sha256:2ad16880ccd971ac8e570550fbdef1385e094b022d6fc85ef3ce7df400dddad3",
-                "sha256:2be4e6294c53f2ec8ea36486b56390e3bcaa052bf3a9a47005687ccf376745d1",
-                "sha256:2c55040d8ea65414de7c47f1a23823cd9f3fad0dc93e6b6b728fee81230f817b",
-                "sha256:352df882088a55293f621328ec33b6ffca936ad7f23013b22520542e1ab6ad1b",
-                "sha256:3823dda635988e6744d4417e13f2e2b5fe76c4bf29dd67e95f98717e1b094cad",
-                "sha256:38ef80328e3fee2be0a1abe3fe9445d3a2e52a1282ba342d0dab6edf1fef4707",
-                "sha256:39b02b645632c5fe46b8dd30755682f629ffbb62ff317ecc14c998c21b2896ff",
-                "sha256:3b0cd89a7bd03f57ae58263d0f828a072d1b440c8c2949f38f3b446148321171",
-                "sha256:3ec7a0ed9b32afdf337172678a4a0e6419775ba4e649b66f49415615fa47efbd",
-                "sha256:3f0ef620ecbab46e81035cf3dedfb412a7da35340500ba470f9ce43a1e6c423b",
-                "sha256:50e074aea505f4427151c286955ea025f51752fa42f9939749336672e0674c81",
-                "sha256:55e699466106d09f028ab78d3c2e1f621b5ef2c8694598242259e4515715da7c",
-                "sha256:5e180fff133d21a800c4f050733d59340f40d42364fcb9d14f6a67764bdc48d2",
-                "sha256:6cacc0b2dd7d22a918a9642fc89840a5d3cee18a0e1fe41080b1141b23b10916",
-                "sha256:7af40425ac535cbda129d9915edcaa002afe35d84609fd3b9d6a8c46732e02ee",
-                "sha256:7d8139ca0b9f93890ab899da678816518af74312bb8cd71fb721436a93a93298",
-                "sha256:7deeae5071930abb3669b5185abb6c33ddfd2398f87660fafdb9e6a5fb0f3f2f",
-                "sha256:86a22143a4001f53bf58027b044da1fb10d67b62a785fc1390b5c7f089d9838c",
-                "sha256:8ca484ca11c65e05639ffe80f20d45e6be81fbec7683d6c9a15cd421e6e8b340",
-                "sha256:8d1d7d63e5d2f4e92a39ae1e897a5d551720179bb8d1254883e7113d3826d43c",
-                "sha256:8e702e7489f39375601c7ea5a0bef207256828a2bc5986c65cb15cd0cf097a87",
-                "sha256:a055ba17f4675aadcda3005df2e28a86feb731fdcc865e1f6b4f209ed1225cba",
-                "sha256:a33cb3f095e7d776ec76e79d92d83117438b6153510770fcd57b9c96f9ef623d",
-                "sha256:a61184c7289146c8cff06b6b41807c6994c6d437278e72cf00ff7fe1c7a263d1",
-                "sha256:af55cc207865d641a57f7044e98b08b09220da3d1b13a46f26487cc2f898a072",
-                "sha256:b00cf0471888823b7a9f722c6c41eb6985cf34f077edcf62695ac4bed6ec01ee",
-                "sha256:b03850c290c765b87102959ea53299dc9addf76ca08a06ea98383348ae205c99",
-                "sha256:b97fd5bb6b7c1a64b7ac0632f7ce389b8ab362e7bd5f60654c2a418496be5d7f",
-                "sha256:c37bc677690fd33932182b85d37433845de612962ed080c3e4d92f758d1bd894",
-                "sha256:cecb66492440ae8592797dd705a0cbaa6abe0555f4fa6c5f40b078bd2740fc6b",
-                "sha256:d0a83afab5e062abffcdcbcc74f9d3ba37b2385294dd0927ad65fc6ebe04e054",
-                "sha256:d3cf56cc36d42908495760b223ca9c2c0f9f0002b4eddc994b24db5fcb86a9e4",
-                "sha256:e646b19f47d655261b22df9976e572f588185279970efba3d45c377127d35349",
-                "sha256:e7908c2025eb18394e32d65dd02d2e37e17d733cdbe7d78231c2b6d7eb20cdb9",
-                "sha256:e8f2df79a46e130235bc5e1bbef4de0583fb19d481eaa0bffa76e8347ea45ec6",
-                "sha256:eaeeb2464019765bc4340214fca1143081d49972864773f3f1e95dba5c7edc7d",
-                "sha256:eb18549b770351b54e1ab5da37d22bc530b8bfe2ee31e22b9ebe650640d2ef12",
-                "sha256:f2e5b6f5cf7c18df66d082604a1d9c7a2d18f7d1dbe9514a2afaccbb51cc4fc3",
-                "sha256:f8cafa6f885a0ff5e39efa9325195217bb47d5929ab0051636610d24aef45ade"
+                "sha256:03e08af7a5f9386a43919eda9de33ffda16b44eb11f3b313e6822243770e9763",
+                "sha256:0572f4bd6f94752167adfd7c1bed84f4b240ee6203a95e05d1e208d488d0d436",
+                "sha256:07b441f7d03b9a66299ce7ccf3ef2900abc81c0db434f42a5694a37bd73870f2",
+                "sha256:1bc330d9d29c7f06f003ab10e1eaced295e87940405afe1b110f2eb93a233588",
+                "sha256:1e0d612a17581b6616ff03c8e3d5eff7452f34655c901f75d62bd86449d9750e",
+                "sha256:23623166bfefe1487d81b698c423f8678e80df8b54614c2bf4b4cfcd7c711959",
+                "sha256:2519f3a5d0517fc159afab1015e54bb81b4406c278749779be57a569d8d1bb0d",
+                "sha256:28120ef39c92c2dd60f2721af9328479516844c6b550b077ca450c7d7dc68575",
+                "sha256:37350015056a553e442ff672c2d20e6f4b6d0b2495691fa239d8aa18bb3bc908",
+                "sha256:39769a115f730d683b0eb7b694db9789267bcd027326cccc3125e862eb03bfd8",
+                "sha256:3c01117dd36800f2ecaa238c65365b7b16497adc1522bf84906e5710ee9ba0e8",
+                "sha256:3d6718667da04294d7df1670d70eeddd414f313738d20a6f1d1f379e3139a545",
+                "sha256:3dbb986bad3ed5ceaf090200eba750b5245150bd97d3e67343a3cfed06feecf7",
+                "sha256:4557e1f11c5f653ebfdd924f3f9d5ebfc718283b0b9beebaa5dd6b77ec290971",
+                "sha256:46331b00096a6db1fdc052d55b101dbbfc99155a548e20a0e4a8e5e4d1362855",
+                "sha256:4a121d62ebe7d26fec9155f83f8be5189ef1405f5973ea4874a26fab9f1e262c",
+                "sha256:4f5e9cd989b45b73bd359f693b935364f7e1f79486e29015813c338450aa5a71",
+                "sha256:50aae840ebbd6cdd41af1c14590e5741665e5272d2fee999306673a1bb1fdb4d",
+                "sha256:59b1ee96617135f6e1d6f275bbe988f419c5178016f3d41d3c0abb0c819f75bb",
+                "sha256:59b8f3adb3971929a3e660337f5dacc5942c2cdb760afcabb2614ffbda9f9f72",
+                "sha256:66bffbad8d6271bb1cc2f9a4ea4f86f80fe5e2e3e501a5ae2a3dc6a76e604e6f",
+                "sha256:69f93723edbca7342624d09f6704e7126b152eaed3cdbb634cb657a54332a3c5",
+                "sha256:6a440293d802d3011028e14e4226da1434b373cbaf4a4bbb63f845761a708346",
+                "sha256:72c28b84b174ce8af8504ca28ae9347d317f9dba3999e5981a3cd441f3712e24",
+                "sha256:79d2e78abc26d871875b419e1fd3c0bca31a1cb0043277d0d850014599626c2e",
+                "sha256:7f2767680b6d2398aea7082e45a774b2b0767b5c8d8ffb9c8b683088ea9b29c5",
+                "sha256:8318f4776c85abc3f40ab185e388bee7a6ea99e7fa3a30686580b209eaa35c08",
+                "sha256:8958b10490125124463095bbdadda5aa22ec799f91958e410438ad6c97a7b793",
+                "sha256:8c78ac40bde930c60e0f78b3cd184c580f89456dd87fc08f9e3ee3ce8765ce88",
+                "sha256:90812a8933df713fdf748b355527e3af257a11e415b613dd794512461eb8a686",
+                "sha256:9bc633f4ee4b4c46e7adcb3a9b5ec083bf1d9a97c1d3854b92749d935de40b9b",
+                "sha256:9e46ed38affdfc95d2c958de328d037d87801cfcbea6d421000859e9789e61c2",
+                "sha256:9fe53b404f24789b5ea9003fc25b9a3988feddebd7e7b369c8fac27ad6f52f28",
+                "sha256:a4e46a888b54be23d03a89be510f24a7652fe6ff660787b96cd0e57a4ebcb46d",
+                "sha256:a86bfab2ef46d63300c0f06936bd6e6c0105faa11d509083ba8f2f9d237fb5b5",
+                "sha256:ac9dfa18ff2a67b09b372d5db8743c27966abf0e5344c555d86cc7199f7ad83a",
+                "sha256:af148a33ff0349f53512a049c6406923e4e02bf2f26c5fb285f143faf4f0e46a",
+                "sha256:b11d0cfdd2b095e7b0686cf5fabeb9c67fae5b06d265d8180715b8cfa86522e3",
+                "sha256:b2985c0b06e989c043f1dc09d4fe89e1616aadd35392aea2844f0458a989eacf",
+                "sha256:b544ad1935a8541d177cb402948b94e871067656b3a0b9e91dbec136b06a2ff5",
+                "sha256:b5cc79df7f4bc3d11e4b542596c03826063092611e481fcf1c9dfee3c94355ef",
+                "sha256:b817d41d692bf286abc181f8af476c4fbef3fd05e798777492618378448ee689",
+                "sha256:b81ee3d84803fd42d0b154cb6892ae57ea6b7c55d8359a02379965706c7efe6c",
+                "sha256:be9812b766cad94a25bc63bec11f88c4ad3629a0cec1cd5d4ba48dc23860486b",
+                "sha256:c245b1fbade9c35e5bd3b64270ab49ce990369018289ecfde3f9c318411aaa07",
+                "sha256:c3f3631693003d8e585d4200730616b78fafd5a01ef8b698f6967da5c605b3fa",
+                "sha256:c4ae3005ed83f5967f961fd091f2f8c5329161f69ce8480aa8168b2d7fe37f06",
+                "sha256:c54a1e53a0c308a8e8a7dffb59097bff7facda27c70c286f005327f21b2bd6b1",
+                "sha256:d0ddd9db6e59c44875211bc4c7953a9f6638b937b0a88ae6d09eb46cced54eff",
+                "sha256:dc022184d3e5cacc9579e41805a681187650e170eb2fd70e28b86192a479dcaa",
+                "sha256:e32092c47011d113dc01ab3e1d3ce9f006a47223b18422c5c0d150af13a00687",
+                "sha256:f7b64e6ec3f02c35647be6b4851008b26cff592a95ecb13b6788a54ef80bbdd4",
+                "sha256:f942a799516184c855e1a32fbc7b29d7e571b52612647866d4ec1c3242578fcb",
+                "sha256:f9511d8dd4a6e9271d07d150fb2f81874a3c8c95e11ff9af3a2dfc35fe42ee44",
+                "sha256:fd3a55deef00f689ce931d4d1b23fa9f04c880a48ee97af488fd215cf24e2a6c",
+                "sha256:fddbe92b4760c6f5d48162aef14824add991aeda8ddadb3c31d56eb15ca69f8e",
+                "sha256:fdf3386a801ea5aba17c6410dd1dc8d39cf454ca2565541b5ac42a84e1e28f53"
             ],
             "index": "pypi",
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
-            "version": "==1.4.51"
+            "markers": "python_version >= '3.7'",
+            "version": "==2.0.36"
         },
         "starlette": {
             "hashes": [
-                "sha256:13d429aa93a61dc40bf503e8c801db1f1bca3dc706b10ef2434a36123568f044",
-                "sha256:90a671733cfb35771d8cc605e0b679d23b992f8dcfad48cc60b38cb29aeb7080"
+                "sha256:6fe59f29268538e5d0d182f2791a479a0c64638e6935d1c6989e63fb2699c6ee",
+                "sha256:9af890290133b79fc3db55474ade20f6220a364a0402e0b556e7cd5e1e093823"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.36.3"
-        },
-        "strenum": {
-            "hashes": [
-                "sha256:878fb5ab705442070e4dd1929bb5e2249511c0bcf2b0eeacf3bcd80875c82eff",
-                "sha256:a30cda4af7cc6b5bf52c8055bc4bf4b2b6b14a93b574626da33df53cf7740659"
-            ],
-            "version": "==0.4.15"
+            "version": "==0.37.2"
         },
         "telethon": {
             "hashes": [
-                "sha256:99d7a2e161e9af1cdf03feef7a3fea6eef304a9caf620fe13aefc53099845555"
+                "sha256:e5e43cff1c1b34e2f9c2b395215beb6e9bda706b69def7efff4f55b23c9c4374"
             ],
             "markers": "python_version >= '3.5'",
-            "version": "==1.35.0"
+            "version": "==1.37.0"
         },
         "text-unidecode": {
             "hashes": [
@@ -2087,6 +2762,14 @@
             ],
             "version": "==0.3.5"
         },
+        "tomli": {
+            "hashes": [
+                "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc",
+                "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==2.0.1"
+        },
         "tornado": {
             "hashes": [
                 "sha256:02ccefc7d8211e5a7f9e8bc3f9e5b0ad6262ba2fbb683a6443ecc804e5224ce0",
@@ -2106,19 +2789,19 @@
         },
         "tqdm": {
             "hashes": [
-                "sha256:b75ca56b413b030bc3f00af51fd2c1a1a5eac6a0c1cca83cbb37a5c52abce644",
-                "sha256:e4d936c9de8727928f3be6079590e97d9abfe8d39a590be678eb5919ffc186bb"
+                "sha256:90279a3770753eafc9194a0364852159802111925aa30eb3f9d85b0e805ac7cd",
+                "sha256:e1020aef2e5096702d8a025ac7d16b1577279c9d63f8375b63083e9a5f0fcbad"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==4.66.4"
+            "version": "==4.66.5"
         },
         "trio": {
             "hashes": [
-                "sha256:9b41f5993ad2c0e5f62d0acca320ec657fdb6b2a2c22b8c7aed6caf154475c4e",
-                "sha256:e6458efe29cc543e557a91e614e2b51710eba2961669329ce9c862d50c6e8e81"
+                "sha256:0346c3852c15e5c7d40ea15972c4805689ef2cb8b5206f794c9c19450119f3a4",
+                "sha256:c5237e8133eb0a1d72f09a971a55c28ebe69e351c783fc64bc37db8db8bbe1d0"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.25.0"
+            "version": "==0.26.2"
         },
         "trio-websocket": {
             "hashes": [
@@ -2135,13 +2818,21 @@
             ],
             "version": "==0.2.0"
         },
-        "typing-extensions": {
+        "twine": {
             "hashes": [
-                "sha256:83f085bd5ca59c80295fc2a82ab5dac679cbe02b9f33f7d83af68e241bea51b0",
-                "sha256:c1f94d72897edaf4ce775bb7558d5b79d8126906a14ea5ed1635921406c0387a"
+                "sha256:215dbe7b4b94c2c50a7315c0275d2258399280fbb7d04182c7e55e24b5f93997",
+                "sha256:9aa0825139c02b3434d913545c7b847a21c835e11597f5255842d457da2322db"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==4.11.0"
+            "version": "==5.1.1"
+        },
+        "typing-extensions": {
+            "hashes": [
+                "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d",
+                "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==4.12.2"
         },
         "typing-inspect": {
             "hashes": [
@@ -2171,20 +2862,20 @@
                 "socks"
             ],
             "hashes": [
-                "sha256:450b20ec296a467077128bff42b73080516e71b56ff59a60a02bef2232c4fa9d",
-                "sha256:d0570876c61ab9e520d776c38acbbb5b05a776d3f9ff98a5c8fd5162a444cf19"
+                "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472",
+                "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2.2.1"
+            "version": "==2.2.2"
         },
         "uvicorn": {
             "hashes": [
-                "sha256:3d9a267296243532db80c83a959a3400502165ade2c1338dea4e67915fd4745a",
-                "sha256:5c89da2f3895767472a35556e539fd59f7edbe9b1e9c0e1c99eebeadc61838e4"
+                "sha256:cd17daa7f3b9d7a24de3617820e634d0933b69eed8e33a516071174427238c81",
+                "sha256:d46cd8e0fd80240baffbcd9ec1012a712938754afcf81bce56c024c1656aece8"
             ],
             "index": "pypi",
             "markers": "python_version >= '3.8'",
-            "version": "==0.27.1"
+            "version": "==0.30.1"
         },
         "vine": {
             "hashes": [
@@ -2203,11 +2894,11 @@
         },
         "vk-url-scraper": {
             "hashes": [
-                "sha256:133d252ee94ceb1ee9515fb448d410ba471cbccc19e303b548076cd44cc81f30",
-                "sha256:c1c001b66b80343a991628080398d8a923e8753183b952f99f40ecafe1087070"
+                "sha256:2e83e690844bb9b04772fae56bed2d9654780ca23132155e63de4ed9bde70c23",
+                "sha256:6616c8fbe6ea6f8cbe4605898a89d1173a579ed6a9da5410dba80269d708fcb1"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==0.3.27"
+            "markers": "python_version >= '3.10'",
+            "version": "==0.3.30"
         },
         "warcio": {
             "hashes": [
@@ -2216,6 +2907,102 @@
             ],
             "version": "==1.7.4"
         },
+        "watchfiles": {
+            "hashes": [
+                "sha256:00095dd368f73f8f1c3a7982a9801190cc88a2f3582dd395b289294f8975172b",
+                "sha256:00ad0bcd399503a84cc688590cdffbe7a991691314dde5b57b3ed50a41319a31",
+                "sha256:00f39592cdd124b4ec5ed0b1edfae091567c72c7da1487ae645426d1b0ffcad1",
+                "sha256:030bc4e68d14bcad2294ff68c1ed87215fbd9a10d9dea74e7cfe8a17869785ab",
+                "sha256:052d668a167e9fc345c24203b104c313c86654dd6c0feb4b8a6dfc2462239249",
+                "sha256:067dea90c43bf837d41e72e546196e674f68c23702d3ef80e4e816937b0a3ffd",
+                "sha256:0b04a2cbc30e110303baa6d3ddce8ca3664bc3403be0f0ad513d1843a41c97d1",
+                "sha256:0bc3b2f93a140df6806c8467c7f51ed5e55a931b031b5c2d7ff6132292e803d6",
+                "sha256:0c8e0aa0e8cc2a43561e0184c0513e291ca891db13a269d8d47cb9841ced7c71",
+                "sha256:103622865599f8082f03af4214eaff90e2426edff5e8522c8f9e93dc17caee13",
+                "sha256:1235c11510ea557fe21be5d0e354bae2c655a8ee6519c94617fe63e05bca4171",
+                "sha256:1cc0cba54f47c660d9fa3218158b8963c517ed23bd9f45fe463f08262a4adae1",
+                "sha256:1d9188979a58a096b6f8090e816ccc3f255f137a009dd4bbec628e27696d67c1",
+                "sha256:213792c2cd3150b903e6e7884d40660e0bcec4465e00563a5fc03f30ea9c166c",
+                "sha256:25c817ff2a86bc3de3ed2df1703e3d24ce03479b27bb4527c57e722f8554d971",
+                "sha256:2627a91e8110b8de2406d8b2474427c86f5a62bf7d9ab3654f541f319ef22bcb",
+                "sha256:280a4afbc607cdfc9571b9904b03a478fc9f08bbeec382d648181c695648202f",
+                "sha256:28324d6b28bcb8d7c1041648d7b63be07a16db5510bea923fc80b91a2a6cbed6",
+                "sha256:28585744c931576e535860eaf3f2c0ec7deb68e3b9c5a85ca566d69d36d8dd27",
+                "sha256:28f393c1194b6eaadcdd8f941307fc9bbd7eb567995232c830f6aef38e8a6e88",
+                "sha256:2abeb79209630da981f8ebca30a2c84b4c3516a214451bfc5f106723c5f45843",
+                "sha256:2bdadf6b90c099ca079d468f976fd50062905d61fae183f769637cb0f68ba59a",
+                "sha256:2f350cbaa4bb812314af5dab0eb8d538481e2e2279472890864547f3fe2281ed",
+                "sha256:3218a6f908f6a276941422b035b511b6d0d8328edd89a53ae8c65be139073f84",
+                "sha256:3973145235a38f73c61474d56ad6199124e7488822f3a4fc97c72009751ae3b0",
+                "sha256:3a0d883351a34c01bd53cfa75cd0292e3f7e268bacf2f9e33af4ecede7e21d1d",
+                "sha256:425440e55cd735386ec7925f64d5dde392e69979d4c8459f6bb4e920210407f2",
+                "sha256:4b9f2a128a32a2c273d63eb1fdbf49ad64852fc38d15b34eaa3f7ca2f0d2b797",
+                "sha256:4cc382083afba7918e32d5ef12321421ef43d685b9a67cc452a6e6e18920890e",
+                "sha256:52fc9b0dbf54d43301a19b236b4a4614e610605f95e8c3f0f65c3a456ffd7d35",
+                "sha256:55b7cc10261c2786c41d9207193a85c1db1b725cf87936df40972aab466179b6",
+                "sha256:581f0a051ba7bafd03e17127735d92f4d286af941dacf94bcf823b101366249e",
+                "sha256:5834e1f8b71476a26df97d121c0c0ed3549d869124ed2433e02491553cb468c2",
+                "sha256:5e45fb0d70dda1623a7045bd00c9e036e6f1f6a85e4ef2c8ae602b1dfadf7550",
+                "sha256:61af9efa0733dc4ca462347becb82e8ef4945aba5135b1638bfc20fad64d4f0e",
+                "sha256:68fe0c4d22332d7ce53ad094622b27e67440dacefbaedd29e0794d26e247280c",
+                "sha256:72a44e9481afc7a5ee3291b09c419abab93b7e9c306c9ef9108cb76728ca58d2",
+                "sha256:7a74436c415843af2a769b36bf043b6ccbc0f8d784814ba3d42fc961cdb0a9dc",
+                "sha256:8597b6f9dc410bdafc8bb362dac1cbc9b4684a8310e16b1ff5eee8725d13dcd6",
+                "sha256:8c39987a1397a877217be1ac0fb1d8b9f662c6077b90ff3de2c05f235e6a8f96",
+                "sha256:8c3e3675e6e39dc59b8fe5c914a19d30029e36e9f99468dddffd432d8a7b1c93",
+                "sha256:8dc1fc25a1dedf2dd952909c8e5cb210791e5f2d9bc5e0e8ebc28dd42fed7562",
+                "sha256:8fdebb655bb1ba0122402352b0a4254812717a017d2dc49372a1d47e24073795",
+                "sha256:9165bcab15f2b6d90eedc5c20a7f8a03156b3773e5fb06a790b54ccecdb73385",
+                "sha256:94ebe84a035993bb7668f58a0ebf998174fb723a39e4ef9fce95baabb42b787f",
+                "sha256:9624a68b96c878c10437199d9a8b7d7e542feddda8d5ecff58fdc8e67b460848",
+                "sha256:96eec15e5ea7c0b6eb5bfffe990fc7c6bd833acf7e26704eb18387fb2f5fd087",
+                "sha256:97b94e14b88409c58cdf4a8eaf0e67dfd3ece7e9ce7140ea6ff48b0407a593ec",
+                "sha256:988e981aaab4f3955209e7e28c7794acdb690be1efa7f16f8ea5aba7ffdadacb",
+                "sha256:a8a31bfd98f846c3c284ba694c6365620b637debdd36e46e1859c897123aa232",
+                "sha256:a927b3034d0672f62fb2ef7ea3c9fc76d063c4b15ea852d1db2dc75fe2c09696",
+                "sha256:ace7d060432acde5532e26863e897ee684780337afb775107c0a90ae8dbccfd2",
+                "sha256:aec83c3ba24c723eac14225194b862af176d52292d271c98820199110e31141e",
+                "sha256:b44b70850f0073b5fcc0b31ede8b4e736860d70e2dbf55701e05d3227a154a67",
+                "sha256:b610fb5e27825b570554d01cec427b6620ce9bd21ff8ab775fc3a32f28bba63e",
+                "sha256:b810a2c7878cbdecca12feae2c2ae8af59bea016a78bc353c184fa1e09f76b68",
+                "sha256:bbf8a20266136507abf88b0df2328e6a9a7c7309e8daff124dda3803306a9fdb",
+                "sha256:bd4c06100bce70a20c4b81e599e5886cf504c9532951df65ad1133e508bf20be",
+                "sha256:c2444dc7cb9d8cc5ab88ebe792a8d75709d96eeef47f4c8fccb6df7c7bc5be71",
+                "sha256:c49b76a78c156979759d759339fb62eb0549515acfe4fd18bb151cc07366629c",
+                "sha256:c4a65474fd2b4c63e2c18ac67a0c6c66b82f4e73e2e4d940f837ed3d2fd9d4da",
+                "sha256:c5af2347d17ab0bd59366db8752d9e037982e259cacb2ba06f2c41c08af02c39",
+                "sha256:c668228833c5619f6618699a2c12be057711b0ea6396aeaece4ded94184304ea",
+                "sha256:c7b978c384e29d6c7372209cbf421d82286a807bbcdeb315427687f8371c340a",
+                "sha256:d048ad5d25b363ba1d19f92dcf29023988524bee6f9d952130b316c5802069cb",
+                "sha256:d3e1f3cf81f1f823e7874ae563457828e940d75573c8fbf0ee66818c8b6a9099",
+                "sha256:d47e9ef1a94cc7a536039e46738e17cce058ac1593b2eccdede8bf72e45f372a",
+                "sha256:da1e0a8caebf17976e2ffd00fa15f258e14749db5e014660f53114b676e68538",
+                "sha256:dc1b9b56f051209be458b87edb6856a449ad3f803315d87b2da4c93b43a6fe72",
+                "sha256:dc2e8fe41f3cac0660197d95216c42910c2b7e9c70d48e6d84e22f577d106fc1",
+                "sha256:dc92d2d2706d2b862ce0568b24987eba51e17e14b79a1abcd2edc39e48e743c8",
+                "sha256:dd64f3a4db121bc161644c9e10a9acdb836853155a108c2446db2f5ae1778c3d",
+                "sha256:e0f0a874231e2839abbf473256efffe577d6ee2e3bfa5b540479e892e47c172d",
+                "sha256:f7e1f9c5d1160d03b93fc4b68a0aeb82fe25563e12fbcdc8507f8434ab6f823c",
+                "sha256:fe82d13461418ca5e5a808a9e40f79c1879351fcaeddbede094028e74d836e86"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==0.22.0"
+        },
+        "webencodings": {
+            "hashes": [
+                "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78",
+                "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923"
+            ],
+            "version": "==0.5.1"
+        },
+        "websocket-client": {
+            "hashes": [
+                "sha256:17b44cc997f5c498e809b22cdf2d9c7a9e71c02c8cc2b6c56e7c2d1239bfa526",
+                "sha256:3239df9f44da632f96012472805d40a23281a991027ce11d2f45a6f24ac4c3da"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==1.8.0"
+        },
         "websockets": {
             "hashes": [
                 "sha256:00700340c6c7ab788f176d118775202aadea7602c5cc6be6ae127761c16d6b0b",
@@ -2291,16 +3078,16 @@
                 "sha256:fc4e7fa5414512b481a2483775a8e8be7803a35b30ca805afa4998a84f9fd9e8",
                 "sha256:ffefa1374cd508d633646d51a8e9277763a9b78ae71324183693959cf94635a7"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_version >= '3.8'",
             "version": "==12.0"
         },
         "werkzeug": {
             "hashes": [
-                "sha256:097e5bfda9f0aba8da6b8545146def481d06aa7d3266e7448e2cccf67dd8bd18",
-                "sha256:fc9645dc43e03e4d630d23143a04a7f947a9a3b5727cd535fdfe155a17cc48c8"
+                "sha256:02c9eb92b7d6c06f31a782811505d2157837cea66aaede3e217c7c27c039476c",
+                "sha256:34f2371506b250df4d4f84bfe7b0921e4762525762bbd936614909fe25cd7306"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==3.0.3"
+            "version": "==3.0.4"
         },
         "wsproto": {
             "hashes": [
@@ -2408,21 +3195,77 @@
         },
         "yt-dlp": {
             "hashes": [
-                "sha256:7ee90572b4d313b582b99c89e4eccf779b57ff54edc331873c6b3fba77faa8b0",
-                "sha256:d6ff6798bd114cc48763564fcb2f296464ec1604f731e69b07a8814c89b170a2"
+                "sha256:2a59d9e65ef6dadb1ff318346d04403664c3fa395e098fcd0d7ad626ef9f8a89",
+                "sha256:f4614e1c710fcb387bf152d2162868c565ed3f675647ecaa19dab54e581780eb"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==2024.4.9"
+            "version": "==2024.7.15.232803.dev0"
+        },
+        "zipp": {
+            "hashes": [
+                "sha256:bf1dcf6450f873a13e952a29504887c89e6de7506209e5b1bcc3460135d4de19",
+                "sha256:f091755f667055f2d02b32c53771a7a6c8b47e1fdbc4b72a8b9072b3eef8015c"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==3.19.2"
         }
     },
     "develop": {
-        "attrs": {
+        "anyio": {
             "hashes": [
-                "sha256:935dc3b529c262f6cf76e50877d35a4bd3c1de194fd41f47a2b7ae8f19971f30",
-                "sha256:99b87a485a5820b23b879f04c2305b44b951b502fd64be915879d77a7e8fc6f1"
+                "sha256:5aadc6a1bbb7cdb0bede386cac5e2940f5e2ff3aa20277e991cf028e0585ce94",
+                "sha256:c1b2d8f46a8a812513012e1107cb0e68c17159a7a594208005a57dc776e1bdc7"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==4.4.0"
+        },
+        "certifi": {
+            "hashes": [
+                "sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b",
+                "sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==2024.7.4"
+        },
+        "exceptiongroup": {
+            "hashes": [
+                "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b",
+                "sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc"
+            ],
+            "markers": "python_version < '3.11'",
+            "version": "==1.2.2"
+        },
+        "h11": {
+            "hashes": [
+                "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d",
+                "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==23.2.0"
+            "version": "==0.14.0"
+        },
+        "httpcore": {
+            "hashes": [
+                "sha256:27b59625743b85577a8c0e10e55b50b5368a4f2cfe8cc7bcfa9cf00829c2682f",
+                "sha256:73f6dbd6eb8c21bbf7ef8efad555481853f5f6acdeaff1edb0694289269ee17f"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==1.0.6"
+        },
+        "httpx": {
+            "hashes": [
+                "sha256:7bb2708e112d8fdd7829cd4243970f0c223274051cb35ee80c03301ee29a3df0",
+                "sha256:f7c2be1d2f3c3c3160d441802406b206c2b76f5947b11115e6df10c6c65e66c2"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==0.27.2"
+        },
+        "idna": {
+            "hashes": [
+                "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc",
+                "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"
+            ],
+            "markers": "python_version >= '3.5'",
+            "version": "==3.7"
         },
         "iniconfig": {
             "hashes": [
@@ -2434,80 +3277,89 @@
         },
         "packaging": {
             "hashes": [
-                "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5",
-                "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"
+                "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002",
+                "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==23.2"
+            "markers": "python_version >= '3.8'",
+            "version": "==24.1"
         },
         "pluggy": {
             "hashes": [
-                "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0",
-                "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"
+                "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1",
+                "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==0.13.1"
-        },
-        "py": {
-            "hashes": [
-                "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719",
-                "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378"
-            ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
-            "version": "==1.11.0"
+            "markers": "python_version >= '3.8'",
+            "version": "==1.5.0"
         },
         "pytest": {
             "hashes": [
-                "sha256:50bcad0a0b9c5a72c8e4e7c9855a3ad496ca6a881a3641b4260605450772c54b",
-                "sha256:91ef2131a9bd6be8f76f1f08eac5c5317221d6ad1e143ae03894b862e8976890"
-            ],
-            "index": "pypi",
-            "markers": "python_version >= '3.6'",
-            "version": "==6.2.4"
-        },
-        "toml": {
-            "hashes": [
-                "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b",
-                "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"
-            ],
-            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==0.10.2"
-        },
-        "watchdog": {
-            "hashes": [
-                "sha256:11e12fafb13372e18ca1bbf12d50f593e7280646687463dd47730fd4f4d5d257",
-                "sha256:2895bf0518361a9728773083908801a376743bcc37dfa252b801af8fd281b1ca",
-                "sha256:39cb34b1f1afbf23e9562501673e7146777efe95da24fab5707b88f7fb11649b",
-                "sha256:45cc09cc4c3b43fb10b59ef4d07318d9a3ecdbff03abd2e36e77b6dd9f9a5c85",
-                "sha256:4986db5e8880b0e6b7cd52ba36255d4793bf5cdc95bd6264806c233173b1ec0b",
-                "sha256:5369136a6474678e02426bd984466343924d1df8e2fd94a9b443cb7e3aa20d19",
-                "sha256:557ba04c816d23ce98a06e70af6abaa0485f6d94994ec78a42b05d1c03dcbd50",
-                "sha256:6a4db54edea37d1058b08947c789a2354ee02972ed5d1e0dca9b0b820f4c7f92",
-                "sha256:6a80d5cae8c265842c7419c560b9961561556c4361b297b4c431903f8c33b269",
-                "sha256:6a9c71a0b02985b4b0b6d14b875a6c86ddea2fdbebd0c9a720a806a8bbffc69f",
-                "sha256:6c47bdd680009b11c9ac382163e05ca43baf4127954c5f6d0250e7d772d2b80c",
-                "sha256:6e949a8a94186bced05b6508faa61b7adacc911115664ccb1923b9ad1f1ccf7b",
-                "sha256:73c7a935e62033bd5e8f0da33a4dcb763da2361921a69a5a95aaf6c93aa03a87",
-                "sha256:76ad8484379695f3fe46228962017a7e1337e9acadafed67eb20aabb175df98b",
-                "sha256:8350d4055505412a426b6ad8c521bc7d367d1637a762c70fdd93a3a0d595990b",
-                "sha256:87e9df830022488e235dd601478c15ad73a0389628588ba0b028cb74eb72fed8",
-                "sha256:8f9a542c979df62098ae9c58b19e03ad3df1c9d8c6895d96c0d51da17b243b1c",
-                "sha256:8fec441f5adcf81dd240a5fe78e3d83767999771630b5ddfc5867827a34fa3d3",
-                "sha256:9a03e16e55465177d416699331b0f3564138f1807ecc5f2de9d55d8f188d08c7",
-                "sha256:ba30a896166f0fee83183cec913298151b73164160d965af2e93a20bbd2ab605",
-                "sha256:c17d98799f32e3f55f181f19dd2021d762eb38fdd381b4a748b9f5a36738e935",
-                "sha256:c522392acc5e962bcac3b22b9592493ffd06d1fc5d755954e6be9f4990de932b",
-                "sha256:d0f9bd1fd919134d459d8abf954f63886745f4660ef66480b9d753a7c9d40927",
-                "sha256:d18d7f18a47de6863cd480734613502904611730f8def45fc52a5d97503e5101",
-                "sha256:d31481ccf4694a8416b681544c23bd271f5a123162ab603c7d7d2dd7dd901a07",
-                "sha256:e3e7065cbdabe6183ab82199d7a4f6b3ba0a438c5a512a68559846ccb76a78ec",
-                "sha256:eed82cdf79cd7f0232e2fdc1ad05b06a5e102a43e331f7d041e5f0e0a34a51c4",
-                "sha256:f970663fa4f7e80401a7b0cbeec00fa801bf0287d93d48368fc3e6fa32716245",
-                "sha256:f9b2fdca47dc855516b2d66eef3c39f2672cbf7e7a42e7e67ad2cbfcd6ba107d"
+                "sha256:70b98107bd648308a7952b06e6ca9a50bc660be218d53c257cc1fc94fda10181",
+                "sha256:a6853c7375b2663155079443d2e45de913a911a11d669df02a50814944db57b2"
             ],
             "index": "pypi",
             "markers": "python_version >= '3.8'",
-            "version": "==4.0.0"
+            "version": "==8.3.3"
+        },
+        "sniffio": {
+            "hashes": [
+                "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2",
+                "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==1.3.1"
+        },
+        "tomli": {
+            "hashes": [
+                "sha256:2ebe24485c53d303f690b0ec092806a085f07af5a5aa1464f3931eec36caaa38",
+                "sha256:d46d457a85337051c36524bc5349dd91b1877838e2979ac5ced3e710ed8a60ed"
+            ],
+            "markers": "python_version < '3.11'",
+            "version": "==2.0.2"
+        },
+        "typing-extensions": {
+            "hashes": [
+                "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d",
+                "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==4.12.2"
+        },
+        "watchdog": {
+            "hashes": [
+                "sha256:0f9332243355643d567697c3e3fa07330a1d1abf981611654a1f2bf2175612b7",
+                "sha256:1021223c08ba8d2d38d71ec1704496471ffd7be42cfb26b87cd5059323a389a1",
+                "sha256:108f42a7f0345042a854d4d0ad0834b741d421330d5f575b81cb27b883500176",
+                "sha256:1e9679245e3ea6498494b3028b90c7b25dbb2abe65c7d07423ecfc2d6218ff7c",
+                "sha256:223160bb359281bb8e31c8f1068bf71a6b16a8ad3d9524ca6f523ac666bb6a1e",
+                "sha256:26dd201857d702bdf9d78c273cafcab5871dd29343748524695cecffa44a8d97",
+                "sha256:294b7a598974b8e2c6123d19ef15de9abcd282b0fbbdbc4d23dfa812959a9e05",
+                "sha256:349c9488e1d85d0a58e8cb14222d2c51cbc801ce11ac3936ab4c3af986536926",
+                "sha256:49f4d36cb315c25ea0d946e018c01bb028048023b9e103d3d3943f58e109dd45",
+                "sha256:53a3f10b62c2d569e260f96e8d966463dec1a50fa4f1b22aec69e3f91025060e",
+                "sha256:53adf73dcdc0ef04f7735066b4a57a4cd3e49ef135daae41d77395f0b5b692cb",
+                "sha256:560135542c91eaa74247a2e8430cf83c4342b29e8ad4f520ae14f0c8a19cfb5b",
+                "sha256:720ef9d3a4f9ca575a780af283c8fd3a0674b307651c1976714745090da5a9e8",
+                "sha256:752fb40efc7cc8d88ebc332b8f4bcbe2b5cc7e881bccfeb8e25054c00c994ee3",
+                "sha256:78864cc8f23dbee55be34cc1494632a7ba30263951b5b2e8fc8286b95845f82c",
+                "sha256:85527b882f3facda0579bce9d743ff7f10c3e1e0db0a0d0e28170a7d0e5ce2ea",
+                "sha256:90a67d7857adb1d985aca232cc9905dd5bc4803ed85cfcdcfcf707e52049eda7",
+                "sha256:91b522adc25614cdeaf91f7897800b82c13b4b8ac68a42ca959f992f6990c490",
+                "sha256:9413384f26b5d050b6978e6fcd0c1e7f0539be7a4f1a885061473c5deaa57221",
+                "sha256:94d11b07c64f63f49876e0ab8042ae034674c8653bfcdaa8c4b32e71cfff87e8",
+                "sha256:950f531ec6e03696a2414b6308f5c6ff9dab7821a768c9d5788b1314e9a46ca7",
+                "sha256:a2e8f3f955d68471fa37b0e3add18500790d129cc7efe89971b8a4cc6fdeb0b2",
+                "sha256:ae6deb336cba5d71476caa029ceb6e88047fc1dc74b62b7c4012639c0b563906",
+                "sha256:b8ca4d854adcf480bdfd80f46fdd6fb49f91dd020ae11c89b3a79e19454ec627",
+                "sha256:c66f80ee5b602a9c7ab66e3c9f36026590a0902db3aea414d59a2f55188c1f49",
+                "sha256:d52db5beb5e476e6853da2e2d24dbbbed6797b449c8bf7ea118a4ee0d2c9040e",
+                "sha256:dd021efa85970bd4824acacbb922066159d0f9e546389a4743d56919b6758b91",
+                "sha256:e25adddab85f674acac303cf1f5835951345a56c5f7f582987d266679979c75b",
+                "sha256:f00b4cf737f568be9665563347a910f8bdc76f88c2970121c86243c8cfdf90e9",
+                "sha256:f01f4a3565a387080dc49bdd1fefe4ecc77f894991b88ef927edbfa45eb10818"
+            ],
+            "index": "pypi",
+            "markers": "python_version >= '3.9'",
+            "version": "==5.0.3"
         }
     }
 }
diff --git a/src/core/__init__.py b/src/core/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/src/core/config.py b/src/core/config.py
new file mode 100644
index 0000000..1aef56c
--- /dev/null
+++ b/src/core/config.py
@@ -0,0 +1,27 @@
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
+
+VERSION = "0.7.0"
+API_DESCRIPTION = """
+#### API for the Auto-Archiver project, a tool to archive web pages and Google Sheets.
+
+**Usage notes:**
+- The API requires a Bearer token for most operations, which you can obtain by logging in with your Google account.
+- You can use this API to archive single URLs or entire Google Sheets. 
+- Once you submit a URL or Sheet for archiving, the API will return a task_id that you can use to check the status of the archiving process. It works asynchronously.
+"""
+
+ALLOWED_ORIGINS = os.environ.get("ALLOWED_ORIGINS", "chrome-extension://ondkcheoicfckabcnkdgbepofpjmjcmb,chrome-extension://ojcimmjndnlmmlgnjaeojoebaceokpdp").split(",")
+
+BREAKING_CHANGES = {"minVersion": "0.3.1", "message": "The latest update has breaking changes, please update the extension to the most recent version."}
+
+SERVE_LOCAL_ARCHIVE = os.environ.get("SERVE_LOCAL_ARCHIVE", "")
+
+SQLALCHEMY_DATABASE_URL = os.environ.get("DATABASE_PATH")
+
+REPEAT_COUNT_METRICS_SECONDS = 15
+
+CHROME_APP_IDS = set([app_id.strip() for app_id in os.environ.get("CHROME_APP_IDS", "").split(",")])
+BLOCKED_EMAILS = set([e.strip().lower() for e in os.environ.get("BLOCKED_EMAILS", "").split(",")])
\ No newline at end of file
diff --git a/src/core/events.py b/src/core/events.py
new file mode 100644
index 0000000..a5e483c
--- /dev/null
+++ b/src/core/events.py
@@ -0,0 +1,44 @@
+import asyncio
+import logging
+import alembic.config
+from fastapi import FastAPI
+from sqlalchemy.orm import Session
+from contextlib import asynccontextmanager
+from fastapi_utils.tasks import repeat_every
+from loguru import logger
+
+from db import crud, models
+from db.database import get_db, engine
+from utils.metrics import measure_regular_metrics, redis_subscribe_worker_exceptions
+from core.config import REPEAT_COUNT_METRICS_SECONDS
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    # see https://fastapi.tiangolo.com/advanced/events/#lifespan
+
+    # STARTUP
+    models.Base.metadata.create_all(bind=engine)
+    alembic.config.main(argv=['--raiseerr', 'upgrade', 'head'])
+    # disabling uvicorn logger since we use loguru in logging_middleware
+    logging.getLogger("uvicorn.access").disabled = True
+    asyncio.create_task(redis_subscribe_worker_exceptions())
+    asyncio.create_task(refresh_user_groups())
+    asyncio.create_task(measure_regular_metrics())
+
+    yield  # separates startup from shutdown instructions
+
+    # SHUTDOWN
+    logger.info("shutting down")
+
+
+# CRON JOBS
+
+@repeat_every(seconds=60 * 60)  # 1 hour
+async def refresh_user_groups():
+    db: Session = next(get_db())
+    crud.upsert_user_groups(db)
+
+@repeat_every(seconds=REPEAT_COUNT_METRICS_SECONDS)
+async def repeat_measure_regular_metrics():
+    measure_regular_metrics()
diff --git a/src/core/logging.py b/src/core/logging.py
new file mode 100644
index 0000000..1798f0c
--- /dev/null
+++ b/src/core/logging.py
@@ -0,0 +1,15 @@
+from loguru import logger
+from fastapi import Request
+from utils.metrics import EXCEPTION_COUNTER
+
+
+# logging configurations
+logger.add("logs/api_logs.log", retention="30 days", rotation="3 days")
+async def logging_middleware(request: Request, call_next):
+    try:
+        response = await call_next(request)
+        logger.info(f"{request.client.host}:{request.client.port} {request.method} {request.url._url} - HTTP {response.status_code}")
+        return response
+    except Exception as e:
+        EXCEPTION_COUNTER.labels(type(e).__name__).inc()
+        raise e
\ No newline at end of file
diff --git a/src/db/crud.py b/src/db/crud.py
index a619190..765ebd3 100644
--- a/src/db/crud.py
+++ b/src/db/crud.py
@@ -15,7 +15,7 @@ MAX_LIMIT = 100
 # --------------- TASK = Archive
 
 
-def get_task(db: Session, task_id: str, email: str):
+def get_archive(db: Session, task_id: str, email: str):
     email = email.lower()
     query = base_query(db).filter(models.Archive.id == task_id)
     if email != ALLOW_ANY_EMAIL:
@@ -24,7 +24,7 @@ def get_task(db: Session, task_id: str, email: str):
     return query.first()
 
 
-def search_tasks_by_url(db: Session, url: str, email: str, skip: int = 0, limit: int = 100, archived_after: datetime = None, archived_before: datetime = None, absolute_search: bool = False):
+def search_archives_by_url(db: Session, url: str, email: str, skip: int = 0, limit: int = 100, archived_after: datetime = None, archived_before: datetime = None, absolute_search: bool = False):
     # searches for partial URLs, if email is * no ownership filtering happens
     query = base_query(db)
     if email != ALLOW_ANY_EMAIL:
@@ -42,7 +42,7 @@ def search_tasks_by_url(db: Session, url: str, email: str, skip: int = 0, limit:
     return query.order_by(models.Archive.created_at.desc()).offset(skip).limit(min(limit, MAX_LIMIT)).all()
 
 
-def search_tasks_by_email(db: Session, email: str, skip: int = 0, limit: int = 100):
+def search_archives_by_email(db: Session, email: str, skip: int = 0, limit: int = 100):
     email = email.lower()
     return base_query(db).filter(models.Archive.author.has(email=email)).offset(skip).limit(min(limit, MAX_LIMIT)).all()
 
diff --git a/src/db/database.py b/src/db/database.py
index 8c425a0..c212d20 100644
--- a/src/db/database.py
+++ b/src/db/database.py
@@ -1,10 +1,7 @@
 from sqlalchemy import create_engine
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy.orm import sessionmaker
-import os
+from sqlalchemy.orm import sessionmaker, declarative_base
+from core.config import SQLALCHEMY_DATABASE_URL
 
-SQLALCHEMY_DATABASE_URL = os.environ.get("DATABASE_PATH")#"sqlite:///./auto-archiver.db"
-# SQLALCHEMY_DATABASE_URL = "postgresql://user:password@postgresserver/db"
 
 engine = create_engine(
     SQLALCHEMY_DATABASE_URL, connect_args={"check_same_thread": False}
@@ -13,3 +10,8 @@ SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
 
 Base = declarative_base()
 
+
+def get_db():
+    session = SessionLocal()
+    try: yield session
+    finally: session.close()
diff --git a/src/db/schemas.py b/src/db/schemas.py
index 28823cc..fff03f9 100644
--- a/src/db/schemas.py
+++ b/src/db/schemas.py
@@ -19,9 +19,7 @@ class Archive(ArchiveCreate):
     updated_at: datetime | None
     deleted: bool
 
-    class Config:
-        orm_mode = True
-
+    model_config = { "from_attributes": True }
 
 class SubmitSheet(BaseModel):
     sheet_name: str | None = None
@@ -39,3 +37,13 @@ class SubmitManual(BaseModel):
     author_id: str | None = None
     group_id: str | None = None
     tags: set | None = set()
+
+class Task(BaseModel):
+    id: str
+
+class TaskResult(Task):
+    status: str
+    result: str
+
+class TaskDelete(Task):
+    deleted: bool
\ No newline at end of file
diff --git a/src/endpoints/__init__.py b/src/endpoints/__init__.py
new file mode 100644
index 0000000..1551fae
--- /dev/null
+++ b/src/endpoints/__init__.py
@@ -0,0 +1,5 @@
+from endpoints.default import default_router
+from endpoints.url import url_router
+from endpoints.task import task_router
+from endpoints.interoperability import interoperability_router
+from endpoints.sheet import sheet_router
\ No newline at end of file
diff --git a/src/endpoints/default.py b/src/endpoints/default.py
new file mode 100644
index 0000000..0cdf180
--- /dev/null
+++ b/src/endpoints/default.py
@@ -0,0 +1,36 @@
+
+from fastapi import APIRouter, Depends, Request, HTTPException
+from fastapi.responses import FileResponse, JSONResponse
+from loguru import logger
+from sqlalchemy.orm import Session
+
+from core.config import VERSION, BREAKING_CHANGES
+from db import crud
+from db.database import get_db
+from security import get_user_auth, bearer_security
+
+
+default_router = APIRouter()
+
+
+@default_router.get("/")
+async def home(request: Request):
+    # TODO: maybe split into 2 routes: one non authenticated and one authenticated for the groups info only
+    status = {"version": VERSION, "breakingChanges": BREAKING_CHANGES}
+    try:
+        email = await get_user_auth(await bearer_security(request))
+        db: Session = next(get_db())
+        status["groups"] = crud.get_user_groups(db, email)
+    except HTTPException: pass  # not authenticated is fine
+    except Exception as e: logger.error(e)
+    return JSONResponse(status)
+
+
+@default_router.get("/groups", response_model=list[str])
+def get_user_groups(db: Session = Depends(get_db), email=Depends(get_user_auth)):
+    return crud.get_user_groups(db, email)
+
+
+@default_router.get('/favicon.ico', include_in_schema=False)
+async def favicon():
+    return FileResponse("static/favicon.ico")
diff --git a/src/endpoints/interoperability.py b/src/endpoints/interoperability.py
new file mode 100644
index 0000000..e59702d
--- /dev/null
+++ b/src/endpoints/interoperability.py
@@ -0,0 +1,26 @@
+from fastapi import APIRouter, Depends, HTTPException
+from fastapi.responses import JSONResponse
+from auto_archiver import Metadata
+from loguru import logger
+import sqlalchemy
+
+from security import token_api_key_auth
+from db import models, schemas
+from worker import insert_result_into_db
+
+
+interoperability_router = APIRouter(prefix="/interop", tags=["Interoperability endpoints."])
+
+
+# ----- endpoint to submit data archived elsewhere
+@interoperability_router.post("/submit-archive", status_code=201, summary="Submit a manual archive entry, for data that was archived elsewhere.")
+def submit_manual_archive(manual: schemas.SubmitManual, auth=Depends(token_api_key_auth)):
+    result = Metadata.from_json(manual.result)
+    logger.info(f"MANUAL SUBMIT {result.get_url()} {manual.author_id}")
+    manual.tags.add("manual")
+    try:
+        archive_id = insert_result_into_db(result, manual.tags, manual.public, manual.group_id, manual.author_id, models.generate_uuid())
+    except sqlalchemy.exc.IntegrityError as e:
+        logger.error(e)
+        raise HTTPException(status_code=422, detail=f"Cannot insert into DB due to integrity error")
+    return JSONResponse({"id": archive_id})
diff --git a/src/endpoints/sheet.py b/src/endpoints/sheet.py
new file mode 100644
index 0000000..4db344c
--- /dev/null
+++ b/src/endpoints/sheet.py
@@ -0,0 +1,23 @@
+
+from fastapi import APIRouter, Depends, HTTPException
+from fastapi.responses import JSONResponse
+
+from loguru import logger
+
+from security import ALLOW_ANY_EMAIL, get_token_or_user_auth
+from db import schemas
+from worker import create_sheet_task
+
+sheet_router = APIRouter(prefix="/sheet", tags=["Google Spreadsheet operations"])
+
+
+@sheet_router.post("/archive", status_code=201, summary="Submit a Google Sheet archive request, starts a sheet archiving task.", response_model=schemas.Task, response_description="task_id for the archiving task.")
+def archive_sheet(sheet:schemas.SubmitSheet, email = Depends(get_token_or_user_auth)):
+    logger.info(f"SHEET TASK for {sheet=}")
+    if email == ALLOW_ANY_EMAIL:
+        email = sheet.author_id or "api-endpoint"
+    sheet.author_id = email
+    if not sheet.sheet_name and not sheet.sheet_id:
+        raise HTTPException(status_code=422, detail=f"sheet name or id is required")
+    task = create_sheet_task.delay(sheet.model_dump_json())
+    return JSONResponse({"id": task.id})
\ No newline at end of file
diff --git a/src/endpoints/task.py b/src/endpoints/task.py
new file mode 100644
index 0000000..a6b09b4
--- /dev/null
+++ b/src/endpoints/task.py
@@ -0,0 +1,47 @@
+import traceback
+from celery.result import AsyncResult
+from fastapi import APIRouter, Depends
+from fastapi.encoders import jsonable_encoder
+from fastapi.responses import JSONResponse
+
+from loguru import logger
+from security import get_token_or_user_auth
+
+from db import schemas
+
+from worker import celery
+
+
+task_router = APIRouter(prefix="/task", tags=["Async task operations"])
+
+
+@task_router.get("/{task_id}", response_model=schemas.TaskResult, summary="Check the status of an async task by its id, works for URLs and Sheet tasks.")
+def get_status(task_id, email=Depends(get_token_or_user_auth)):
+    logger.info(f"status check for user {email} task {task_id}")
+    task = AsyncResult(task_id, app=celery)
+    try:
+        if task.status == "FAILURE":
+            # *FAILURE* The task raised an exception, or has exceeded the retry limit.
+            # The :attr:`result` attribute then contains the exception raised by the task.
+            # https://docs.celeryq.dev/en/stable/_modules/celery/result.html#AsyncResult
+            raise task.result
+        # TODO: refactor to use schema?
+        # response = schemas.TaskResult(id=task_id, status=task.status, result=task.result)
+
+        response = {
+            "id": task_id,
+            "status": task.status,
+            "result": task.result
+        }
+        return JSONResponse(jsonable_encoder(response, exclude_unset=True))
+
+    except Exception as e:
+        logger.error(e)
+        logger.error(traceback.format_exc())
+        # TODO: refactor to use schema?
+        # response = schemas.TaskResult(id=task_id, status="FAILURE", result={"error": str(e)})
+        return JSONResponse({
+            "id": task_id,
+            "status": "FAILURE",
+            "result": {"error": str(e)}
+        })
diff --git a/src/endpoints/url.py b/src/endpoints/url.py
new file mode 100644
index 0000000..199be59
--- /dev/null
+++ b/src/endpoints/url.py
@@ -0,0 +1,57 @@
+
+from fastapi import APIRouter, Depends, HTTPException
+from fastapi.responses import JSONResponse
+from datetime import datetime
+
+from loguru import logger
+from security import get_user_auth, get_token_or_user_auth
+from sqlalchemy.orm import Session
+
+from db import crud, schemas
+from db.database import get_db
+
+from worker import create_archive_task
+
+url_router = APIRouter(prefix="/url", tags=["Single URL operations"])
+
+
+@url_router.post("/archive", status_code=201, summary="Submit a single URL archive request, starts an archiving task.", response_model=schemas.Task, response_description="task_id for the archiving task, will match the archive id.")
+def archive_url(archive: schemas.ArchiveCreate, email=Depends(get_token_or_user_auth)):
+    archive.author_id = email
+    url = archive.url
+    logger.info(f"new {archive.public=} task for {email=} and {archive.group_id=}: {url}")
+    if type(url) != str or len(url) <= 5:
+        raise HTTPException(status_code=422, detail=f"Invalid URL received: {url}")
+    logger.info("creating task")
+    task = create_archive_task.delay(archive.model_dump_json())
+    task_response = schemas.Task(id=task.id)
+    return JSONResponse(task_response.model_dump())
+
+
+@url_router.get("/search", response_model=list[schemas.Archive], summary="Search for archive entries by URL.")
+def search_by_url(
+        url: str, skip: int = 0, limit: int = 25,
+        archived_after: datetime = None, archived_before: datetime = None,
+        db: Session = Depends(get_db),
+        email=Depends(get_token_or_user_auth)):
+    return crud.search_archives_by_url(db, url.strip(), email, skip=skip, limit=limit, archived_after=archived_after, archived_before=archived_before)
+
+
+@url_router.get("/latest", response_model=list[schemas.Archive], summary="Fetch latest URL archives for the authenticated user.")
+def latest(skip: int = 0, limit: int = 25, db: Session = Depends(get_db), email=Depends(get_user_auth)):
+    return crud.search_archives_by_email(db, email, skip=skip, limit=limit)
+
+
+@url_router.get("/{id}", response_model=schemas.Archive, summary="Fetch a single URL archive by the associated id.")
+def lookup(id, db: Session = Depends(get_db), email=Depends(get_token_or_user_auth)):
+    return crud.get_archive(db, id, email)
+
+
+@url_router.delete("/{id}", response_model=schemas.TaskDelete, summary="Delete a single URL archive by id.")
+def delete_task(id, db: Session = Depends(get_db), email=Depends(get_user_auth)):
+    logger.info(f"deleting url archive task {id} request by {email}")
+    #TODO: use response model?
+    return JSONResponse({
+        "id": id,
+        "deleted": crud.soft_delete_task(db, id, email)
+    })
diff --git a/src/main.py b/src/main.py
index 801af50..190cff5 100644
--- a/src/main.py
+++ b/src/main.py
@@ -1,63 +1,41 @@
+import traceback, os
 from celery.result import AsyncResult
-from fastapi import FastAPI, Depends, Request, HTTPException
+from fastapi import FastAPI, Depends, HTTPException
 from fastapi.encoders import jsonable_encoder
 from fastapi.responses import JSONResponse
 from fastapi.staticfiles import StaticFiles
 from fastapi.middleware.cors import CORSMiddleware
-from fastapi_utils.tasks import repeat_every
-import alembic.config
 from dotenv import load_dotenv
-import traceback, os, logging
-from loguru import logger
+from prometheus_fastapi_instrumentator import Instrumentator
 from datetime import datetime
 import sqlalchemy
-from prometheus_fastapi_instrumentator import Instrumentator
-from prometheus_client import Counter, Gauge
-from contextlib import asynccontextmanager
-import asyncio, json
-import shutil
+from sqlalchemy.orm import Session
+from loguru import logger
 
-from worker import REDIS_EXCEPTIONS_CHANNEL, create_archive_task, create_sheet_task, celery, insert_result_into_db, Rdis
+from core.logging import logging_middleware
+from worker import create_archive_task, create_sheet_task, celery, insert_result_into_db
 
 from db import crud, models, schemas
-from db.database import engine, SessionLocal, SQLALCHEMY_DATABASE_URL
-from sqlalchemy.orm import Session
-from security import get_user_auth, token_api_key_auth, bearer_security, get_token_or_user_auth
+from security import get_user_auth, token_api_key_auth, get_token_or_user_auth
+from core.config import ALLOWED_ORIGINS, VERSION, SERVE_LOCAL_ARCHIVE, API_DESCRIPTION
+from db.database import get_db
+from core.events import lifespan
+
 from auto_archiver import Metadata
 
+from endpoints import default_router, url_router, sheet_router, task_router, interoperability_router
+
 load_dotenv()
 
-# Configuration
-ALLOWED_ORIGINS = os.environ.get("ALLOWED_ORIGINS", "chrome-extension://ondkcheoicfckabcnkdgbepofpjmjcmb,chrome-extension://ojcimmjndnlmmlgnjaeojoebaceokpdp").split(",")
-VERSION = "0.6.3"
-# min-version refers to the version of auto-archiver-extension on the webstore
-BREAKING_CHANGES = {"minVersion": "0.3.1", "message": "The latest update has breaking changes, please update the extension to the most recent version."}
 
-@repeat_every(seconds=60 * 60) # 1 hour
-async def refresh_user_groups():
-    db: Session = next(get_db())
-    crud.upsert_user_groups(db)
+app = FastAPI(
+    title="Auto-Archiver API",
+    description=API_DESCRIPTION,
+    version=VERSION,
+    contact={"name": "GitHub", "url": "https://github.com/bellingcat/auto-archiver-api"},
+    lifespan=lifespan
+)
 
-@asynccontextmanager
-async def lifespan(app: FastAPI):
-    # see https://fastapi.tiangolo.com/advanced/events/#lifespan
-    # STARTUP
-
-    models.Base.metadata.create_all(bind=engine)
-    alembic.config.main(argv=['--raiseerr', 'upgrade', 'head'])
-    # disabling uvicorn logger since we use loguru in logging_middleware
-    logging.getLogger("uvicorn.access").disabled = True
-    asyncio.create_task(redis_subscribe_worker_exceptions())
-    asyncio.create_task(refresh_user_groups())
-    asyncio.create_task(measure_regular_metrics())
-    
-    yield # separates startup from shutdown instructions
-
-    # SHUTDOWN
-    logger.info("shutting down")
-
-
-app = FastAPI(title="Auto-Archiver API", version=VERSION, contact={"name":"Bellingcat", "url":"https://github.com/bellingcat/auto-archiver-api"}, lifespan=lifespan) 
 app.add_middleware(
     CORSMiddleware,
     allow_origins=ALLOWED_ORIGINS,
@@ -66,83 +44,56 @@ app.add_middleware(
     allow_headers=["*"],
 )
 
-EXCEPTION_COUNTER = Counter(
-    "exceptions",
-    "Number of times a certain exception has occurred.",
-    labelnames=("types",)
-)
+app.include_router(default_router)
+app.include_router(url_router)
+app.include_router(sheet_router)
+app.include_router(task_router)
+app.include_router(interoperability_router)
+
 # prometheus exposed in /metrics with authentication
 Instrumentator(should_group_status_codes=False, excluded_handlers=["/metrics"]).instrument(app).expose(app, dependencies=[Depends(token_api_key_auth)])
 
-app.mount("/static", StaticFiles(directory="static"), name="static")
-
+# used mostly for development in combination with local_archive
 SERVE_LOCAL_ARCHIVE = os.environ.get("SERVE_LOCAL_ARCHIVE", "")
 if len(SERVE_LOCAL_ARCHIVE) > 1 and os.path.isdir(SERVE_LOCAL_ARCHIVE):
     logger.info(f"mounting local archive {SERVE_LOCAL_ARCHIVE}")
     app.mount(SERVE_LOCAL_ARCHIVE, StaticFiles(directory=SERVE_LOCAL_ARCHIVE), name=SERVE_LOCAL_ARCHIVE)
 
-def get_db():
-    session = SessionLocal()
-    try: yield session
-    finally: session.close()
+
+app.middleware("http")(logging_middleware)
+
+# -----Submit URL and manipulate tasks. Bearer protected below
 
 
-# logging configurations
-logger.add("logs/api_logs.log", retention="30 days", rotation="3 days")
-@app.middleware("http")
-async def logging_middleware(request: Request, call_next):
-    try:
-        response = await call_next(request)
-        logger.info(f"{request.client.host}:{request.client.port} {request.method} {request.url._url} - HTTP {response.status_code}")
-        return response
-    except Exception as e:
-        EXCEPTION_COUNTER.labels(type(e).__name__).inc()
-        raise e
-
-@app.get("/")
-async def home(request: Request): 
-    status = {"version": VERSION, "breakingChanges": BREAKING_CHANGES}
-    try:
-        # if authenticated will load available groups
-        email = await get_user_auth(await bearer_security(request))
-        db: Session = next(get_db())
-        status["groups"] = crud.get_user_groups(db, email)
-    except HTTPException: pass
-    except Exception as e: logger.error(e)
-    return JSONResponse(status)
+@app.get("/tasks/search-url", response_model=list[schemas.Archive], deprecated=True)  # DEPRECATED
+def search_by_url(url: str, skip: int = 0, limit: int = 100, archived_after: datetime = None, archived_before: datetime = None, db: Session = Depends(get_db), email=Depends(get_token_or_user_auth)):
+    return crud.search_archives_by_url(db, url.strip(), email, skip=skip, limit=limit, archived_after=archived_after, archived_before=archived_before)
 
 
-#-----Submit URL and manipulate tasks. Bearer protected below
+@app.get("/tasks/sync", response_model=list[schemas.Archive], deprecated=True)  # DEPRECATED
+def search(skip: int = 0, limit: int = 100, db: Session = Depends(get_db), email=Depends(get_user_auth)):
+    return crud.search_archives_by_email(db, email, skip=skip, limit=limit)
 
-@app.get("/groups", response_model=list[str])
-def get_user_groups(db: Session = Depends(get_db), email = Depends(get_user_auth)):
-    return crud.get_user_groups(db, email)
 
-@app.get("/tasks/search-url", response_model=list[schemas.Archive])
-def search_by_url(url:str, skip: int = 0, limit: int = 100, archived_after:datetime=None, archived_before:datetime=None, db: Session = Depends(get_db), email = Depends(get_token_or_user_auth)):
-    return crud.search_tasks_by_url(db, url.strip(), email, skip=skip, limit=limit, archived_after=archived_after, archived_before=archived_before)
-    
-@app.get("/tasks/sync", response_model=list[schemas.Archive])
-def search(skip: int = 0, limit: int = 100, db: Session = Depends(get_db), email = Depends(get_user_auth)):
-    return crud.search_tasks_by_email(db, email, skip=skip, limit=limit)
-
-@app.post("/tasks", status_code=201)
-def archive_tasks(archive:schemas.ArchiveCreate, email = Depends(get_token_or_user_auth)):
+@app.post("/tasks", status_code=201, deprecated=True)  # DEPRECATED
+def archive_tasks(archive: schemas.ArchiveCreate, email=Depends(get_token_or_user_auth)):
     archive.author_id = email
     url = archive.url
     logger.info(f"new {archive.public=} task for {email=} and {archive.group_id=}: {url}")
-    if type(url)!=str or len(url)<=5:
+    if type(url) != str or len(url) <= 5:
         raise HTTPException(status_code=422, detail=f"Invalid URL received: {url}")
     logger.info("creating task")
-    task = create_archive_task.delay(archive.json())
+    task = create_archive_task.delay(archive.model_dump_json())
     return JSONResponse({"id": task.id})
 
-@app.get("/archive/{task_id}")
-def lookup(task_id, db: Session = Depends(get_db), email = Depends(get_token_or_user_auth)):
-    return crud.get_task(db, task_id, email)
 
-@app.get("/tasks/{task_id}")
-def get_status(task_id, email = Depends(get_token_or_user_auth)):
+@app.get("/archive/{task_id}", deprecated=True)  # DEPRECATED
+def lookup(task_id, db: Session = Depends(get_db), email=Depends(get_token_or_user_auth)):
+    return crud.get_archive(db, task_id, email)
+
+
+@app.get("/tasks/{task_id}", deprecated=True)  # DEPRECATED
+def get_status(task_id, email=Depends(get_token_or_user_auth)):
     logger.info(f"status check for user {email} task {task_id}")
     task = AsyncResult(task_id, app=celery)
     try:
@@ -168,36 +119,42 @@ def get_status(task_id, email = Depends(get_token_or_user_auth)):
             "result": {"error": str(e)}
         })
 
-@app.delete("/tasks/{task_id}")
-def delete_task(task_id, db: Session = Depends(get_db), email = Depends(get_user_auth)):
+
+@app.delete("/tasks/{task_id}", deprecated=True)  # DEPRECATED
+def delete_task(task_id, db: Session = Depends(get_db), email=Depends(get_user_auth)):
     logger.info(f"deleting task {task_id} request by {email}")
     return JSONResponse({
         "id": task_id,
         "deleted": crud.soft_delete_task(db, task_id, email)
     })
 
-#----- Google Sheets Logic
-@app.post("/sheet", status_code=201)
-def archive_sheet(sheet:schemas.SubmitSheet, email = Depends(get_user_auth)):
+# ----- Google Sheets Logic
+
+
+@app.post("/sheet", status_code=201, deprecated=True)  # DEPRECATED
+def archive_sheet(sheet: schemas.SubmitSheet, email=Depends(get_user_auth)):
     logger.info(f"SHEET TASK for {sheet=}")
     sheet.author_id = email
     if not sheet.sheet_name and not sheet.sheet_id:
         raise HTTPException(status_code=422, detail=f"sheet name or id is required")
-    task = create_sheet_task.delay(sheet.json())
+    task = create_sheet_task.delay(sheet.model_dump_json())
     return JSONResponse({"id": task.id})
 
-@app.post("/sheet_service", status_code=201)
-def archive_sheet_service(sheet:schemas.SubmitSheet, auth = Depends(token_api_key_auth)):
+
+@app.post("/sheet_service", status_code=201, deprecated=True)  # DEPRECATED
+def archive_sheet_service(sheet: schemas.SubmitSheet, auth=Depends(token_api_key_auth)):
     logger.info(f"SHEET TASK for {sheet=}")
     sheet.author_id = sheet.author_id or "api-endpoint"
     if not sheet.sheet_name and not sheet.sheet_id:
         raise HTTPException(status_code=422, detail=f"sheet name or id is required")
-    task = create_sheet_task.delay(sheet.json())
+    task = create_sheet_task.delay(sheet.model_dump_json())
     return JSONResponse({"id": task.id})
 
-#----- endpoint to submit data archived elsewhere
-@app.post("/submit-archive", status_code=201)
-def submit_manual_archive(manual:schemas.SubmitManual, auth = Depends(token_api_key_auth)):
+# ----- endpoint to submit data archived elsewhere
+
+
+@app.post("/submit-archive", status_code=201, deprecated=True)  # DEPRECATED
+def submit_manual_archive(manual: schemas.SubmitManual, auth=Depends(token_api_key_auth)):
     result = Metadata.from_json(manual.result)
     logger.info(f"MANUAL SUBMIT {result.get_url()} {manual.author_id}")
     manual.tags.add("manual")
@@ -207,51 +164,3 @@ def submit_manual_archive(manual:schemas.SubmitManual, auth = Depends(token_api_
         logger.error(e)
         raise HTTPException(status_code=422, detail=f"Cannot insert into DB due to integrity error")
     return JSONResponse({"id": archive_id})
-
-# --------- Prometheus metrics
-
-WORKER_EXCEPTION = Counter(
-    "worker_exceptions_total",
-    "Number of times a certain exception has occurred on the worker.",
-    labelnames=("exception", "task",)
-)
-async def redis_subscribe_worker_exceptions():
-    PubSubExceptions = Rdis.pubsub()
-    PubSubExceptions.subscribe(REDIS_EXCEPTIONS_CHANNEL)
-    while True:
-        message = PubSubExceptions.get_message()
-        if message and message["type"] == "message":
-            data = json.loads(message["data"].decode("utf-8"))
-            WORKER_EXCEPTION.labels(exception=data["exception"], task=data["task"]).inc()
-        await asyncio.sleep(1)
-
-DISK_UTILIZATION = Gauge(
-    "disk_utilization",
-    "Disk utilization in GB",
-    labelnames=("type",)
-)
-DATABASE_METRICS = Gauge(
-    "database_metrics",
-    "Useful database metrics from queries",
-    labelnames=("query", "user")
-)
-
-REPEAT_COUNT_METRICS_SECONDS = 15
-@repeat_every(seconds=REPEAT_COUNT_METRICS_SECONDS)
-async def measure_regular_metrics():
-    _total, used, free = shutil.disk_usage("/")
-    DISK_UTILIZATION.labels(type="used").set(used / (2**30))
-    DISK_UTILIZATION.labels(type="free").set(free / (2**30))
-    try: 
-        fs = os.stat(SQLALCHEMY_DATABASE_URL.replace("sqlite:///", ""))
-        DISK_UTILIZATION.labels(type="database").set(fs.st_size / (2**30))
-    except Exception as e: logger.error(e)
-
-    session: Session = next(get_db())
-    count_archives = crud.count_archives(session)
-    count_archive_urls = crud.count_archive_urls(session)
-    DATABASE_METRICS.labels(query="count_archives", user="-").set(count_archives)
-    DATABASE_METRICS.labels(query="count_archive_urls", user="-").set(count_archive_urls)
-
-    for user in crud.count_by_user_since(session, REPEAT_COUNT_METRICS_SECONDS):
-        DATABASE_METRICS.labels(query="count_by_user", user=user.author_id).set(user.total)
diff --git a/src/security.py b/src/security.py
index 021b9be..f1e70f3 100644
--- a/src/security.py
+++ b/src/security.py
@@ -2,18 +2,16 @@ from loguru import logger
 import requests, os, secrets
 from fastapi import HTTPException, status, Depends
 from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
+from core.config import CHROME_APP_IDS, BLOCKED_EMAILS
 
-
-# Configuration
-CHROME_APP_IDS = set([app_id.strip() for app_id in os.environ.get("CHROME_APP_IDS", "").split(",")])
+# Configuration checks
 assert len(CHROME_APP_IDS) > 0, "CHROME_APP_IDS env variable not properly set, it's a csv"
 for app_id in CHROME_APP_IDS:
     assert len(app_id) > 10, f"CHROME_APP_IDS got invalid id: {app_id} env variable not set"
 logger.info(f"{CHROME_APP_IDS=}")
-
-BLOCKED_EMAILS = set([e.strip().lower() for e in os.environ.get("BLOCKED_EMAILS", "").split(",")])
 logger.info(f"{len(BLOCKED_EMAILS)=}")
 
+# Auth logic
 bearer_security = HTTPBearer()
 
 ALLOW_ANY_EMAIL = "*"
diff --git a/src/static/favicon.ico b/src/static/favicon.ico
new file mode 100644
index 0000000000000000000000000000000000000000..949a10781862a710f27b6b989df973b36cde718b
GIT binary patch
literal 95499
zcmeI5d8iiE8^^DyS(dn_p_W^OScp++*dmokX}D#%fKld>R!}HzSSCt|l__R!<X(#Y
zWtpX@Xf8=2SXkLYR%BMTXt^()-~F5)&oFoHy!XEEdl%oi@At@Y=AG@HIp;Iq=Q+!q
zDJZB`_;3FFg2Mk>7MvVYt)Ov1K|$NXSh%U6L$&{JFI+7s7+LtEcyNWA6qN^JKn#ch
zF(3xSfEW-1Vn7Ut0Wly3#DEwO17bi7hygJm2E>3E5CdXB42S_SAO^&M7!U(uKn#ch
zF(3xSfEW-1V&D=mP_0@uXR&(q>W+BXWtaJFEW_&@XZzW%alje}T@nNEQlmx<&rHpl
zHG?JVmtTIlyW)x~g8g<JOIt7L+S*sO*)vUwTh^*o%Qb4$$lZ3^ZSIjr9&uyGj&*Oo
z`KDX8Y?*ui{rBC*jT_z0ojctppM2ui-+S*pw|MblH*@Ap_w>_GyYAh)yBlt}!PTi#
zC(yp=-w|I`n=kAaI%B$Y%{AA!haP&!GqP*fF8AxNzq&JL&bW&gFS-jCF1Y{x`!B7?
zvBVQ6PPnhW`pUig?z?XI@Zp{}{+8*R_^R4`;V0FtTi5mM+0(6BwaWeW+i#wy^XJby
zB3xO7t#kuyh0n8V{qxU1ZriqPp0%b;n+Ccj&XUg=w#oDguIkmR=SGYe;XeKJQ+Mv%
zxj>iFWhmR@;1u?L_~8dPZQ3-~qD6~9&r}~sE@$usSNP)(Km4%!;)^dlSEeo5@{-MY
z#u!}x^2;x7;>3xre*OC1PD{&@#Tc3ZSNJ@wTeo&=*RFME&z|*q6}Dl<iTPOkHV$l?
zEyL&8HnWNLJG!*%+miiu9T?xYZ=dVhwX097bSxQ+p;=d6d8O9{%I>@Gz6;p0a*i&o
z%*{tfWAM|z{PIg*Ph7Zgp_?>ml6(I7=iO6JJ>`ZC8|LlobI(2JCQqL1R<2y>KK}S)
zcl78{ua8^@eo6}bO^*5Z-+$eN2@`ysQ;a2pG1{n}c;X3n{P=NShUt_wJni1S+r9kq
z%U)9&G-%-U4;`f3t^Q}aXiN54y!z^^-R-yE?m67Lb*pC-&58?bH=aq~nl)>D+X&Bx
z>}Bq8M!CQMZCGdwY@rd+5Y{Tn_2-{|cCWqmn!EMZTisPxUFGX{<x#$*<Ca@)aj(Dr
zx;uRMaKKVDJ@y;kwr}6=uDk9!`DB-%zoxV@M?+|5>eHu>J9g}tw^OP3LN_?()mLA2
z&6+g}+UL=v(*8~+`>oxpRjXF+jW^zKv{jT8;C;)MEv|9n#^suhOwz1li)yFU5z#Je
z(~%=b0v|eTo5HpTow)htn}a??u2E5TiR&8s9Xob(d-m+{=_76MA5J6evF(EoKJfk+
z_oVVJTHckNA69ns$?V;`H>EF6{gHP0x88coHEh_hQf-hu1EiJyg8B332W`4>8q2dV
zW5x_`>nn=;$}Xp*{ym;?{P`DOe6g5KL>nx^*DJ5Q;@iT^XRpkAhAkq0+tPdL)Ty8k
zDr`4NGyD4X?W^|wOYDomSKGF2eZOJ2jLg>H)4<k(1q(b|lyPNpmFjQdn@uik+aPUG
zI|*N;`{2QYu4T)XK|MCrwY0tBY$&`i#sQlW4U`*gSvz*@@IEuTkt=+W4%+Fkm83Il
zCyg)In>%-|=POszoo_7Dm-qYczZdh9Va@7^cieGD5nnzR@(90*NSg@beZqW^cC+gz
zPo8ugI&{dH=OP#KuCmfV`Ot=rEs4`Fw#}P2uTp*aOz)k0=+vo``}5B~W1kVYqkjPH
z&GgzjK6|@Kx!}Vv_65Ge&p*#TziDr3-@d)~sdG86xsU17rw6`1>9@c-ZFTM1wexx>
zJDEK*=`sLY)Wc{S#Yc$-*h0J3uV3#OEbn&Hbkmv6aoh`alqOA@xNpDxHt4^!d*VKb
z^aF4oxqL6_+;38jA)g<8^ieB@B1`m)`1s?Gd%kk{nYU*J4I4LZTu`618?tBR`|rQ^
z<9WDm&Phr+B{7{=CiEM|1#HS+fBoghCUb1AmxrYhJ!T9>)Yfx9)Po;*-~q37Ni4G@
zrJ3@;reOc#0_C%R|9)TpPttv%v&)w+kFBe~*zDP}y?xB}=Osybo+_>M2YvS0XK{FW
z+sN4ZT(*mG_eFYp_3Gu*AH6a7`uy|HeH(Gyx!P8~en0(qhYlTz7K8tde$juIG``R!
z-T`6!bTq&h{ioPau~fb+Me~z#AfMyPfpVcNMvoqyIKFruXy@9rX;ZPbFz$)xhIi?r
z`JvzC&l`L(7AG!H4vb$-nx6t+)DP*0=N@3ng7L*zYVlP5JVo;qzIYclE+m;R>Mi~H
z^-IauiWMso_uNFwE(yQ!Qy6y~7vPJwucYZ0=|#hspTNdd*!`J)F}@WBlTb!UP9tN8
z50&8GKl|*niQ~)4gE=sGuZVVg3-Cpx&dhrxcC3~aV=H;b|G<F*mi<MRlmqWcC$0Wy
ztiTU-7j)PH_XKZ;3ZE(XvtzZa7{2IF*}Qpk5g*vff^uNoL{j-;T-euNe_iC>;0xY(
z-|5<GuN7YvIIa~x1s{aDk>Ud7Kpj7+ehTv$(3Tq2FMgwIY6Ghn%`#g6U)1AypE@p}
zU*CN5jh`pS&Pmb|zOen=2TY+|reET#V%dTd<Eu-TE=6pJu|}Ejegxw_lf;#!l{Qx9
zwBtQG<{bz}=Cq^l!j9Fl{CP9Jc+ZS+5<J`CV9z>rNPIAkPf|IMUeb+5vy8?h-AfYF
zO8(<p@eck+AAJ<qkSJr!0YN>0a!TTTD$V7uP3h#R=@*QR8a2xMDNzlBHQJDFyzxfQ
zSUT6$F_)%1b2Z=L?E34k_x(h1eBp;NACB}(++@ozj4fHRq)1)f>?C~s_~VbRdGqG7
zj}_Qnlx^81;XJg7_q&+CD_oz4G5BH}{?w^c10NY4lW<>3V{T|JeBtNwj-b8QVfuy5
zWZNHq{Nd;KqFsr)n943U$}S1z05ACY%(p~a1sZ0-+9_a+x}<&gAPM)SH0IVb0B0|}
z@PdEO#`v-&obmn*bL^mX@F<@@x5_WqpC|ZYTmkQM#|8Le&dh-W2YSZDm-xzrY5aC{
zi9R8|$$^~=w^{I;$Ugd&I956)#?oaBzR)CM|Ni~`oDKM0(EwwNabd1X`uR9sbxLto
z8fV5Ab`l>8U(DvF2<sTk@tzmsEAF}H9<O1XBgVv7X^h!^`WzVp%G_=EUBqaBImYkv
zz85;iIoM0PhL++j6$5YrFSJL{E_V9#=_p>_H$%tp`I#@DF{9#3Or>Hfyxr^_yzmVv
z<_f@8VjIJ@l6BhnY5SitWlB)5gw62vv@Y)A7=SYv<6AH9yz`E?lVQG0)Xau5cRBBk
zU@zmYr)}aa%m92*KQw=A_3G80F*J*40luhLvRt=rou2~&e_uYocnb4W(mFcEc-JL`
zV=eH<@V%k|ZNrZ8on+<(N5^2ar0Z&*7=!_I4905LuI*;cnib0zng(A)=ALKXX=#}F
zNk>D>-oY8;D40V8zdp|AH@}~D*ashc(A!t(m^e#^GdRH?qdkIe6k{jje15j!lQFi3
zZ!5!PI@i}R`OP6ZX10=c{%4+f#_JfiG0Yk3M7A@IpKsQ39#J+kzZptbLtq74`PgHR
zxxfGZ+v`=-=Z7zBD0Ar#9Xiyvk;ztyuhJMZ9WyPn?~=yxWo;LH=U~8q0jg7KEPqD6
z5{zLhyLId4$5zDkSHc}y#&=xqyYD{l!*O0|9BaR{4tr-h20P605cb7P)67=VcES8(
z>PHeg>F5W1g&FHoIR21tvSAm)oSDy$KelMmBCmPrTwlkCLm0p(qYakGw=!s(fU#&m
z#~61(`>WUyJLxijhQSs~Y~RR{BYoX5$`{NLllTpXbn`;TRqS&EU-Ts{UAok_!J=8w
z!257*+O$!dnCg=m+cEro`nTbSdC=qRB-@4z8RE50ZLSsjyjL|ng%|Vt=ggTC%NPFs
z+i$-uzNB9n>z6$*FxI(q=h!jU*vq|p_xgII^h^3>%w)2JFXp4<+p=+iIwtRuh%fP#
z30HP3+~9|8*s!5UTbQwi->><f#Fp`r=@P#1^%pEyP>e5YZ{=NG>6aMGlr4^j8zN)H
z;sT7(Zz;aSSGIiNgFX4=lg0E4-r%cm-@f8Yb;zvhkmwR^`hx}yO34@dq+imnvgjAz
z#f}T;m-rG}*|25)7xuA7j~;P&`7vR`hY!z68?#N5&Z}BHWBXF^g?{lJB-zKRRpyoX
z441?gq9(q?R#|uyU*e}M{FHXC_!3{GF;bTO@WniIjK`rrC>+@~dh}@dUuyr#s*Xq9
zk+Ft+llaa%?<^MYy6Z0g4J3{$3&T1$KQ+?cD}piB!;yAZwiD4fF_NE*l%?F^25m~k
z58Ex4<?rZRu_*?`fEW-1Vn7Ut0Wly3#DEwO17bi7hygJm2E>3E5CdW$eFkWow>}~I
zgLoIx`i%I^dP(j3%Ioy=Iv?{2cHoG4fN#3#CO3HSU^jN`ST|<O7{AVU<(ca|RsT^w
z=9%uf^k)jL&?e?(z4zXG-G?83=)Z{@9;3rL^G<KvxY6Bz|NX(-Vd{@8J%96^K{Si`
zzg~LjB|ldob7H0njKOFAa*pL3v#0r<hr0jx=M4Qpx28>-=1!eD<-e<jR>cL_BC?-v
zf$-fK^U;*Q@%fvt`2ttwv-7?Xnv{yGIM&$4H;sDs?CCWuU-MD-9sgVzU-;|H>xE6s
zlrOYw-@bjmKA?P#&*yy27pp7YefQnLch%q}lfdzOpMi1H@S3lAsr!z9w(!lzk00;f
z;mCxm@Udtf*AQRvd7Q88!2^8FojbQod|_9)hWHX+;b+P0<HCgtv*ru?2w%)cC%(j2
zm@jys{y25&)Y!S0!Zt9~y75KZhWHX+QNDQBjJcT5ptx@;q~gl98(+P9_ZDAz=e9`w
z&h)E6g9h%WpMDDJkj73rOZYl`_^{&|(l6;(RKM`=(Xesj#<{a+&-(T+m?4%FY(I5J
z&ZP}se2K3xU*Yw7_3F9x>(_e?OQ)TcHZLOcm^W<LQ0@1!jYVu@m@U@PpQcTlx-DC_
z1Ugode!<|@ty^8QX3aEbfY>ULEn^M)R;NxKH)YBccjU;CpdZ`xj3trpUs30*Teq&S
zf66Y(E*e9nEol?OA8XaBmH!^(iWMuorlD)Bj~X?~wP?}8kGJ4B{I}Assr};57(k1N
zaAr#wBC;*IUD_pfj2$h-f*243Vn7Ut0Wly3#DEwO17bi7hygJm2E>3E5CdXB42S_S
zAO^&M7!U(uKn#chF(3xSfEW-1Vn7Ut0Wly3#DEy6vJ4a!t%7nuIiMU+4k!nd1IhvA
MfO0@NkhmQ9AA1&&T>t<8

literal 0
HcmV?d00001

diff --git a/src/utils/__init__.py b/src/utils/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/src/utils/metrics.py b/src/utils/metrics.py
new file mode 100644
index 0000000..70ff157
--- /dev/null
+++ b/src/utils/metrics.py
@@ -0,0 +1,66 @@
+import asyncio
+import json
+import os
+import shutil
+from loguru import logger
+from prometheus_client import Counter, Gauge
+from sqlalchemy.orm import Session
+
+from core.config import REPEAT_COUNT_METRICS_SECONDS
+from db import crud
+from db.database import get_db
+from core.config import SQLALCHEMY_DATABASE_URL
+from worker import REDIS_EXCEPTIONS_CHANNEL, Rdis
+
+
+# Custom metrics
+EXCEPTION_COUNTER = Counter(
+    "exceptions",
+    "Number of times a certain exception has occurred.",
+    labelnames=("types",)
+)
+WORKER_EXCEPTION = Counter(
+    "worker_exceptions_total",
+    "Number of times a certain exception has occurred on the worker.",
+    labelnames=("exception", "task",)
+)
+DISK_UTILIZATION = Gauge(
+    "disk_utilization",
+    "Disk utilization in GB",
+    labelnames=("type",)
+)
+DATABASE_METRICS = Gauge(
+    "database_metrics",
+    "Useful database metrics from queries",
+    labelnames=("query", "user")
+)
+
+
+async def redis_subscribe_worker_exceptions():
+    # Subscribe to Redis channel and increment the counter for each exception with info on the exception and task
+    PubSubExceptions = Rdis.pubsub()
+    PubSubExceptions.subscribe(REDIS_EXCEPTIONS_CHANNEL)
+    while True:
+        message = PubSubExceptions.get_message()
+        if message and message["type"] == "message":
+            data = json.loads(message["data"].decode("utf-8"))
+            WORKER_EXCEPTION.labels(exception=data["exception"], task=data["task"]).inc()
+        await asyncio.sleep(1)
+
+async def measure_regular_metrics():
+    _total, used, free = shutil.disk_usage("/")
+    DISK_UTILIZATION.labels(type="used").set(used / (2**30))
+    DISK_UTILIZATION.labels(type="free").set(free / (2**30))
+    try: 
+        fs = os.stat(SQLALCHEMY_DATABASE_URL.replace("sqlite:///", ""))
+        DISK_UTILIZATION.labels(type="database").set(fs.st_size / (2**30))
+    except Exception as e: logger.error(e)
+
+    session: Session = next(get_db())
+    count_archives = crud.count_archives(session)
+    count_archive_urls = crud.count_archive_urls(session)
+    DATABASE_METRICS.labels(query="count_archives", user="-").set(count_archives)
+    DATABASE_METRICS.labels(query="count_archive_urls", user="-").set(count_archive_urls)
+
+    for user in crud.count_by_user_since(session, REPEAT_COUNT_METRICS_SECONDS):
+        DATABASE_METRICS.labels(query="count_by_user", user=user.author_id).set(user.total)
\ No newline at end of file
diff --git a/src/worker.py b/src/worker.py
index 8b3e78c..a2220d7 100644
--- a/src/worker.py
+++ b/src/worker.py
@@ -1,9 +1,9 @@
 
-import os, traceback, yaml, datetime
+import os, traceback, yaml, datetime, sys
 from typing import List, Set
 
 from celery import Celery
-from celery.signals import task_failure
+from celery.signals import task_failure, worker_init
 from auto_archiver import Config, ArchivingOrchestrator, Metadata
 from auto_archiver.core import Media
 from loguru import logger
@@ -42,7 +42,7 @@ def create_archive_task(self, archive_json: str):
 
     if not archive.rearchive:
         with get_db() as session:
-            archives = crud.search_tasks_by_url(session, url, archive.author_id, absolute_search=True)
+            archives = crud.search_archives_by_url(session, url, archive.author_id, absolute_search=True)
             if len(archives):
                 logger.info(f"Skipping {url=} as it was already archived")
                 return Metadata.choose_most_complete([a.result for a in archives])
@@ -212,6 +212,9 @@ def redis_publish_exception(exception, task_name):
         logger.error(f"Could not publish to {REDIS_EXCEPTIONS_CHANNEL}")
 
 
-# INIT
-ORCHESTRATORS = {}
-load_orchestrators()
+@worker_init.connect
+def at_start(sender, **kwargs):
+    global ORCHESTRATORS
+    ORCHESTRATORS = {}
+    load_orchestrators()
+    logger.info("Orchestrators loaded successfully.")

From 1a3273da7e0e0b2463620de4e6ab60ad01ef0f83 Mon Sep 17 00:00:00 2001
From: msramalho <19508417+msramalho@users.noreply.github.com>
Date: Thu, 17 Oct 2024 11:29:20 +0100
Subject: [PATCH 02/33] first CI test

---
 .github/workflows/ci.yml | 41 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 41 insertions(+)
 create mode 100644 .github/workflows/ci.yml

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
new file mode 100644
index 0000000..b08c02f
--- /dev/null
+++ b/.github/workflows/ci.yml
@@ -0,0 +1,41 @@
+name: CI
+on:
+  push:
+    branches:
+      - main
+      - dev
+  pull_request:
+    branches:
+      - main
+      - dev
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    services:
+      redis:
+        image: redis:6-alpine
+        ports:
+          - 6379:6379
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v2
+
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.10'
+
+    - name: Install pipenv
+      run: pip install pipenv
+      working-directory: src
+
+    - name: Install dependencies
+      run: pipenv install --dev
+      working-directory: src
+
+    - name: Run tests
+      run: pipenv run PYTHONPATH=. pytest -v --color=yes tests/
+      working-directory: src
\ No newline at end of file

From f2a14abb17e90dee3a5c16b381711dd9de734741 Mon Sep 17 00:00:00 2001
From: msramalho <19508417+msramalho@users.noreply.github.com>
Date: Thu, 17 Oct 2024 13:09:50 +0100
Subject: [PATCH 03/33] adds health check and new env logic

---
 .gitignore               |  3 +++
 README.md                |  4 +++-
 docker-compose.dev.yml   |  4 ++++
 docker-compose.yml       | 19 +++++++++++++++++--
 src/endpoints/default.py |  4 ++++
 5 files changed, 31 insertions(+), 3 deletions(-)

diff --git a/.gitignore b/.gitignore
index 4311303..271c3d9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -7,11 +7,14 @@ secrets
 __pycache
 .pytest_cach
 .env
+.env.dev
+.env.prod
 *.db
 redis/data/*
 .ipynb_checkpoints*
 #temp
 tests
 src/user-groups.yaml
+src/user-groups.dev.yaml
 wit*
 src/crawls
\ No newline at end of file
diff --git a/README.md b/README.md
index 013f455..0ec9362 100644
--- a/README.md
+++ b/README.md
@@ -6,6 +6,8 @@ An api that uses celery workers to process URL archive requests via [bellingcat/
 ## Development
 http://localhost:8004
 
+TODO: update .env file instructions, should use .env.prod and .env.dev and only use .env for always overwriting dev/prod settings.
+
 requires `src/.env`
 
 cd /src
@@ -37,7 +39,7 @@ Auto-archiver orchestrator files configurations. For each archiving task an orch
 orchestrators:
   group1: secrets/orchestration-group1.yaml
   group2: secrets/orchestration-group2.yaml
-  default: secrets/orchestration-default:.yaml
+  default: secrets/orchestration-default:orchestration.yaml
 ```
 
 ## Database migrations
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index aa15f48..7814296 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -1,15 +1,19 @@
 services:
   web:
     restart: "no"
+    env_file: src/.env.dev
     environment:
       - SERVE_LOCAL_ARCHIVE=/app/local_archive # See orchestration.yaml local_storage.save_to
       - ALLOWED_ORIGINS=http://localhost:8004,chrome-extension://ojcimmjndnlmmlgnjaeojoebaceokpdp
       - API_BEARER_TOKEN=dev-api-bearer-token
+      - USER_GROUPS_FILENAME=user-groups.dev.yaml
 
   worker:
     restart: "no"
+    env_file: src/.env.dev
 
   redis:
     restart: "no"
+    env_file: src/.env.dev
     ports:
       - 6379:6379
diff --git a/docker-compose.yml b/docker-compose.yml
index 6549b28..3c13e08 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -4,7 +4,7 @@ x-broker-url: &broker-url "redis://:${REDIS_PASSWORD}@redis:6379/0"
 x-base-setup: &base-setup
   build: ./src
   restart: always
-  env_file: src/.env
+  env_file: src/.env.prod
   environment:
     CELERY_BROKER_URL: *broker-url
     CELERY_RESULT_BACKEND: *broker-url
@@ -23,10 +23,15 @@ services:
       - ./src:/app
     depends_on:
       - redis
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
 
   worker:
     <<: *base-setup
-    command: celery worker --app=worker.celery --loglevel=info --logfile=logs/celery.log
+    command: celery --app=worker.celery worker --loglevel=info --logfile=logs/celery.log
     volumes:
       - ./src:/app
       - /var/run/docker.sock:/var/run/docker.sock
@@ -41,6 +46,11 @@ services:
     depends_on:
       - web
       - redis
+    healthcheck:
+      test: ["CMD", "pipenv", "run", "celery", "-A", "worker.celery", "status"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
 
   redis:
     image: redis:6-alpine
@@ -49,6 +59,11 @@ services:
     volumes:
       - "./redis/data:/data"
       - "./redis/config:/conf"
+    healthcheck:
+      test: ["CMD", "redis-cli", "-a", "${REDIS_PASSWORD}", "ping"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
 
   # dashboard service will only launch the dashboard if "--profile flower" is passed to docker compose; or if explicitly called "docker compose up dashboard"
   dashboard:
diff --git a/src/endpoints/default.py b/src/endpoints/default.py
index 0cdf180..5580324 100644
--- a/src/endpoints/default.py
+++ b/src/endpoints/default.py
@@ -26,6 +26,10 @@ async def home(request: Request):
     return JSONResponse(status)
 
 
+@default_router.get("/health")
+async def health(request: Request):
+    return JSONResponse({"status": "ok"})
+
 @default_router.get("/groups", response_model=list[str])
 def get_user_groups(db: Session = Depends(get_db), email=Depends(get_user_auth)):
     return crud.get_user_groups(db, email)

From 94f7be3fd85e40a456d534b6fde72d919cabdea1 Mon Sep 17 00:00:00 2001
From: msramalho <19508417+msramalho@users.noreply.github.com>
Date: Thu, 17 Oct 2024 13:10:19 +0100
Subject: [PATCH 04/33] removing load_env

---
 src/core/config.py    | 3 ---
 src/main.py           | 3 ---
 src/migrations/env.py | 3 ---
 3 files changed, 9 deletions(-)

diff --git a/src/core/config.py b/src/core/config.py
index 1aef56c..9671eeb 100644
--- a/src/core/config.py
+++ b/src/core/config.py
@@ -1,7 +1,4 @@
 import os
-from dotenv import load_dotenv
-
-load_dotenv()
 
 VERSION = "0.7.0"
 API_DESCRIPTION = """
diff --git a/src/main.py b/src/main.py
index 190cff5..7424dee 100644
--- a/src/main.py
+++ b/src/main.py
@@ -5,7 +5,6 @@ from fastapi.encoders import jsonable_encoder
 from fastapi.responses import JSONResponse
 from fastapi.staticfiles import StaticFiles
 from fastapi.middleware.cors import CORSMiddleware
-from dotenv import load_dotenv
 from prometheus_fastapi_instrumentator import Instrumentator
 from datetime import datetime
 import sqlalchemy
@@ -25,8 +24,6 @@ from auto_archiver import Metadata
 
 from endpoints import default_router, url_router, sheet_router, task_router, interoperability_router
 
-load_dotenv()
-
 
 app = FastAPI(
     title="Auto-Archiver API",
diff --git a/src/migrations/env.py b/src/migrations/env.py
index baf6ff6..9b95d75 100644
--- a/src/migrations/env.py
+++ b/src/migrations/env.py
@@ -4,9 +4,6 @@ from sqlalchemy import engine_from_config
 from sqlalchemy import pool
 
 from alembic import context
-from dotenv import load_dotenv
-
-load_dotenv()
 
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.

From 3ba20d24733f5ebe5116450de9c89c661e730bf5 Mon Sep 17 00:00:00 2001
From: msramalho <19508417+msramalho@users.noreply.github.com>
Date: Thu, 17 Oct 2024 13:13:06 +0100
Subject: [PATCH 05/33] removes flower and updates celery>5

---
 .example.env       |   2 -
 README.md          |   2 +-
 docker-compose.yml |  15 +----
 src/Pipfile        |   3 +-
 src/Pipfile.lock   | 147 +++++++++++++++++++++++++++------------------
 5 files changed, 92 insertions(+), 77 deletions(-)

diff --git a/.example.env b/.example.env
index 5e6f3c2..e8970b2 100644
--- a/.example.env
+++ b/.example.env
@@ -1,3 +1 @@
-FLOWER_USERNAME=TODO
-FLOWER_PASSWORD=TODO
 REDIS_PASSWORD=TODO
\ No newline at end of file
diff --git a/README.md b/README.md
index 0ec9362..b026676 100644
--- a/README.md
+++ b/README.md
@@ -12,7 +12,7 @@ requires `src/.env`
 
 cd /src
 <!-- * `pipenv install --editable ../../auto-archiver` -->
-* console 1 - `docker compose up redis` optionally add `dashboard` for flower dashboard and `web` if not running uvicorn locally
+* console 1 - `docker compose up redis` optionally add `web` if not running uvicorn locally
 * console 2 - `pipenv shell` + `celery worker --app=worker.celery --loglevel=info --logfile=logs/celery_dev.log`
   * `celery --app=worker.celery worker --loglevel=info --logfile=logs/celery_dev.log` celery 5
   * or with watchdog for dev auto-reload `watchmedo auto-restart -d ./  -- celery --app=worker.celery worker --loglevel=info --logfile=logs/celery_dev.log`
diff --git a/docker-compose.yml b/docker-compose.yml
index 3c13e08..3fd6d2a 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -63,17 +63,4 @@ services:
       test: ["CMD", "redis-cli", "-a", "${REDIS_PASSWORD}", "ping"]
       interval: 30s
       timeout: 10s
-      retries: 3
-
-  # dashboard service will only launch the dashboard if "--profile flower" is passed to docker compose; or if explicitly called "docker compose up dashboard"
-  dashboard:
-    <<: *base-setup
-    profiles:
-      - flower
-    command: ["flower", "--app=worker.celery", "--port=5555", "--broker", *broker-url, "--basic_auth=${FLOWER_USERNAME}:${FLOWER_PASSWORD}"]
-    ports:
-      - 5556:5555
-    depends_on:
-      - web
-      - redis
-      - worker
+      retries: 3
\ No newline at end of file
diff --git a/src/Pipfile b/src/Pipfile
index 32e26d4..6b5cfbe 100644
--- a/src/Pipfile
+++ b/src/Pipfile
@@ -5,9 +5,8 @@ name = "pypi"
 
 [packages]
 aiofiles = "==0.6.0"
-celery = "==4.4.7"
+celery = ">=5.0"
 fastapi = "*"
-flower = "==0.9.7"
 jinja2 = "*"
 redis = "==3.5.3"
 requests = ">=2.25.1"
diff --git a/src/Pipfile.lock b/src/Pipfile.lock
index 1d2a301..4fdcb68 100644
--- a/src/Pipfile.lock
+++ b/src/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "d601bfa04bab584dcde6b8b7b619538bd03b7d2e13de53f802eacc6427703996"
+            "sha256": "dd6fbd0e75f6bbcb4e17ff6fd971349e25ce6d96a16dff34e7e39a625ca66244"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -142,11 +142,11 @@
         },
         "amqp": {
             "hashes": [
-                "sha256:70cdb10628468ff14e57ec2f751c7aa9e48e7e3651cfd62d431213c0c4e58f21",
-                "sha256:aa7f313fb887c91f15474c1229907a04dac0b8135822d6603437803424c0aa59"
+                "sha256:827cb12fb0baa892aad844fd95258143bce4027fdac4fccddbc43330fd281637",
+                "sha256:a1ecff425ad063ad42a486c902807d1482311481c8ad95a72694b2975e75f7fd"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
-            "version": "==2.6.1"
+            "markers": "python_version >= '3.6'",
+            "version": "==5.2.0"
         },
         "annotated-types": {
             "hashes": [
@@ -237,10 +237,11 @@
         },
         "billiard": {
             "hashes": [
-                "sha256:299de5a8da28a783d51b197d496bef4f1595dd023a93a4f59dde1886ae905547",
-                "sha256:87103ea78fa6ab4d5c751c4909bcff74617d985de7fa8b672cf8618afd5a875b"
+                "sha256:12b641b0c539073fc8d3f5b8b7be998956665c4233c7c1fcd66a7e677c4fb36f",
+                "sha256:40b59a4ac8806ba2c2369ea98d876bc6108b051c227baffd928c644d15d8f3cb"
             ],
-            "version": "==3.6.4.0"
+            "markers": "python_version >= '3.7'",
+            "version": "==4.2.1"
         },
         "black": {
             "hashes": [
@@ -288,19 +289,19 @@
         },
         "boto3": {
             "hashes": [
-                "sha256:2bf7e7f376aee52155fc4ae4487f29333a6bcdf3a05c3bc4fede10b972d951a6",
-                "sha256:e74bc6d69c04ca611b7f58afe08e2ded6cb6504a4a80557b656abeefee395f88"
+                "sha256:a5b00f8b82dce62870759f04861747944da834d64a64355970120c475efdafc0",
+                "sha256:e1f36f8be453505cebcc3da178ea081b2a06c0e5e1cdee774f1067599b8d9c3e"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==1.35.41"
+            "version": "==1.35.42"
         },
         "botocore": {
             "hashes": [
-                "sha256:8a09a32136df8768190a6c92f0240cd59c30deb99c89026563efadbbed41fa00",
-                "sha256:915c4d81e3a0be3b793c1e2efdf19af1d0a9cd4a2d8de08ee18216c14d67764b"
+                "sha256:05af0bb8b9cea7ce7bc589c332348d338a21b784e9d088a588fd10ec145007ff",
+                "sha256:af348636f73dc24b7e2dc760a34d08c8f2f94366e9b4c78d877307b128abecef"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==1.35.41"
+            "version": "==1.35.42"
         },
         "brotli": {
             "hashes": [
@@ -408,12 +409,12 @@
         },
         "celery": {
             "hashes": [
-                "sha256:a92e1d56e650781fb747032a3997d16236d037c8199eacd5217d1a72893bca45",
-                "sha256:d220b13a8ed57c78149acf82c006785356071844afe0b27012a4991d44026f9f"
+                "sha256:369631eb580cf8c51a82721ec538684994f8277637edde2dfc0dacd73ed97f64",
+                "sha256:504a19140e8d3029d5acad88330c541d4c3f64c789d85f94756762d8bca7e706"
             ],
             "index": "pypi",
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
-            "version": "==4.4.7"
+            "markers": "python_version >= '3.8'",
+            "version": "==5.4.0"
         },
         "certifi": {
             "hashes": [
@@ -607,6 +608,29 @@
             "markers": "python_version >= '3.7'",
             "version": "==8.1.7"
         },
+        "click-didyoumean": {
+            "hashes": [
+                "sha256:4f82fdff0dbe64ef8ab2279bd6aa3f6a99c3b28c05aa09cbfc07c9d7fbb5a463",
+                "sha256:5c4bb6007cfea5f2fd6583a2fb6701a22a41eb98957e63d0fac41c10e7c3117c"
+            ],
+            "markers": "python_full_version >= '3.6.2'",
+            "version": "==0.3.1"
+        },
+        "click-plugins": {
+            "hashes": [
+                "sha256:46ab999744a9d831159c3411bb0c79346d94a444df9a3a3742e9ed63645f264b",
+                "sha256:5d262006d3222f5057fd81e1623d4443e41dcda5dc815c06b442aa3c02889fc8"
+            ],
+            "version": "==1.1.1"
+        },
+        "click-repl": {
+            "hashes": [
+                "sha256:17849c23dba3d667247dc4defe1757fff98694e90fe37474f3feebb69ced26a9",
+                "sha256:fb7e06deb8da8de86180a33a9da97ac316751c094c6899382da7feeeeb51b812"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==0.3.0"
+        },
         "cloudscraper": {
             "hashes": [
                 "sha256:429c6e8aa6916d5bad5c8a5eac50f3ea53c9ac22616f6cb21b18dcc71517d0d3",
@@ -809,14 +833,6 @@
             "markers": "python_version >= '3.8'",
             "version": "==3.0.3"
         },
-        "flower": {
-            "hashes": [
-                "sha256:8d6d6ac03e60b3a4227d156da489eb435e2442d82e89922d413df9054b9221eb",
-                "sha256:cf27a254268bb06fd4972408d0518237fcd847f7da4b4cd8055e228150ace8f3"
-            ],
-            "index": "pypi",
-            "version": "==0.9.7"
-        },
         "frozenlist": {
             "hashes": [
                 "sha256:04ced3e6a46b4cfffe20f9ae482818e34eba9b5fb0ce4056e4cc9b6e212d09b7",
@@ -1082,14 +1098,6 @@
             "markers": "python_version >= '3.8'",
             "version": "==0.27.2"
         },
-        "humanize": {
-            "hashes": [
-                "sha256:b53caaec8532bcb2fff70c8826f904c35943f8cecaca29d272d9df38092736c0",
-                "sha256:e66f36020a2d5a974c504bd2555cf770621dbdbb6d82f94a6857c0b1ea2608be"
-            ],
-            "markers": "python_version >= '3.9'",
-            "version": "==4.11.0"
-        },
         "idna": {
             "hashes": [
                 "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc",
@@ -1212,11 +1220,11 @@
         },
         "kombu": {
             "hashes": [
-                "sha256:be48cdffb54a2194d93ad6533d73f69408486483d189fe9f5990ee24255b0e0a",
-                "sha256:ca1b45faac8c0b18493d02a8571792f3c40291cf2bcf1f55afed3d8f3aa7ba74"
+                "sha256:14212f5ccf022fc0a70453bb025a1dcc32782a588c49ea866884047d66e14763",
+                "sha256:eef572dd2fd9fc614b37580e3caeafdd5af46c1eff31e7fba89138cdb406f2cf"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
-            "version": "==4.6.11"
+            "markers": "python_version >= '3.8'",
+            "version": "==5.4.2"
         },
         "livereload": {
             "hashes": [
@@ -1917,10 +1925,11 @@
         },
         "prometheus-client": {
             "hashes": [
-                "sha256:983c7ac4b47478720db338f1491ef67a100b474e3bc7dafcbaefb7d0b8f9b01c",
-                "sha256:c6e6b706833a6bd1fd51711299edee907857be10ece535126a158f911ee80915"
+                "sha256:4fa6b4dd0ac16d58bb587c04b1caae65b8c5043e85f778f42f5f632f6af2e166",
+                "sha256:96c83c606b71ff2b0a433c98889d275f51ffec6c5e267de37c7a2b5c9aa9233e"
             ],
-            "version": "==0.8.0"
+            "markers": "python_version >= '3.8'",
+            "version": "==0.21.0"
         },
         "prometheus-fastapi-instrumentator": {
             "hashes": [
@@ -1931,6 +1940,14 @@
             "markers": "python_full_version >= '3.8.1' and python_full_version < '4.0.0'",
             "version": "==7.0.0"
         },
+        "prompt-toolkit": {
+            "hashes": [
+                "sha256:d6623ab0477a80df74e646bdbc93621143f5caf104206aa29294d53de1a03d90",
+                "sha256:f49a827f90062e411f1ce1f854f2aedb3c23353244f8108b89283587397ac10e"
+            ],
+            "markers": "python_full_version >= '3.7.0'",
+            "version": "==3.0.48"
+        },
         "proto-plus": {
             "hashes": [
                 "sha256:30b72a5ecafe4406b0d339db35b56c4059064e69227b8c3bda7462397f966445",
@@ -2784,7 +2801,7 @@
                 "sha256:f7894c581ecdcf91666a0912f18ce5e757213999e183ebfc2c3fdbf4d5bd764e",
                 "sha256:fd03192e287fbd0899dd8f81c6fb9cbbc69194d2074b38f384cb6fa72b80e9c2"
             ],
-            "markers": "python_full_version >= '3.5.2'",
+            "markers": "python_version >= '3.8'",
             "version": "==6.4"
         },
         "tqdm": {
@@ -2797,11 +2814,11 @@
         },
         "trio": {
             "hashes": [
-                "sha256:0346c3852c15e5c7d40ea15972c4805689ef2cb8b5206f794c9c19450119f3a4",
-                "sha256:c5237e8133eb0a1d72f09a971a55c28ebe69e351c783fc64bc37db8db8bbe1d0"
+                "sha256:1dcc95ab1726b2da054afea8fd761af74bad79bd52381b84eae408e983c76831",
+                "sha256:68eabbcf8f457d925df62da780eff15ff5dc68fd6b367e2dde59f7aaf2a0b884"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==0.26.2"
+            "version": "==0.27.0"
         },
         "trio-websocket": {
             "hashes": [
@@ -2841,6 +2858,14 @@
             ],
             "version": "==0.9.0"
         },
+        "tzdata": {
+            "hashes": [
+                "sha256:7d85cc416e9382e69095b7bdf4afd9e3880418a2413feec7069d533d6b4e31cc",
+                "sha256:a48093786cdcde33cad18c2555e8532f34422074448fbc874186f0abd79565cd"
+            ],
+            "markers": "python_version >= '2'",
+            "version": "==2024.2"
+        },
         "tzlocal": {
             "hashes": [
                 "sha256:49816ef2fe65ea8ac19d19aa7a1ae0551c834303d5014c6d5a62e4cbda8047b8",
@@ -2879,11 +2904,11 @@
         },
         "vine": {
             "hashes": [
-                "sha256:133ee6d7a9016f177ddeaf191c1f58421a1dcc6ee9a42c58b34bed40e1d2cd87",
-                "sha256:ea4947cc56d1fd6f2095c8d543ee25dad966f78692528e68b4fada11ba3f98af"
+                "sha256:40fdf3c48b2cfe1c38a49e9ae2da6fda88e4794c810050a728bd7413811fb1dc",
+                "sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==1.3.0"
+            "markers": "python_version >= '3.6'",
+            "version": "==5.1.0"
         },
         "vk-api": {
             "hashes": [
@@ -2988,6 +3013,13 @@
             "markers": "python_version >= '3.8'",
             "version": "==0.22.0"
         },
+        "wcwidth": {
+            "hashes": [
+                "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859",
+                "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5"
+            ],
+            "version": "==0.2.13"
+        },
         "webencodings": {
             "hashes": [
                 "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78",
@@ -3232,7 +3264,7 @@
                 "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b",
                 "sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc"
             ],
-            "markers": "python_version < '3.11'",
+            "markers": "python_version >= '3.7'",
             "version": "==1.2.2"
         },
         "h11": {
@@ -3293,12 +3325,11 @@
         },
         "pytest": {
             "hashes": [
-                "sha256:70b98107bd648308a7952b06e6ca9a50bc660be218d53c257cc1fc94fda10181",
-                "sha256:a6853c7375b2663155079443d2e45de913a911a11d669df02a50814944db57b2"
+                "sha256:c434598117762e2bd304e526244f67bf66bbd7b5d6cf22138be51ff661980343",
+                "sha256:de4bb8104e201939ccdc688b27a89a7be2079b22e2bd2b07f806b6ba71117977"
             ],
-            "index": "pypi",
             "markers": "python_version >= '3.8'",
-            "version": "==8.3.3"
+            "version": "==8.2.2"
         },
         "sniffio": {
             "hashes": [
@@ -3310,11 +3341,11 @@
         },
         "tomli": {
             "hashes": [
-                "sha256:2ebe24485c53d303f690b0ec092806a085f07af5a5aa1464f3931eec36caaa38",
-                "sha256:d46d457a85337051c36524bc5349dd91b1877838e2979ac5ced3e710ed8a60ed"
+                "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc",
+                "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"
             ],
-            "markers": "python_version < '3.11'",
-            "version": "==2.0.2"
+            "markers": "python_version >= '3.7'",
+            "version": "==2.0.1"
         },
         "typing-extensions": {
             "hashes": [

From a39ecc07773164cc5e287cf7a53567d1aec94e84 Mon Sep 17 00:00:00 2001
From: msramalho <19508417+msramalho@users.noreply.github.com>
Date: Thu, 17 Oct 2024 16:07:50 +0100
Subject: [PATCH 06/33] adding tests for CI

---
 .github/workflows/ci.yml |  2 +-
 .gitignore               |  2 --
 src/tests/conftest.py    | 17 +++++++++++++++++
 src/tests/test_start.py  | 13 +++++++++++++
 4 files changed, 31 insertions(+), 3 deletions(-)
 create mode 100644 src/tests/conftest.py
 create mode 100644 src/tests/test_start.py

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index b08c02f..7dc87d2 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -37,5 +37,5 @@ jobs:
       working-directory: src
 
     - name: Run tests
-      run: pipenv run PYTHONPATH=. pytest -v --color=yes tests/
+      run: PYTHONPATH=. pipenv run pytest -v --color=yes tests/
       working-directory: src
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index 271c3d9..4ae5a9c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,8 +12,6 @@ __pycache
 *.db
 redis/data/*
 .ipynb_checkpoints*
-#temp
-tests
 src/user-groups.yaml
 src/user-groups.dev.yaml
 wit*
diff --git a/src/tests/conftest.py b/src/tests/conftest.py
new file mode 100644
index 0000000..94f5358
--- /dev/null
+++ b/src/tests/conftest.py
@@ -0,0 +1,17 @@
+import pytest
+from unittest.mock import patch
+from sqlalchemy import create_engine
+
+# from sqlalchemy_utils import create_database, database_exists, drop_database
+from sqlalchemy.orm import sessionmaker
+
+@pytest.fixture(autouse=True)
+def mock_logger_add():
+    """Fixture to mock loguru.logger.add for all tests."""
+    with patch('loguru.logger.add') as mock_add:
+        yield mock_add  # This makes the mock available to tests
+
+@pytest.fixture(autouse=True)
+def mock_database_url():
+    with patch('core.config.SQLALCHEMY_DATABASE_URL', "sqlite:////app/auto-archiver.test.db") as mock_wb_url:
+        yield mock_wb_url
diff --git a/src/tests/test_start.py b/src/tests/test_start.py
new file mode 100644
index 0000000..4fec658
--- /dev/null
+++ b/src/tests/test_start.py
@@ -0,0 +1,13 @@
+import pytest
+from fastapi.testclient import TestClient
+from core.config import VERSION
+
+def test_mock_logger():
+    from main import app
+
+    client = TestClient(app)
+    
+    response = client.get("/")
+    assert response.status_code == 200
+    r = response.json()
+    assert "version" in r and r["version"] == VERSION
\ No newline at end of file

From 036049fdc373ae7dc921cdaf4bf9c9dbcf3606e3 Mon Sep 17 00:00:00 2001
From: msramalho <19508417+msramalho@users.noreply.github.com>
Date: Thu, 17 Oct 2024 16:30:50 +0100
Subject: [PATCH 07/33] setting env vars for tests

---
 src/tests/conftest.py | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/src/tests/conftest.py b/src/tests/conftest.py
index 94f5358..6d8a561 100644
--- a/src/tests/conftest.py
+++ b/src/tests/conftest.py
@@ -1,9 +1,6 @@
 import pytest
+import os
 from unittest.mock import patch
-from sqlalchemy import create_engine
-
-# from sqlalchemy_utils import create_database, database_exists, drop_database
-from sqlalchemy.orm import sessionmaker
 
 @pytest.fixture(autouse=True)
 def mock_logger_add():
@@ -11,7 +8,6 @@ def mock_logger_add():
     with patch('loguru.logger.add') as mock_add:
         yield mock_add  # This makes the mock available to tests
 
-@pytest.fixture(autouse=True)
-def mock_database_url():
-    with patch('core.config.SQLALCHEMY_DATABASE_URL', "sqlite:////app/auto-archiver.test.db") as mock_wb_url:
-        yield mock_wb_url
+os.environ["CHROME_APP_IDS"] = 'test_app_id_1,test_app_id_2'
+os.environ["DATABASE_PATH"] = "sqlite:////app/auto-archiver.test.db"
+os.environ["BLOCKED_EMAILS"] = "blocked@example.com"
\ No newline at end of file

From 879c0cb87956a0f6308f6b4ea02fa6788fe5c2b5 Mon Sep 17 00:00:00 2001
From: msramalho <19508417+msramalho@users.noreply.github.com>
Date: Thu, 17 Oct 2024 16:44:32 +0100
Subject: [PATCH 08/33] adding code coverage to test execution

---
 .github/workflows/ci.yml |  8 +++--
 .gitignore               |  5 +++-
 README.md                | 20 +++++++++++++
 src/Pipfile              |  1 +
 src/Pipfile.lock         | 63 +++++++++++++++++++++++++++++++++++++++-
 5 files changed, 93 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 7dc87d2..bc3c029 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -36,6 +36,10 @@ jobs:
       run: pipenv install --dev
       working-directory: src
 
-    - name: Run tests
-      run: PYTHONPATH=. pipenv run pytest -v --color=yes tests/
+    - name: Run tests with coverage
+      run: PYTHONPATH=. pipenv run coverage run -m pytest -v --color=yes tests/
+      working-directory: src
+
+    - name: Report coverage
+      run: pipenv run coverage report
       working-directory: src
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index 4ae5a9c..7df1354 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,4 +15,7 @@ redis/data/*
 src/user-groups.yaml
 src/user-groups.dev.yaml
 wit*
-src/crawls
\ No newline at end of file
+src/crawls
+.coverage
+.pytest_cache/*
+htmlcov
\ No newline at end of file
diff --git a/README.md b/README.md
index b026676..236aba6 100644
--- a/README.md
+++ b/README.md
@@ -68,4 +68,24 @@ Run `pipenv update auto-archiver` inside `src` to update the auto-archiver versi
 # CALL /sheet POST endpoint
 curl -XPOST -H "Authorization: Bearer GOOGLE_OAUTH_TOKEN" -H "Content-type: application/json" -d '{"sheet_id": "SHEET_ID", "header": 1}' 'http://localhost:8004/sheet'
 
+```
+
+
+### Testing
+```bash
+# can be done from top level but let's do it from the src folder for consistency with CI etc
+cd src
+# run tests 
+PYTHONPATH=. pipenv run pytest -v --color=yes tests/
+
+# TO GET COVERAGE
+# run tests with coverage instead 
+PYTHONPATH=. pipenv run coverage run -m pytest -v --color=yes tests/
+
+# get coverage
+pipenv run coverage report
+# get coverage HTML
+
+pipenv run coverage html
+# > open/run server on htmlcov/index.html to navigate through line coverage
 ```
\ No newline at end of file
diff --git a/src/Pipfile b/src/Pipfile
index 6b5cfbe..dec6834 100644
--- a/src/Pipfile
+++ b/src/Pipfile
@@ -24,6 +24,7 @@ auto-archiver = "*"
 watchdog = "*"
 pytest = "*"
 httpx = "*"
+coverage = "*"
 
 [requires]
 python_version = "3.10"
diff --git a/src/Pipfile.lock b/src/Pipfile.lock
index 4fdcb68..068420f 100644
--- a/src/Pipfile.lock
+++ b/src/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "dd6fbd0e75f6bbcb4e17ff6fd971349e25ce6d96a16dff34e7e39a625ca66244"
+            "sha256": "359638472cb3c3914fac7040bc702463c0ed1ae2e5cee00abddd59d9b34e923e"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -3259,6 +3259,67 @@
             "markers": "python_version >= '3.6'",
             "version": "==2024.7.4"
         },
+        "coverage": {
+            "extras": [
+                "toml"
+            ],
+            "hashes": [
+                "sha256:0086cd4fc71b7d485ac93ca4239c8f75732c2ae3ba83f6be1c9be59d9e2c6382",
+                "sha256:01c322ef2bbe15057bc4bf132b525b7e3f7206f071799eb8aa6ad1940bcf5fb1",
+                "sha256:03cafe82c1b32b770a29fd6de923625ccac3185a54a5e66606da26d105f37dac",
+                "sha256:044a0985a4f25b335882b0966625270a8d9db3d3409ddc49a4eb00b0ef5e8cee",
+                "sha256:07ed352205574aad067482e53dd606926afebcb5590653121063fbf4e2175166",
+                "sha256:0d1b923fc4a40c5832be4f35a5dab0e5ff89cddf83bb4174499e02ea089daf57",
+                "sha256:0e7b27d04131c46e6894f23a4ae186a6a2207209a05df5b6ad4caee6d54a222c",
+                "sha256:1fad32ee9b27350687035cb5fdf9145bc9cf0a094a9577d43e909948ebcfa27b",
+                "sha256:289cc803fa1dc901f84701ac10c9ee873619320f2f9aff38794db4a4a0268d51",
+                "sha256:3c59105f8d58ce500f348c5b56163a4113a440dad6daa2294b5052a10db866da",
+                "sha256:46c3d091059ad0b9c59d1034de74a7f36dcfa7f6d3bde782c49deb42438f2450",
+                "sha256:482855914928c8175735a2a59c8dc5806cf7d8f032e4820d52e845d1f731dca2",
+                "sha256:49c76cdfa13015c4560702574bad67f0e15ca5a2872c6a125f6327ead2b731dd",
+                "sha256:4b03741e70fb811d1a9a1d75355cf391f274ed85847f4b78e35459899f57af4d",
+                "sha256:4bea27c4269234e06f621f3fac3925f56ff34bc14521484b8f66a580aacc2e7d",
+                "sha256:4d5fae0a22dc86259dee66f2cc6c1d3e490c4a1214d7daa2a93d07491c5c04b6",
+                "sha256:543ef9179bc55edfd895154a51792b01c017c87af0ebaae092720152e19e42ca",
+                "sha256:54dece71673b3187c86226c3ca793c5f891f9fc3d8aa183f2e3653da18566169",
+                "sha256:6379688fb4cfa921ae349c76eb1a9ab26b65f32b03d46bb0eed841fd4cb6afb1",
+                "sha256:65fa405b837060db569a61ec368b74688f429b32fa47a8929a7a2f9b47183713",
+                "sha256:6616d1c9bf1e3faea78711ee42a8b972367d82ceae233ec0ac61cc7fec09fa6b",
+                "sha256:6fe885135c8a479d3e37a7aae61cbd3a0fb2deccb4dda3c25f92a49189f766d6",
+                "sha256:7221f9ac9dad9492cecab6f676b3eaf9185141539d5c9689d13fd6b0d7de840c",
+                "sha256:76d5f82213aa78098b9b964ea89de4617e70e0d43e97900c2778a50856dac605",
+                "sha256:7792f0ab20df8071d669d929c75c97fecfa6bcab82c10ee4adb91c7a54055463",
+                "sha256:831b476d79408ab6ccfadaaf199906c833f02fdb32c9ab907b1d4aa0713cfa3b",
+                "sha256:9146579352d7b5f6412735d0f203bbd8d00113a680b66565e205bc605ef81bc6",
+                "sha256:9cc44bf0315268e253bf563f3560e6c004efe38f76db03a1558274a6e04bf5d5",
+                "sha256:a73d18625f6a8a1cbb11eadc1d03929f9510f4131879288e3f7922097a429f63",
+                "sha256:a8659fd33ee9e6ca03950cfdcdf271d645cf681609153f218826dd9805ab585c",
+                "sha256:a94925102c89247530ae1dab7dc02c690942566f22e189cbd53579b0693c0783",
+                "sha256:ad4567d6c334c46046d1c4c20024de2a1c3abc626817ae21ae3da600f5779b44",
+                "sha256:b2e16f4cd2bc4d88ba30ca2d3bbf2f21f00f382cf4e1ce3b1ddc96c634bc48ca",
+                "sha256:bbdf9a72403110a3bdae77948b8011f644571311c2fb35ee15f0f10a8fc082e8",
+                "sha256:beb08e8508e53a568811016e59f3234d29c2583f6b6e28572f0954a6b4f7e03d",
+                "sha256:c4cbe651f3904e28f3a55d6f371203049034b4ddbce65a54527a3f189ca3b390",
+                "sha256:c7b525ab52ce18c57ae232ba6f7010297a87ced82a2383b1afd238849c1ff933",
+                "sha256:ca5d79cfdae420a1d52bf177de4bc2289c321d6c961ae321503b2ca59c17ae67",
+                "sha256:cdab02a0a941af190df8782aafc591ef3ad08824f97850b015c8c6a8b3877b0b",
+                "sha256:d17c6a415d68cfe1091d3296ba5749d3d8696e42c37fca5d4860c5bf7b729f03",
+                "sha256:d39bd10f0ae453554798b125d2f39884290c480f56e8a02ba7a6ed552005243b",
+                "sha256:d4b3cd1ca7cd73d229487fa5caca9e4bc1f0bca96526b922d61053ea751fe791",
+                "sha256:d50a252b23b9b4dfeefc1f663c568a221092cbaded20a05a11665d0dbec9b8fb",
+                "sha256:da8549d17489cd52f85a9829d0e1d91059359b3c54a26f28bec2c5d369524807",
+                "sha256:dcd070b5b585b50e6617e8972f3fbbee786afca71b1936ac06257f7e178f00f6",
+                "sha256:ddaaa91bfc4477d2871442bbf30a125e8fe6b05da8a0015507bfbf4718228ab2",
+                "sha256:df423f351b162a702c053d5dddc0fc0ef9a9e27ea3f449781ace5f906b664428",
+                "sha256:dff044f661f59dace805eedb4a7404c573b6ff0cdba4a524141bc63d7be5c7fd",
+                "sha256:e7e128f85c0b419907d1f38e616c4f1e9f1d1b37a7949f44df9a73d5da5cd53c",
+                "sha256:ed8d1d1821ba5fc88d4a4f45387b65de52382fa3ef1f0115a4f7a20cdfab0e94",
+                "sha256:f2501d60d7497fd55e391f423f965bbe9e650e9ffc3c627d5f0ac516026000b8",
+                "sha256:f7db0b6ae1f96ae41afe626095149ecd1b212b424626175a6633c2999eaad45b"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==7.6.0"
+        },
         "exceptiongroup": {
             "hashes": [
                 "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b",

From aefa88d3b13f22d91d7d3341026063f2c7a7518d Mon Sep 17 00:00:00 2001
From: msramalho <19508417+msramalho@users.noreply.github.com>
Date: Thu, 17 Oct 2024 19:04:36 +0100
Subject: [PATCH 09/33] default endpoints tested

---
 .gitignore                          |  3 +-
 Pipfile                             | 11 ++++
 src/endpoints/default.py            |  3 +-
 src/main.py                         | 15 ++++--
 src/security.py                     |  2 -
 src/tests/endpoints/test_default.py | 79 +++++++++++++++++++++++++++++
 src/tests/test_main.py              | 22 ++++++++
 src/tests/test_start.py             | 13 -----
 8 files changed, 125 insertions(+), 23 deletions(-)
 create mode 100644 Pipfile
 create mode 100644 src/tests/endpoints/test_default.py
 create mode 100644 src/tests/test_main.py
 delete mode 100644 src/tests/test_start.py

diff --git a/.gitignore b/.gitignore
index 7df1354..e9ae646 100644
--- a/.gitignore
+++ b/.gitignore
@@ -18,4 +18,5 @@ wit*
 src/crawls
 .coverage
 .pytest_cache/*
-htmlcov
\ No newline at end of file
+htmlcov
+local_archive
\ No newline at end of file
diff --git a/Pipfile b/Pipfile
new file mode 100644
index 0000000..c398b0d
--- /dev/null
+++ b/Pipfile
@@ -0,0 +1,11 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+
+[dev-packages]
+
+[requires]
+python_version = "3.10"
diff --git a/src/endpoints/default.py b/src/endpoints/default.py
index 5580324..091202e 100644
--- a/src/endpoints/default.py
+++ b/src/endpoints/default.py
@@ -12,7 +12,6 @@ from security import get_user_auth, bearer_security
 
 default_router = APIRouter()
 
-
 @default_router.get("/")
 async def home(request: Request):
     # TODO: maybe split into 2 routes: one non authenticated and one authenticated for the groups info only
@@ -27,7 +26,7 @@ async def home(request: Request):
 
 
 @default_router.get("/health")
-async def health(request: Request):
+async def health():
     return JSONResponse({"status": "ok"})
 
 @default_router.get("/groups", response_model=list[str])
diff --git a/src/main.py b/src/main.py
index 7424dee..369a206 100644
--- a/src/main.py
+++ b/src/main.py
@@ -50,11 +50,16 @@ app.include_router(interoperability_router)
 # prometheus exposed in /metrics with authentication
 Instrumentator(should_group_status_codes=False, excluded_handlers=["/metrics"]).instrument(app).expose(app, dependencies=[Depends(token_api_key_auth)])
 
-# used mostly for development in combination with local_archive
-SERVE_LOCAL_ARCHIVE = os.environ.get("SERVE_LOCAL_ARCHIVE", "")
-if len(SERVE_LOCAL_ARCHIVE) > 1 and os.path.isdir(SERVE_LOCAL_ARCHIVE):
-    logger.info(f"mounting local archive {SERVE_LOCAL_ARCHIVE}")
-    app.mount(SERVE_LOCAL_ARCHIVE, StaticFiles(directory=SERVE_LOCAL_ARCHIVE), name=SERVE_LOCAL_ARCHIVE)
+def setup_local_archive_serve():
+    # if env SERVE_LOCAL_ARCHIVE is set it serves files from that dir, useful for development and using local_archive
+    SERVE_LOCAL_ARCHIVE = os.environ.get("SERVE_LOCAL_ARCHIVE", "")
+    local_dir = SERVE_LOCAL_ARCHIVE
+    if not os.path.isdir(local_dir) and os.path.isdir(local_dir.replace("/app", ".")):
+        local_dir = local_dir.replace("/app", ".")
+    if len(SERVE_LOCAL_ARCHIVE) > 1 and os.path.isdir(local_dir):
+        logger.warning(f"MOUNTing local archive {SERVE_LOCAL_ARCHIVE}")
+        app.mount(SERVE_LOCAL_ARCHIVE, StaticFiles(directory=local_dir), name=SERVE_LOCAL_ARCHIVE)
+setup_local_archive_serve()
 
 
 app.middleware("http")(logging_middleware)
diff --git a/src/security.py b/src/security.py
index f1e70f3..860a396 100644
--- a/src/security.py
+++ b/src/security.py
@@ -8,8 +8,6 @@ from core.config import CHROME_APP_IDS, BLOCKED_EMAILS
 assert len(CHROME_APP_IDS) > 0, "CHROME_APP_IDS env variable not properly set, it's a csv"
 for app_id in CHROME_APP_IDS:
     assert len(app_id) > 10, f"CHROME_APP_IDS got invalid id: {app_id} env variable not set"
-logger.info(f"{CHROME_APP_IDS=}")
-logger.info(f"{len(BLOCKED_EMAILS)=}")
 
 # Auth logic
 bearer_security = HTTPBearer()
diff --git a/src/tests/endpoints/test_default.py b/src/tests/endpoints/test_default.py
new file mode 100644
index 0000000..12a5e44
--- /dev/null
+++ b/src/tests/endpoints/test_default.py
@@ -0,0 +1,79 @@
+from unittest.mock import AsyncMock, patch
+from fastapi.testclient import TestClient
+from core.config import VERSION
+
+
+def test_endpoint_home():
+    from main import app
+    client = TestClient(app)
+
+    r = client.get("/")
+    assert r.status_code == 200
+    j = r.json()
+    assert "version" in j and j["version"] == VERSION
+    assert "breakingChanges" in j
+    assert "groups" not in j
+
+
+@patch("endpoints.default.bearer_security", new_callable=AsyncMock)
+@patch("endpoints.default.get_user_auth", new_callable=AsyncMock, return_value="test@example.com")
+@patch("endpoints.default.crud.get_user_groups", return_value=["group1", "group2"])
+def test_endpoint_home_with_groups(m1, m2, m3):
+    from main import app
+    client = TestClient(app)
+
+    r = client.get("/")
+    assert r.status_code == 200
+    j = r.json()
+    assert "version" in j and j["version"] == VERSION
+    assert "breakingChanges" in j
+    assert "groups" in j
+    assert j["groups"] == ["group1", "group2"]
+
+
+def test_endpoint_health():
+    from main import app
+    client = TestClient(app)
+
+    r = client.get("/health")
+    assert r.status_code == 200
+    assert r.json() == {"status": "ok"}
+
+
+def test_endpoint_groups_403():
+    from main import app
+    client = TestClient(app)
+    r = client.get("/groups")
+    assert r.status_code == 403
+
+
+@patch("endpoints.default.crud.get_user_groups", return_value=["group1", "group2"])
+def test_endpoint_groups(m1):
+    async def mock_get_user_auth(): return True
+    from main import app
+    from security import get_user_auth
+    app.dependency_overrides[get_user_auth] = mock_get_user_auth
+    
+    client = TestClient(app)
+    r = client.get("/groups")
+
+    assert r.status_code == 200
+    j = r.json()
+    assert j == ["group1", "group2"]
+
+
+def test_no_serve_local_archive_by_default():
+    from main import app
+    client = TestClient(app)
+
+    r = client.get("/app/local_archive_test/temp.txt")
+    assert r.status_code == 404
+
+
+def test_favicon():
+    from main import app
+    client = TestClient(app)
+
+    r = client.get("/favicon.ico")
+    assert r.status_code == 200
+    assert r.headers["content-type"] == "image/vnd.microsoft.icon"
diff --git a/src/tests/test_main.py b/src/tests/test_main.py
new file mode 100644
index 0000000..4c26073
--- /dev/null
+++ b/src/tests/test_main.py
@@ -0,0 +1,22 @@
+import os
+from fastapi.testclient import TestClient
+
+
+def test_serve_local_archive_logic():
+    os.environ["SERVE_LOCAL_ARCHIVE"] = "/app/local_archive_test"
+
+    # create a test file
+    os.makedirs("local_archive_test", exist_ok=True)
+    with open("local_archive_test/temp.txt", "w") as f:
+        f.write("test")
+
+    from main import app, setup_local_archive_serve
+    setup_local_archive_serve()
+    client = TestClient(app)
+
+    r = client.get("/app/local_archive_test/temp.txt")
+    assert r.status_code == 200
+    assert r.text == "test"
+
+    os.remove("local_archive_test/temp.txt")
+    os.rmdir("local_archive_test")
diff --git a/src/tests/test_start.py b/src/tests/test_start.py
deleted file mode 100644
index 4fec658..0000000
--- a/src/tests/test_start.py
+++ /dev/null
@@ -1,13 +0,0 @@
-import pytest
-from fastapi.testclient import TestClient
-from core.config import VERSION
-
-def test_mock_logger():
-    from main import app
-
-    client = TestClient(app)
-    
-    response = client.get("/")
-    assert response.status_code == 200
-    r = response.json()
-    assert "version" in r and r["version"] == VERSION
\ No newline at end of file

From 11a8e6f4e0f8dc1775012912139008bd3d80635c Mon Sep 17 00:00:00 2001
From: msramalho <19508417+msramalho@users.noreply.github.com>
Date: Thu, 17 Oct 2024 19:17:20 +0100
Subject: [PATCH 10/33] testing task endpoints

---
 src/tests/endpoints/test_default.py |  1 +
 src/tests/endpoints/test_task.py    | 64 +++++++++++++++++++++++++++++
 2 files changed, 65 insertions(+)
 create mode 100644 src/tests/endpoints/test_task.py

diff --git a/src/tests/endpoints/test_default.py b/src/tests/endpoints/test_default.py
index 12a5e44..f05d34f 100644
--- a/src/tests/endpoints/test_default.py
+++ b/src/tests/endpoints/test_default.py
@@ -60,6 +60,7 @@ def test_endpoint_groups(m1):
     assert r.status_code == 200
     j = r.json()
     assert j == ["group1", "group2"]
+    app.dependency_overrides = {}
 
 
 def test_no_serve_local_archive_by_default():
diff --git a/src/tests/endpoints/test_task.py b/src/tests/endpoints/test_task.py
new file mode 100644
index 0000000..87a5489
--- /dev/null
+++ b/src/tests/endpoints/test_task.py
@@ -0,0 +1,64 @@
+from unittest.mock import AsyncMock, patch
+from fastapi.testclient import TestClient
+
+
+
+def setup_client():
+    from main import app
+    from security import get_token_or_user_auth
+    async def mock_get_token_or_user_auth(): return "example@email.com"
+    app.dependency_overrides[get_token_or_user_auth] = mock_get_token_or_user_auth
+    return TestClient(app), app
+
+@patch("endpoints.task.AsyncResult")
+def test_get_status_success(mock_async_result):
+    client, app = setup_client()
+
+    mock_async_result.return_value.status = "SUCCESS"
+    mock_async_result.return_value.result = {"data": "some result"}
+
+    response = client.get("/task/test-task-id")
+
+    assert response.status_code == 200
+    assert response.json() == {
+        "id": "test-task-id",
+        "status": "SUCCESS",
+        "result": {"data": "some result"}
+    }
+    app.dependency_overrides = {}
+
+
+@patch("endpoints.task.AsyncResult")
+def test_get_status_failure(mock_async_result):
+    client, app = setup_client()
+
+    mock_async_result.return_value.status = "FAILURE"
+    mock_async_result.return_value.result = Exception("Some error")
+
+    response = client.get("/task/test-task-id")
+
+    assert response.status_code == 200
+    assert response.json() == {
+        "id": "test-task-id",
+        "status": "FAILURE",
+        "result": {"error": "Some error"}
+    }
+    app.dependency_overrides = {}
+
+
+@patch("endpoints.task.AsyncResult")
+def test_get_status_pending(mock_async_result):
+    client, app = setup_client()
+
+    mock_async_result.return_value.status = "PENDING"
+    mock_async_result.return_value.result = None
+
+    response = client.get("/task/test-task-id")
+
+    assert response.status_code == 200
+    assert response.json() == {
+        "id": "test-task-id",
+        "status": "PENDING",
+        "result": None
+    }
+    app.dependency_overrides = {}

From ca7e862855e72dc5fb2551b31486f5476ed2a070 Mon Sep 17 00:00:00 2001
From: msramalho <19508417+msramalho@users.noreply.github.com>
Date: Fri, 18 Oct 2024 11:08:51 +0100
Subject: [PATCH 11/33] refactor to use pydantic settings and WAL sqlite mode

---
 .gitignore                       |  4 ++-
 src/.env.test                    |  7 ++++
 src/Pipfile                      |  1 +
 src/Pipfile.lock                 | 19 ++++++++---
 src/core/config.py               | 16 +---------
 src/core/events.py               | 19 ++++++-----
 src/db/crud.py                   | 15 +++++----
 src/db/database.py               | 35 +++++++++++++++-----
 src/db/models.py                 |  7 ++--
 src/db/schemas.py                | 13 ++++++--
 src/endpoints/default.py         | 11 ++++---
 src/endpoints/url.py             | 10 +++---
 src/main.py                      | 28 ++++++++--------
 src/migrations/env.py            |  4 ++-
 src/security.py                  | 27 ++++++++--------
 src/shared/settings.py           | 31 ++++++++++++++++++
 src/tests/conftest.py            | 55 +++++++++++++++++++++++++++++---
 src/tests/endpoints/test_task.py |  4 +--
 src/tests/test_main.py           | 27 ++++++++--------
 src/utils/metrics.py             | 20 ++++++------
 src/worker.py                    | 28 +++++++---------
 21 files changed, 246 insertions(+), 135 deletions(-)
 create mode 100644 src/.env.test
 create mode 100644 src/shared/settings.py

diff --git a/.gitignore b/.gitignore
index e9ae646..01ac3f7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -19,4 +19,6 @@ src/crawls
 .coverage
 .pytest_cache/*
 htmlcov
-local_archive
\ No newline at end of file
+local_archive
+*db-wal
+*db-shm
\ No newline at end of file
diff --git a/src/.env.test b/src/.env.test
new file mode 100644
index 0000000..fca6b15
--- /dev/null
+++ b/src/.env.test
@@ -0,0 +1,7 @@
+CHROME_APP_IDS='["test_app_id_1","test_app_id_2"]'
+ALLOWED_ORIGINS='["chrome-extension://example1","chrome-extension://example2","http://localhost:8081"]'
+BLOCKED_EMAILS='["blocked@example.com"]'
+
+
+DATABASE_PATH="sqlite:////app/auto-archiver.test.db"
+API_BEARER_TOKEN=this_is_the_test_api_token
\ No newline at end of file
diff --git a/src/Pipfile b/src/Pipfile
index dec6834..4423b6f 100644
--- a/src/Pipfile
+++ b/src/Pipfile
@@ -19,6 +19,7 @@ alembic = "*"
 fastapi-utils = "*"
 prometheus-fastapi-instrumentator = "*"
 auto-archiver = "*"
+pydantic-settings = "*"
 
 [dev-packages]
 watchdog = "*"
diff --git a/src/Pipfile.lock b/src/Pipfile.lock
index 068420f..6e6c0f2 100644
--- a/src/Pipfile.lock
+++ b/src/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "359638472cb3c3914fac7040bc702463c0ed1ae2e5cee00abddd59d9b34e923e"
+            "sha256": "c34b5745f3a6f67222d3f26e6c7f2d13615a3301d0ca4d1f2b0ec58474b1d43a"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -1470,11 +1470,11 @@
         },
         "marshmallow": {
             "hashes": [
-                "sha256:4972f529104a220bb8637d595aa4c9762afbe7f7a77d82dc58c1615d70c5823e",
-                "sha256:71a2dce49ef901c3f97ed296ae5051135fd3febd2bf43afe0ae9a82143a494d9"
+                "sha256:82f20a2397834fe6d9611b241f2f7e7b680ed89c49f84728a1ad937be6b4bdf4",
+                "sha256:98d8827a9f10c03d44ead298d2e99c6aea8197df18ccfad360dae7f89a50da2e"
             ],
-            "markers": "python_version >= '3.8'",
-            "version": "==3.22.0"
+            "markers": "python_version >= '3.9'",
+            "version": "==3.23.0"
         },
         "mccabe": {
             "hashes": [
@@ -2182,6 +2182,15 @@
             "markers": "python_version >= '3.8'",
             "version": "==2.23.4"
         },
+        "pydantic-settings": {
+            "hashes": [
+                "sha256:44a1804abffac9e6a30372bb45f6cafab945ef5af25e66b1c634c01dd39e0188",
+                "sha256:4a819166f119b74d7f8c765196b165f95cc7487ce58ea27dec8a5a26be0970e0"
+            ],
+            "index": "pypi",
+            "markers": "python_version >= '3.8'",
+            "version": "==2.6.0"
+        },
         "pyflakes": {
             "hashes": [
                 "sha256:1c61603ff154621fb2a9172037d84dca3500def8c8b630657d1701f026f8af3f",
diff --git a/src/core/config.py b/src/core/config.py
index 9671eeb..0d33cf6 100644
--- a/src/core/config.py
+++ b/src/core/config.py
@@ -1,5 +1,3 @@
-import os
-
 VERSION = "0.7.0"
 API_DESCRIPTION = """
 #### API for the Auto-Archiver project, a tool to archive web pages and Google Sheets.
@@ -9,16 +7,4 @@ API_DESCRIPTION = """
 - You can use this API to archive single URLs or entire Google Sheets. 
 - Once you submit a URL or Sheet for archiving, the API will return a task_id that you can use to check the status of the archiving process. It works asynchronously.
 """
-
-ALLOWED_ORIGINS = os.environ.get("ALLOWED_ORIGINS", "chrome-extension://ondkcheoicfckabcnkdgbepofpjmjcmb,chrome-extension://ojcimmjndnlmmlgnjaeojoebaceokpdp").split(",")
-
-BREAKING_CHANGES = {"minVersion": "0.3.1", "message": "The latest update has breaking changes, please update the extension to the most recent version."}
-
-SERVE_LOCAL_ARCHIVE = os.environ.get("SERVE_LOCAL_ARCHIVE", "")
-
-SQLALCHEMY_DATABASE_URL = os.environ.get("DATABASE_PATH")
-
-REPEAT_COUNT_METRICS_SECONDS = 15
-
-CHROME_APP_IDS = set([app_id.strip() for app_id in os.environ.get("CHROME_APP_IDS", "").split(",")])
-BLOCKED_EMAILS = set([e.strip().lower() for e in os.environ.get("BLOCKED_EMAILS", "").split(",")])
\ No newline at end of file
+BREAKING_CHANGES = {"minVersion": "0.3.1", "message": "The latest update has breaking changes, please update the extension to the most recent version."}
\ No newline at end of file
diff --git a/src/core/events.py b/src/core/events.py
index a5e483c..abad52a 100644
--- a/src/core/events.py
+++ b/src/core/events.py
@@ -2,15 +2,16 @@ import asyncio
 import logging
 import alembic.config
 from fastapi import FastAPI
-from sqlalchemy.orm import Session
 from contextlib import asynccontextmanager
 from fastapi_utils.tasks import repeat_every
 from loguru import logger
 
 from db import crud, models
-from db.database import get_db, engine
+from db.database import get_db, make_engine
+from shared.settings import Settings
 from utils.metrics import measure_regular_metrics, redis_subscribe_worker_exceptions
-from core.config import REPEAT_COUNT_METRICS_SECONDS
+
+settings = Settings()
 
 
 @asynccontextmanager
@@ -18,13 +19,14 @@ async def lifespan(app: FastAPI):
     # see https://fastapi.tiangolo.com/advanced/events/#lifespan
 
     # STARTUP
+    engine = make_engine(settings.DATABASE_PATH)
     models.Base.metadata.create_all(bind=engine)
     alembic.config.main(argv=['--raiseerr', 'upgrade', 'head'])
     # disabling uvicorn logger since we use loguru in logging_middleware
     logging.getLogger("uvicorn.access").disabled = True
     asyncio.create_task(redis_subscribe_worker_exceptions())
     asyncio.create_task(refresh_user_groups())
-    asyncio.create_task(measure_regular_metrics())
+    asyncio.create_task(repeat_measure_regular_metrics())
 
     yield  # separates startup from shutdown instructions
 
@@ -36,9 +38,10 @@ async def lifespan(app: FastAPI):
 
 @repeat_every(seconds=60 * 60)  # 1 hour
 async def refresh_user_groups():
-    db: Session = next(get_db())
-    crud.upsert_user_groups(db)
+    with get_db() as db:
+        crud.upsert_user_groups(db)
 
-@repeat_every(seconds=REPEAT_COUNT_METRICS_SECONDS)
+
+@repeat_every(seconds=settings.REPEAT_COUNT_METRICS_SECONDS)
 async def repeat_measure_regular_metrics():
-    measure_regular_metrics()
+    measure_regular_metrics(settings.DATABASE_PATH, settings.REPEAT_COUNT_METRICS_SECONDS)
diff --git a/src/db/crud.py b/src/db/crud.py
index 765ebd3..a62113f 100644
--- a/src/db/crud.py
+++ b/src/db/crud.py
@@ -5,12 +5,13 @@ from loguru import logger
 from datetime import datetime, timedelta
 
 from security import ALLOW_ANY_EMAIL
+from shared.settings import Settings
 from . import models, schemas
-import yaml, os
+import yaml
 
 DOMAIN_GROUPS = {}
 DOMAIN_GROUPS_LOADED = False
-MAX_LIMIT = 100
+DATABASE_QUERY_LIMIT = Settings().DATABASE_QUERY_LIMIT
 
 # --------------- TASK = Archive
 
@@ -39,12 +40,12 @@ def search_archives_by_url(db: Session, url: str, email: str, skip: int = 0, lim
         query = query.filter(models.Archive.created_at >= archived_after)
     if archived_before:
         query = query.filter(models.Archive.created_at <= archived_before)
-    return query.order_by(models.Archive.created_at.desc()).offset(skip).limit(min(limit, MAX_LIMIT)).all()
+    return query.order_by(models.Archive.created_at.desc()).offset(skip).limit(min(limit, DATABASE_QUERY_LIMIT)).all()
 
 
 def search_archives_by_email(db: Session, email: str, skip: int = 0, limit: int = 100):
     email = email.lower()
-    return base_query(db).filter(models.Archive.author.has(email=email)).offset(skip).limit(min(limit, MAX_LIMIT)).all()
+    return base_query(db).filter(models.Archive.author.has(email=email)).offset(skip).limit(min(limit, DATABASE_QUERY_LIMIT)).all()
 
 
 def create_task(db: Session, task: schemas.ArchiveCreate, tags: list[models.Tag], urls: list[models.ArchiveUrl]):
@@ -76,7 +77,7 @@ def count_by_user_since(db:Session, seconds_delta: int = 15):
     return db.query(models.Archive.author_id,func.count().label('total'))\
         .filter(models.Archive.created_at >= time_threshold)\
         .group_by(models.Archive.author_id)\
-        .order_by(func.count().desc()).limit(5 * MAX_LIMIT).all()
+        .order_by(func.count().desc()).limit(5 * DATABASE_QUERY_LIMIT).all()
 
 def base_query(db: Session):
     # allow only some fields to be returned, for example author should remain hidden
@@ -98,7 +99,7 @@ def create_tag(db: Session, tag: str):
 
 
 def search_tags(db: Session, tag: str, skip: int = 0, limit: int = 100):
-    return db.query(models.Tag).filter(models.Tag.url.like(f'%{tag}%')).offset(skip).limit(min(limit, MAX_LIMIT)).all()
+    return db.query(models.Tag).filter(models.Tag.url.like(f'%{tag}%')).offset(skip).limit(min(limit, DATABASE_QUERY_LIMIT)).all()
 
 
 def is_user_in_group(db: Session, group_name: str, email: str) -> models.Group:
@@ -148,7 +149,7 @@ def upsert_user_groups(db: Session):
     along with new participation of users in groups
     """
     logger.debug("Updating user-groups configuration.")
-    filename = os.environ.get("USER_GROUPS_FILENAME", "user-groups.yaml")
+    filename = Settings().USER_GROUPS_FILENAME
 
     # read yaml safely
     with open(filename) as inf:
diff --git a/src/db/database.py b/src/db/database.py
index c212d20..8b72f70 100644
--- a/src/db/database.py
+++ b/src/db/database.py
@@ -1,17 +1,36 @@
-from sqlalchemy import create_engine
+from sqlalchemy import Engine, create_engine, event
 from sqlalchemy.orm import sessionmaker, declarative_base
-from core.config import SQLALCHEMY_DATABASE_URL
+from shared.settings import Settings
+from contextlib import contextmanager
 
 
-engine = create_engine(
-    SQLALCHEMY_DATABASE_URL, connect_args={"check_same_thread": False}
-)
-SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
+settings = Settings()
 
-Base = declarative_base()
+def make_engine(database_url: str):
+    engine = create_engine(database_url, connect_args={"check_same_thread": False})
+
+    @event.listens_for(engine, "connect")
+    def set_sqlite_pragma(conn, _) -> None:
+        cursor = conn.cursor()
+        cursor.execute("PRAGMA journal_mode=WAL")
+        cursor.close()
+
+    return engine
 
 
+def make_session_local(engine: Engine):
+    session_local = sessionmaker(autocommit=False, autoflush=False, bind=engine)
+    return session_local
+
+
+@contextmanager
 def get_db():
-    session = SessionLocal()
+    session = make_session_local(make_engine(settings.DATABASE_PATH))()
     try: yield session
     finally: session.close()
+
+
+def get_db_dependency():
+    # to use with Depends and ensure proper session closing
+    with get_db() as db:
+        yield db
\ No newline at end of file
diff --git a/src/db/models.py b/src/db/models.py
index 604bbcc..29f0a05 100644
--- a/src/db/models.py
+++ b/src/db/models.py
@@ -1,8 +1,10 @@
 from sqlalchemy import Column, String, JSON, DateTime, Boolean, Table, ForeignKey
 from sqlalchemy.sql import func
-from sqlalchemy.orm import relationship
+from sqlalchemy.orm import relationship, declarative_base
 import uuid
-from .database import Base
+
+
+Base = declarative_base()
 
 def generate_uuid():
     return str(uuid.uuid4())
@@ -59,7 +61,6 @@ class Tag(Base):
 
     archives = relationship("Archive", back_populates="tags", secondary=association_table_archive_tags)
 
-
 class User(Base):
     __tablename__ = "users"
 
diff --git a/src/db/schemas.py b/src/db/schemas.py
index fff03f9..ef06fe7 100644
--- a/src/db/schemas.py
+++ b/src/db/schemas.py
@@ -2,6 +2,13 @@ from pydantic import BaseModel
 from datetime import datetime
 
 
+class Tag(BaseModel):
+    id: str
+    created_at: datetime
+
+    model_config = { "from_attributes": True }
+    __hash__ = object.__hash__
+
 class ArchiveCreate(BaseModel):
     id: str | None = None
     url: str
@@ -9,7 +16,7 @@ class ArchiveCreate(BaseModel):
     public: bool = True
     author_id: str | None = None
     group_id: str | None = None
-    tags: set = set()
+    tags: set[Tag] | None = set()
     rearchive: bool = True
     # urls: list = []
 
@@ -28,7 +35,7 @@ class SubmitSheet(BaseModel):
     public: bool = False
     author_id: str | None = None
     group_id: str | None = None
-    tags: set | None = set()
+    tags: set[Tag] | None = set()
     columns: dict | None = {} # TODO: implement
 
 class SubmitManual(BaseModel):
@@ -36,7 +43,7 @@ class SubmitManual(BaseModel):
     public: bool = False
     author_id: str | None = None
     group_id: str | None = None
-    tags: set | None = set()
+    tags: set[Tag] | None = set()
 
 class Task(BaseModel):
     id: str
diff --git a/src/endpoints/default.py b/src/endpoints/default.py
index 091202e..7a6a9ea 100644
--- a/src/endpoints/default.py
+++ b/src/endpoints/default.py
@@ -6,20 +6,20 @@ from sqlalchemy.orm import Session
 
 from core.config import VERSION, BREAKING_CHANGES
 from db import crud
-from db.database import get_db
+from db.database import get_db_dependency, get_db
 from security import get_user_auth, bearer_security
 
-
 default_router = APIRouter()
 
+
 @default_router.get("/")
 async def home(request: Request):
     # TODO: maybe split into 2 routes: one non authenticated and one authenticated for the groups info only
     status = {"version": VERSION, "breakingChanges": BREAKING_CHANGES}
     try:
         email = await get_user_auth(await bearer_security(request))
-        db: Session = next(get_db())
-        status["groups"] = crud.get_user_groups(db, email)
+        with get_db() as db:
+            status["groups"] = crud.get_user_groups(db, email)
     except HTTPException: pass  # not authenticated is fine
     except Exception as e: logger.error(e)
     return JSONResponse(status)
@@ -29,8 +29,9 @@ async def home(request: Request):
 async def health():
     return JSONResponse({"status": "ok"})
 
+
 @default_router.get("/groups", response_model=list[str])
-def get_user_groups(db: Session = Depends(get_db), email=Depends(get_user_auth)):
+def get_user_groups(db: Session = Depends(get_db_dependency), email=Depends(get_user_auth)):
     return crud.get_user_groups(db, email)
 
 
diff --git a/src/endpoints/url.py b/src/endpoints/url.py
index 199be59..dc1ca68 100644
--- a/src/endpoints/url.py
+++ b/src/endpoints/url.py
@@ -8,7 +8,7 @@ from security import get_user_auth, get_token_or_user_auth
 from sqlalchemy.orm import Session
 
 from db import crud, schemas
-from db.database import get_db
+from db.database import get_db_dependency
 
 from worker import create_archive_task
 
@@ -32,23 +32,23 @@ def archive_url(archive: schemas.ArchiveCreate, email=Depends(get_token_or_user_
 def search_by_url(
         url: str, skip: int = 0, limit: int = 25,
         archived_after: datetime = None, archived_before: datetime = None,
-        db: Session = Depends(get_db),
+        db: Session = Depends(get_db_dependency),
         email=Depends(get_token_or_user_auth)):
     return crud.search_archives_by_url(db, url.strip(), email, skip=skip, limit=limit, archived_after=archived_after, archived_before=archived_before)
 
 
 @url_router.get("/latest", response_model=list[schemas.Archive], summary="Fetch latest URL archives for the authenticated user.")
-def latest(skip: int = 0, limit: int = 25, db: Session = Depends(get_db), email=Depends(get_user_auth)):
+def latest(skip: int = 0, limit: int = 25, db: Session = Depends(get_db_dependency), email=Depends(get_user_auth)):
     return crud.search_archives_by_email(db, email, skip=skip, limit=limit)
 
 
 @url_router.get("/{id}", response_model=schemas.Archive, summary="Fetch a single URL archive by the associated id.")
-def lookup(id, db: Session = Depends(get_db), email=Depends(get_token_or_user_auth)):
+def lookup(id, db: Session = Depends(get_db_dependency), email=Depends(get_token_or_user_auth)):
     return crud.get_archive(db, id, email)
 
 
 @url_router.delete("/{id}", response_model=schemas.TaskDelete, summary="Delete a single URL archive by id.")
-def delete_task(id, db: Session = Depends(get_db), email=Depends(get_user_auth)):
+def delete_task(id, db: Session = Depends(get_db_dependency), email=Depends(get_user_auth)):
     logger.info(f"deleting url archive task {id} request by {email}")
     #TODO: use response model?
     return JSONResponse({
diff --git a/src/main.py b/src/main.py
index 369a206..a4c2adf 100644
--- a/src/main.py
+++ b/src/main.py
@@ -16,14 +16,16 @@ from worker import create_archive_task, create_sheet_task, celery, insert_result
 
 from db import crud, models, schemas
 from security import get_user_auth, token_api_key_auth, get_token_or_user_auth
-from core.config import ALLOWED_ORIGINS, VERSION, SERVE_LOCAL_ARCHIVE, API_DESCRIPTION
-from db.database import get_db
+from core.config import VERSION, API_DESCRIPTION
+from db.database import get_db_dependency
 from core.events import lifespan
+from shared.settings import Settings
 
 from auto_archiver import Metadata
 
 from endpoints import default_router, url_router, sheet_router, task_router, interoperability_router
 
+settings = Settings()
 
 app = FastAPI(
     title="Auto-Archiver API",
@@ -35,7 +37,7 @@ app = FastAPI(
 
 app.add_middleware(
     CORSMiddleware,
-    allow_origins=ALLOWED_ORIGINS,
+    allow_origins=settings.ALLOWED_ORIGINS,
     allow_credentials=True,
     allow_methods=["*"],
     allow_headers=["*"],
@@ -48,17 +50,15 @@ app.include_router(task_router)
 app.include_router(interoperability_router)
 
 # prometheus exposed in /metrics with authentication
-Instrumentator(should_group_status_codes=False, excluded_handlers=["/metrics"]).instrument(app).expose(app, dependencies=[Depends(token_api_key_auth)])
+Instrumentator(should_group_status_codes=False, excluded_handlers=["/metrics", "/health"]).instrument(app).expose(app, dependencies=[Depends(token_api_key_auth)])
 
 def setup_local_archive_serve():
-    # if env SERVE_LOCAL_ARCHIVE is set it serves files from that dir, useful for development and using local_archive
-    SERVE_LOCAL_ARCHIVE = os.environ.get("SERVE_LOCAL_ARCHIVE", "")
-    local_dir = SERVE_LOCAL_ARCHIVE
+    local_dir = settings.SERVE_LOCAL_ARCHIVE
     if not os.path.isdir(local_dir) and os.path.isdir(local_dir.replace("/app", ".")):
         local_dir = local_dir.replace("/app", ".")
-    if len(SERVE_LOCAL_ARCHIVE) > 1 and os.path.isdir(local_dir):
-        logger.warning(f"MOUNTing local archive {SERVE_LOCAL_ARCHIVE}")
-        app.mount(SERVE_LOCAL_ARCHIVE, StaticFiles(directory=local_dir), name=SERVE_LOCAL_ARCHIVE)
+    if len(settings.SERVE_LOCAL_ARCHIVE) > 1 and os.path.isdir(local_dir):
+        logger.warning(f"MOUNTing local archive {settings.SERVE_LOCAL_ARCHIVE}")
+        app.mount(settings.SERVE_LOCAL_ARCHIVE, StaticFiles(directory=local_dir), name=settings.SERVE_LOCAL_ARCHIVE)
 setup_local_archive_serve()
 
 
@@ -68,12 +68,12 @@ app.middleware("http")(logging_middleware)
 
 
 @app.get("/tasks/search-url", response_model=list[schemas.Archive], deprecated=True)  # DEPRECATED
-def search_by_url(url: str, skip: int = 0, limit: int = 100, archived_after: datetime = None, archived_before: datetime = None, db: Session = Depends(get_db), email=Depends(get_token_or_user_auth)):
+def search_by_url(url: str, skip: int = 0, limit: int = 100, archived_after: datetime = None, archived_before: datetime = None, db: Session = Depends(get_db_dependency), email=Depends(get_token_or_user_auth)):
     return crud.search_archives_by_url(db, url.strip(), email, skip=skip, limit=limit, archived_after=archived_after, archived_before=archived_before)
 
 
 @app.get("/tasks/sync", response_model=list[schemas.Archive], deprecated=True)  # DEPRECATED
-def search(skip: int = 0, limit: int = 100, db: Session = Depends(get_db), email=Depends(get_user_auth)):
+def search(skip: int = 0, limit: int = 100, db: Session = Depends(get_db_dependency), email=Depends(get_user_auth)):
     return crud.search_archives_by_email(db, email, skip=skip, limit=limit)
 
 
@@ -90,7 +90,7 @@ def archive_tasks(archive: schemas.ArchiveCreate, email=Depends(get_token_or_use
 
 
 @app.get("/archive/{task_id}", deprecated=True)  # DEPRECATED
-def lookup(task_id, db: Session = Depends(get_db), email=Depends(get_token_or_user_auth)):
+def lookup(task_id, db: Session = Depends(get_db_dependency), email=Depends(get_token_or_user_auth)):
     return crud.get_archive(db, task_id, email)
 
 
@@ -123,7 +123,7 @@ def get_status(task_id, email=Depends(get_token_or_user_auth)):
 
 
 @app.delete("/tasks/{task_id}", deprecated=True)  # DEPRECATED
-def delete_task(task_id, db: Session = Depends(get_db), email=Depends(get_user_auth)):
+def delete_task(task_id, db: Session = Depends(get_db_dependency), email=Depends(get_user_auth)):
     logger.info(f"deleting task {task_id} request by {email}")
     return JSONResponse({
         "id": task_id,
diff --git a/src/migrations/env.py b/src/migrations/env.py
index 9b95d75..cabd992 100644
--- a/src/migrations/env.py
+++ b/src/migrations/env.py
@@ -5,10 +5,12 @@ from sqlalchemy import pool
 
 from alembic import context
 
+from shared.settings import Settings
+
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
 config = context.config
-config.set_main_option('sqlalchemy.url', os.environ.get("DATABASE_PATH"))
+config.set_main_option('sqlalchemy.url', Settings().DATABASE_PATH)
 # Interpret the config file for Python logging.
 # This line sets up loggers basically.
 if config.config_file_name is not None:
diff --git a/src/security.py b/src/security.py
index 860a396..03ef4d5 100644
--- a/src/security.py
+++ b/src/security.py
@@ -2,21 +2,18 @@ from loguru import logger
 import requests, os, secrets
 from fastapi import HTTPException, status, Depends
 from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
-from core.config import CHROME_APP_IDS, BLOCKED_EMAILS
-
-# Configuration checks
-assert len(CHROME_APP_IDS) > 0, "CHROME_APP_IDS env variable not properly set, it's a csv"
-for app_id in CHROME_APP_IDS:
-    assert len(app_id) > 10, f"CHROME_APP_IDS got invalid id: {app_id} env variable not set"
-
-# Auth logic
-bearer_security = HTTPBearer()
+from shared.settings import Settings
 
 ALLOW_ANY_EMAIL = "*"
 
+settings = Settings()
+bearer_security = HTTPBearer()
+
+
 def secure_compare(token, api_key):
     return secrets.compare_digest(token.encode("utf8"), api_key.encode("utf8"))
 
+
 # Factory method to create an authentication dependency for a specific key
 def api_key_auth(api_key):
 
@@ -35,9 +32,10 @@ def api_key_auth(api_key):
 
     return auth
 
+
 # --------------------- Token Auth for AA itself to query the API, AA setup tool and Prometheus
-API_BEARER_TOKEN = os.environ.get("API_BEARER_TOKEN", "")  # min length is 20 chars
-token_api_key_auth = api_key_auth(API_BEARER_TOKEN)
+token_api_key_auth = api_key_auth(settings.API_BEARER_TOKEN)
+
 
 async def get_token_or_user_auth(credentials: HTTPAuthorizationCredentials = Depends(bearer_security)):
     # tries to use the static API_KEY and defaults to google JWT auth
@@ -45,6 +43,7 @@ async def get_token_or_user_auth(credentials: HTTPAuthorizationCredentials = Dep
     if token_api_key_auth(access_token, auto_error=False): return ALLOW_ANY_EMAIL
     return await get_user_auth(credentials)
 
+
 async def get_user_auth(credentials: HTTPAuthorizationCredentials = Depends(bearer_security)):
     # validates the Bearer token in the case that it requires it
     valid_user, info = authenticate_user(credentials.credentials)
@@ -56,6 +55,7 @@ async def get_user_auth(credentials: HTTPAuthorizationCredentials = Depends(bear
         headers={"WWW-Authenticate": "Bearer"},
     )
 
+
 def authenticate_user(access_token):
     # https://cloud.google.com/docs/authentication/token-types#access
     if type(access_token) != str or len(access_token) < 10: return False, "invalid access_token"
@@ -63,9 +63,9 @@ def authenticate_user(access_token):
     if r.status_code != 200: return False, "error occurred"
     try:
         j = r.json()
-        if j.get("azp") not in CHROME_APP_IDS and j.get("aud") not in CHROME_APP_IDS:
+        if j.get("azp") not in settings.CHROME_APP_IDS and j.get("aud") not in settings.CHROME_APP_IDS:
             return False, f"token does not belong to valid APP_ID"
-        if j.get("email") in BLOCKED_EMAILS:
+        if j.get("email") in settings.BLOCKED_EMAILS:
             return False, f"email '{j.get('email')}' not allowed"
         if j.get("email_verified") != "true":
             return False, f"email '{j.get('email')}' not verified"
@@ -75,4 +75,3 @@ def authenticate_user(access_token):
     except Exception as e:
         logger.warning(f"EXCEPTION occurred: {e}")
         return False, f"EXCEPTION occurred"
-
diff --git a/src/shared/settings.py b/src/shared/settings.py
new file mode 100644
index 0000000..dc216d3
--- /dev/null
+++ b/src/shared/settings.py
@@ -0,0 +1,31 @@
+
+from pydantic_settings import BaseSettings
+from pydantic import ConfigDict
+from typing import Annotated, Set
+from annotated_types import Len
+
+
+class Settings(BaseSettings):
+    model_config = ConfigDict(extra='ignore', str_strip_whitespace=True)
+    
+	# general
+    SERVE_LOCAL_ARCHIVE: str = ""
+    USER_GROUPS_FILENAME: str = "user-groups.yaml"
+    
+	# database
+    DATABASE_PATH: str
+    DATABASE_QUERY_LIMIT: int = 100
+    # redis
+    CELERY_BROKER_URL: str = "redis://localhost:6379"
+    CELERY_RESULT_BACKEND: str = "redis://localhost:6379"
+    REDIS_EXCEPTIONS_CHANNEL: str = "exceptions-channel"
+    
+	# observability
+    REPEAT_COUNT_METRICS_SECONDS: int = 15
+
+	# security
+    API_BEARER_TOKEN: Annotated[str, Len(min_length=20)]
+    ALLOWED_ORIGINS: Annotated[set[str], Len(min_length=1)]
+    CHROME_APP_IDS: Annotated[set[Annotated[str, Len(min_length=10)]], Len(min_length=1)]
+    BLOCKED_EMAILS: Annotated[Set[str], Len(min_length=0)] = set()
+
diff --git a/src/tests/conftest.py b/src/tests/conftest.py
index 6d8a561..62c57bd 100644
--- a/src/tests/conftest.py
+++ b/src/tests/conftest.py
@@ -1,6 +1,7 @@
-import pytest
 import os
+import pytest
 from unittest.mock import patch
+from shared.settings import Settings
 
 @pytest.fixture(autouse=True)
 def mock_logger_add():
@@ -8,6 +9,52 @@ def mock_logger_add():
     with patch('loguru.logger.add') as mock_add:
         yield mock_add  # This makes the mock available to tests
 
-os.environ["CHROME_APP_IDS"] = 'test_app_id_1,test_app_id_2'
-os.environ["DATABASE_PATH"] = "sqlite:////app/auto-archiver.test.db"
-os.environ["BLOCKED_EMAILS"] = "blocked@example.com"
\ No newline at end of file
+# @pytest.fixture(autouse=True)
+# def settings():
+#     return Settings(_env_file=".env.test")
+
+
+@pytest.fixture(autouse=True)
+def settings():
+    with patch('shared.settings.Settings', return_value=Settings(_env_file=".env.test")) as mock_settings:
+        yield mock_settings
+
+
+@pytest.fixture()
+def test_db(settings):
+    from db.database import make_engine, make_session_local
+    from db import models
+
+    engine = make_engine(settings.DATABASE_PATH)
+
+    if not os.path.exists(settings.DATABASE_PATH):
+        open(settings.DATABASE_PATH, 'w').close()
+
+    models.Base.metadata.create_all(engine)
+
+    connection = engine.connect()
+    yield connection
+    connection.close()
+
+    models.Base.metadata.drop_all(bind=engine)
+    os.remove(settings.DATABASE_PATH)
+
+# @pytest.fixture()
+# def db_session(test_db):
+#     session_local = make_session_local(test_db)
+#     with session_local() as session:
+#         yield session
+
+# # create test data and insert it into the database
+# def create_test_data():
+#     from db.database import SessionLocal
+#     from db.models import Task
+
+#     db = SessionLocal()
+#     task = Task(id="test-task-id", status="PENDING")
+#     db.add(task)
+#     db.commit()
+#     db.refresh(task)
+#     db.close()
+
+#     return task.id
\ No newline at end of file
diff --git a/src/tests/endpoints/test_task.py b/src/tests/endpoints/test_task.py
index 87a5489..7510a34 100644
--- a/src/tests/endpoints/test_task.py
+++ b/src/tests/endpoints/test_task.py
@@ -1,8 +1,7 @@
-from unittest.mock import AsyncMock, patch
+from unittest.mock import patch
 from fastapi.testclient import TestClient
 
 
-
 def setup_client():
     from main import app
     from security import get_token_or_user_auth
@@ -10,6 +9,7 @@ def setup_client():
     app.dependency_overrides[get_token_or_user_auth] = mock_get_token_or_user_auth
     return TestClient(app), app
 
+
 @patch("endpoints.task.AsyncResult")
 def test_get_status_success(mock_async_result):
     client, app = setup_client()
diff --git a/src/tests/test_main.py b/src/tests/test_main.py
index 4c26073..f19f6ae 100644
--- a/src/tests/test_main.py
+++ b/src/tests/test_main.py
@@ -1,22 +1,23 @@
 import os
+from unittest.mock import patch
 from fastapi.testclient import TestClient
 
 
 def test_serve_local_archive_logic():
-    os.environ["SERVE_LOCAL_ARCHIVE"] = "/app/local_archive_test"
+    with patch("main.settings.SERVE_LOCAL_ARCHIVE", "/app/local_archive_test"):
 
-    # create a test file
-    os.makedirs("local_archive_test", exist_ok=True)
-    with open("local_archive_test/temp.txt", "w") as f:
-        f.write("test")
+        # create a test file
+        os.makedirs("local_archive_test", exist_ok=True)
+        with open("local_archive_test/temp.txt", "w") as f:
+            f.write("test")
 
-    from main import app, setup_local_archive_serve
-    setup_local_archive_serve()
-    client = TestClient(app)
+        from main import app, setup_local_archive_serve
+        setup_local_archive_serve()
+        client = TestClient(app)
 
-    r = client.get("/app/local_archive_test/temp.txt")
-    assert r.status_code == 200
-    assert r.text == "test"
+        r = client.get("/app/local_archive_test/temp.txt")
+        assert r.status_code == 200
+        assert r.text == "test"
 
-    os.remove("local_archive_test/temp.txt")
-    os.rmdir("local_archive_test")
+        os.remove("local_archive_test/temp.txt")
+        os.rmdir("local_archive_test")
diff --git a/src/utils/metrics.py b/src/utils/metrics.py
index 70ff157..f706154 100644
--- a/src/utils/metrics.py
+++ b/src/utils/metrics.py
@@ -6,10 +6,8 @@ from loguru import logger
 from prometheus_client import Counter, Gauge
 from sqlalchemy.orm import Session
 
-from core.config import REPEAT_COUNT_METRICS_SECONDS
 from db import crud
 from db.database import get_db
-from core.config import SQLALCHEMY_DATABASE_URL
 from worker import REDIS_EXCEPTIONS_CHANNEL, Rdis
 
 
@@ -47,20 +45,20 @@ async def redis_subscribe_worker_exceptions():
             WORKER_EXCEPTION.labels(exception=data["exception"], task=data["task"]).inc()
         await asyncio.sleep(1)
 
-async def measure_regular_metrics():
+async def measure_regular_metrics(sqlite_db_url:str, repeat_in_seconds:int):
     _total, used, free = shutil.disk_usage("/")
     DISK_UTILIZATION.labels(type="used").set(used / (2**30))
     DISK_UTILIZATION.labels(type="free").set(free / (2**30))
     try: 
-        fs = os.stat(SQLALCHEMY_DATABASE_URL.replace("sqlite:///", ""))
+        fs = os.stat(sqlite_db_url.replace("sqlite:///", ""))
         DISK_UTILIZATION.labels(type="database").set(fs.st_size / (2**30))
     except Exception as e: logger.error(e)
 
-    session: Session = next(get_db())
-    count_archives = crud.count_archives(session)
-    count_archive_urls = crud.count_archive_urls(session)
-    DATABASE_METRICS.labels(query="count_archives", user="-").set(count_archives)
-    DATABASE_METRICS.labels(query="count_archive_urls", user="-").set(count_archive_urls)
+    with get_db as db:
+        count_archives = crud.count_archives(db)
+        count_archive_urls = crud.count_archive_urls(db)
+        DATABASE_METRICS.labels(query="count_archives", user="-").set(count_archives)
+        DATABASE_METRICS.labels(query="count_archive_urls", user="-").set(count_archive_urls)
 
-    for user in crud.count_by_user_since(session, REPEAT_COUNT_METRICS_SECONDS):
-        DATABASE_METRICS.labels(query="count_by_user", user=user.author_id).set(user.total)
\ No newline at end of file
+        for user in crud.count_by_user_since(db, repeat_in_seconds):
+            DATABASE_METRICS.labels(query="count_by_user", user=user.author_id).set(user.total)
\ No newline at end of file
diff --git a/src/worker.py b/src/worker.py
index a2220d7..513ebf1 100644
--- a/src/worker.py
+++ b/src/worker.py
@@ -1,5 +1,5 @@
 
-import os, traceback, yaml, datetime, sys
+import traceback, yaml, datetime
 from typing import List, Set
 
 from celery import Celery
@@ -9,29 +9,25 @@ from auto_archiver.core import Media
 from loguru import logger
 
 from db import crud, schemas, models
-from db.database import SessionLocal
-from contextlib import contextmanager
+from db.database import get_db
+from shared.settings import Settings
 import json
 import redis
 from sqlalchemy import exc
 
+settings = Settings()
+
 celery = Celery(__name__)
-celery.conf.broker_url = os.environ.get("CELERY_BROKER_URL", "redis://localhost:6379")
-celery.conf.result_backend = os.environ.get("CELERY_RESULT_BACKEND", "redis://localhost:6379")
-USER_GROUPS_FILENAME = os.environ.get("USER_GROUPS_FILENAME", "user-groups.yaml")
-REDIS_EXCEPTIONS_CHANNEL = "exceptions-channel"
+celery.conf.broker_url = settings.CELERY_BROKER_URL
+celery.conf.result_backend = settings.CELERY_RESULT_BACKEND
+USER_GROUPS_FILENAME = settings.USER_GROUPS_FILENAME
+REDIS_EXCEPTIONS_CHANNEL = settings.REDIS_EXCEPTIONS_CHANNEL
+
 Rdis = redis.Redis.from_url(celery.conf.broker_url)
 
-@contextmanager
-def get_db():
-    session = SessionLocal()
-    try: yield session
-    finally: session.close()
-
-
 @celery.task(name="create_archive_task", bind=True, autoretry_for=(Exception,), retry_backoff=True, retry_kwargs={'max_retries': 3})
 def create_archive_task(self, archive_json: str):
-    archive = schemas.ArchiveCreate.parse_raw(archive_json)
+    archive = schemas.ArchiveCreate.model_validate_json(archive_json)
     logger.info(f"Archiving {archive.url=} {archive.tags=} {archive.public=} {archive.group_id=} {archive.author_id=}")
     invalid = is_group_invalid_for_user(archive.public, archive.group_id, archive.author_id)
     if invalid:
@@ -63,7 +59,7 @@ def create_archive_task(self, archive_json: str):
 
 @celery.task(name="create_sheet_task", bind=True, autoretry_for=(Exception,), retry_backoff=True, retry_kwargs={'max_retries': 0})
 def create_sheet_task(self, sheet_json: str):
-    sheet = schemas.SubmitSheet.parse_raw(sheet_json)
+    sheet = schemas.SubmitSheet.model_validate_json(sheet_json)
     sheet.tags.add("gsheet")
     logger.info(f"SHEET START {sheet=}")
 

From ea8e06dfee510d90c5bcc3ae1e9df3ee9927bee7 Mon Sep 17 00:00:00 2001
From: msramalho <19508417+msramalho@users.noreply.github.com>
Date: Fri, 18 Oct 2024 16:35:12 +0100
Subject: [PATCH 12/33] refactoring with app_factory

---
 .gitignore                          |   1 +
 docker-compose.dev.yml              |   2 +-
 docker-compose.yml                  |   2 +-
 src/.env.test                       |   2 +-
 src/main.py                         | 168 ---------------------------
 src/security.py                     |   2 +-
 src/shared/settings.py              |   1 -
 src/tests/conftest.py               |  56 ++++++---
 src/tests/endpoints/test_default.py |  34 ++----
 src/tests/endpoints/test_task.py    |  22 +---
 src/tests/endpoints/test_url.py     |   6 +
 src/tests/test_main.py              |  23 ----
 src/tests/web/test_main.py          |  28 +++++
 src/web/__init__.py                 |   4 +
 src/web/main.py                     | 170 ++++++++++++++++++++++++++++
 15 files changed, 265 insertions(+), 256 deletions(-)
 delete mode 100644 src/main.py
 create mode 100644 src/tests/endpoints/test_url.py
 delete mode 100644 src/tests/test_main.py
 create mode 100644 src/tests/web/test_main.py
 create mode 100644 src/web/__init__.py
 create mode 100644 src/web/main.py

diff --git a/.gitignore b/.gitignore
index 01ac3f7..16d4ed8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -20,5 +20,6 @@ src/crawls
 .pytest_cache/*
 htmlcov
 local_archive
+local_archive_test
 *db-wal
 *db-shm
\ No newline at end of file
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 7814296..3782477 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -5,8 +5,8 @@ services:
     environment:
       - SERVE_LOCAL_ARCHIVE=/app/local_archive # See orchestration.yaml local_storage.save_to
       - ALLOWED_ORIGINS=http://localhost:8004,chrome-extension://ojcimmjndnlmmlgnjaeojoebaceokpdp
-      - API_BEARER_TOKEN=dev-api-bearer-token
       - USER_GROUPS_FILENAME=user-groups.dev.yaml
+      - DATABASE_PATH=sqlite:////app/auto-archiver.db
 
   worker:
     restart: "no"
diff --git a/docker-compose.yml b/docker-compose.yml
index 3fd6d2a..5650f74 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -18,7 +18,7 @@ services:
     <<: *base-setup
     ports:
       - "127.0.0.1:8004:8000"
-    command: uvicorn main:app --host 0.0.0.0 --reload
+    command: uvicorn web:app --host 0.0.0.0 --reload
     volumes:
       - ./src:/app
     depends_on:
diff --git a/src/.env.test b/src/.env.test
index fca6b15..9beb17a 100644
--- a/src/.env.test
+++ b/src/.env.test
@@ -3,5 +3,5 @@ ALLOWED_ORIGINS='["chrome-extension://example1","chrome-extension://example2","h
 BLOCKED_EMAILS='["blocked@example.com"]'
 
 
-DATABASE_PATH="sqlite:////app/auto-archiver.test.db"
+DATABASE_PATH="sqlite:///auto-archiver.test.db"
 API_BEARER_TOKEN=this_is_the_test_api_token
\ No newline at end of file
diff --git a/src/main.py b/src/main.py
deleted file mode 100644
index a4c2adf..0000000
--- a/src/main.py
+++ /dev/null
@@ -1,168 +0,0 @@
-import traceback, os
-from celery.result import AsyncResult
-from fastapi import FastAPI, Depends, HTTPException
-from fastapi.encoders import jsonable_encoder
-from fastapi.responses import JSONResponse
-from fastapi.staticfiles import StaticFiles
-from fastapi.middleware.cors import CORSMiddleware
-from prometheus_fastapi_instrumentator import Instrumentator
-from datetime import datetime
-import sqlalchemy
-from sqlalchemy.orm import Session
-from loguru import logger
-
-from core.logging import logging_middleware
-from worker import create_archive_task, create_sheet_task, celery, insert_result_into_db
-
-from db import crud, models, schemas
-from security import get_user_auth, token_api_key_auth, get_token_or_user_auth
-from core.config import VERSION, API_DESCRIPTION
-from db.database import get_db_dependency
-from core.events import lifespan
-from shared.settings import Settings
-
-from auto_archiver import Metadata
-
-from endpoints import default_router, url_router, sheet_router, task_router, interoperability_router
-
-settings = Settings()
-
-app = FastAPI(
-    title="Auto-Archiver API",
-    description=API_DESCRIPTION,
-    version=VERSION,
-    contact={"name": "GitHub", "url": "https://github.com/bellingcat/auto-archiver-api"},
-    lifespan=lifespan
-)
-
-app.add_middleware(
-    CORSMiddleware,
-    allow_origins=settings.ALLOWED_ORIGINS,
-    allow_credentials=True,
-    allow_methods=["*"],
-    allow_headers=["*"],
-)
-
-app.include_router(default_router)
-app.include_router(url_router)
-app.include_router(sheet_router)
-app.include_router(task_router)
-app.include_router(interoperability_router)
-
-# prometheus exposed in /metrics with authentication
-Instrumentator(should_group_status_codes=False, excluded_handlers=["/metrics", "/health"]).instrument(app).expose(app, dependencies=[Depends(token_api_key_auth)])
-
-def setup_local_archive_serve():
-    local_dir = settings.SERVE_LOCAL_ARCHIVE
-    if not os.path.isdir(local_dir) and os.path.isdir(local_dir.replace("/app", ".")):
-        local_dir = local_dir.replace("/app", ".")
-    if len(settings.SERVE_LOCAL_ARCHIVE) > 1 and os.path.isdir(local_dir):
-        logger.warning(f"MOUNTing local archive {settings.SERVE_LOCAL_ARCHIVE}")
-        app.mount(settings.SERVE_LOCAL_ARCHIVE, StaticFiles(directory=local_dir), name=settings.SERVE_LOCAL_ARCHIVE)
-setup_local_archive_serve()
-
-
-app.middleware("http")(logging_middleware)
-
-# -----Submit URL and manipulate tasks. Bearer protected below
-
-
-@app.get("/tasks/search-url", response_model=list[schemas.Archive], deprecated=True)  # DEPRECATED
-def search_by_url(url: str, skip: int = 0, limit: int = 100, archived_after: datetime = None, archived_before: datetime = None, db: Session = Depends(get_db_dependency), email=Depends(get_token_or_user_auth)):
-    return crud.search_archives_by_url(db, url.strip(), email, skip=skip, limit=limit, archived_after=archived_after, archived_before=archived_before)
-
-
-@app.get("/tasks/sync", response_model=list[schemas.Archive], deprecated=True)  # DEPRECATED
-def search(skip: int = 0, limit: int = 100, db: Session = Depends(get_db_dependency), email=Depends(get_user_auth)):
-    return crud.search_archives_by_email(db, email, skip=skip, limit=limit)
-
-
-@app.post("/tasks", status_code=201, deprecated=True)  # DEPRECATED
-def archive_tasks(archive: schemas.ArchiveCreate, email=Depends(get_token_or_user_auth)):
-    archive.author_id = email
-    url = archive.url
-    logger.info(f"new {archive.public=} task for {email=} and {archive.group_id=}: {url}")
-    if type(url) != str or len(url) <= 5:
-        raise HTTPException(status_code=422, detail=f"Invalid URL received: {url}")
-    logger.info("creating task")
-    task = create_archive_task.delay(archive.model_dump_json())
-    return JSONResponse({"id": task.id})
-
-
-@app.get("/archive/{task_id}", deprecated=True)  # DEPRECATED
-def lookup(task_id, db: Session = Depends(get_db_dependency), email=Depends(get_token_or_user_auth)):
-    return crud.get_archive(db, task_id, email)
-
-
-@app.get("/tasks/{task_id}", deprecated=True)  # DEPRECATED
-def get_status(task_id, email=Depends(get_token_or_user_auth)):
-    logger.info(f"status check for user {email} task {task_id}")
-    task = AsyncResult(task_id, app=celery)
-    try:
-        if task.status == "FAILURE":
-            # *FAILURE* The task raised an exception, or has exceeded the retry limit.
-            # The :attr:`result` attribute then contains the exception raised by the task.
-            # https://docs.celeryq.dev/en/stable/_modules/celery/result.html#AsyncResult
-            raise task.result
-
-        response = {
-            "id": task_id,
-            "status": task.status,
-            "result": task.result
-        }
-        return JSONResponse(jsonable_encoder(response, exclude_unset=True))
-
-    except Exception as e:
-        logger.error(e)
-        logger.error(traceback.format_exc())
-        return JSONResponse({
-            "id": task_id,
-            "status": "FAILURE",
-            "result": {"error": str(e)}
-        })
-
-
-@app.delete("/tasks/{task_id}", deprecated=True)  # DEPRECATED
-def delete_task(task_id, db: Session = Depends(get_db_dependency), email=Depends(get_user_auth)):
-    logger.info(f"deleting task {task_id} request by {email}")
-    return JSONResponse({
-        "id": task_id,
-        "deleted": crud.soft_delete_task(db, task_id, email)
-    })
-
-# ----- Google Sheets Logic
-
-
-@app.post("/sheet", status_code=201, deprecated=True)  # DEPRECATED
-def archive_sheet(sheet: schemas.SubmitSheet, email=Depends(get_user_auth)):
-    logger.info(f"SHEET TASK for {sheet=}")
-    sheet.author_id = email
-    if not sheet.sheet_name and not sheet.sheet_id:
-        raise HTTPException(status_code=422, detail=f"sheet name or id is required")
-    task = create_sheet_task.delay(sheet.model_dump_json())
-    return JSONResponse({"id": task.id})
-
-
-@app.post("/sheet_service", status_code=201, deprecated=True)  # DEPRECATED
-def archive_sheet_service(sheet: schemas.SubmitSheet, auth=Depends(token_api_key_auth)):
-    logger.info(f"SHEET TASK for {sheet=}")
-    sheet.author_id = sheet.author_id or "api-endpoint"
-    if not sheet.sheet_name and not sheet.sheet_id:
-        raise HTTPException(status_code=422, detail=f"sheet name or id is required")
-    task = create_sheet_task.delay(sheet.model_dump_json())
-    return JSONResponse({"id": task.id})
-
-# ----- endpoint to submit data archived elsewhere
-
-
-@app.post("/submit-archive", status_code=201, deprecated=True)  # DEPRECATED
-def submit_manual_archive(manual: schemas.SubmitManual, auth=Depends(token_api_key_auth)):
-    result = Metadata.from_json(manual.result)
-    logger.info(f"MANUAL SUBMIT {result.get_url()} {manual.author_id}")
-    manual.tags.add("manual")
-    try:
-        archive_id = insert_result_into_db(result, manual.tags, manual.public, manual.group_id, manual.author_id, models.generate_uuid())
-    except sqlalchemy.exc.IntegrityError as e:
-        logger.error(e)
-        raise HTTPException(status_code=422, detail=f"Cannot insert into DB due to integrity error")
-    return JSONResponse({"id": archive_id})
diff --git a/src/security.py b/src/security.py
index 03ef4d5..26abadc 100644
--- a/src/security.py
+++ b/src/security.py
@@ -60,7 +60,7 @@ def authenticate_user(access_token):
     # https://cloud.google.com/docs/authentication/token-types#access
     if type(access_token) != str or len(access_token) < 10: return False, "invalid access_token"
     r = requests.get("https://oauth2.googleapis.com/tokeninfo", {"access_token": access_token})
-    if r.status_code != 200: return False, "error occurred"
+    if r.status_code != 200: return False, "invalid token"
     try:
         j = r.json()
         if j.get("azp") not in settings.CHROME_APP_IDS and j.get("aud") not in settings.CHROME_APP_IDS:
diff --git a/src/shared/settings.py b/src/shared/settings.py
index dc216d3..2190726 100644
--- a/src/shared/settings.py
+++ b/src/shared/settings.py
@@ -28,4 +28,3 @@ class Settings(BaseSettings):
     ALLOWED_ORIGINS: Annotated[set[str], Len(min_length=1)]
     CHROME_APP_IDS: Annotated[set[Annotated[str, Len(min_length=10)]], Len(min_length=1)]
     BLOCKED_EMAILS: Annotated[Set[str], Len(min_length=0)] = set()
-
diff --git a/src/tests/conftest.py b/src/tests/conftest.py
index 62c57bd..6a417de 100644
--- a/src/tests/conftest.py
+++ b/src/tests/conftest.py
@@ -1,34 +1,38 @@
 import os
+from fastapi.testclient import TestClient
 import pytest
 from unittest.mock import patch
 from shared.settings import Settings
 
+
 @pytest.fixture(autouse=True)
 def mock_logger_add():
     """Fixture to mock loguru.logger.add for all tests."""
     with patch('loguru.logger.add') as mock_add:
         yield mock_add  # This makes the mock available to tests
 
-# @pytest.fixture(autouse=True)
-# def settings():
-#     return Settings(_env_file=".env.test")
+
+@pytest.fixture()
+def settings():
+    return Settings(_env_file=".env.test")
 
 
 @pytest.fixture(autouse=True)
-def settings():
+def mock_settings():
     with patch('shared.settings.Settings', return_value=Settings(_env_file=".env.test")) as mock_settings:
         yield mock_settings
 
 
 @pytest.fixture()
-def test_db(settings):
-    from db.database import make_engine, make_session_local
+def test_db(settings: Settings):
+    from db.database import make_engine
     from db import models
 
     engine = make_engine(settings.DATABASE_PATH)
 
-    if not os.path.exists(settings.DATABASE_PATH):
-        open(settings.DATABASE_PATH, 'w').close()
+    fs = settings.DATABASE_PATH.replace("sqlite:///", "")
+    if not os.path.exists(fs):
+        open(fs, 'w').close()
 
     models.Base.metadata.create_all(engine)
 
@@ -37,13 +41,35 @@ def test_db(settings):
     connection.close()
 
     models.Base.metadata.drop_all(bind=engine)
-    os.remove(settings.DATABASE_PATH)
+    for suffix in ["", "-wal", "-shm"]:
+        new_fs = fs + suffix
+        if os.path.exists(new_fs): 
+            os.remove(new_fs)
 
-# @pytest.fixture()
-# def db_session(test_db):
-#     session_local = make_session_local(test_db)
-#     with session_local() as session:
-#         yield session
+
+@pytest.fixture()
+def db_session(test_db):
+    from db.database import make_session_local
+    session_local = make_session_local(test_db)
+    with session_local() as session:
+        yield session
+
+
+@pytest.fixture()
+def app(db_session, settings):
+    from web.main import app_factory
+    app = app_factory()
+    from security import get_token_or_user_auth
+    app.dependency_overrides[get_token_or_user_auth] = lambda: "example@email.com"
+    # app.dependency_overrides[settings] = lambda: settings
+    # app.dependency_overrides[get_session] = lambda: db_session
+    return app
+
+
+@pytest.fixture()
+def client(app):
+    client = TestClient(app)
+    return client
 
 # # create test data and insert it into the database
 # def create_test_data():
@@ -57,4 +83,4 @@ def test_db(settings):
 #     db.refresh(task)
 #     db.close()
 
-#     return task.id
\ No newline at end of file
+#     return task.id
diff --git a/src/tests/endpoints/test_default.py b/src/tests/endpoints/test_default.py
index f05d34f..4e049c6 100644
--- a/src/tests/endpoints/test_default.py
+++ b/src/tests/endpoints/test_default.py
@@ -3,10 +3,7 @@ from fastapi.testclient import TestClient
 from core.config import VERSION
 
 
-def test_endpoint_home():
-    from main import app
-    client = TestClient(app)
-
+def test_endpoint_home(client):
     r = client.get("/")
     assert r.status_code == 200
     j = r.json()
@@ -18,10 +15,7 @@ def test_endpoint_home():
 @patch("endpoints.default.bearer_security", new_callable=AsyncMock)
 @patch("endpoints.default.get_user_auth", new_callable=AsyncMock, return_value="test@example.com")
 @patch("endpoints.default.crud.get_user_groups", return_value=["group1", "group2"])
-def test_endpoint_home_with_groups(m1, m2, m3):
-    from main import app
-    client = TestClient(app)
-
+def test_endpoint_home_with_groups(m1, m2, m3, client):
     r = client.get("/")
     assert r.status_code == 200
     j = r.json()
@@ -31,30 +25,24 @@ def test_endpoint_home_with_groups(m1, m2, m3):
     assert j["groups"] == ["group1", "group2"]
 
 
-def test_endpoint_health():
-    from main import app
-    client = TestClient(app)
-
+def test_endpoint_health(client):
     r = client.get("/health")
     assert r.status_code == 200
     assert r.json() == {"status": "ok"}
 
 
-def test_endpoint_groups_403():
-    from main import app
-    client = TestClient(app)
+def test_endpoint_groups_403(client):
     r = client.get("/groups")
     assert r.status_code == 403
 
 
 @patch("endpoints.default.crud.get_user_groups", return_value=["group1", "group2"])
-def test_endpoint_groups(m1):
+def test_endpoint_groups(m1, app):
     async def mock_get_user_auth(): return True
-    from main import app
     from security import get_user_auth
     app.dependency_overrides[get_user_auth] = mock_get_user_auth
-    
     client = TestClient(app)
+
     r = client.get("/groups")
 
     assert r.status_code == 200
@@ -63,18 +51,12 @@ def test_endpoint_groups(m1):
     app.dependency_overrides = {}
 
 
-def test_no_serve_local_archive_by_default():
-    from main import app
-    client = TestClient(app)
-
+def test_no_serve_local_archive_by_default(client):
     r = client.get("/app/local_archive_test/temp.txt")
     assert r.status_code == 404
 
 
-def test_favicon():
-    from main import app
-    client = TestClient(app)
-
+def test_favicon(client):
     r = client.get("/favicon.ico")
     assert r.status_code == 200
     assert r.headers["content-type"] == "image/vnd.microsoft.icon"
diff --git a/src/tests/endpoints/test_task.py b/src/tests/endpoints/test_task.py
index 7510a34..e6ff8be 100644
--- a/src/tests/endpoints/test_task.py
+++ b/src/tests/endpoints/test_task.py
@@ -2,18 +2,8 @@ from unittest.mock import patch
 from fastapi.testclient import TestClient
 
 
-def setup_client():
-    from main import app
-    from security import get_token_or_user_auth
-    async def mock_get_token_or_user_auth(): return "example@email.com"
-    app.dependency_overrides[get_token_or_user_auth] = mock_get_token_or_user_auth
-    return TestClient(app), app
-
-
 @patch("endpoints.task.AsyncResult")
-def test_get_status_success(mock_async_result):
-    client, app = setup_client()
-
+def test_get_status_success(mock_async_result, client):
     mock_async_result.return_value.status = "SUCCESS"
     mock_async_result.return_value.result = {"data": "some result"}
 
@@ -25,12 +15,10 @@ def test_get_status_success(mock_async_result):
         "status": "SUCCESS",
         "result": {"data": "some result"}
     }
-    app.dependency_overrides = {}
 
 
 @patch("endpoints.task.AsyncResult")
-def test_get_status_failure(mock_async_result):
-    client, app = setup_client()
+def test_get_status_failure(mock_async_result, client):
 
     mock_async_result.return_value.status = "FAILURE"
     mock_async_result.return_value.result = Exception("Some error")
@@ -43,13 +31,10 @@ def test_get_status_failure(mock_async_result):
         "status": "FAILURE",
         "result": {"error": "Some error"}
     }
-    app.dependency_overrides = {}
 
 
 @patch("endpoints.task.AsyncResult")
-def test_get_status_pending(mock_async_result):
-    client, app = setup_client()
-
+def test_get_status_pending(mock_async_result, client):
     mock_async_result.return_value.status = "PENDING"
     mock_async_result.return_value.result = None
 
@@ -61,4 +46,3 @@ def test_get_status_pending(mock_async_result):
         "status": "PENDING",
         "result": None
     }
-    app.dependency_overrides = {}
diff --git a/src/tests/endpoints/test_url.py b/src/tests/endpoints/test_url.py
new file mode 100644
index 0000000..911554a
--- /dev/null
+++ b/src/tests/endpoints/test_url.py
@@ -0,0 +1,6 @@
+
+# def test_archive_url(client):
+#     response = client.get("/archive/url")
+#     assert response.status_code == 200
+#     assert response.json() == {"message": "Archive URL"}
+
diff --git a/src/tests/test_main.py b/src/tests/test_main.py
deleted file mode 100644
index f19f6ae..0000000
--- a/src/tests/test_main.py
+++ /dev/null
@@ -1,23 +0,0 @@
-import os
-from unittest.mock import patch
-from fastapi.testclient import TestClient
-
-
-def test_serve_local_archive_logic():
-    with patch("main.settings.SERVE_LOCAL_ARCHIVE", "/app/local_archive_test"):
-
-        # create a test file
-        os.makedirs("local_archive_test", exist_ok=True)
-        with open("local_archive_test/temp.txt", "w") as f:
-            f.write("test")
-
-        from main import app, setup_local_archive_serve
-        setup_local_archive_serve()
-        client = TestClient(app)
-
-        r = client.get("/app/local_archive_test/temp.txt")
-        assert r.status_code == 200
-        assert r.text == "test"
-
-        os.remove("local_archive_test/temp.txt")
-        os.rmdir("local_archive_test")
diff --git a/src/tests/web/test_main.py b/src/tests/web/test_main.py
new file mode 100644
index 0000000..333e0e6
--- /dev/null
+++ b/src/tests/web/test_main.py
@@ -0,0 +1,28 @@
+import os
+from unittest.mock import patch
+from fastapi.testclient import TestClient
+from shared.settings import Settings
+
+
+import shutil
+
+def test_serve_local_archive_logic(settings: Settings):
+    # create a test file first
+    os.makedirs("local_archive_test", exist_ok=True)
+    with open("local_archive_test/temp.txt", "w") as f:
+        f.write("test")
+
+    try:
+        # modify the settings
+        settings.SERVE_LOCAL_ARCHIVE = "/app/local_archive_test"
+        from web.main import app_factory
+        app = app_factory(settings)
+        
+        # test
+        client = TestClient(app)
+        r = client.get("/app/local_archive_test/temp.txt")
+        assert r.status_code == 200
+        assert r.text == "test"
+    finally:
+        # cleanup
+        shutil.rmtree("local_archive_test")
diff --git a/src/web/__init__.py b/src/web/__init__.py
new file mode 100644
index 0000000..cdcd353
--- /dev/null
+++ b/src/web/__init__.py
@@ -0,0 +1,4 @@
+from web.main import app_factory
+
+
+app = app_factory
\ No newline at end of file
diff --git a/src/web/main.py b/src/web/main.py
new file mode 100644
index 0000000..babfb3a
--- /dev/null
+++ b/src/web/main.py
@@ -0,0 +1,170 @@
+import traceback, os
+from celery.result import AsyncResult
+from fastapi import FastAPI, Depends, HTTPException
+from fastapi.encoders import jsonable_encoder
+from fastapi.responses import JSONResponse
+from fastapi.staticfiles import StaticFiles
+from fastapi.middleware.cors import CORSMiddleware
+from prometheus_fastapi_instrumentator import Instrumentator
+from datetime import datetime
+import sqlalchemy
+from sqlalchemy.orm import Session
+from loguru import logger
+
+from core.logging import logging_middleware
+from worker import create_archive_task, create_sheet_task, celery, insert_result_into_db
+
+from db import crud, models, schemas
+from security import get_user_auth, token_api_key_auth, get_token_or_user_auth
+from core.config import VERSION, API_DESCRIPTION
+from db.database import get_db_dependency
+from core.events import lifespan
+from shared.settings import Settings
+
+from auto_archiver import Metadata
+
+from endpoints import default_router, url_router, sheet_router, task_router, interoperability_router
+
+
+def app_factory(settings = Settings()):
+    
+
+    app = FastAPI(
+        title="Auto-Archiver API",
+        description=API_DESCRIPTION,
+        version=VERSION,
+        contact={"name": "GitHub", "url": "https://github.com/bellingcat/auto-archiver-api"},
+        lifespan=lifespan
+    )
+
+    app.add_middleware(
+        CORSMiddleware,
+        allow_origins=settings.ALLOWED_ORIGINS,
+        allow_credentials=True,
+        allow_methods=["*"],
+        allow_headers=["*"],
+    )
+
+    app.include_router(default_router)
+    app.include_router(url_router)
+    app.include_router(sheet_router)
+    app.include_router(task_router)
+    app.include_router(interoperability_router)
+
+    # prometheus exposed in /metrics with authentication
+    Instrumentator(should_group_status_codes=False, excluded_handlers=["/metrics", "/health"]).instrument(app).expose(app, dependencies=[Depends(token_api_key_auth)])
+
+    local_dir = settings.SERVE_LOCAL_ARCHIVE
+    if not os.path.isdir(local_dir) and os.path.isdir(local_dir.replace("/app", ".")):
+        local_dir = local_dir.replace("/app", ".")
+    if len(settings.SERVE_LOCAL_ARCHIVE) > 1 and os.path.isdir(local_dir):
+        logger.warning(f"MOUNTing local archive {settings.SERVE_LOCAL_ARCHIVE}")
+        app.mount(settings.SERVE_LOCAL_ARCHIVE, StaticFiles(directory=local_dir), name=settings.SERVE_LOCAL_ARCHIVE)
+
+
+    app.middleware("http")(logging_middleware)
+
+    # -----Submit URL and manipulate tasks. Bearer protected below
+
+
+    @app.get("/tasks/search-url", response_model=list[schemas.Archive], deprecated=True)  # DEPRECATED
+    def search_by_url(url: str, skip: int = 0, limit: int = 100, archived_after: datetime = None, archived_before: datetime = None, db: Session = Depends(get_db_dependency), email=Depends(get_token_or_user_auth)):
+        return crud.search_archives_by_url(db, url.strip(), email, skip=skip, limit=limit, archived_after=archived_after, archived_before=archived_before)
+
+
+    @app.get("/tasks/sync", response_model=list[schemas.Archive], deprecated=True)  # DEPRECATED
+    def search(skip: int = 0, limit: int = 100, db: Session = Depends(get_db_dependency), email=Depends(get_user_auth)):
+        return crud.search_archives_by_email(db, email, skip=skip, limit=limit)
+
+
+    @app.post("/tasks", status_code=201, deprecated=True)  # DEPRECATED
+    def archive_tasks(archive: schemas.ArchiveCreate, email=Depends(get_token_or_user_auth)):
+        archive.author_id = email
+        url = archive.url
+        logger.info(f"new {archive.public=} task for {email=} and {archive.group_id=}: {url}")
+        if type(url) != str or len(url) <= 5:
+            raise HTTPException(status_code=422, detail=f"Invalid URL received: {url}")
+        logger.info("creating task")
+        task = create_archive_task.delay(archive.model_dump_json())
+        return JSONResponse({"id": task.id})
+
+
+    @app.get("/archive/{task_id}", deprecated=True)  # DEPRECATED
+    def lookup(task_id, db: Session = Depends(get_db_dependency), email=Depends(get_token_or_user_auth)):
+        return crud.get_archive(db, task_id, email)
+
+
+    @app.get("/tasks/{task_id}", deprecated=True)  # DEPRECATED
+    def get_status(task_id, email=Depends(get_token_or_user_auth)):
+        logger.info(f"status check for user {email} task {task_id}")
+        task = AsyncResult(task_id, app=celery)
+        try:
+            if task.status == "FAILURE":
+                # *FAILURE* The task raised an exception, or has exceeded the retry limit.
+                # The :attr:`result` attribute then contains the exception raised by the task.
+                # https://docs.celeryq.dev/en/stable/_modules/celery/result.html#AsyncResult
+                raise task.result
+
+            response = {
+                "id": task_id,
+                "status": task.status,
+                "result": task.result
+            }
+            return JSONResponse(jsonable_encoder(response, exclude_unset=True))
+
+        except Exception as e:
+            logger.error(e)
+            logger.error(traceback.format_exc())
+            return JSONResponse({
+                "id": task_id,
+                "status": "FAILURE",
+                "result": {"error": str(e)}
+            })
+
+
+    @app.delete("/tasks/{task_id}", deprecated=True)  # DEPRECATED
+    def delete_task(task_id, db: Session = Depends(get_db_dependency), email=Depends(get_user_auth)):
+        logger.info(f"deleting task {task_id} request by {email}")
+        return JSONResponse({
+            "id": task_id,
+            "deleted": crud.soft_delete_task(db, task_id, email)
+        })
+
+    # ----- Google Sheets Logic
+
+
+    @app.post("/sheet", status_code=201, deprecated=True)  # DEPRECATED
+    def archive_sheet(sheet: schemas.SubmitSheet, email=Depends(get_user_auth)):
+        logger.info(f"SHEET TASK for {sheet=}")
+        sheet.author_id = email
+        if not sheet.sheet_name and not sheet.sheet_id:
+            raise HTTPException(status_code=422, detail=f"sheet name or id is required")
+        task = create_sheet_task.delay(sheet.model_dump_json())
+        return JSONResponse({"id": task.id})
+
+
+    @app.post("/sheet_service", status_code=201, deprecated=True)  # DEPRECATED
+    def archive_sheet_service(sheet: schemas.SubmitSheet, auth=Depends(token_api_key_auth)):
+        logger.info(f"SHEET TASK for {sheet=}")
+        sheet.author_id = sheet.author_id or "api-endpoint"
+        if not sheet.sheet_name and not sheet.sheet_id:
+            raise HTTPException(status_code=422, detail=f"sheet name or id is required")
+        task = create_sheet_task.delay(sheet.model_dump_json())
+        return JSONResponse({"id": task.id})
+
+    # ----- endpoint to submit data archived elsewhere
+
+
+    @app.post("/submit-archive", status_code=201, deprecated=True)  # DEPRECATED
+    def submit_manual_archive(manual: schemas.SubmitManual, auth=Depends(token_api_key_auth)):
+        result = Metadata.from_json(manual.result)
+        logger.info(f"MANUAL SUBMIT {result.get_url()} {manual.author_id}")
+        manual.tags.add("manual")
+        try:
+            archive_id = insert_result_into_db(result, manual.tags, manual.public, manual.group_id, manual.author_id, models.generate_uuid())
+        except sqlalchemy.exc.IntegrityError as e:
+            logger.error(e)
+            raise HTTPException(status_code=422, detail=f"Cannot insert into DB due to integrity error")
+        return JSONResponse({"id": archive_id})
+
+    return app
\ No newline at end of file

From 56a81f6ec0809bbaedec24b502a8d0e078245434 Mon Sep 17 00:00:00 2001
From: msramalho <19508417+msramalho@users.noreply.github.com>
Date: Fri, 18 Oct 2024 19:16:05 +0100
Subject: [PATCH 13/33] adds new tests and improvements

---
 src/db/crud.py                      |   6 +-
 src/endpoints/url.py                |   8 +-
 src/tests/conftest.py               |  30 +++--
 src/tests/endpoints/test_default.py |  44 +++++---
 src/tests/endpoints/test_task.py    |  12 +-
 src/tests/endpoints/test_url.py     | 165 +++++++++++++++++++++++++++-
 6 files changed, 217 insertions(+), 48 deletions(-)

diff --git a/src/db/crud.py b/src/db/crud.py
index a62113f..097e556 100644
--- a/src/db/crud.py
+++ b/src/db/crud.py
@@ -16,9 +16,9 @@ DATABASE_QUERY_LIMIT = Settings().DATABASE_QUERY_LIMIT
 # --------------- TASK = Archive
 
 
-def get_archive(db: Session, task_id: str, email: str):
+def get_archive(db: Session, id: str, email: str):
     email = email.lower()
-    query = base_query(db).filter(models.Archive.id == task_id)
+    query = base_query(db).filter(models.Archive.id == id)
     if email != ALLOW_ANY_EMAIL:
         groups = get_user_groups(db, email)
         query = query.filter(or_(models.Archive.public == True, models.Archive.author_id == email, models.Archive.group_id.in_(groups)))
@@ -45,7 +45,7 @@ def search_archives_by_url(db: Session, url: str, email: str, skip: int = 0, lim
 
 def search_archives_by_email(db: Session, email: str, skip: int = 0, limit: int = 100):
     email = email.lower()
-    return base_query(db).filter(models.Archive.author.has(email=email)).offset(skip).limit(min(limit, DATABASE_QUERY_LIMIT)).all()
+    return base_query(db).filter(models.Archive.author.has(email=email)).order_by(models.Archive.created_at.desc()).offset(skip).limit(min(limit, DATABASE_QUERY_LIMIT)).all()
 
 
 def create_task(db: Session, task: schemas.ArchiveCreate, tags: list[models.Tag], urls: list[models.ArchiveUrl]):
diff --git a/src/endpoints/url.py b/src/endpoints/url.py
index dc1ca68..0771ed6 100644
--- a/src/endpoints/url.py
+++ b/src/endpoints/url.py
@@ -25,7 +25,7 @@ def archive_url(archive: schemas.ArchiveCreate, email=Depends(get_token_or_user_
     logger.info("creating task")
     task = create_archive_task.delay(archive.model_dump_json())
     task_response = schemas.Task(id=task.id)
-    return JSONResponse(task_response.model_dump())
+    return JSONResponse(task_response.model_dump(), status_code=201)
 
 
 @url_router.get("/search", response_model=list[schemas.Archive], summary="Search for archive entries by URL.")
@@ -44,13 +44,15 @@ def latest(skip: int = 0, limit: int = 25, db: Session = Depends(get_db_dependen
 
 @url_router.get("/{id}", response_model=schemas.Archive, summary="Fetch a single URL archive by the associated id.")
 def lookup(id, db: Session = Depends(get_db_dependency), email=Depends(get_token_or_user_auth)):
-    return crud.get_archive(db, id, email)
+    archive = crud.get_archive(db, id, email)
+    if archive is None:
+        raise HTTPException(status_code=404, detail="Archive not found")
+    return archive
 
 
 @url_router.delete("/{id}", response_model=schemas.TaskDelete, summary="Delete a single URL archive by id.")
 def delete_task(id, db: Session = Depends(get_db_dependency), email=Depends(get_user_auth)):
     logger.info(f"deleting url archive task {id} request by {email}")
-    #TODO: use response model?
     return JSONResponse({
         "id": id,
         "deleted": crud.soft_delete_task(db, id, email)
diff --git a/src/tests/conftest.py b/src/tests/conftest.py
index 6a417de..a63b5ba 100644
--- a/src/tests/conftest.py
+++ b/src/tests/conftest.py
@@ -43,7 +43,7 @@ def test_db(settings: Settings):
     models.Base.metadata.drop_all(bind=engine)
     for suffix in ["", "-wal", "-shm"]:
         new_fs = fs + suffix
-        if os.path.exists(new_fs): 
+        if os.path.exists(new_fs):
             os.remove(new_fs)
 
 
@@ -56,13 +56,9 @@ def db_session(test_db):
 
 
 @pytest.fixture()
-def app(db_session, settings):
+def app(db_session):
     from web.main import app_factory
     app = app_factory()
-    from security import get_token_or_user_auth
-    app.dependency_overrides[get_token_or_user_auth] = lambda: "example@email.com"
-    # app.dependency_overrides[settings] = lambda: settings
-    # app.dependency_overrides[get_session] = lambda: db_session
     return app
 
 
@@ -71,16 +67,16 @@ def client(app):
     client = TestClient(app)
     return client
 
-# # create test data and insert it into the database
-# def create_test_data():
-#     from db.database import SessionLocal
-#     from db.models import Task
 
-#     db = SessionLocal()
-#     task = Task(id="test-task-id", status="PENDING")
-#     db.add(task)
-#     db.commit()
-#     db.refresh(task)
-#     db.close()
+@pytest.fixture()
+def app_with_auth(app):
+    from security import get_token_or_user_auth, get_user_auth
+    app.dependency_overrides[get_token_or_user_auth] = lambda: "rick@example.com"
+    app.dependency_overrides[get_user_auth] = lambda: "morty@example.com"
+    return app
 
-#     return task.id
+
+@pytest.fixture()
+def client_with_auth(app_with_auth):
+    client = TestClient(app_with_auth)
+    return client
diff --git a/src/tests/endpoints/test_default.py b/src/tests/endpoints/test_default.py
index 4e049c6..0d46a99 100644
--- a/src/tests/endpoints/test_default.py
+++ b/src/tests/endpoints/test_default.py
@@ -3,8 +3,8 @@ from fastapi.testclient import TestClient
 from core.config import VERSION
 
 
-def test_endpoint_home(client):
-    r = client.get("/")
+def test_endpoint_home(client_with_auth):
+    r = client_with_auth.get("/")
     assert r.status_code == 200
     j = r.json()
     assert "version" in j and j["version"] == VERSION
@@ -15,8 +15,8 @@ def test_endpoint_home(client):
 @patch("endpoints.default.bearer_security", new_callable=AsyncMock)
 @patch("endpoints.default.get_user_auth", new_callable=AsyncMock, return_value="test@example.com")
 @patch("endpoints.default.crud.get_user_groups", return_value=["group1", "group2"])
-def test_endpoint_home_with_groups(m1, m2, m3, client):
-    r = client.get("/")
+def test_endpoint_home_with_groups(m1, m2, m3, client_with_auth):
+    r = client_with_auth.get("/")
     assert r.status_code == 200
     j = r.json()
     assert "version" in j and j["version"] == VERSION
@@ -24,9 +24,20 @@ def test_endpoint_home_with_groups(m1, m2, m3, client):
     assert "groups" in j
     assert j["groups"] == ["group1", "group2"]
 
+@patch("endpoints.default.bearer_security", new_callable=AsyncMock)
+@patch("endpoints.default.get_user_auth", new_callable=AsyncMock, return_value="test@example.com")
+@patch("endpoints.default.crud.get_user_groups", side_effect=Exception('mocked error'))
+def test_endpoint_home_with_groups_exception(m1, m2, m3, client_with_auth): # mocks call that triggers an internal error
+    r = client_with_auth.get("/")
+    assert r.status_code == 200
+    j = r.json()
+    assert "version" in j and j["version"] == VERSION
+    assert "breakingChanges" in j
+    assert "groups" not in j
 
-def test_endpoint_health(client):
-    r = client.get("/health")
+
+def test_endpoint_health(client_with_auth):
+    r = client_with_auth.get("/health")
     assert r.status_code == 200
     assert r.json() == {"status": "ok"}
 
@@ -36,27 +47,30 @@ def test_endpoint_groups_403(client):
     assert r.status_code == 403
 
 
+def test_endpoint_groups_empty(client_with_auth):
+    r = client_with_auth.get("/groups")
+    assert r.status_code == 200
+    assert r.json() == []
+
+
 @patch("endpoints.default.crud.get_user_groups", return_value=["group1", "group2"])
 def test_endpoint_groups(m1, app):
-    async def mock_get_user_auth(): return True
     from security import get_user_auth
-    app.dependency_overrides[get_user_auth] = mock_get_user_auth
+    app.dependency_overrides[get_user_auth] = lambda: True
     client = TestClient(app)
 
     r = client.get("/groups")
 
     assert r.status_code == 200
-    j = r.json()
-    assert j == ["group1", "group2"]
-    app.dependency_overrides = {}
+    assert r.json() == ["group1", "group2"]
 
 
-def test_no_serve_local_archive_by_default(client):
-    r = client.get("/app/local_archive_test/temp.txt")
+def test_no_serve_local_archive_by_default(client_with_auth):
+    r = client_with_auth.get("/app/local_archive_test/temp.txt")
     assert r.status_code == 404
 
 
-def test_favicon(client):
-    r = client.get("/favicon.ico")
+def test_favicon(client_with_auth):
+    r = client_with_auth.get("/favicon.ico")
     assert r.status_code == 200
     assert r.headers["content-type"] == "image/vnd.microsoft.icon"
diff --git a/src/tests/endpoints/test_task.py b/src/tests/endpoints/test_task.py
index e6ff8be..36285f5 100644
--- a/src/tests/endpoints/test_task.py
+++ b/src/tests/endpoints/test_task.py
@@ -3,11 +3,11 @@ from fastapi.testclient import TestClient
 
 
 @patch("endpoints.task.AsyncResult")
-def test_get_status_success(mock_async_result, client):
+def test_get_status_success(mock_async_result, client_with_auth):
     mock_async_result.return_value.status = "SUCCESS"
     mock_async_result.return_value.result = {"data": "some result"}
 
-    response = client.get("/task/test-task-id")
+    response = client_with_auth.get("/task/test-task-id")
 
     assert response.status_code == 200
     assert response.json() == {
@@ -18,12 +18,12 @@ def test_get_status_success(mock_async_result, client):
 
 
 @patch("endpoints.task.AsyncResult")
-def test_get_status_failure(mock_async_result, client):
+def test_get_status_failure(mock_async_result, client_with_auth):
 
     mock_async_result.return_value.status = "FAILURE"
     mock_async_result.return_value.result = Exception("Some error")
 
-    response = client.get("/task/test-task-id")
+    response = client_with_auth.get("/task/test-task-id")
 
     assert response.status_code == 200
     assert response.json() == {
@@ -34,11 +34,11 @@ def test_get_status_failure(mock_async_result, client):
 
 
 @patch("endpoints.task.AsyncResult")
-def test_get_status_pending(mock_async_result, client):
+def test_get_status_pending(mock_async_result, client_with_auth):
     mock_async_result.return_value.status = "PENDING"
     mock_async_result.return_value.result = None
 
-    response = client.get("/task/test-task-id")
+    response = client_with_auth.get("/task/test-task-id")
 
     assert response.status_code == 200
     assert response.json() == {
diff --git a/src/tests/endpoints/test_url.py b/src/tests/endpoints/test_url.py
index 911554a..e5f0afa 100644
--- a/src/tests/endpoints/test_url.py
+++ b/src/tests/endpoints/test_url.py
@@ -1,6 +1,163 @@
 
-# def test_archive_url(client):
-#     response = client.get("/archive/url")
-#     assert response.status_code == 200
-#     assert response.json() == {"message": "Archive URL"}
 
+import json
+import time
+from unittest.mock import patch
+
+from db.schemas import ArchiveCreate, TaskResult
+
+NO_AUTH = {'detail': 'Not authenticated'}
+
+
+def test_archive_url_unauthenticated(client):
+    response = client.post("/url/archive")
+    assert response.status_code == 403
+    assert response.json() == NO_AUTH
+
+    # this will call archive/{id}
+    response = client.get("/url/archive")
+    assert response.status_code == 403
+    assert response.json() == NO_AUTH
+
+
+@patch("worker.create_archive_task.delay", return_value=TaskResult(id="123-456-789", status="PENDING", result=""))
+def test_archive_url(m1, client_with_auth):
+    response = client_with_auth.post("/url/archive", json={"url": "bad"})
+    assert response.status_code == 422
+    assert response.json() == {'detail': 'Invalid URL received: bad'}
+    m1.assert_not_called()
+
+    response = client_with_auth.post("/url/archive", json={"url": "https://example.com"})
+    assert response.status_code == 201
+    assert response.json() == {'id': '123-456-789'}
+
+    m1.assert_called_once()
+    called_val = m1.call_args.args[0]
+    assert json.loads(called_val) == {"id": None, "url": "https://example.com", "result": None, "public": True, "author_id": "rick@example.com", "group_id": None, "tags": [], "rearchive": True}
+
+
+def test_search_by_url_unauthenticated(client):
+    response = client.get("/url/search")
+    assert response.status_code == 403
+    assert response.json() == NO_AUTH
+
+
+def test_search_by_url(client_with_auth, db_session):
+    # tests the search endpoint, including through some db data for the endpoint params
+    response = client_with_auth.get("/url/search")
+    assert response.status_code == 422
+    assert response.json()["detail"][0]["msg"] == "Field required"
+
+    response = client_with_auth.get("/url/search?url=https://example.com")
+    assert response.status_code == 200
+    assert response.json() == []
+
+    from db import crud
+    for i in range(11):
+        crud.create_task(db_session, ArchiveCreate(id=f"url-456-{i}", url="https://example.com" if i < 10 else "https://something-else.com", result={}, public=True, author_id="rick@example.com", group_id=None), [], [])
+        #NB: this insertion is too fast for the ordering to be correct as they are within the same second
+
+    response = client_with_auth.get("/url/search?url=https://example.com")
+    assert response.status_code == 200
+    assert len(j := response.json()) == 10
+    assert "url-456-0" in [i["id"] for i in j]
+    assert "url-456-9" in [i["id"] for i in j]
+    assert "url-456-10" not in [i["id"] for i in j]
+
+    response = client_with_auth.get("/url/search?url=https://example.com&limit=5")
+    assert response.status_code == 200
+    assert len(response.json()) == 5
+
+    response = client_with_auth.get("/url/search?url=https://example.com&skip=5&limit=2")
+    assert response.status_code == 200
+    assert len(response.json()) == 2
+
+    response = client_with_auth.get("/url/search?url=https://example.com&archived_before=2010-01-01")
+    assert response.status_code == 200
+    assert len(response.json()) == 0
+
+    response = client_with_auth.get("/url/search?url=https://example.com&archived_affter=2010-01-01")
+    assert response.status_code == 200
+    assert len(response.json()) == 10
+
+
+def test_latest_unauthenticated(client):
+    response = client.get("/url/latest")
+    assert response.status_code == 403
+    assert response.json() == NO_AUTH
+
+
+def test_latest(client_with_auth, db_session):
+    response = client_with_auth.get("/url/latest")
+    assert response.status_code == 200
+    assert response.json() == []
+
+    from db import crud
+    for i in range(11):
+        crud.create_task(db_session, ArchiveCreate(id=f"latest-456-{i}", url="https://example.com", result={}, public=True, author_id="morty@example.com" if i < 10 else "rick@example.com", group_id=None), [], [])
+        #NB: this insertion is too fast for the ordering to be correct as they are within the same second
+
+    # user must exist for /latest to work
+    crud.get_user(db_session, "morty@example.com")
+
+    response = client_with_auth.get("/url/latest")
+    assert response.status_code == 200
+    assert len(j := response.json()) == 10
+    assert "latest-456-0" in [i["id"] for i in j]
+    assert "latest-456-9" in [i["id"] for i in j]
+    assert "latest-456-10" not in [i["id"] for i in j]
+
+    response = client_with_auth.get("/url/latest?limit=5")
+    assert response.status_code == 200
+    assert len(response.json()) == 5
+
+    response = client_with_auth.get("/url/latest?skip=5&limit=2")
+    assert response.status_code == 200
+    assert len(response.json()) == 2
+
+
+def test_lookup_unauthenticated(client):
+    response = client.get("/url/123-456-789")
+    assert response.status_code == 403
+    assert response.json() == NO_AUTH
+
+
+def test_lookup(client_with_auth, db_session):
+    response = client_with_auth.get("/url/lookup-123-456-789")
+    assert response.status_code == 404
+    assert response.json() == {"detail": "Archive not found"}
+
+    from db import crud
+    crud.create_task(db_session, ArchiveCreate(id="lookup-123-456-789", url="https://example.com", result={}, public=True, author_id="rick@example.com", group_id=None), [], [])
+
+    response = client_with_auth.get("/url/lookup-123-456-789")
+    assert response.status_code == 200
+    j = response.json()
+    assert j["id"] == "lookup-123-456-789"
+    assert j["url"] == "https://example.com"
+    assert j["result"] == {}
+    assert j["public"] == True
+    assert j["author_id"] == "rick@example.com"
+    assert j["group_id"] == None
+    assert j["tags"] == []
+    assert j["updated_at"] == None
+    assert j["rearchive"] == True
+
+
+def test_delete_task_unauthenticated(client):
+    response = client.delete("/url/123-456-789")
+    assert response.status_code == 403
+    assert response.json() == NO_AUTH
+
+
+def test_delete_task(client_with_auth, db_session):
+    response = client_with_auth.delete("/url/delete-123-456-789")
+    assert response.status_code == 200
+    assert response.json() == {"id": "delete-123-456-789", "deleted": False}
+
+    from db import crud
+    crud.create_task(db_session, ArchiveCreate(id="delete-123-456-789", url="https://example.com", result={}, public=True, author_id="morty@example.com", group_id=None), [], [])
+
+    response = client_with_auth.delete("/url/delete-123-456-789")
+    assert response.status_code == 200
+    assert response.json() == {"id": "delete-123-456-789", "deleted": True}

From 6b9d6e224566765ace5388f77255d2832f80f232 Mon Sep 17 00:00:00 2001
From: msramalho <19508417+msramalho@users.noreply.github.com>
Date: Fri, 18 Oct 2024 19:46:56 +0100
Subject: [PATCH 14/33] new tests and abstractions

---
 src/db/schemas.py                            |  4 +-
 src/endpoints/interoperability.py            |  2 +-
 src/tests/conftest.py                        | 13 ++++++-
 src/tests/endpoints/test_interopreability.py | 19 ++++++++++
 src/tests/endpoints/test_url.py              | 39 ++++++--------------
 5 files changed, 45 insertions(+), 32 deletions(-)
 create mode 100644 src/tests/endpoints/test_interopreability.py

diff --git a/src/db/schemas.py b/src/db/schemas.py
index ef06fe7..2f04462 100644
--- a/src/db/schemas.py
+++ b/src/db/schemas.py
@@ -35,7 +35,7 @@ class SubmitSheet(BaseModel):
     public: bool = False
     author_id: str | None = None
     group_id: str | None = None
-    tags: set[Tag] | None = set()
+    tags: set[str] | None = set()
     columns: dict | None = {} # TODO: implement
 
 class SubmitManual(BaseModel):
@@ -43,7 +43,7 @@ class SubmitManual(BaseModel):
     public: bool = False
     author_id: str | None = None
     group_id: str | None = None
-    tags: set[Tag] | None = set()
+    tags: set[str] | None = set()
 
 class Task(BaseModel):
     id: str
diff --git a/src/endpoints/interoperability.py b/src/endpoints/interoperability.py
index e59702d..7885b08 100644
--- a/src/endpoints/interoperability.py
+++ b/src/endpoints/interoperability.py
@@ -23,4 +23,4 @@ def submit_manual_archive(manual: schemas.SubmitManual, auth=Depends(token_api_k
     except sqlalchemy.exc.IntegrityError as e:
         logger.error(e)
         raise HTTPException(status_code=422, detail=f"Cannot insert into DB due to integrity error")
-    return JSONResponse({"id": archive_id})
+    return JSONResponse({"id": archive_id}, status_code=201)
diff --git a/src/tests/conftest.py b/src/tests/conftest.py
index a63b5ba..d21fc41 100644
--- a/src/tests/conftest.py
+++ b/src/tests/conftest.py
@@ -70,9 +70,10 @@ def client(app):
 
 @pytest.fixture()
 def app_with_auth(app):
-    from security import get_token_or_user_auth, get_user_auth
+    from security import get_token_or_user_auth, get_user_auth, token_api_key_auth
     app.dependency_overrides[get_token_or_user_auth] = lambda: "rick@example.com"
     app.dependency_overrides[get_user_auth] = lambda: "morty@example.com"
+    app.dependency_overrides[token_api_key_auth] = lambda: "jerry@example.com"
     return app
 
 
@@ -80,3 +81,13 @@ def app_with_auth(app):
 def client_with_auth(app_with_auth):
     client = TestClient(app_with_auth)
     return client
+
+
+@pytest.fixture()
+def test_no_auth():
+    # reusable code to ensure a method/endpoint combination is unauthorized
+    def no_auth(http_method, endpoint):
+        response = http_method(endpoint)
+        assert response.status_code == 403
+        assert response.json() == {"detail": "Not authenticated"}
+    return no_auth
\ No newline at end of file
diff --git a/src/tests/endpoints/test_interopreability.py b/src/tests/endpoints/test_interopreability.py
new file mode 100644
index 0000000..82136f0
--- /dev/null
+++ b/src/tests/endpoints/test_interopreability.py
@@ -0,0 +1,19 @@
+import json
+
+
+def test_submit_manual_archive_unauthenticated(client, test_no_auth):
+    test_no_auth(client.post, "/interop/submit-archive")
+
+
+def test_submit_manual_archive(client_with_auth):
+    aa_metadata = json.dumps({"status": "test: success", "metadata": {"url": "http://example.com"}, "media": []})
+
+    r = client_with_auth.post("/interop/submit-archive", json={"result": aa_metadata, "public": False, "author_id": "jerry@gmail.com", "group_id": None, "tags": ["test"]})
+    assert r.status_code == 201
+    assert "id" in r.json()
+
+    # cannot have the same URL twice
+    aa_metadata = json.dumps({"status": "test: success", "metadata": {"url": "http://example.com"}, "media": [{"filename": "fn1", "urls": ["http://example.com", "http://example.com"]}]})
+    r = client_with_auth.post("/interop/submit-archive", json={"result": aa_metadata, "public": False, "author_id": "jerry@gmail.com", "group_id": None, "tags": ["test"]})
+    assert r.status_code == 422
+    assert r.json() == {"detail": "Cannot insert into DB due to integrity error"}
diff --git a/src/tests/endpoints/test_url.py b/src/tests/endpoints/test_url.py
index e5f0afa..8353287 100644
--- a/src/tests/endpoints/test_url.py
+++ b/src/tests/endpoints/test_url.py
@@ -6,18 +6,9 @@ from unittest.mock import patch
 
 from db.schemas import ArchiveCreate, TaskResult
 
-NO_AUTH = {'detail': 'Not authenticated'}
-
-
-def test_archive_url_unauthenticated(client):
-    response = client.post("/url/archive")
-    assert response.status_code == 403
-    assert response.json() == NO_AUTH
-
-    # this will call archive/{id}
-    response = client.get("/url/archive")
-    assert response.status_code == 403
-    assert response.json() == NO_AUTH
+def test_archive_url_unauthenticated(client, test_no_auth):
+    test_no_auth(client.post, "/url/archive")
+    test_no_auth(client.get, "/url/archive")
 
 
 @patch("worker.create_archive_task.delay", return_value=TaskResult(id="123-456-789", status="PENDING", result=""))
@@ -36,10 +27,8 @@ def test_archive_url(m1, client_with_auth):
     assert json.loads(called_val) == {"id": None, "url": "https://example.com", "result": None, "public": True, "author_id": "rick@example.com", "group_id": None, "tags": [], "rearchive": True}
 
 
-def test_search_by_url_unauthenticated(client):
-    response = client.get("/url/search")
-    assert response.status_code == 403
-    assert response.json() == NO_AUTH
+def test_search_by_url_unauthenticated(client, test_no_auth):
+    test_no_auth(client.get, "/url/search")
 
 
 def test_search_by_url(client_with_auth, db_session):
@@ -81,10 +70,8 @@ def test_search_by_url(client_with_auth, db_session):
     assert len(response.json()) == 10
 
 
-def test_latest_unauthenticated(client):
-    response = client.get("/url/latest")
-    assert response.status_code == 403
-    assert response.json() == NO_AUTH
+def test_latest_unauthenticated(client, test_no_auth):
+    test_no_auth(client.get, "/url/latest")
 
 
 def test_latest(client_with_auth, db_session):
@@ -116,10 +103,8 @@ def test_latest(client_with_auth, db_session):
     assert len(response.json()) == 2
 
 
-def test_lookup_unauthenticated(client):
-    response = client.get("/url/123-456-789")
-    assert response.status_code == 403
-    assert response.json() == NO_AUTH
+def test_lookup_unauthenticated(client, test_no_auth):
+    test_no_auth(client.get, "/url/123-456-789")
 
 
 def test_lookup(client_with_auth, db_session):
@@ -144,10 +129,8 @@ def test_lookup(client_with_auth, db_session):
     assert j["rearchive"] == True
 
 
-def test_delete_task_unauthenticated(client):
-    response = client.delete("/url/123-456-789")
-    assert response.status_code == 403
-    assert response.json() == NO_AUTH
+def test_delete_task_unauthenticated(client, test_no_auth):
+    test_no_auth(client.delete, "/url/123-456-789")
 
 
 def test_delete_task(client_with_auth, db_session):

From a41a33c4ff141e2be9904e4d0789cfadd38b3bda Mon Sep 17 00:00:00 2001
From: msramalho <19508417+msramalho@users.noreply.github.com>
Date: Fri, 18 Oct 2024 19:48:33 +0100
Subject: [PATCH 15/33] missing test files

---
 src/.env.test             |  3 ++-
 src/user-groups.test.yaml | 18 ++++++++++++++++++
 2 files changed, 20 insertions(+), 1 deletion(-)
 create mode 100644 src/user-groups.test.yaml

diff --git a/src/.env.test b/src/.env.test
index 9beb17a..fdbff38 100644
--- a/src/.env.test
+++ b/src/.env.test
@@ -4,4 +4,5 @@ BLOCKED_EMAILS='["blocked@example.com"]'
 
 
 DATABASE_PATH="sqlite:///auto-archiver.test.db"
-API_BEARER_TOKEN=this_is_the_test_api_token
\ No newline at end of file
+API_BEARER_TOKEN=this_is_the_test_api_token
+USER_GROUPS_FILENAME=user-groups.test.yaml
\ No newline at end of file
diff --git a/src/user-groups.test.yaml b/src/user-groups.test.yaml
new file mode 100644
index 0000000..97225db
--- /dev/null
+++ b/src/user-groups.test.yaml
@@ -0,0 +1,18 @@
+# NOTE: all emails should be lower-cased
+users:
+  rick@example.com:
+    - spaceship
+    - interdimensional
+  morty@example.com:
+    - spaceship
+  jerry@example.com:
+
+domains:
+  example.com:
+    - animated-characters
+
+
+orchestrators:
+  spaceship: secrets/orchestration-dev.yaml
+  interdimensional: secrets/orchestration-dev.yaml
+  default: secrets/orchestration-dev.yaml

From 145d92a1ae14a63fc566083a7f13fb50923fa5e1 Mon Sep 17 00:00:00 2001
From: msramalho <19508417+msramalho@users.noreply.github.com>
Date: Fri, 18 Oct 2024 19:51:09 +0100
Subject: [PATCH 16/33] fixing test with new test files

---
 src/tests/endpoints/test_default.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/tests/endpoints/test_default.py b/src/tests/endpoints/test_default.py
index 0d46a99..cfa6fcf 100644
--- a/src/tests/endpoints/test_default.py
+++ b/src/tests/endpoints/test_default.py
@@ -24,10 +24,11 @@ def test_endpoint_home_with_groups(m1, m2, m3, client_with_auth):
     assert "groups" in j
     assert j["groups"] == ["group1", "group2"]
 
+
 @patch("endpoints.default.bearer_security", new_callable=AsyncMock)
 @patch("endpoints.default.get_user_auth", new_callable=AsyncMock, return_value="test@example.com")
 @patch("endpoints.default.crud.get_user_groups", side_effect=Exception('mocked error'))
-def test_endpoint_home_with_groups_exception(m1, m2, m3, client_with_auth): # mocks call that triggers an internal error
+def test_endpoint_home_with_groups_exception(m1, m2, m3, client_with_auth):  # mocks call that triggers an internal error
     r = client_with_auth.get("/")
     assert r.status_code == 200
     j = r.json()
@@ -47,10 +48,12 @@ def test_endpoint_groups_403(client):
     assert r.status_code == 403
 
 
-def test_endpoint_groups_empty(client_with_auth):
+def test_endpoint_groups_rick_and_morty(client_with_auth):
     r = client_with_auth.get("/groups")
     assert r.status_code == 200
-    assert r.json() == []
+    assert len(j := r.json()) == 2
+    assert 'animated-characters' in j
+    assert 'spaceship' in j
 
 
 @patch("endpoints.default.crud.get_user_groups", return_value=["group1", "group2"])

From 8e1152ab339f506a388f574025a68e184099ddbd Mon Sep 17 00:00:00 2001
From: msramalho <19508417+msramalho@users.noreply.github.com>
Date: Sat, 19 Oct 2024 00:56:47 +0100
Subject: [PATCH 17/33] adds 403 tests

---
 src/tests/endpoints/test_default.py | 5 ++---
 src/tests/endpoints/test_task.py    | 5 ++++-
 src/tests/endpoints/test_url.py     | 1 -
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/tests/endpoints/test_default.py b/src/tests/endpoints/test_default.py
index cfa6fcf..4242a12 100644
--- a/src/tests/endpoints/test_default.py
+++ b/src/tests/endpoints/test_default.py
@@ -43,9 +43,8 @@ def test_endpoint_health(client_with_auth):
     assert r.json() == {"status": "ok"}
 
 
-def test_endpoint_groups_403(client):
-    r = client.get("/groups")
-    assert r.status_code == 403
+def test_endpoint_groups_no_auth(client, test_no_auth):
+    test_no_auth(client.get, "/groups")
 
 
 def test_endpoint_groups_rick_and_morty(client_with_auth):
diff --git a/src/tests/endpoints/test_task.py b/src/tests/endpoints/test_task.py
index 36285f5..9585c39 100644
--- a/src/tests/endpoints/test_task.py
+++ b/src/tests/endpoints/test_task.py
@@ -1,5 +1,8 @@
 from unittest.mock import patch
-from fastapi.testclient import TestClient
+
+
+def test_endpoint_task_status_no_auth(client, test_no_auth):
+    test_no_auth(client.get, "/task/test-task-id")
 
 
 @patch("endpoints.task.AsyncResult")
diff --git a/src/tests/endpoints/test_url.py b/src/tests/endpoints/test_url.py
index 8353287..e091bc0 100644
--- a/src/tests/endpoints/test_url.py
+++ b/src/tests/endpoints/test_url.py
@@ -1,7 +1,6 @@
 
 
 import json
-import time
 from unittest.mock import patch
 
 from db.schemas import ArchiveCreate, TaskResult

From 17950ecff707e29d8bc3581fb8b950a2e372c6f9 Mon Sep 17 00:00:00 2001
From: msramalho <19508417+msramalho@users.noreply.github.com>
Date: Sat, 19 Oct 2024 01:16:36 +0100
Subject: [PATCH 18/33] all endpoints tested

---
 src/endpoints/sheet.py            |  2 +-
 src/tests/endpoints/test_sheet.py | 46 +++++++++++++++++++++++++++++++
 2 files changed, 47 insertions(+), 1 deletion(-)
 create mode 100644 src/tests/endpoints/test_sheet.py

diff --git a/src/endpoints/sheet.py b/src/endpoints/sheet.py
index 4db344c..74cee62 100644
--- a/src/endpoints/sheet.py
+++ b/src/endpoints/sheet.py
@@ -20,4 +20,4 @@ def archive_sheet(sheet:schemas.SubmitSheet, email = Depends(get_token_or_user_a
     if not sheet.sheet_name and not sheet.sheet_id:
         raise HTTPException(status_code=422, detail=f"sheet name or id is required")
     task = create_sheet_task.delay(sheet.model_dump_json())
-    return JSONResponse({"id": task.id})
\ No newline at end of file
+    return JSONResponse({"id": task.id}, status_code=201)
\ No newline at end of file
diff --git a/src/tests/endpoints/test_sheet.py b/src/tests/endpoints/test_sheet.py
new file mode 100644
index 0000000..1c0838f
--- /dev/null
+++ b/src/tests/endpoints/test_sheet.py
@@ -0,0 +1,46 @@
+import json
+from unittest.mock import patch
+
+from db.schemas import TaskResult
+
+
+def test_sheet_no_auth(client, test_no_auth):
+    test_no_auth(client.post, "/sheet/archive")
+
+
+@patch("worker.create_sheet_task.delay", return_value=TaskResult(id="123-456-789", status="PENDING", result=""))
+def test_sheet_rick(m1, client_with_auth):
+
+    response = client_with_auth.post("/sheet/archive", json={"sheet_id": "123-sheet-id"})
+    assert response.status_code == 201
+    assert response.json() == {'id': '123-456-789'}
+
+    m1.assert_called_once()
+    called_val = m1.call_args.args[0]
+    assert json.loads(called_val) == {"sheet_id": "123-sheet-id", "sheet_name": None, "public": False, "author_id": "rick@example.com", "group_id": None, "tags": [], "columns": {}, "header": 1}
+
+
+def test_sheet_missing_sheet_data(client_with_auth):
+    r = client_with_auth.post("/sheet/archive", json={})
+    assert r.status_code == 422
+    assert r.json() == {"detail": "sheet name or id is required"}
+
+
+@patch("worker.create_sheet_task.delay", return_value=TaskResult(id="123-API-789", status="PENDING", result=""))
+def test_sheet_api(m1, client):
+
+    response = client.post("/sheet/archive", json={"sheet_name": "456-sheet_name-id"}, headers={"Authorization": "Bearer this_is_the_test_api_token"})
+    assert response.status_code == 201
+    assert response.json() == {'id': '123-API-789'}
+
+    m1.assert_called_once()
+    called_val = m1.call_args.args[0]
+    assert json.loads(called_val) == {"sheet_name": "456-sheet_name-id", "sheet_id": None, "public": False, "author_id": "api-endpoint", "group_id": None, "tags": [], "columns": {}, "header": 1}
+
+    response = client.post("/sheet/archive", json={"sheet_id": "456-sheet-id", "author_id": "custom-author"}, headers={"Authorization": "Bearer this_is_the_test_api_token"})
+    assert response.status_code == 201
+    assert response.json() == {'id': '123-API-789'}
+
+    assert m1.call_count == 2
+    called_val = m1.call_args.args[0]
+    assert json.loads(called_val) == {"sheet_id": "456-sheet-id", "sheet_name": None, "public": False, "author_id": "custom-author", "group_id": None, "tags": [], "columns": {}, "header": 1}

From 0486f0d9efedc068dc6818176692027ad92ee4f0 Mon Sep 17 00:00:00 2001
From: msramalho <19508417+msramalho@users.noreply.github.com>
Date: Sat, 19 Oct 2024 01:30:59 +0100
Subject: [PATCH 19/33] typo fix

---
 src/tests/endpoints/test_url.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/tests/endpoints/test_url.py b/src/tests/endpoints/test_url.py
index e091bc0..11ade9b 100644
--- a/src/tests/endpoints/test_url.py
+++ b/src/tests/endpoints/test_url.py
@@ -64,7 +64,7 @@ def test_search_by_url(client_with_auth, db_session):
     assert response.status_code == 200
     assert len(response.json()) == 0
 
-    response = client_with_auth.get("/url/search?url=https://example.com&archived_affter=2010-01-01")
+    response = client_with_auth.get("/url/search?url=https://example.com&archived_after=2010-01-01")
     assert response.status_code == 200
     assert len(response.json()) == 10
 

From 45d76e6db85bbe29254b3bcfe720013653091751 Mon Sep 17 00:00:00 2001
From: msramalho <19508417+msramalho@users.noreply.github.com>
Date: Sun, 20 Oct 2024 18:12:48 +0100
Subject: [PATCH 20/33] crud tests ready

---
 src/.env.test                          |   2 +-
 src/db/crud.py                         |  46 +--
 src/tests/conftest.py                  |   3 +-
 src/tests/db/test_crud.py              | 383 +++++++++++++++++++++++++
 src/tests/db/test_models.py            |   6 +
 src/tests/endpoints/test_url.py        |   2 +-
 src/tests/user-groups.test.broken.yaml |   6 +
 src/{ => tests}/user-groups.test.yaml  |   3 +-
 src/tests/web/test_main.py             |   1 -
 src/web/main.py                        |   2 -
 src/worker.py                          |   2 +-
 11 files changed, 427 insertions(+), 29 deletions(-)
 create mode 100644 src/tests/db/test_crud.py
 create mode 100644 src/tests/db/test_models.py
 create mode 100644 src/tests/user-groups.test.broken.yaml
 rename src/{ => tests}/user-groups.test.yaml (89%)

diff --git a/src/.env.test b/src/.env.test
index fdbff38..bef1d88 100644
--- a/src/.env.test
+++ b/src/.env.test
@@ -5,4 +5,4 @@ BLOCKED_EMAILS='["blocked@example.com"]'
 
 DATABASE_PATH="sqlite:///auto-archiver.test.db"
 API_BEARER_TOKEN=this_is_the_test_api_token
-USER_GROUPS_FILENAME=user-groups.test.yaml
\ No newline at end of file
+USER_GROUPS_FILENAME=tests/user-groups.test.yaml
\ No newline at end of file
diff --git a/src/db/crud.py b/src/db/crud.py
index 097e556..4a4c383 100644
--- a/src/db/crud.py
+++ b/src/db/crud.py
@@ -37,15 +37,15 @@ def search_archives_by_url(db: Session, url: str, email: str, skip: int = 0, lim
     else:
         query = query.filter(models.Archive.url.like(f'%{url}%'))
     if archived_after:
-        query = query.filter(models.Archive.created_at >= archived_after)
+        query = query.filter(models.Archive.created_at > archived_after)
     if archived_before:
-        query = query.filter(models.Archive.created_at <= archived_before)
+        query = query.filter(models.Archive.created_at < archived_before)
     return query.order_by(models.Archive.created_at.desc()).offset(skip).limit(min(limit, DATABASE_QUERY_LIMIT)).all()
 
 
 def search_archives_by_email(db: Session, email: str, skip: int = 0, limit: int = 100):
     email = email.lower()
-    return base_query(db).filter(models.Archive.author.has(email=email)).order_by(models.Archive.created_at.desc()).offset(skip).limit(min(limit, DATABASE_QUERY_LIMIT)).all()
+    return base_query(db).filter(models.Archive.author_id == email).order_by(models.Archive.created_at.desc()).offset(skip).limit(min(limit, DATABASE_QUERY_LIMIT)).all()
 
 
 def create_task(db: Session, task: schemas.ArchiveCreate, tags: list[models.Tag], urls: list[models.ArchiveUrl]):
@@ -66,21 +66,25 @@ def soft_delete_task(db: Session, task_id: str, email: str) -> bool:
         db.commit()
     return db_task is not None
 
-def count_archives(db:Session):
+
+def count_archives(db: Session):
     return db.query(func.count(models.Archive.id)).scalar()
 
-def count_archive_urls(db:Session):
+
+def count_archive_urls(db: Session):
     return db.query(func.count(models.ArchiveUrl.url)).scalar()
 
-def count_by_user_since(db:Session, seconds_delta: int = 15):
+
+def count_by_user_since(db: Session, seconds_delta: int = 15):
     time_threshold = datetime.now() - timedelta(seconds=seconds_delta)
-    return db.query(models.Archive.author_id,func.count().label('total'))\
+    return db.query(models.Archive.author_id, func.count().label('total'))\
         .filter(models.Archive.created_at >= time_threshold)\
         .group_by(models.Archive.author_id)\
         .order_by(func.count().desc()).limit(5 * DATABASE_QUERY_LIMIT).all()
 
+
 def base_query(db: Session):
-    # allow only some fields to be returned, for example author should remain hidden
+    # TODO: allow only some fields to be returned, for example author should remain hidden
     return db.query(models.Archive)\
         .options(load_only(models.Archive.id, models.Archive.created_at, models.Archive.url, models.Archive.result))\
         .filter(models.Archive.deleted == False)
@@ -98,10 +102,6 @@ def create_tag(db: Session, tag: str):
     return db_tag
 
 
-def search_tags(db: Session, tag: str, skip: int = 0, limit: int = 100):
-    return db.query(models.Tag).filter(models.Tag.url.like(f'%{tag}%')).offset(skip).limit(min(limit, DATABASE_QUERY_LIMIT)).all()
-
-
 def is_user_in_group(db: Session, group_name: str, email: str) -> models.Group:
     if email == ALLOW_ANY_EMAIL: return True
     return len(group_name) and len(email) and group_name in get_user_groups(db, email)
@@ -109,6 +109,7 @@ def is_user_in_group(db: Session, group_name: str, email: str) -> models.Group:
 
 def get_user_groups(db: Session, email: str):
     email = email.lower()
+    if "@" not in email: return []
     global DOMAIN_GROUPS, DOMAIN_GROUPS_LOADED
     if not DOMAIN_GROUPS_LOADED: upsert_user_groups(db)
     # given an email retrieves the user groups from the DB and then the email-domain groups from a global variable
@@ -122,8 +123,8 @@ def get_user_groups(db: Session, email: str):
 # --------------- INIT User-Groups
 
 
-def get_user(db: Session, author_id: str):
-    if type(author_id)==str: author_id = author_id.lower()
+def create_or_get_user(db: Session, author_id: str):
+    if type(author_id) == str: author_id = author_id.lower()
     db_user = db.query(models.User).filter(models.User.email == author_id).first()
     if not db_user:
         db_user = models.User(email=author_id)
@@ -134,11 +135,13 @@ def get_user(db: Session, author_id: str):
 
 
 @cache
-def get_group(db: Session, group_name: str) -> models.Group:
+def create_or_get_group(db: Session, group_name: str) -> models.Group:
     db_group = db.query(models.Group).filter(models.Group.id == group_name).first()
     if db_group is None:
         db_group = models.Group(id=group_name)
         db.add(db_group)
+        db.commit()
+        db.refresh(db_group)
     return db_group
 
 
@@ -152,12 +155,13 @@ def upsert_user_groups(db: Session):
     filename = Settings().USER_GROUPS_FILENAME
 
     # read yaml safely
-    with open(filename) as inf:
-        try:
+    try:
+        with open(filename) as inf:
             user_groups_yaml = yaml.safe_load(inf)
-        except yaml.YAMLError as e:
-            logger.error(f"could not open user groups filename {filename}: {e}")
-            raise e
+            logger.error(user_groups_yaml)
+    except Exception as e:
+        logger.error(f"could not open user groups filename {filename}: {e}")
+        raise e
     # updating domain->groups access
     DOMAIN_GROUPS = user_groups_yaml.get("domains", {})
 
@@ -176,7 +180,7 @@ def upsert_user_groups(db: Session):
             db.add(db_user)
         if not groups: continue  # avoid hanging in for x in None:
         for group in groups:
-            db_group = get_group(db, group)
+            db_group = create_or_get_group(db, group)
             db_group.users.append(db_user)
 
     db.commit()
diff --git a/src/tests/conftest.py b/src/tests/conftest.py
index d21fc41..bfe52cb 100644
--- a/src/tests/conftest.py
+++ b/src/tests/conftest.py
@@ -16,7 +16,6 @@ def mock_logger_add():
 def settings():
     return Settings(_env_file=".env.test")
 
-
 @pytest.fixture(autouse=True)
 def mock_settings():
     with patch('shared.settings.Settings', return_value=Settings(_env_file=".env.test")) as mock_settings:
@@ -58,7 +57,9 @@ def db_session(test_db):
 @pytest.fixture()
 def app(db_session):
     from web.main import app_factory
+    from db import crud
     app = app_factory()
+    crud.upsert_user_groups(db_session)
     return app
 
 
diff --git a/src/tests/db/test_crud.py b/src/tests/db/test_crud.py
new file mode 100644
index 0000000..7603191
--- /dev/null
+++ b/src/tests/db/test_crud.py
@@ -0,0 +1,383 @@
+from datetime import datetime
+from unittest.mock import patch
+
+import pytest
+import yaml
+from db import models
+from shared.settings import Settings
+
+authors = ["rick@example.com", "morty@example.com", "jerry@example.com"]
+
+
+@pytest.fixture()
+def test_data(db_session):
+
+    # creates 3 users
+    for email in authors:
+        db_session.add(models.User(email=email))
+    db_session.commit()
+    assert db_session.query(models.User).count() == 3
+
+    # creates 100 archives for 3 users over 2 months with repeating URLs
+    for i in range(100):
+        author = authors[i % 3]
+        archive = models.Archive(
+            id=f"archive-id-456-{i}",
+            url=f"https://example-{i%3}.com",
+            result={},
+            public=author == "jerry@example.com",
+            author_id=author,
+            group_id="spaceship" if author == "morty@example.com" and i % 2 == 0 else None,
+            created_at=datetime(2021, (i % 2) + 1, (i % 25) + 1)
+        )
+        if i % 5 == 0:
+            archive.tags.append(models.Tag(id=f"tag-{i}"))
+        if i % 10 == 0:
+            archive.tags.append(models.Tag(id=f"tag-second-{i}"))
+        if i % 4 == 0:
+            archive.tags.append(models.Tag(id=f"tag-third-{i}"))
+        for j in range(10):
+            archive.urls.append(models.ArchiveUrl(url=f"https://example-{i}.com/{j}", key=f"media_{j}"))
+        db_session.add(archive)
+
+    db_session.commit()
+
+    assert db_session.query(models.Archive).count() == 100
+    assert db_session.query(models.Tag).count() == 20 + 10 + 25
+    assert db_session.query(models.ArchiveUrl).count() == 1000
+    assert db_session.query(models.ArchiveUrl).filter(models.ArchiveUrl.archive_id == "archive-id-456-0").count() == 10
+
+    # setup groups
+    assert db_session.query(models.Group).count() == 0
+    from db import crud
+    crud.upsert_user_groups(db_session)
+    assert db_session.query(models.Group).count() == 3
+    assert db_session.query(models.User).count() == 4
+
+
+def test_get_archive(test_data, db_session):
+    from db import crud
+    from security import ALLOW_ANY_EMAIL
+
+    print(db_session.query(models.Group).all())
+
+    # each author's archives work
+    assert (a0 := crud.get_archive(db_session, "archive-id-456-0", authors[0])) is not None
+    assert a0.id == "archive-id-456-0"
+    assert a0.url == "https://example-0.com"
+    assert a0.author_id == authors[0]
+    assert a0.public == False
+
+    assert crud.get_archive(db_session, "archive-id-456-1", authors[1]) is not None
+    assert crud.get_archive(db_session, "archive-id-456-2", authors[2]) is not None
+
+    # ALLOW_ANY_EMAIL
+    assert crud.get_archive(db_session, "archive-id-456-0", ALLOW_ANY_EMAIL) is not None
+    assert crud.get_archive(db_session, "archive-id-456-1", ALLOW_ANY_EMAIL) is not None
+
+    # not found
+    assert crud.get_archive(db_session, "archive-missing", authors[0]) is None
+
+    # public
+    assert (a_public := crud.get_archive(db_session, "archive-id-456-2", authors[0])) is not None
+    assert a_public.public == True
+
+    # not public - rick's
+    assert crud.get_archive(db_session, "archive-id-456-0", authors[1]) is None
+
+
+def test_search_archives_by_url(test_data, db_session):
+    from db import crud
+    from security import ALLOW_ANY_EMAIL
+
+    # rick's archives are private
+    assert len(crud.search_archives_by_url(db_session, "https://example-0.com", "rick@example.com")) == 34
+    assert len(crud.search_archives_by_url(db_session, "https://example-0.com", ALLOW_ANY_EMAIL)) == 34
+    assert len(crud.search_archives_by_url(db_session, "https://example-0.com", "morty@example.com")) == 0
+
+    # morty's archives are public but half are in spaceship group
+    assert len(crud.search_archives_by_url(db_session, "https://example-1.com", "rick@example.com")) == 16
+
+    # jerry's archives are public
+    assert len(crud.search_archives_by_url(db_session, "https://example-2.com", "jerry@example.com")) == 33
+    assert len(crud.search_archives_by_url(db_session, "https://example-2.com", "rick@example.com")) == 33
+
+    # fuzzy search
+    assert len(crud.search_archives_by_url(db_session, "https://example", ALLOW_ANY_EMAIL)) == 100
+    assert len(crud.search_archives_by_url(db_session, "https://EXAMPLE", ALLOW_ANY_EMAIL)) == 100
+    assert len(crud.search_archives_by_url(db_session, "2.com", ALLOW_ANY_EMAIL)) == 33
+
+    # absolute search
+    assert len(crud.search_archives_by_url(db_session, "example-2.com", ALLOW_ANY_EMAIL, absolute_search=True)) == 0
+    assert len(crud.search_archives_by_url(db_session, "https://example-2.com", ALLOW_ANY_EMAIL, absolute_search=True)) == 33
+
+    # archived_after
+    assert len(crud.search_archives_by_url(db_session, "https://example", ALLOW_ANY_EMAIL, archived_after=datetime(2010, 1, 1))) == 100
+    assert len(crud.search_archives_by_url(db_session, "https://example", ALLOW_ANY_EMAIL, archived_after=datetime(2021, 1, 15))) == 70
+    assert len(crud.search_archives_by_url(db_session, "https://example", ALLOW_ANY_EMAIL, archived_after=datetime(2031, 1, 1))) == 0
+
+    # archived before
+    assert len(crud.search_archives_by_url(db_session, "https://example", ALLOW_ANY_EMAIL, archived_before=datetime(2010, 1, 1))) == 0
+    assert len(crud.search_archives_by_url(db_session, "https://example", ALLOW_ANY_EMAIL, archived_before=datetime(2021, 1, 15))) == 28
+    assert len(crud.search_archives_by_url(db_session, "https://example", ALLOW_ANY_EMAIL, archived_before=datetime(2031, 1, 1))) == 100
+
+    # archived before and after
+    assert len(crud.search_archives_by_url(db_session, "https://example", ALLOW_ANY_EMAIL, archived_after=datetime(2001, 1, 1), archived_before=datetime(2031, 1, 11))) == 100
+    assert len(crud.search_archives_by_url(db_session, "https://example", ALLOW_ANY_EMAIL, archived_after=datetime(2021, 1, 14), archived_before=datetime(2021, 1, 16))) == 2
+
+    # limit
+    assert len(crud.search_archives_by_url(db_session, "https://example", ALLOW_ANY_EMAIL, limit=10)) == 10
+
+    # skip
+    assert len(crud.search_archives_by_url(db_session, "https://example", ALLOW_ANY_EMAIL, skip=10)) == 90
+
+
+def test_search_archives_by_email(test_data, db_session):
+    from security import ALLOW_ANY_EMAIL
+    from db import crud
+    from security import ALLOW_ANY_EMAIL
+
+    # lower/upper case
+    assert len(crud.search_archives_by_email(db_session, "rick@example.com")) == 34
+    assert len(crud.search_archives_by_email(db_session, "RICK@example.com")) == 34
+
+    # ALLOW_ANY_EMAIL is not a user
+    assert len(crud.search_archives_by_email(db_session, ALLOW_ANY_EMAIL)) == 0
+
+    # most recent first
+    a1 = crud.search_archives_by_email(db_session, "rick@example.com", limit=1)
+    assert len(a1) == 1
+    assert a1[0].created_at == datetime(2021, 2, 25)
+
+    # earliest is the last
+    a2 = crud.search_archives_by_email(db_session, "rick@example.com", skip=33)
+    assert len(a2) == 1
+    assert a2[0].created_at == datetime(2021, 1, 1)
+
+
+@patch("db.crud.DATABASE_QUERY_LIMIT", new=25)
+def test_max_query_limit(test_data, db_session):
+    from db import crud
+    from security import ALLOW_ANY_EMAIL
+
+    assert len(crud.search_archives_by_url(db_session, "https://example", ALLOW_ANY_EMAIL)) == 25
+    assert len(crud.search_archives_by_url(db_session, "https://example", ALLOW_ANY_EMAIL, limit=1000)) == 25
+
+    assert len(crud.search_archives_by_email(db_session, "rick@example.com")) == 25
+    assert len(crud.search_archives_by_email(db_session, "rick@example.com", limit=1000)) == 25
+
+
+def test_create_task(db_session):
+    from db import crud
+    from db import schemas
+
+    task = schemas.ArchiveCreate(
+        id="archive-id-456-101",
+        url="https://example-0.com",
+        result={},
+        public=False,
+        author_id="rick@example.com",
+        group_id="spaceship",
+        tags=[],
+        urls=[]
+    )
+
+    # with tags and urls
+    nt = crud.create_task(db_session, task, [models.Tag(id="tag-101")], [models.ArchiveUrl(url="https://example-0.com/0", key="media_0")])
+
+    assert nt is not None
+    assert nt.id == "archive-id-456-101"
+    assert nt.url == "https://example-0.com"
+    assert nt.author_id == "rick@example.com"
+    assert nt.public == False
+    assert nt.group_id == "spaceship"
+    assert len(nt.tags) == 1
+    assert nt.tags[0].id == "tag-101"
+    assert len(nt.urls) == 1
+    assert nt.urls[0].url == "https://example-0.com/0"
+    assert nt.urls[0].key == "media_0"
+    assert nt.created_at is not None
+
+    # without tags and urls
+    task.id = "archive-id-456-102"
+    nt = crud.create_task(db_session, task, [], [])
+    assert nt is not None
+    assert nt.id == "archive-id-456-102"
+    assert nt.url == "https://example-0.com"
+    assert nt.author_id == "rick@example.com"
+    assert nt.public == False
+    assert nt.group_id == "spaceship"
+    assert len(nt.tags) == 0
+    assert len(nt.urls) == 0
+    assert nt.created_at is not None
+
+
+def test_soft_delete(test_data, db_session):
+    from db import crud
+
+    # none deleted yet
+    assert crud.get_archive(db_session, "archive-id-456-0", "rick@example.com") is not None
+    assert db_session.query(models.Archive).filter(models.Archive.deleted == True).count() == 0
+
+    # delete
+    assert crud.soft_delete_task(db_session, "archive-id-456-0", "rick@example.com") == True
+
+    # ensure soft delete
+    assert db_session.query(models.Archive).filter(models.Archive.deleted == True).count() == 1
+    assert crud.get_archive(db_session, "archive-id-456-0", "rick@example.com") is None
+
+    # already deleted
+    assert crud.soft_delete_task(db_session, "archive-id-456-0", "rick@example.com") == False
+
+
+def test_count_archives(test_data, db_session):
+    from db import crud
+
+    assert crud.count_archives(db_session) == 100
+    db_session.query(models.Archive).filter(models.Archive.id == "archive-id-456-0").delete()
+    db_session.commit()
+    assert crud.count_archives(db_session) == 99
+
+
+def test_count_archive_urls(test_data, db_session):
+    from db import crud
+
+    assert crud.count_archive_urls(db_session) == 1000
+    db_session.query(models.ArchiveUrl).filter(models.ArchiveUrl.url == "https://example-0.com/0").delete()
+    db_session.commit()
+    assert crud.count_archive_urls(db_session) == 999
+
+    db_session.query(models.Archive).filter(models.Archive.id == "archive-id-456-0").delete()
+    db_session.commit()
+    # no Cascade is enabled
+    assert crud.count_archives(db_session) == 99
+    assert crud.count_archive_urls(db_session) == 999
+
+
+def test_count_by_users_since(test_data, db_session):
+    from db import crud
+
+    # 100y window
+    assert len(cu := crud.count_by_user_since(db_session, 60 * 60 * 24 * 31 * 12 * 100)) == 3
+    assert cu[0].total == 34
+    assert cu[1].total == 33
+    assert cu[2].total == 33
+
+
+def test_create_tag(db_session):
+    from db import crud
+
+    assert db_session.query(models.Tag).count() == 0
+
+    # create first
+    create_tag = crud.create_tag(db_session, "tag-101")
+    assert create_tag is not None
+    assert create_tag.id == "tag-101"
+    assert db_session.query(models.Tag).count() == 1
+    assert db_session.query(models.Tag).filter(models.Tag.id == "tag-101").first() == create_tag
+
+    # same id does not add new db entry
+    existing_tag = crud.create_tag(db_session, "tag-101")
+    assert existing_tag == create_tag
+    assert db_session.query(models.Tag).count() == 1
+
+    # create second
+    second_tag = crud.create_tag(db_session, "tag-102")
+    assert second_tag is not None
+    assert second_tag.id == "tag-102"
+    assert db_session.query(models.Tag).count() == 2
+
+
+def test_is_user_in_group(test_data, db_session):
+    from db import crud
+    from security import ALLOW_ANY_EMAIL
+
+    # see user-groups.test.yaml
+    test_pairs = [
+        (ALLOW_ANY_EMAIL, "spaceship", True),
+        (ALLOW_ANY_EMAIL, "non-existant!@#!%!", True),
+
+        ("rick@example.com", "spaceship", True),
+        ("rick@example.com", "SPACESHIP", False),
+        ("RICK@example.com", "interdimensional", True),
+        ("rick@example.com", "the-jerrys-club", False),
+
+        ("morty@example.com", "spaceship", True),
+        ("morty@example.com", "interdimensional", False),
+        ("morty@example.com", "the-jerrys-club", False),
+
+        ("jerry@example.com", "spaceship", False),
+        ("jerry@example.com", "interdimensional", False),
+        ("jerry@example.com", "the-jerrys-club", True),
+
+        ("rick@example.com", "animated-characters", True),
+        ("morty@example.com", "animated-characters", True),
+        ("jerry@example.com", "animated-characters", True),
+
+        ("rick@example.com", "", False),
+        ("", "spaceship", False),
+        ("BADEMAILexample.com", "spaceship", False),
+    ]
+    for email, group, expected in test_pairs:
+        assert crud.is_user_in_group(db_session, group, email) == expected
+
+
+def test_create_or_get_user(test_data, db_session):
+    from db import crud
+
+    assert db_session.query(models.User).count() == 4
+
+    assert (u1 := crud.create_or_get_user(db_session, "rick@example.com")) is not None
+    assert u1.email == "rick@example.com"
+    assert u1.is_active == True
+
+    assert (u2 := crud.create_or_get_user(db_session, "beth@example.com")) is not None
+    assert u2.email == "beth@example.com"
+    assert u2.is_active == True
+
+    assert db_session.query(models.User).count() == 5
+
+
+def test_get_group(test_data, db_session):
+    from db import crud
+
+    assert db_session.query(models.Group).count() == 3
+
+    assert (g1 := crud.create_or_get_group(db_session, "spaceship")) is not None
+    assert g1.id == "spaceship"
+    assert len(g1.users) == 2
+    assert [u.email for u in g1.users] == ["rick@example.com", "morty@example.com"]
+
+    assert (g2 := crud.create_or_get_group(db_session, "the-jerrys-club")) is not None
+    assert g2.id == "the-jerrys-club"
+    assert len(g2.users) == 1
+    assert g2.users[0].email == "jerry@example.com"
+
+    assert (g3 := crud.create_or_get_group(db_session, "this-is-a-new-group")) is not None
+    assert g3.id == "this-is-a-new-group"
+    assert len(g3.users) == 0
+
+    assert db_session.query(models.Group).count() == 4
+
+
+def test_upsert_user_groups(db_session):
+    from db import crud
+
+    @patch('db.crud.Settings', new = lambda: bad_setings)
+    def test_missing_yaml(db_session):
+        with pytest.raises(FileNotFoundError):
+            crud.upsert_user_groups(db_session)
+
+
+    @patch('db.crud.Settings', new = lambda: bad_setings)
+    def test_broken_yaml(db_session):
+        with pytest.raises(yaml.YAMLError):
+            crud.upsert_user_groups(db_session)
+
+    bad_setings = Settings(_env_file=".env.test")
+
+    bad_setings.USER_GROUPS_FILENAME = "tests/user-groups.test.missing.yaml"
+    test_missing_yaml(db_session)
+
+    bad_setings.USER_GROUPS_FILENAME = "tests/user-groups.test.broken.yaml"
+    test_broken_yaml(db_session)
\ No newline at end of file
diff --git a/src/tests/db/test_models.py b/src/tests/db/test_models.py
new file mode 100644
index 0000000..d5ced1e
--- /dev/null
+++ b/src/tests/db/test_models.py
@@ -0,0 +1,6 @@
+def test_generate_uuid():
+	from db.models import generate_uuid
+
+	assert generate_uuid() != generate_uuid()
+	assert len(generate_uuid()) == 36
+	assert generate_uuid().count("-") == 4
\ No newline at end of file
diff --git a/src/tests/endpoints/test_url.py b/src/tests/endpoints/test_url.py
index 11ade9b..b128493 100644
--- a/src/tests/endpoints/test_url.py
+++ b/src/tests/endpoints/test_url.py
@@ -84,7 +84,7 @@ def test_latest(client_with_auth, db_session):
         #NB: this insertion is too fast for the ordering to be correct as they are within the same second
 
     # user must exist for /latest to work
-    crud.get_user(db_session, "morty@example.com")
+    crud.create_or_get_user(db_session, "morty@example.com")
 
     response = client_with_auth.get("/url/latest")
     assert response.status_code == 200
diff --git a/src/tests/user-groups.test.broken.yaml b/src/tests/user-groups.test.broken.yaml
new file mode 100644
index 0000000..8bc59c5
--- /dev/null
+++ b/src/tests/user-groups.test.broken.yaml
@@ -0,0 +1,6 @@
+broken: True
+This is just an invalid yaml for testing
+
+still broken: True
+  - one
+  - two
\ No newline at end of file
diff --git a/src/user-groups.test.yaml b/src/tests/user-groups.test.yaml
similarity index 89%
rename from src/user-groups.test.yaml
rename to src/tests/user-groups.test.yaml
index 97225db..7c5cfcc 100644
--- a/src/user-groups.test.yaml
+++ b/src/tests/user-groups.test.yaml
@@ -6,12 +6,13 @@ users:
   morty@example.com:
     - spaceship
   jerry@example.com:
+    - the-jerrys-club
+  birdman@example.com:
 
 domains:
   example.com:
     - animated-characters
 
-
 orchestrators:
   spaceship: secrets/orchestration-dev.yaml
   interdimensional: secrets/orchestration-dev.yaml
diff --git a/src/tests/web/test_main.py b/src/tests/web/test_main.py
index 333e0e6..a8c184f 100644
--- a/src/tests/web/test_main.py
+++ b/src/tests/web/test_main.py
@@ -1,5 +1,4 @@
 import os
-from unittest.mock import patch
 from fastapi.testclient import TestClient
 from shared.settings import Settings
 
diff --git a/src/web/main.py b/src/web/main.py
index babfb3a..51fbb58 100644
--- a/src/web/main.py
+++ b/src/web/main.py
@@ -27,8 +27,6 @@ from endpoints import default_router, url_router, sheet_router, task_router, int
 
 
 def app_factory(settings = Settings()):
-    
-
     app = FastAPI(
         title="Auto-Archiver API",
         description=API_DESCRIPTION,
diff --git a/src/worker.py b/src/worker.py
index 513ebf1..68d8aee 100644
--- a/src/worker.py
+++ b/src/worker.py
@@ -168,7 +168,7 @@ def insert_result_into_db(result: Metadata, tags: Set[str], public: bool, group_
     with get_db() as session:
         # urls are created by get_all_urls
         # create author_id if needed
-        crud.get_user(session, author_id)
+        crud.create_or_get_user(session, author_id)
         # create DB TAGs if needed
         db_tags = [crud.create_tag(session, tag) for tag in tags]
         # insert archive

From d2f9f91a5c5f655e98ff04a8fd73281638b392c8 Mon Sep 17 00:00:00 2001
From: msramalho <19508417+msramalho@users.noreply.github.com>
Date: Sun, 20 Oct 2024 19:14:13 +0100
Subject: [PATCH 21/33] relocating security file

---
 src/db/crud.py                      |  2 +-
 src/endpoints/default.py            |  2 +-
 src/endpoints/interoperability.py   |  2 +-
 src/endpoints/sheet.py              |  2 +-
 src/endpoints/task.py               |  2 +-
 src/endpoints/url.py                |  2 +-
 src/tests/conftest.py               |  2 +-
 src/tests/db/test_crud.py           | 12 ++++++------
 src/tests/endpoints/test_default.py |  2 +-
 src/web/main.py                     |  2 +-
 src/{ => web}/security.py           |  0
 11 files changed, 15 insertions(+), 15 deletions(-)
 rename src/{ => web}/security.py (100%)

diff --git a/src/db/crud.py b/src/db/crud.py
index 4a4c383..b251e6d 100644
--- a/src/db/crud.py
+++ b/src/db/crud.py
@@ -4,7 +4,7 @@ from sqlalchemy import Column, or_, func
 from loguru import logger
 from datetime import datetime, timedelta
 
-from security import ALLOW_ANY_EMAIL
+from web.security import ALLOW_ANY_EMAIL
 from shared.settings import Settings
 from . import models, schemas
 import yaml
diff --git a/src/endpoints/default.py b/src/endpoints/default.py
index 7a6a9ea..da2fc97 100644
--- a/src/endpoints/default.py
+++ b/src/endpoints/default.py
@@ -7,7 +7,7 @@ from sqlalchemy.orm import Session
 from core.config import VERSION, BREAKING_CHANGES
 from db import crud
 from db.database import get_db_dependency, get_db
-from security import get_user_auth, bearer_security
+from web.security import get_user_auth, bearer_security
 
 default_router = APIRouter()
 
diff --git a/src/endpoints/interoperability.py b/src/endpoints/interoperability.py
index 7885b08..d1bcfa4 100644
--- a/src/endpoints/interoperability.py
+++ b/src/endpoints/interoperability.py
@@ -4,7 +4,7 @@ from auto_archiver import Metadata
 from loguru import logger
 import sqlalchemy
 
-from security import token_api_key_auth
+from web.security import token_api_key_auth
 from db import models, schemas
 from worker import insert_result_into_db
 
diff --git a/src/endpoints/sheet.py b/src/endpoints/sheet.py
index 74cee62..5c6a6dd 100644
--- a/src/endpoints/sheet.py
+++ b/src/endpoints/sheet.py
@@ -4,7 +4,7 @@ from fastapi.responses import JSONResponse
 
 from loguru import logger
 
-from security import ALLOW_ANY_EMAIL, get_token_or_user_auth
+from web.security import ALLOW_ANY_EMAIL, get_token_or_user_auth
 from db import schemas
 from worker import create_sheet_task
 
diff --git a/src/endpoints/task.py b/src/endpoints/task.py
index a6b09b4..7544aca 100644
--- a/src/endpoints/task.py
+++ b/src/endpoints/task.py
@@ -5,7 +5,7 @@ from fastapi.encoders import jsonable_encoder
 from fastapi.responses import JSONResponse
 
 from loguru import logger
-from security import get_token_or_user_auth
+from web.security import get_token_or_user_auth
 
 from db import schemas
 
diff --git a/src/endpoints/url.py b/src/endpoints/url.py
index 0771ed6..9a59b77 100644
--- a/src/endpoints/url.py
+++ b/src/endpoints/url.py
@@ -4,7 +4,7 @@ from fastapi.responses import JSONResponse
 from datetime import datetime
 
 from loguru import logger
-from security import get_user_auth, get_token_or_user_auth
+from web.security import get_user_auth, get_token_or_user_auth
 from sqlalchemy.orm import Session
 
 from db import crud, schemas
diff --git a/src/tests/conftest.py b/src/tests/conftest.py
index bfe52cb..407616d 100644
--- a/src/tests/conftest.py
+++ b/src/tests/conftest.py
@@ -71,7 +71,7 @@ def client(app):
 
 @pytest.fixture()
 def app_with_auth(app):
-    from security import get_token_or_user_auth, get_user_auth, token_api_key_auth
+    from web.security import get_token_or_user_auth, get_user_auth, token_api_key_auth
     app.dependency_overrides[get_token_or_user_auth] = lambda: "rick@example.com"
     app.dependency_overrides[get_user_auth] = lambda: "morty@example.com"
     app.dependency_overrides[token_api_key_auth] = lambda: "jerry@example.com"
diff --git a/src/tests/db/test_crud.py b/src/tests/db/test_crud.py
index 7603191..a4fdeff 100644
--- a/src/tests/db/test_crud.py
+++ b/src/tests/db/test_crud.py
@@ -57,7 +57,7 @@ def test_data(db_session):
 
 def test_get_archive(test_data, db_session):
     from db import crud
-    from security import ALLOW_ANY_EMAIL
+    from web.security import ALLOW_ANY_EMAIL
 
     print(db_session.query(models.Group).all())
 
@@ -88,7 +88,7 @@ def test_get_archive(test_data, db_session):
 
 def test_search_archives_by_url(test_data, db_session):
     from db import crud
-    from security import ALLOW_ANY_EMAIL
+    from web.security import ALLOW_ANY_EMAIL
 
     # rick's archives are private
     assert len(crud.search_archives_by_url(db_session, "https://example-0.com", "rick@example.com")) == 34
@@ -133,9 +133,9 @@ def test_search_archives_by_url(test_data, db_session):
 
 
 def test_search_archives_by_email(test_data, db_session):
-    from security import ALLOW_ANY_EMAIL
+    from web.security import ALLOW_ANY_EMAIL
     from db import crud
-    from security import ALLOW_ANY_EMAIL
+    from web.security import ALLOW_ANY_EMAIL
 
     # lower/upper case
     assert len(crud.search_archives_by_email(db_session, "rick@example.com")) == 34
@@ -158,7 +158,7 @@ def test_search_archives_by_email(test_data, db_session):
 @patch("db.crud.DATABASE_QUERY_LIMIT", new=25)
 def test_max_query_limit(test_data, db_session):
     from db import crud
-    from security import ALLOW_ANY_EMAIL
+    from web.security import ALLOW_ANY_EMAIL
 
     assert len(crud.search_archives_by_url(db_session, "https://example", ALLOW_ANY_EMAIL)) == 25
     assert len(crud.search_archives_by_url(db_session, "https://example", ALLOW_ANY_EMAIL, limit=1000)) == 25
@@ -290,7 +290,7 @@ def test_create_tag(db_session):
 
 def test_is_user_in_group(test_data, db_session):
     from db import crud
-    from security import ALLOW_ANY_EMAIL
+    from web.security import ALLOW_ANY_EMAIL
 
     # see user-groups.test.yaml
     test_pairs = [
diff --git a/src/tests/endpoints/test_default.py b/src/tests/endpoints/test_default.py
index 4242a12..c9a4149 100644
--- a/src/tests/endpoints/test_default.py
+++ b/src/tests/endpoints/test_default.py
@@ -57,7 +57,7 @@ def test_endpoint_groups_rick_and_morty(client_with_auth):
 
 @patch("endpoints.default.crud.get_user_groups", return_value=["group1", "group2"])
 def test_endpoint_groups(m1, app):
-    from security import get_user_auth
+    from web.security import get_user_auth
     app.dependency_overrides[get_user_auth] = lambda: True
     client = TestClient(app)
 
diff --git a/src/web/main.py b/src/web/main.py
index 51fbb58..be125c4 100644
--- a/src/web/main.py
+++ b/src/web/main.py
@@ -15,7 +15,7 @@ from core.logging import logging_middleware
 from worker import create_archive_task, create_sheet_task, celery, insert_result_into_db
 
 from db import crud, models, schemas
-from security import get_user_auth, token_api_key_auth, get_token_or_user_auth
+from web.security import get_user_auth, token_api_key_auth, get_token_or_user_auth
 from core.config import VERSION, API_DESCRIPTION
 from db.database import get_db_dependency
 from core.events import lifespan
diff --git a/src/security.py b/src/web/security.py
similarity index 100%
rename from src/security.py
rename to src/web/security.py

From 84cbf720a9863db3948236e80da1cf8ec566291e Mon Sep 17 00:00:00 2001
From: msramalho <19508417+msramalho@users.noreply.github.com>
Date: Sun, 20 Oct 2024 21:30:00 +0100
Subject: [PATCH 22/33] refactors settings and adds security tests

---
 .github/workflows/ci.yml       |   2 +-
 src/Pipfile                    |   1 +
 src/Pipfile.lock               |  65 ++++++++++++++++++---
 src/core/events.py             |  10 ++--
 src/db/crud.py                 |   6 +-
 src/db/database.py             |   8 +--
 src/migrations/env.py          |   4 +-
 src/shared/settings.py         |   5 ++
 src/tests/conftest.py          |   8 +--
 src/tests/db/test_crud.py      |   5 +-
 src/tests/web/test_main.py     |   8 +--
 src/tests/web/test_security.py | 104 +++++++++++++++++++++++++++++++++
 src/web/main.py                |   4 +-
 src/web/security.py            |  16 ++---
 src/worker.py                  |   4 +-
 15 files changed, 203 insertions(+), 47 deletions(-)
 create mode 100644 src/tests/web/test_security.py

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index bc3c029..bf9ca3b 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -37,7 +37,7 @@ jobs:
       working-directory: src
 
     - name: Run tests with coverage
-      run: PYTHONPATH=. pipenv run coverage run -m pytest -v --color=yes tests/
+      run: PYTHONPATH=. PIPENV_DOTENV_LOCATION=.env.test pipenv run coverage run -m pytest -v --color=yes tests/
       working-directory: src
 
     - name: Report coverage
diff --git a/src/Pipfile b/src/Pipfile
index 4423b6f..dabe443 100644
--- a/src/Pipfile
+++ b/src/Pipfile
@@ -26,6 +26,7 @@ watchdog = "*"
 pytest = "*"
 httpx = "*"
 coverage = "*"
+pytest-asyncio = "*"
 
 [requires]
 python_version = "3.10"
diff --git a/src/Pipfile.lock b/src/Pipfile.lock
index 6e6c0f2..840330c 100644
--- a/src/Pipfile.lock
+++ b/src/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "c34b5745f3a6f67222d3f26e6c7f2d13615a3301d0ca4d1f2b0ec58474b1d43a"
+            "sha256": "da25332a2152541157c6873ec43ac771c5491bff7d60bb2714c26c4e6b40577f"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -289,25 +289,26 @@
         },
         "boto3": {
             "hashes": [
-                "sha256:a5b00f8b82dce62870759f04861747944da834d64a64355970120c475efdafc0",
-                "sha256:e1f36f8be453505cebcc3da178ea081b2a06c0e5e1cdee774f1067599b8d9c3e"
+                "sha256:18416d07b41e6094101a44f8b881047dcec6b846dad0b9f83b9bbf2f0cd93d07",
+                "sha256:7f8e8a252458d584d8cf7877c372c4f74ec103356eedf43d2dd9e479f47f3639"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==1.35.42"
+            "version": "==1.35.44"
         },
         "botocore": {
             "hashes": [
-                "sha256:05af0bb8b9cea7ce7bc589c332348d338a21b784e9d088a588fd10ec145007ff",
-                "sha256:af348636f73dc24b7e2dc760a34d08c8f2f94366e9b4c78d877307b128abecef"
+                "sha256:1fcd97b966ad8a88de4106fe1bd3bbd6d8dadabe99bbd4a6aadcf11cb6c66b39",
+                "sha256:55388e80624401d017a9a2b8109afd94814f7e666b53e28fce51375cfa8d9326"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==1.35.42"
+            "version": "==1.35.44"
         },
         "brotli": {
             "hashes": [
                 "sha256:03d20af184290887bdea3f0f78c4f737d126c74dc2f3ccadf07e54ceca3bf208",
                 "sha256:0541e747cce78e24ea12d69176f6a7ddb690e62c425e01d31cc065e69ce55b48",
                 "sha256:069a121ac97412d1fe506da790b3e69f52254b9df4eb665cd42460c837193354",
+                "sha256:0737ddb3068957cf1b054899b0883830bb1fec522ec76b1098f9b6e0f02d9419",
                 "sha256:0b63b949ff929fbc2d6d3ce0e924c9b93c9785d877a21a1b678877ffbbc4423a",
                 "sha256:0c6244521dda65ea562d5a69b9a26120769b7a9fb3db2fe9545935ed6735b128",
                 "sha256:11d00ed0a83fa22d29bc6b64ef636c4552ebafcef57154b4ddd132f5638fbd1c",
@@ -315,43 +316,67 @@
                 "sha256:19c116e796420b0cee3da1ccec3b764ed2952ccfcc298b55a10e5610ad7885f9",
                 "sha256:1ab4fbee0b2d9098c74f3057b2bc055a8bd92ccf02f65944a241b4349229185a",
                 "sha256:1ae56aca0402a0f9a3431cddda62ad71666ca9d4dc3a10a142b9dce2e3c0cda3",
+                "sha256:1b2c248cd517c222d89e74669a4adfa5577e06ab68771a529060cf5a156e9757",
+                "sha256:1e9a65b5736232e7a7f91ff3d02277f11d339bf34099a56cdab6a8b3410a02b2",
                 "sha256:224e57f6eac61cc449f498cc5f0e1725ba2071a3d4f48d5d9dffba42db196438",
                 "sha256:22fc2a8549ffe699bfba2256ab2ed0421a7b8fadff114a3d201794e45a9ff578",
                 "sha256:23032ae55523cc7bccb4f6a0bf368cd25ad9bcdcc1990b64a647e7bbcce9cb5b",
                 "sha256:2333e30a5e00fe0fe55903c8832e08ee9c3b1382aacf4db26664a16528d51b4b",
                 "sha256:2954c1c23f81c2eaf0b0717d9380bd348578a94161a65b3a2afc62c86467dd68",
+                "sha256:2a24c50840d89ded6c9a8fdc7b6ed3692ed4e86f1c4a4a938e1e92def92933e0",
                 "sha256:2de9d02f5bda03d27ede52e8cfe7b865b066fa49258cbab568720aa5be80a47d",
+                "sha256:2feb1d960f760a575dbc5ab3b1c00504b24caaf6986e2dc2b01c09c87866a943",
                 "sha256:30924eb4c57903d5a7526b08ef4a584acc22ab1ffa085faceb521521d2de32dd",
                 "sha256:316cc9b17edf613ac76b1f1f305d2a748f1b976b033b049a6ecdfd5612c70409",
+                "sha256:32d95b80260d79926f5fab3c41701dbb818fde1c9da590e77e571eefd14abe28",
                 "sha256:38025d9f30cf4634f8309c6874ef871b841eb3c347e90b0851f63d1ded5212da",
                 "sha256:39da8adedf6942d76dc3e46653e52df937a3c4d6d18fdc94a7c29d263b1f5b50",
+                "sha256:3c0ef38c7a7014ffac184db9e04debe495d317cc9c6fb10071f7fefd93100a4f",
                 "sha256:3d7954194c36e304e1523f55d7042c59dc53ec20dd4e9ea9d151f1b62b4415c0",
+                "sha256:3ee8a80d67a4334482d9712b8e83ca6b1d9bc7e351931252ebef5d8f7335a547",
                 "sha256:4093c631e96fdd49e0377a9c167bfd75b6d0bad2ace734c6eb20b348bc3ea180",
+                "sha256:43395e90523f9c23a3d5bdf004733246fba087f2948f87ab28015f12359ca6a0",
                 "sha256:43ce1b9935bfa1ede40028054d7f48b5469cd02733a365eec8a329ffd342915d",
+                "sha256:4410f84b33374409552ac9b6903507cdb31cd30d2501fc5ca13d18f73548444a",
+                "sha256:494994f807ba0b92092a163a0a283961369a65f6cbe01e8891132b7a320e61eb",
                 "sha256:4d4a848d1837973bf0f4b5e54e3bec977d99be36a7895c61abb659301b02c112",
                 "sha256:4ed11165dd45ce798d99a136808a794a748d5dc38511303239d4e2363c0695dc",
+                "sha256:4f3607b129417e111e30637af1b56f24f7a49e64763253bbc275c75fa887d4b2",
                 "sha256:510b5b1bfbe20e1a7b3baf5fed9e9451873559a976c1a78eebaa3b86c57b4265",
                 "sha256:524f35912131cc2cabb00edfd8d573b07f2d9f21fa824bd3fb19725a9cf06327",
                 "sha256:587ca6d3cef6e4e868102672d3bd9dc9698c309ba56d41c2b9c85bbb903cdb95",
+                "sha256:58d4b711689366d4a03ac7957ab8c28890415e267f9b6589969e74b6e42225ec",
                 "sha256:5b3cc074004d968722f51e550b41a27be656ec48f8afaeeb45ebf65b561481dd",
+                "sha256:5dab0844f2cf82be357a0eb11a9087f70c5430b2c241493fc122bb6f2bb0917c",
+                "sha256:5e55da2c8724191e5b557f8e18943b1b4839b8efc3ef60d65985bcf6f587dd38",
                 "sha256:5eeb539606f18a0b232d4ba45adccde4125592f3f636a6182b4a8a436548b914",
                 "sha256:5f4d5ea15c9382135076d2fb28dde923352fe02951e66935a9efaac8f10e81b0",
                 "sha256:5fb2ce4b8045c78ebbc7b8f3c15062e435d47e7393cc57c25115cfd49883747a",
                 "sha256:6172447e1b368dcbc458925e5ddaf9113477b0ed542df258d84fa28fc45ceea7",
+                "sha256:6967ced6730aed543b8673008b5a391c3b1076d834ca438bbd70635c73775368",
+                "sha256:6974f52a02321b36847cd19d1b8e381bf39939c21efd6ee2fc13a28b0d99348c",
                 "sha256:6c3020404e0b5eefd7c9485ccf8393cfb75ec38ce75586e046573c9dc29967a0",
+                "sha256:6c6e0c425f22c1c719c42670d561ad682f7bfeeef918edea971a79ac5252437f",
                 "sha256:70051525001750221daa10907c77830bc889cb6d865cc0b813d9db7fefc21451",
                 "sha256:7905193081db9bfa73b1219140b3d315831cbff0d8941f22da695832f0dd188f",
+                "sha256:7bc37c4d6b87fb1017ea28c9508b36bbcb0c3d18b4260fcdf08b200c74a6aee8",
                 "sha256:7c4855522edb2e6ae7fdb58e07c3ba9111e7621a8956f481c68d5d979c93032e",
                 "sha256:7e4c4629ddad63006efa0ef968c8e4751c5868ff0b1c5c40f76524e894c50248",
+                "sha256:7eedaa5d036d9336c95915035fb57422054014ebdeb6f3b42eac809928e40d0c",
                 "sha256:7f4bf76817c14aa98cc6697ac02f3972cb8c3da93e9ef16b9c66573a68014f91",
                 "sha256:81de08ac11bcb85841e440c13611c00b67d3bf82698314928d0b676362546724",
+                "sha256:832436e59afb93e1836081a20f324cb185836c617659b07b129141a8426973c7",
                 "sha256:861bf317735688269936f755fa136a99d1ed526883859f86e41a5d43c61d8966",
+                "sha256:87a3044c3a35055527ac75e419dfa9f4f3667a1e887ee80360589eb8c90aabb9",
                 "sha256:890b5a14ce214389b2cc36ce82f3093f96f4cc730c1cffdbefff77a7c71f2a97",
                 "sha256:89f4988c7203739d48c6f806f1e87a1d96e0806d44f0fba61dba81392c9e474d",
+                "sha256:8bf32b98b75c13ec7cf774164172683d6e7891088f6316e54425fde1efc276d5",
                 "sha256:8dadd1314583ec0bf2d1379f7008ad627cd6336625d6679cf2f8e67081b83acf",
                 "sha256:901032ff242d479a0efa956d853d16875d42157f98951c0230f69e69f9c09bac",
+                "sha256:9011560a466d2eb3f5a6e4929cf4a09be405c64154e12df0dd72713f6500e32b",
                 "sha256:906bc3a79de8c4ae5b86d3d75a8b77e44404b0f4261714306e3ad248d8ab0951",
                 "sha256:919e32f147ae93a09fe064d77d5ebf4e35502a8df75c29fb05788528e330fe74",
+                "sha256:91d7cc2a76b5567591d12c01f019dd7afce6ba8cba6571187e21e2fc418ae648",
                 "sha256:929811df5462e182b13920da56c6e0284af407d1de637d8e536c5cd00a7daf60",
                 "sha256:949f3b7c29912693cee0afcf09acd6ebc04c57af949d9bf77d6101ebb61e388c",
                 "sha256:a090ca607cbb6a34b0391776f0cb48062081f5f60ddcce5d11838e67a01928d1",
@@ -362,27 +387,44 @@
                 "sha256:a599669fd7c47233438a56936988a2478685e74854088ef5293802123b5b2460",
                 "sha256:a743e5a28af5f70f9c080380a5f908d4d21d40e8f0e0c8901604d15cfa9ba751",
                 "sha256:a77def80806c421b4b0af06f45d65a136e7ac0bdca3c09d9e2ea4e515367c7e9",
+                "sha256:a7e53012d2853a07a4a79c00643832161a910674a893d296c9f1259859a289d2",
+                "sha256:a93dde851926f4f2678e704fadeb39e16c35d8baebd5252c9fd94ce8ce68c4a0",
                 "sha256:aac0411d20e345dc0920bdec5548e438e999ff68d77564d5e9463a7ca9d3e7b1",
                 "sha256:ae15b066e5ad21366600ebec29a7ccbc86812ed267e4b28e860b8ca16a2bc474",
+                "sha256:aea440a510e14e818e67bfc4027880e2fb500c2ccb20ab21c7a7c8b5b4703d75",
+                "sha256:af6fa6817889314555aede9a919612b23739395ce767fe7fcbea9a80bf140fe5",
+                "sha256:b760c65308ff1e462f65d69c12e4ae085cff3b332d894637f6273a12a482d09f",
                 "sha256:be36e3d172dc816333f33520154d708a2657ea63762ec16b62ece02ab5e4daf2",
+                "sha256:c247dd99d39e0338a604f8c2b3bc7061d5c2e9e2ac7ba9cc1be5a69cb6cd832f",
+                "sha256:c5529b34c1c9d937168297f2c1fde7ebe9ebdd5e121297ff9c043bdb2ae3d6fb",
                 "sha256:c8146669223164fc87a7e3de9f81e9423c67a79d6b3447994dfb9c95da16e2d6",
                 "sha256:c8fd5270e906eef71d4a8d19b7c6a43760c6abcfcc10c9101d14eb2357418de9",
+                "sha256:ca63e1890ede90b2e4454f9a65135a4d387a4585ff8282bb72964fab893f2111",
                 "sha256:caf9ee9a5775f3111642d33b86237b05808dafcd6268faa492250e9b78046eb2",
+                "sha256:cb1dac1770878ade83f2ccdf7d25e494f05c9165f5246b46a621cc849341dc01",
                 "sha256:cdad5b9014d83ca68c25d2e9444e28e967ef16e80f6b436918c700c117a85467",
                 "sha256:cdbc1fc1bc0bff1cef838eafe581b55bfbffaed4ed0318b724d0b71d4d377619",
                 "sha256:ceb64bbc6eac5a140ca649003756940f8d6a7c444a68af170b3187623b43bebf",
                 "sha256:d0c5516f0aed654134a2fc936325cc2e642f8a0e096d075209672eb321cff408",
                 "sha256:d143fd47fad1db3d7c27a1b1d66162e855b5d50a89666af46e1679c496e8e579",
                 "sha256:d192f0f30804e55db0d0e0a35d83a9fead0e9a359a9ed0285dbacea60cc10a84",
+                "sha256:d2b35ca2c7f81d173d2fadc2f4f31e88cc5f7a39ae5b6db5513cf3383b0e0ec7",
+                "sha256:d342778ef319e1026af243ed0a07c97acf3bad33b9f29e7ae6a1f68fd083e90c",
+                "sha256:d487f5432bf35b60ed625d7e1b448e2dc855422e87469e3f450aa5552b0eb284",
+                "sha256:d7702622a8b40c49bffb46e1e3ba2e81268d5c04a34f460978c6b5517a34dd52",
                 "sha256:db85ecf4e609a48f4b29055f1e144231b90edc90af7481aa731ba2d059226b1b",
                 "sha256:de6551e370ef19f8de1807d0a9aa2cdfdce2e85ce88b122fe9f6b2b076837e59",
                 "sha256:e1140c64812cb9b06c922e77f1c26a75ec5e3f0fb2bf92cc8c58720dec276752",
+                "sha256:e4fe605b917c70283db7dfe5ada75e04561479075761a0b3866c081d035b01c1",
                 "sha256:e6a904cb26bfefc2f0a6f240bdf5233be78cd2488900a2f846f3c3ac8489ab80",
+                "sha256:e79e6520141d792237c70bcd7a3b122d00f2613769ae0cb61c52e89fd3443839",
                 "sha256:e84799f09591700a4154154cab9787452925578841a94321d5ee8fb9a9a328f0",
                 "sha256:e93dfc1a1165e385cc8239fab7c036fb2cd8093728cbd85097b284d7b99249a2",
                 "sha256:efa8b278894b14d6da122a72fefcebc28445f2d3f880ac59d46c90f4c13be9a3",
                 "sha256:f0d8a7a6b5983c2496e364b969f0e526647a06b075d034f3297dc66f3b360c64",
+                "sha256:f0db75f47be8b8abc8d9e31bc7aad0547ca26f24a54e6fd10231d623f183d089",
                 "sha256:f296c40e23065d0d6650c4aefe7470d2a25fffda489bcc3eb66083f3ac9f6643",
+                "sha256:f31859074d57b4639318523d6ffdca586ace54271a73ad23ad021acd807eb14b",
                 "sha256:f66b5337fa213f1da0d9000bc8dc0cb5b896b726eefd9c6046f699b169c41b9e",
                 "sha256:f733d788519c7e3e71f0855c96618720f5d3d60c3cb829d8bbb722dddce37985",
                 "sha256:fce1473f3ccc4187f75b4690cfc922628aed4d3dd013d047f95a9b3919a86596",
@@ -3401,6 +3443,15 @@
             "markers": "python_version >= '3.8'",
             "version": "==8.2.2"
         },
+        "pytest-asyncio": {
+            "hashes": [
+                "sha256:a811296ed596b69bf0b6f3dc40f83bcaf341b155a269052d82efa2b25ac7037b",
+                "sha256:d081d828e576d85f875399194281e92bf8a68d60d72d1a2faf2feddb6c46b276"
+            ],
+            "index": "pypi",
+            "markers": "python_version >= '3.8'",
+            "version": "==0.24.0"
+        },
         "sniffio": {
             "hashes": [
                 "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2",
diff --git a/src/core/events.py b/src/core/events.py
index abad52a..2fb649e 100644
--- a/src/core/events.py
+++ b/src/core/events.py
@@ -8,18 +8,16 @@ from loguru import logger
 
 from db import crud, models
 from db.database import get_db, make_engine
-from shared.settings import Settings
+from shared.settings import get_settings
 from utils.metrics import measure_regular_metrics, redis_subscribe_worker_exceptions
 
-settings = Settings()
-
 
 @asynccontextmanager
 async def lifespan(app: FastAPI):
     # see https://fastapi.tiangolo.com/advanced/events/#lifespan
 
     # STARTUP
-    engine = make_engine(settings.DATABASE_PATH)
+    engine = make_engine(get_settings().DATABASE_PATH)
     models.Base.metadata.create_all(bind=engine)
     alembic.config.main(argv=['--raiseerr', 'upgrade', 'head'])
     # disabling uvicorn logger since we use loguru in logging_middleware
@@ -42,6 +40,6 @@ async def refresh_user_groups():
         crud.upsert_user_groups(db)
 
 
-@repeat_every(seconds=settings.REPEAT_COUNT_METRICS_SECONDS)
+@repeat_every(seconds=get_settings().REPEAT_COUNT_METRICS_SECONDS)
 async def repeat_measure_regular_metrics():
-    measure_regular_metrics(settings.DATABASE_PATH, settings.REPEAT_COUNT_METRICS_SECONDS)
+    measure_regular_metrics(get_settings().DATABASE_PATH, get_settings().REPEAT_COUNT_METRICS_SECONDS)
diff --git a/src/db/crud.py b/src/db/crud.py
index b251e6d..0d98a59 100644
--- a/src/db/crud.py
+++ b/src/db/crud.py
@@ -5,13 +5,13 @@ from loguru import logger
 from datetime import datetime, timedelta
 
 from web.security import ALLOW_ANY_EMAIL
-from shared.settings import Settings
+from shared.settings import get_settings
 from . import models, schemas
 import yaml
 
 DOMAIN_GROUPS = {}
 DOMAIN_GROUPS_LOADED = False
-DATABASE_QUERY_LIMIT = Settings().DATABASE_QUERY_LIMIT
+DATABASE_QUERY_LIMIT = get_settings().DATABASE_QUERY_LIMIT
 
 # --------------- TASK = Archive
 
@@ -152,7 +152,7 @@ def upsert_user_groups(db: Session):
     along with new participation of users in groups
     """
     logger.debug("Updating user-groups configuration.")
-    filename = Settings().USER_GROUPS_FILENAME
+    filename = get_settings().USER_GROUPS_FILENAME
 
     # read yaml safely
     try:
diff --git a/src/db/database.py b/src/db/database.py
index 8b72f70..d18333d 100644
--- a/src/db/database.py
+++ b/src/db/database.py
@@ -1,11 +1,9 @@
 from sqlalchemy import Engine, create_engine, event
-from sqlalchemy.orm import sessionmaker, declarative_base
-from shared.settings import Settings
+from sqlalchemy.orm import sessionmaker
+from shared.settings import get_settings
 from contextlib import contextmanager
 
 
-settings = Settings()
-
 def make_engine(database_url: str):
     engine = create_engine(database_url, connect_args={"check_same_thread": False})
 
@@ -25,7 +23,7 @@ def make_session_local(engine: Engine):
 
 @contextmanager
 def get_db():
-    session = make_session_local(make_engine(settings.DATABASE_PATH))()
+    session = make_session_local(make_engine(get_settings().DATABASE_PATH))()
     try: yield session
     finally: session.close()
 
diff --git a/src/migrations/env.py b/src/migrations/env.py
index cabd992..abd7c6e 100644
--- a/src/migrations/env.py
+++ b/src/migrations/env.py
@@ -5,12 +5,12 @@ from sqlalchemy import pool
 
 from alembic import context
 
-from shared.settings import Settings
+from shared.settings import get_settings
 
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
 config = context.config
-config.set_main_option('sqlalchemy.url', Settings().DATABASE_PATH)
+config.set_main_option('sqlalchemy.url', get_settings().DATABASE_PATH)
 # Interpret the config file for Python logging.
 # This line sets up loggers basically.
 if config.config_file_name is not None:
diff --git a/src/shared/settings.py b/src/shared/settings.py
index 2190726..56e138f 100644
--- a/src/shared/settings.py
+++ b/src/shared/settings.py
@@ -1,4 +1,5 @@
 
+from functools import lru_cache
 from pydantic_settings import BaseSettings
 from pydantic import ConfigDict
 from typing import Annotated, Set
@@ -28,3 +29,7 @@ class Settings(BaseSettings):
     ALLOWED_ORIGINS: Annotated[set[str], Len(min_length=1)]
     CHROME_APP_IDS: Annotated[set[Annotated[str, Len(min_length=10)]], Len(min_length=1)]
     BLOCKED_EMAILS: Annotated[Set[str], Len(min_length=0)] = set()
+
+@lru_cache
+def get_settings():
+    return Settings()
\ No newline at end of file
diff --git a/src/tests/conftest.py b/src/tests/conftest.py
index 407616d..1b69916 100644
--- a/src/tests/conftest.py
+++ b/src/tests/conftest.py
@@ -13,7 +13,7 @@ def mock_logger_add():
 
 
 @pytest.fixture()
-def settings():
+def get_settings():
     return Settings(_env_file=".env.test")
 
 @pytest.fixture(autouse=True)
@@ -23,13 +23,13 @@ def mock_settings():
 
 
 @pytest.fixture()
-def test_db(settings: Settings):
+def test_db(get_settings: Settings):
     from db.database import make_engine
     from db import models
 
-    engine = make_engine(settings.DATABASE_PATH)
+    engine = make_engine(get_settings.DATABASE_PATH)
 
-    fs = settings.DATABASE_PATH.replace("sqlite:///", "")
+    fs = get_settings.DATABASE_PATH.replace("sqlite:///", "")
     if not os.path.exists(fs):
         open(fs, 'w').close()
 
diff --git a/src/tests/db/test_crud.py b/src/tests/db/test_crud.py
index a4fdeff..13352db 100644
--- a/src/tests/db/test_crud.py
+++ b/src/tests/db/test_crud.py
@@ -135,7 +135,6 @@ def test_search_archives_by_url(test_data, db_session):
 def test_search_archives_by_email(test_data, db_session):
     from web.security import ALLOW_ANY_EMAIL
     from db import crud
-    from web.security import ALLOW_ANY_EMAIL
 
     # lower/upper case
     assert len(crud.search_archives_by_email(db_session, "rick@example.com")) == 34
@@ -363,13 +362,13 @@ def test_get_group(test_data, db_session):
 def test_upsert_user_groups(db_session):
     from db import crud
 
-    @patch('db.crud.Settings', new = lambda: bad_setings)
+    @patch('db.crud.get_settings', new = lambda: bad_setings)
     def test_missing_yaml(db_session):
         with pytest.raises(FileNotFoundError):
             crud.upsert_user_groups(db_session)
 
 
-    @patch('db.crud.Settings', new = lambda: bad_setings)
+    @patch('db.crud.get_settings', new = lambda: bad_setings)
     def test_broken_yaml(db_session):
         with pytest.raises(yaml.YAMLError):
             crud.upsert_user_groups(db_session)
diff --git a/src/tests/web/test_main.py b/src/tests/web/test_main.py
index a8c184f..c59b30a 100644
--- a/src/tests/web/test_main.py
+++ b/src/tests/web/test_main.py
@@ -1,11 +1,11 @@
 import os
 from fastapi.testclient import TestClient
-from shared.settings import Settings
+from shared.settings import get_settings
 
 
 import shutil
 
-def test_serve_local_archive_logic(settings: Settings):
+def test_serve_local_archive_logic(get_settings):
     # create a test file first
     os.makedirs("local_archive_test", exist_ok=True)
     with open("local_archive_test/temp.txt", "w") as f:
@@ -13,9 +13,9 @@ def test_serve_local_archive_logic(settings: Settings):
 
     try:
         # modify the settings
-        settings.SERVE_LOCAL_ARCHIVE = "/app/local_archive_test"
+        get_settings.SERVE_LOCAL_ARCHIVE = "/app/local_archive_test"
         from web.main import app_factory
-        app = app_factory(settings)
+        app = app_factory(get_settings)
         
         # test
         client = TestClient(app)
diff --git a/src/tests/web/test_security.py b/src/tests/web/test_security.py
new file mode 100644
index 0000000..29762d1
--- /dev/null
+++ b/src/tests/web/test_security.py
@@ -0,0 +1,104 @@
+from unittest.mock import patch
+
+from fastapi import HTTPException
+from fastapi.security import HTTPAuthorizationCredentials
+import pytest
+
+
+def test_secure_compare():
+    from web.security import secure_compare
+
+    assert secure_compare("test", "test")
+    assert not secure_compare("test", "test2")
+
+
+@pytest.mark.asyncio
+async def test_get_token_or_user_auth_with_api():
+    from web.security import get_token_or_user_auth, ALLOW_ANY_EMAIL
+    mock_api = HTTPAuthorizationCredentials(scheme="lorem", credentials="this_is_the_test_api_token")
+    assert await get_token_or_user_auth(mock_api) == ALLOW_ANY_EMAIL
+
+
+@pytest.mark.asyncio
+async def test_get_token_or_user_auth_with_user():
+    from web.security import get_token_or_user_auth
+    bad_user = HTTPAuthorizationCredentials(scheme="ipsum", credentials="invalid")
+    with pytest.raises(HTTPException) as e:
+        await get_token_or_user_auth(bad_user)
+        assert e.status_code == 401
+        assert e.detail == "invalid access_token"
+
+
+@patch("web.security.authenticate_user", return_value=(True, "summer@example.com"))
+@pytest.mark.asyncio
+async def test_get_user_auth(m1):
+    from web.security import get_user_auth
+    bad_user = HTTPAuthorizationCredentials(scheme="ipsum", credentials="valid-and-good")
+    assert await get_user_auth(bad_user) == "summer@example.com"
+
+
+@patch("web.security.secure_compare", return_value=False)
+@pytest.mark.asyncio
+async def test_token_api_key_auth_exception(m1):
+    from web.security import token_api_key_auth
+
+    with pytest.raises(HTTPException) as e:
+        await token_api_key_auth(HTTPAuthorizationCredentials(scheme="ipsum", credentials="does-not-matter"), auto_error=True)
+        assert e.status_code == 401
+        assert e.detail == "Wrong auth credentials"
+
+
+@pytest.mark.asyncio
+async def test_authenticate_user():
+    from web.security import authenticate_user
+
+    assert authenticate_user("test") == (False, "invalid access_token")
+    assert authenticate_user(123) == (False, "invalid access_token")
+
+    with patch("web.security.requests.get") as mock_get:
+        # bad response from oauth2
+        mock_get.return_value.status_code = 403
+        assert authenticate_user("this-will-call-requests") == (False, "invalid token")
+        assert mock_get.call_count == 1
+
+        # 200 but invalid json
+        mock_get.return_value.status_code = 200
+        assert authenticate_user("this-will-call-requests") == (False, "token does not belong to valid APP_ID")
+        assert mock_get.call_count == 2
+
+        # 200 but invalid azp and aud
+        mock_get.return_value.json.return_value = {"email": "summer@example.com", "azp": "not_an_app"}
+        assert authenticate_user("this-will-call-requests") == (False, "token does not belong to valid APP_ID")
+
+        mock_get.return_value.json.return_value = {"email": "summer@example.com", "aud": "not_an_app"}
+        assert authenticate_user("this-will-call-requests") == (False, "token does not belong to valid APP_ID")
+
+        mock_get.return_value.json.return_value = {"email": "summer@example.com", "azp": "not_an_app", "aud": "not_an_app"}
+        assert authenticate_user("this-will-call-requests") == (False, "token does not belong to valid APP_ID")
+
+        # blocked email
+        mock_get.return_value.json.return_value = {"email": "blocked@example.com", "azp": "test_app_id_1", "aud": "not_an_app"}
+        assert authenticate_user("this-will-call-requests") == (False, "email 'blocked@example.com' not allowed")
+
+        # not verified
+        mock_get.return_value.json.return_value = {"email": "summer@example.com", "azp": "not_an_app", "aud": "test_app_id_1"}
+        assert authenticate_user("this-will-call-requests") == (False, "email 'summer@example.com' not verified")
+
+        # token expired
+        mock_get.return_value.json.return_value = {"email": "summer@example.com", "azp": "test_app_id_2", "email_verified": "true"}
+        assert authenticate_user("this-will-call-requests") == (False, "Token expired")
+
+        # 200 and valid azp and aup and verified
+        mock_get.return_value.json.return_value = {"email": "summer@example.com", "azp": "test_app_id_2", "email_verified": "true", "expires_in": 100}
+        assert authenticate_user("this-will-call-requests") == (True, "summer@example.com")
+        assert mock_get.call_count == 9
+
+
+@pytest.mark.asyncio
+async def test_authenticate_user_exception():
+    from web.security import authenticate_user
+
+    with patch("web.security.requests.get") as mock_get:
+        mock_get.return_value.status_code = 200
+        mock_get.return_value.json.side_effect = Exception("mocked error")
+        assert authenticate_user("this-will-call-requests") == (False, "exception occurred")
diff --git a/src/web/main.py b/src/web/main.py
index be125c4..f70a281 100644
--- a/src/web/main.py
+++ b/src/web/main.py
@@ -19,14 +19,14 @@ from web.security import get_user_auth, token_api_key_auth, get_token_or_user_au
 from core.config import VERSION, API_DESCRIPTION
 from db.database import get_db_dependency
 from core.events import lifespan
-from shared.settings import Settings
+from shared.settings import get_settings
 
 from auto_archiver import Metadata
 
 from endpoints import default_router, url_router, sheet_router, task_router, interoperability_router
 
 
-def app_factory(settings = Settings()):
+def app_factory(settings = get_settings()):
     app = FastAPI(
         title="Auto-Archiver API",
         description=API_DESCRIPTION,
diff --git a/src/web/security.py b/src/web/security.py
index 26abadc..f4fc90e 100644
--- a/src/web/security.py
+++ b/src/web/security.py
@@ -1,12 +1,12 @@
 from loguru import logger
-import requests, os, secrets
+import requests, secrets
 from fastapi import HTTPException, status, Depends
 from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
-from shared.settings import Settings
+from shared.settings import get_settings
 
 ALLOW_ANY_EMAIL = "*"
 
-settings = Settings()
+settings = get_settings()
 bearer_security = HTTPBearer()
 
 
@@ -39,15 +39,15 @@ token_api_key_auth = api_key_auth(settings.API_BEARER_TOKEN)
 
 async def get_token_or_user_auth(credentials: HTTPAuthorizationCredentials = Depends(bearer_security)):
     # tries to use the static API_KEY and defaults to google JWT auth
-    access_token = credentials.credentials
-    if token_api_key_auth(access_token, auto_error=False): return ALLOW_ANY_EMAIL
+    if await token_api_key_auth(credentials, auto_error=False): return ALLOW_ANY_EMAIL
     return await get_user_auth(credentials)
 
 
 async def get_user_auth(credentials: HTTPAuthorizationCredentials = Depends(bearer_security)):
     # validates the Bearer token in the case that it requires it
     valid_user, info = authenticate_user(credentials.credentials)
-    if valid_user: return info
+    if valid_user:
+        return info
     logger.debug(f"TOKEN FAILURE: {valid_user=} {info=}")
     raise HTTPException(
         status_code=status.HTTP_401_UNAUTHORIZED,
@@ -73,5 +73,5 @@ def authenticate_user(access_token):
             return False, "Token expired"
         return True, j.get('email')
     except Exception as e:
-        logger.warning(f"EXCEPTION occurred: {e}")
-        return False, f"EXCEPTION occurred"
+        logger.warning(f"AUTH EXCEPTION occurred: {e}")
+        return False, "exception occurred"
diff --git a/src/worker.py b/src/worker.py
index 68d8aee..94fbb00 100644
--- a/src/worker.py
+++ b/src/worker.py
@@ -10,12 +10,12 @@ from loguru import logger
 
 from db import crud, schemas, models
 from db.database import get_db
-from shared.settings import Settings
+from shared.settings import get_settings
 import json
 import redis
 from sqlalchemy import exc
 
-settings = Settings()
+settings = get_settings()
 
 celery = Celery(__name__)
 celery.conf.broker_url = settings.CELERY_BROKER_URL

From 9542a046469039eb4594adfdbe05331498d990a7 Mon Sep 17 00:00:00 2001
From: msramalho <19508417+msramalho@users.noreply.github.com>
Date: Mon, 21 Oct 2024 11:38:42 +0100
Subject: [PATCH 23/33] refactor shared setting to avoid circular dependency

---
 src/core/config.py             |  5 ++++-
 src/db/crud.py                 |  3 +--
 src/endpoints/sheet.py         |  3 ++-
 src/tests/db/test_crud.py      | 10 +++++-----
 src/tests/web/test_security.py |  4 +++-
 src/web/security.py            |  5 ++---
 6 files changed, 17 insertions(+), 13 deletions(-)

diff --git a/src/core/config.py b/src/core/config.py
index 0d33cf6..6a37fd0 100644
--- a/src/core/config.py
+++ b/src/core/config.py
@@ -7,4 +7,7 @@ API_DESCRIPTION = """
 - You can use this API to archive single URLs or entire Google Sheets. 
 - Once you submit a URL or Sheet for archiving, the API will return a task_id that you can use to check the status of the archiving process. It works asynchronously.
 """
-BREAKING_CHANGES = {"minVersion": "0.3.1", "message": "The latest update has breaking changes, please update the extension to the most recent version."}
\ No newline at end of file
+BREAKING_CHANGES = {"minVersion": "0.3.1", "message": "The latest update has breaking changes, please update the extension to the most recent version."}
+
+# changing this will corrupt the database logic
+ALLOW_ANY_EMAIL = "*"
diff --git a/src/db/crud.py b/src/db/crud.py
index 0d98a59..1bcfc44 100644
--- a/src/db/crud.py
+++ b/src/db/crud.py
@@ -4,7 +4,7 @@ from sqlalchemy import Column, or_, func
 from loguru import logger
 from datetime import datetime, timedelta
 
-from web.security import ALLOW_ANY_EMAIL
+from core.config import ALLOW_ANY_EMAIL
 from shared.settings import get_settings
 from . import models, schemas
 import yaml
@@ -158,7 +158,6 @@ def upsert_user_groups(db: Session):
     try:
         with open(filename) as inf:
             user_groups_yaml = yaml.safe_load(inf)
-            logger.error(user_groups_yaml)
     except Exception as e:
         logger.error(f"could not open user groups filename {filename}: {e}")
         raise e
diff --git a/src/endpoints/sheet.py b/src/endpoints/sheet.py
index 5c6a6dd..ecde4f4 100644
--- a/src/endpoints/sheet.py
+++ b/src/endpoints/sheet.py
@@ -4,7 +4,8 @@ from fastapi.responses import JSONResponse
 
 from loguru import logger
 
-from web.security import ALLOW_ANY_EMAIL, get_token_or_user_auth
+from core.config import ALLOW_ANY_EMAIL
+from web.security import get_token_or_user_auth
 from db import schemas
 from worker import create_sheet_task
 
diff --git a/src/tests/db/test_crud.py b/src/tests/db/test_crud.py
index 13352db..6dbfaf0 100644
--- a/src/tests/db/test_crud.py
+++ b/src/tests/db/test_crud.py
@@ -57,7 +57,7 @@ def test_data(db_session):
 
 def test_get_archive(test_data, db_session):
     from db import crud
-    from web.security import ALLOW_ANY_EMAIL
+    from core.config import ALLOW_ANY_EMAIL
 
     print(db_session.query(models.Group).all())
 
@@ -88,7 +88,7 @@ def test_get_archive(test_data, db_session):
 
 def test_search_archives_by_url(test_data, db_session):
     from db import crud
-    from web.security import ALLOW_ANY_EMAIL
+    from core.config import ALLOW_ANY_EMAIL
 
     # rick's archives are private
     assert len(crud.search_archives_by_url(db_session, "https://example-0.com", "rick@example.com")) == 34
@@ -133,7 +133,7 @@ def test_search_archives_by_url(test_data, db_session):
 
 
 def test_search_archives_by_email(test_data, db_session):
-    from web.security import ALLOW_ANY_EMAIL
+    from core.config import ALLOW_ANY_EMAIL
     from db import crud
 
     # lower/upper case
@@ -157,7 +157,7 @@ def test_search_archives_by_email(test_data, db_session):
 @patch("db.crud.DATABASE_QUERY_LIMIT", new=25)
 def test_max_query_limit(test_data, db_session):
     from db import crud
-    from web.security import ALLOW_ANY_EMAIL
+    from core.config import ALLOW_ANY_EMAIL
 
     assert len(crud.search_archives_by_url(db_session, "https://example", ALLOW_ANY_EMAIL)) == 25
     assert len(crud.search_archives_by_url(db_session, "https://example", ALLOW_ANY_EMAIL, limit=1000)) == 25
@@ -289,7 +289,7 @@ def test_create_tag(db_session):
 
 def test_is_user_in_group(test_data, db_session):
     from db import crud
-    from web.security import ALLOW_ANY_EMAIL
+    from core.config import ALLOW_ANY_EMAIL
 
     # see user-groups.test.yaml
     test_pairs = [
diff --git a/src/tests/web/test_security.py b/src/tests/web/test_security.py
index 29762d1..ecacea6 100644
--- a/src/tests/web/test_security.py
+++ b/src/tests/web/test_security.py
@@ -4,6 +4,8 @@ from fastapi import HTTPException
 from fastapi.security import HTTPAuthorizationCredentials
 import pytest
 
+from core.config import ALLOW_ANY_EMAIL
+
 
 def test_secure_compare():
     from web.security import secure_compare
@@ -14,7 +16,7 @@ def test_secure_compare():
 
 @pytest.mark.asyncio
 async def test_get_token_or_user_auth_with_api():
-    from web.security import get_token_or_user_auth, ALLOW_ANY_EMAIL
+    from web.security import get_token_or_user_auth
     mock_api = HTTPAuthorizationCredentials(scheme="lorem", credentials="this_is_the_test_api_token")
     assert await get_token_or_user_auth(mock_api) == ALLOW_ANY_EMAIL
 
diff --git a/src/web/security.py b/src/web/security.py
index f4fc90e..97ba07b 100644
--- a/src/web/security.py
+++ b/src/web/security.py
@@ -2,10 +2,9 @@ from loguru import logger
 import requests, secrets
 from fastapi import HTTPException, status, Depends
 from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
+from core.config import ALLOW_ANY_EMAIL
 from shared.settings import get_settings
 
-ALLOW_ANY_EMAIL = "*"
-
 settings = get_settings()
 bearer_security = HTTPBearer()
 
@@ -63,7 +62,7 @@ def authenticate_user(access_token):
     if r.status_code != 200: return False, "invalid token"
     try:
         j = r.json()
-        if j.get("azp") not in settings.CHROME_APP_IDS and j.get("aud") not in settings.CHROME_APP_IDS:
+        if j.get("azp") not in settings.CHROME_APP_IDS and j.get("aud") not in seuser_grouttings.CHROME_APP_IDS:
             return False, f"token does not belong to valid APP_ID"
         if j.get("email") in settings.BLOCKED_EMAILS:
             return False, f"email '{j.get('email')}' not allowed"

From 860db588de3b3823df08eeab9df709a552185064 Mon Sep 17 00:00:00 2001
From: msramalho <19508417+msramalho@users.noreply.github.com>
Date: Mon, 21 Oct 2024 11:59:48 +0100
Subject: [PATCH 24/33] bug fix in metrics

---
 src/utils/metrics.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/utils/metrics.py b/src/utils/metrics.py
index f706154..e63297c 100644
--- a/src/utils/metrics.py
+++ b/src/utils/metrics.py
@@ -4,7 +4,6 @@ import os
 import shutil
 from loguru import logger
 from prometheus_client import Counter, Gauge
-from sqlalchemy.orm import Session
 
 from db import crud
 from db.database import get_db
@@ -54,7 +53,7 @@ async def measure_regular_metrics(sqlite_db_url:str, repeat_in_seconds:int):
         DISK_UTILIZATION.labels(type="database").set(fs.st_size / (2**30))
     except Exception as e: logger.error(e)
 
-    with get_db as db:
+    with get_db() as db:
         count_archives = crud.count_archives(db)
         count_archive_urls = crud.count_archive_urls(db)
         DATABASE_METRICS.labels(query="count_archives", user="-").set(count_archives)

From 93e01c9797264d53429248b5420a17780a6d6c1d Mon Sep 17 00:00:00 2001
From: msramalho <19508417+msramalho@users.noreply.github.com>
Date: Mon, 21 Oct 2024 12:00:01 +0100
Subject: [PATCH 25/33] bug fix in security

---
 src/web/security.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/web/security.py b/src/web/security.py
index 97ba07b..b310782 100644
--- a/src/web/security.py
+++ b/src/web/security.py
@@ -62,7 +62,7 @@ def authenticate_user(access_token):
     if r.status_code != 200: return False, "invalid token"
     try:
         j = r.json()
-        if j.get("azp") not in settings.CHROME_APP_IDS and j.get("aud") not in seuser_grouttings.CHROME_APP_IDS:
+        if j.get("azp") not in settings.CHROME_APP_IDS and j.get("aud") not in settings.CHROME_APP_IDS:
             return False, f"token does not belong to valid APP_ID"
         if j.get("email") in settings.BLOCKED_EMAILS:
             return False, f"email '{j.get('email')}' not allowed"

From e58193e47a14927a39af04de7c07318f19d2ca9d Mon Sep 17 00:00:00 2001
From: msramalho <19508417+msramalho@users.noreply.github.com>
Date: Mon, 21 Oct 2024 12:00:12 +0100
Subject: [PATCH 26/33] tests metrics endpoint

---
 src/tests/endpoints/test_default.py | 30 +++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/src/tests/endpoints/test_default.py b/src/tests/endpoints/test_default.py
index c9a4149..dbd9f4f 100644
--- a/src/tests/endpoints/test_default.py
+++ b/src/tests/endpoints/test_default.py
@@ -1,5 +1,6 @@
 from unittest.mock import AsyncMock, patch
 from fastapi.testclient import TestClient
+import pytest
 from core.config import VERSION
 
 
@@ -76,3 +77,32 @@ def test_favicon(client_with_auth):
     r = client_with_auth.get("/favicon.ico")
     assert r.status_code == 200
     assert r.headers["content-type"] == "image/vnd.microsoft.icon"
+
+
+from tests.db.test_crud import test_data
+
+
+@pytest.mark.asyncio
+async def test_prometheus_metrics(test_data, client_with_auth, get_settings):
+    # before metrics calculation
+    r = client_with_auth.get("/metrics")
+    assert r.status_code == 200
+    assert r.headers["content-type"] == "text/plain; version=0.0.4; charset=utf-8"
+    assert "disk_utilization" in r.text
+    assert "database_metrics" in r.text
+    assert "exceptions" in r.text
+    assert "worker_exceptions_total" in r.text
+    assert 'disk_utilization{type="used"}' not in r.text
+
+    # after metrics calculation
+    from utils.metrics import measure_regular_metrics
+    await measure_regular_metrics(get_settings.DATABASE_PATH, 60 * 60 * 24 * 31 * 12 * 100)
+    r2 = client_with_auth.get("/metrics")
+    assert 'disk_utilization{type="used"}' in r2.text
+    assert 'disk_utilization{type="free"}' in r2.text
+    assert 'disk_utilization{type="database"}' in r2.text
+    assert 'database_metrics{query="count_archives",user="-"} 100.0' in r2.text
+    assert 'database_metrics{query="count_archive_urls",user="-"} 1000.0' in r2.text
+    assert 'database_metrics{query="count_by_user",user="rick@example.com"} 34.0' in r2.text
+    assert 'database_metrics{query="count_by_user",user="morty@example.com"} 33.0' in r2.text
+    assert 'database_metrics{query="count_by_user",user="jerry@example.com"} 33.0' in r2.text

From b013e2a17316a9f8479180eb6f0f83c8676768d1 Mon Sep 17 00:00:00 2001
From: msramalho <19508417+msramalho@users.noreply.github.com>
Date: Tue, 22 Oct 2024 12:55:08 +0100
Subject: [PATCH 27/33] refactoring logging, metrics and new tests

---
 src/core/events.py                  |  2 +-
 src/core/logging.py                 | 14 +++++++++-
 src/db/crud.py                      |  5 +++-
 src/endpoints/default.py            |  4 +--
 src/endpoints/interoperability.py   |  3 +-
 src/endpoints/task.py               | 10 ++-----
 src/shared/settings.py              |  2 +-
 src/tests/db/test_crud.py           |  7 +++++
 src/tests/endpoints/test_default.py | 22 +++++++++++----
 src/tests/web/test_main.py          | 26 +++++++++++++++--
 src/tests/web/test_security.py      | 10 ++++---
 src/utils/metrics.py                | 40 +++++++++++++++------------
 src/web/main.py                     | 11 ++++----
 src/worker.py                       | 43 ++++++++++++++---------------
 14 files changed, 128 insertions(+), 71 deletions(-)

diff --git a/src/core/events.py b/src/core/events.py
index 2fb649e..445d027 100644
--- a/src/core/events.py
+++ b/src/core/events.py
@@ -22,7 +22,7 @@ async def lifespan(app: FastAPI):
     alembic.config.main(argv=['--raiseerr', 'upgrade', 'head'])
     # disabling uvicorn logger since we use loguru in logging_middleware
     logging.getLogger("uvicorn.access").disabled = True
-    asyncio.create_task(redis_subscribe_worker_exceptions())
+    asyncio.create_task(redis_subscribe_worker_exceptions(get_settings().REDIS_EXCEPTIONS_CHANNEL, get_settings().CELERY_BROKER_URL))
     asyncio.create_task(refresh_user_groups())
     asyncio.create_task(repeat_measure_regular_metrics())
 
diff --git a/src/core/logging.py b/src/core/logging.py
index 1798f0c..fbaed29 100644
--- a/src/core/logging.py
+++ b/src/core/logging.py
@@ -1,15 +1,27 @@
+import traceback
 from loguru import logger
 from fastapi import Request
-from utils.metrics import EXCEPTION_COUNTER
 
 
 # logging configurations
 logger.add("logs/api_logs.log", retention="30 days", rotation="3 days")
+error_logger = logger.add("logs/error_logs.log", retention="30 days")
+
+    
+def log_error(e: Exception, traceback_str: str = None, extra:str = ""):
+    # EXCEPTION_COUNTER.labels(type(e).__name__).inc()
+    if not traceback_str: traceback_str = traceback.format_exc()
+    if extra: extra = f"{extra}\n"
+    logger.error(f"{extra}{e.__class__.__name__}: {e}")
+    error_logger.error(f"{extra}{e.__class__.__name__}: {e}\n{traceback_str}")
+
 async def logging_middleware(request: Request, call_next):
     try:
         response = await call_next(request)
         logger.info(f"{request.client.host}:{request.client.port} {request.method} {request.url._url} - HTTP {response.status_code}")
         return response
     except Exception as e:
+        from utils.metrics import EXCEPTION_COUNTER
         EXCEPTION_COUNTER.labels(type(e).__name__).inc()
+        log_error(e)
         raise e
\ No newline at end of file
diff --git a/src/db/crud.py b/src/db/crud.py
index 1bcfc44..97dacf3 100644
--- a/src/db/crud.py
+++ b/src/db/crud.py
@@ -74,13 +74,16 @@ def count_archives(db: Session):
 def count_archive_urls(db: Session):
     return db.query(func.count(models.ArchiveUrl.url)).scalar()
 
+def count_users(db: Session):
+    return db.query(func.count(models.User.email)).scalar()
 
 def count_by_user_since(db: Session, seconds_delta: int = 15):
     time_threshold = datetime.now() - timedelta(seconds=seconds_delta)
     return db.query(models.Archive.author_id, func.count().label('total'))\
         .filter(models.Archive.created_at >= time_threshold)\
         .group_by(models.Archive.author_id)\
-        .order_by(func.count().desc()).limit(5 * DATABASE_QUERY_LIMIT).all()
+        .order_by(func.count().desc())\
+        .limit(500).all()
 
 
 def base_query(db: Session):
diff --git a/src/endpoints/default.py b/src/endpoints/default.py
index da2fc97..269fa3f 100644
--- a/src/endpoints/default.py
+++ b/src/endpoints/default.py
@@ -1,10 +1,10 @@
 
 from fastapi import APIRouter, Depends, Request, HTTPException
 from fastapi.responses import FileResponse, JSONResponse
-from loguru import logger
 from sqlalchemy.orm import Session
 
 from core.config import VERSION, BREAKING_CHANGES
+from core.logging import log_error
 from db import crud
 from db.database import get_db_dependency, get_db
 from web.security import get_user_auth, bearer_security
@@ -21,7 +21,7 @@ async def home(request: Request):
         with get_db() as db:
             status["groups"] = crud.get_user_groups(db, email)
     except HTTPException: pass  # not authenticated is fine
-    except Exception as e: logger.error(e)
+    except Exception as e: log_error(e)
     return JSONResponse(status)
 
 
diff --git a/src/endpoints/interoperability.py b/src/endpoints/interoperability.py
index d1bcfa4..b784e03 100644
--- a/src/endpoints/interoperability.py
+++ b/src/endpoints/interoperability.py
@@ -7,6 +7,7 @@ import sqlalchemy
 from web.security import token_api_key_auth
 from db import models, schemas
 from worker import insert_result_into_db
+from core.logging import log_error
 
 
 interoperability_router = APIRouter(prefix="/interop", tags=["Interoperability endpoints."])
@@ -21,6 +22,6 @@ def submit_manual_archive(manual: schemas.SubmitManual, auth=Depends(token_api_k
     try:
         archive_id = insert_result_into_db(result, manual.tags, manual.public, manual.group_id, manual.author_id, models.generate_uuid())
     except sqlalchemy.exc.IntegrityError as e:
-        logger.error(e)
+        log_error(e)
         raise HTTPException(status_code=422, detail=f"Cannot insert into DB due to integrity error")
     return JSONResponse({"id": archive_id}, status_code=201)
diff --git a/src/endpoints/task.py b/src/endpoints/task.py
index 7544aca..532bad6 100644
--- a/src/endpoints/task.py
+++ b/src/endpoints/task.py
@@ -1,4 +1,3 @@
-import traceback
 from celery.result import AsyncResult
 from fastapi import APIRouter, Depends
 from fastapi.encoders import jsonable_encoder
@@ -8,7 +7,7 @@ from loguru import logger
 from web.security import get_token_or_user_auth
 
 from db import schemas
-
+from core.logging import log_error
 from worker import celery
 
 
@@ -25,8 +24,6 @@ def get_status(task_id, email=Depends(get_token_or_user_auth)):
             # The :attr:`result` attribute then contains the exception raised by the task.
             # https://docs.celeryq.dev/en/stable/_modules/celery/result.html#AsyncResult
             raise task.result
-        # TODO: refactor to use schema?
-        # response = schemas.TaskResult(id=task_id, status=task.status, result=task.result)
 
         response = {
             "id": task_id,
@@ -36,10 +33,7 @@ def get_status(task_id, email=Depends(get_token_or_user_auth)):
         return JSONResponse(jsonable_encoder(response, exclude_unset=True))
 
     except Exception as e:
-        logger.error(e)
-        logger.error(traceback.format_exc())
-        # TODO: refactor to use schema?
-        # response = schemas.TaskResult(id=task_id, status="FAILURE", result={"error": str(e)})
+        log_error(e)
         return JSONResponse({
             "id": task_id,
             "status": "FAILURE",
diff --git a/src/shared/settings.py b/src/shared/settings.py
index 56e138f..39e2598 100644
--- a/src/shared/settings.py
+++ b/src/shared/settings.py
@@ -22,7 +22,7 @@ class Settings(BaseSettings):
     REDIS_EXCEPTIONS_CHANNEL: str = "exceptions-channel"
     
 	# observability
-    REPEAT_COUNT_METRICS_SECONDS: int = 15
+    REPEAT_COUNT_METRICS_SECONDS: int = 30
 
 	# security
     API_BEARER_TOKEN: Annotated[str, Len(min_length=20)]
diff --git a/src/tests/db/test_crud.py b/src/tests/db/test_crud.py
index 6dbfaf0..daad766 100644
--- a/src/tests/db/test_crud.py
+++ b/src/tests/db/test_crud.py
@@ -252,6 +252,13 @@ def test_count_archive_urls(test_data, db_session):
     assert crud.count_archives(db_session) == 99
     assert crud.count_archive_urls(db_session) == 999
 
+def test_count_users(test_data, db_session):
+    from db import crud
+
+    assert crud.count_users(db_session) == 4
+    db_session.query(models.User).filter(models.User.email == "rick@example.com").delete()
+    db_session.commit()
+    assert crud.count_users(db_session) == 3
 
 def test_count_by_users_since(test_data, db_session):
     from db import crud
diff --git a/src/tests/endpoints/test_default.py b/src/tests/endpoints/test_default.py
index dbd9f4f..b840260 100644
--- a/src/tests/endpoints/test_default.py
+++ b/src/tests/endpoints/test_default.py
@@ -101,8 +101,20 @@ async def test_prometheus_metrics(test_data, client_with_auth, get_settings):
     assert 'disk_utilization{type="used"}' in r2.text
     assert 'disk_utilization{type="free"}' in r2.text
     assert 'disk_utilization{type="database"}' in r2.text
-    assert 'database_metrics{query="count_archives",user="-"} 100.0' in r2.text
-    assert 'database_metrics{query="count_archive_urls",user="-"} 1000.0' in r2.text
-    assert 'database_metrics{query="count_by_user",user="rick@example.com"} 34.0' in r2.text
-    assert 'database_metrics{query="count_by_user",user="morty@example.com"} 33.0' in r2.text
-    assert 'database_metrics{query="count_by_user",user="jerry@example.com"} 33.0' in r2.text
+    assert 'database_metrics{query="count_archives"} 100.0' in r2.text
+    assert 'database_metrics{query="count_archive_urls"} 1000.0' in r2.text
+    assert 'database_metrics{query="count_users"} 4.0' in r2.text
+    assert 'database_metrics_counter_total{query="count_by_user",user="rick@example.com"} 34.0' in r2.text
+    assert 'database_metrics_counter_total{query="count_by_user",user="morty@example.com"} 33.0' in r2.text
+    assert 'database_metrics_counter_total{query="count_by_user",user="jerry@example.com"} 33.0' in r2.text
+
+    # 30s window, should not change the gauges nor the total in the counters
+    from utils.metrics import measure_regular_metrics
+    await measure_regular_metrics(get_settings.DATABASE_PATH, 30)
+    r3 = client_with_auth.get("/metrics")
+    assert 'database_metrics{query="count_archives"} 100.0' in r3.text
+    assert 'database_metrics{query="count_archive_urls"} 1000.0' in r3.text
+    assert 'database_metrics{query="count_users"} 4.0' in r3.text
+    assert 'database_metrics_counter_total{query="count_by_user",user="rick@example.com"} 34.0' in r3.text
+    assert 'database_metrics_counter_total{query="count_by_user",user="morty@example.com"} 33.0' in r3.text
+    assert 'database_metrics_counter_total{query="count_by_user",user="jerry@example.com"} 33.0' in r3.text
\ No newline at end of file
diff --git a/src/tests/web/test_main.py b/src/tests/web/test_main.py
index c59b30a..e880311 100644
--- a/src/tests/web/test_main.py
+++ b/src/tests/web/test_main.py
@@ -1,10 +1,32 @@
 import os
+from unittest.mock import patch
 from fastapi.testclient import TestClient
-from shared.settings import get_settings
-
 
 import shutil
 
+import pytest
+
+def test_lifespan(app):
+    with TestClient(app) as client:
+        r = client.get("/health")
+        assert r.status_code == 200
+        assert r.json() == {"status": "ok"}
+
+def test_alembic(db_session):
+    import alembic.config
+    alembic.config.main(argv=['--raiseerr', 'upgrade', 'head'])
+    alembic.config.main(argv=['--raiseerr', 'downgrade', 'base'])
+
+@patch("endpoints.default.crud.get_user_groups", side_effect=Exception('mocked error'))
+def test_logging_middleware(m1, client_with_auth):
+    from utils.metrics import EXCEPTION_COUNTER
+    assert len(EXCEPTION_COUNTER.collect()[0].samples) == 0
+    with pytest.raises(Exception, match="mocked error"):
+        client_with_auth.get("/groups")
+    # creates one empty and one from above
+    assert len(EXCEPTION_COUNTER.collect()[0].samples) == 2
+    
+
 def test_serve_local_archive_logic(get_settings):
     # create a test file first
     os.makedirs("local_archive_test", exist_ok=True)
diff --git a/src/tests/web/test_security.py b/src/tests/web/test_security.py
index ecacea6..64fe4d4 100644
--- a/src/tests/web/test_security.py
+++ b/src/tests/web/test_security.py
@@ -25,10 +25,11 @@ async def test_get_token_or_user_auth_with_api():
 async def test_get_token_or_user_auth_with_user():
     from web.security import get_token_or_user_auth
     bad_user = HTTPAuthorizationCredentials(scheme="ipsum", credentials="invalid")
+    e: pytest.ExceptionInfo = None
     with pytest.raises(HTTPException) as e:
         await get_token_or_user_auth(bad_user)
-        assert e.status_code == 401
-        assert e.detail == "invalid access_token"
+    assert e.value.status_code == 401
+    assert e.value.detail == "invalid access_token"
 
 
 @patch("web.security.authenticate_user", return_value=(True, "summer@example.com"))
@@ -44,10 +45,11 @@ async def test_get_user_auth(m1):
 async def test_token_api_key_auth_exception(m1):
     from web.security import token_api_key_auth
 
+    e: pytest.ExceptionInfo = None
     with pytest.raises(HTTPException) as e:
         await token_api_key_auth(HTTPAuthorizationCredentials(scheme="ipsum", credentials="does-not-matter"), auto_error=True)
-        assert e.status_code == 401
-        assert e.detail == "Wrong auth credentials"
+    assert e.value.status_code == 401
+    assert e.value.detail == "Wrong auth credentials"
 
 
 @pytest.mark.asyncio
diff --git a/src/utils/metrics.py b/src/utils/metrics.py
index e63297c..8d513e6 100644
--- a/src/utils/metrics.py
+++ b/src/utils/metrics.py
@@ -2,62 +2,68 @@ import asyncio
 import json
 import os
 import shutil
-from loguru import logger
 from prometheus_client import Counter, Gauge
+import redis
 
 from db import crud
 from db.database import get_db
-from worker import REDIS_EXCEPTIONS_CHANNEL, Rdis
+from core.logging import log_error
 
 
 # Custom metrics
 EXCEPTION_COUNTER = Counter(
     "exceptions",
     "Number of times a certain exception has occurred.",
-    labelnames=("types",)
+    labelnames=["types"]
 )
 WORKER_EXCEPTION = Counter(
     "worker_exceptions_total",
     "Number of times a certain exception has occurred on the worker.",
-    labelnames=("exception", "task",)
+    labelnames=["types", "exception", "task", "traceback"]
 )
 DISK_UTILIZATION = Gauge(
     "disk_utilization",
     "Disk utilization in GB",
-    labelnames=("type",)
+    labelnames=["type"]
 )
 DATABASE_METRICS = Gauge(
     "database_metrics",
-    "Useful database metrics from queries",
-    labelnames=("query", "user")
+    "Database metric readings at a certain point in time",
+    labelnames=["query"]
+)
+DATABASE_METRICS_COUNTER = Counter(
+    "database_metrics_counter",
+    "Database metrics that increase over time",
+    labelnames=["query", "user"]
 )
 
 
-async def redis_subscribe_worker_exceptions():
+async def redis_subscribe_worker_exceptions(REDIS_EXCEPTIONS_CHANNEL, CELERY_BROKER_URL):
     # Subscribe to Redis channel and increment the counter for each exception with info on the exception and task
+    Rdis = redis.Redis.from_url(CELERY_BROKER_URL)
     PubSubExceptions = Rdis.pubsub()
     PubSubExceptions.subscribe(REDIS_EXCEPTIONS_CHANNEL)
     while True:
         message = PubSubExceptions.get_message()
         if message and message["type"] == "message":
             data = json.loads(message["data"].decode("utf-8"))
-            WORKER_EXCEPTION.labels(exception=data["exception"], task=data["task"]).inc()
+            WORKER_EXCEPTION.labels(types=type(data["exception"]).__name__, exception=data["exception"], task=data["task"], traceback=data["traceback"]).inc()
         await asyncio.sleep(1)
 
-async def measure_regular_metrics(sqlite_db_url:str, repeat_in_seconds:int):
+
+async def measure_regular_metrics(sqlite_db_url: str, repeat_in_seconds: int):
     _total, used, free = shutil.disk_usage("/")
     DISK_UTILIZATION.labels(type="used").set(used / (2**30))
     DISK_UTILIZATION.labels(type="free").set(free / (2**30))
-    try: 
+    try:
         fs = os.stat(sqlite_db_url.replace("sqlite:///", ""))
         DISK_UTILIZATION.labels(type="database").set(fs.st_size / (2**30))
-    except Exception as e: logger.error(e)
+    except Exception as e: log_error(e)
 
     with get_db() as db:
-        count_archives = crud.count_archives(db)
-        count_archive_urls = crud.count_archive_urls(db)
-        DATABASE_METRICS.labels(query="count_archives", user="-").set(count_archives)
-        DATABASE_METRICS.labels(query="count_archive_urls", user="-").set(count_archive_urls)
+        DATABASE_METRICS.labels(query="count_archives").set(crud.count_archives(db))
+        DATABASE_METRICS.labels(query="count_archive_urls").set(crud.count_archive_urls(db))
+        DATABASE_METRICS.labels(query="count_users").set(crud.count_users(db))
 
         for user in crud.count_by_user_since(db, repeat_in_seconds):
-            DATABASE_METRICS.labels(query="count_by_user", user=user.author_id).set(user.total)
\ No newline at end of file
+            DATABASE_METRICS_COUNTER.labels(query="count_by_user", user=user.author_id).inc(user.total)
diff --git a/src/web/main.py b/src/web/main.py
index f70a281..addc3a9 100644
--- a/src/web/main.py
+++ b/src/web/main.py
@@ -1,4 +1,4 @@
-import traceback, os
+import os
 from celery.result import AsyncResult
 from fastapi import FastAPI, Depends, HTTPException
 from fastapi.encoders import jsonable_encoder
@@ -11,7 +11,7 @@ import sqlalchemy
 from sqlalchemy.orm import Session
 from loguru import logger
 
-from core.logging import logging_middleware
+from core.logging import logging_middleware, log_error
 from worker import create_archive_task, create_sheet_task, celery, insert_result_into_db
 
 from db import crud, models, schemas
@@ -42,6 +42,7 @@ def app_factory(settings = get_settings()):
         allow_methods=["*"],
         allow_headers=["*"],
     )
+    app.middleware("http")(logging_middleware)
 
     app.include_router(default_router)
     app.include_router(url_router)
@@ -60,7 +61,6 @@ def app_factory(settings = get_settings()):
         app.mount(settings.SERVE_LOCAL_ARCHIVE, StaticFiles(directory=local_dir), name=settings.SERVE_LOCAL_ARCHIVE)
 
 
-    app.middleware("http")(logging_middleware)
 
     # -----Submit URL and manipulate tasks. Bearer protected below
 
@@ -111,8 +111,7 @@ def app_factory(settings = get_settings()):
             return JSONResponse(jsonable_encoder(response, exclude_unset=True))
 
         except Exception as e:
-            logger.error(e)
-            logger.error(traceback.format_exc())
+            log_error(e)
             return JSONResponse({
                 "id": task_id,
                 "status": "FAILURE",
@@ -161,7 +160,7 @@ def app_factory(settings = get_settings()):
         try:
             archive_id = insert_result_into_db(result, manual.tags, manual.public, manual.group_id, manual.author_id, models.generate_uuid())
         except sqlalchemy.exc.IntegrityError as e:
-            logger.error(e)
+            log_error(e)
             raise HTTPException(status_code=422, detail=f"Cannot insert into DB due to integrity error")
         return JSONResponse({"id": archive_id})
 
diff --git a/src/worker.py b/src/worker.py
index 94fbb00..9b6c417 100644
--- a/src/worker.py
+++ b/src/worker.py
@@ -14,6 +14,7 @@ from shared.settings import get_settings
 import json
 import redis
 from sqlalchemy import exc
+from core.logging import log_error
 
 settings = get_settings()
 
@@ -21,17 +22,17 @@ celery = Celery(__name__)
 celery.conf.broker_url = settings.CELERY_BROKER_URL
 celery.conf.result_backend = settings.CELERY_RESULT_BACKEND
 USER_GROUPS_FILENAME = settings.USER_GROUPS_FILENAME
-REDIS_EXCEPTIONS_CHANNEL = settings.REDIS_EXCEPTIONS_CHANNEL
 
 Rdis = redis.Redis.from_url(celery.conf.broker_url)
 
+
 @celery.task(name="create_archive_task", bind=True, autoretry_for=(Exception,), retry_backoff=True, retry_kwargs={'max_retries': 3})
 def create_archive_task(self, archive_json: str):
     archive = schemas.ArchiveCreate.model_validate_json(archive_json)
     logger.info(f"Archiving {archive.url=} {archive.tags=} {archive.public=} {archive.group_id=} {archive.author_id=}")
     invalid = is_group_invalid_for_user(archive.public, archive.group_id, archive.author_id)
     if invalid:
-        raise Exception(invalid) # marks task FAILED, saves the Exception as result
+        raise Exception(invalid)  # marks task FAILED, saves the Exception as result
 
     url = archive.url
     logger.info(f"{url=} {archive=}")
@@ -45,14 +46,13 @@ def create_archive_task(self, archive_json: str):
 
     orchestrator = choose_orchestrator(archive.group_id, archive.author_id)
     result = orchestrator.feed_item(Metadata().set_url(url))
-    
+
     try:
         insert_result_into_db(result, archive.tags, archive.public, archive.group_id, archive.author_id, self.request.id)
     except Exception as e:
         # Log it, then raise again to store the error as the task result
-        logger.error(e)
-        logger.error(traceback.format_exc())
-        redis_publish_exception(e, self.name)
+        log_error(e)
+        redis_publish_exception(e, self.name, traceback.format_exc())
         raise e
     return result.to_dict()
 
@@ -72,7 +72,7 @@ def create_sheet_task(self, sheet_json: str):
 
     stats = {"archived": 0, "failed": 0, "errors": []}
     for result in orchestrator.feed():
-        if not result: 
+        if not result:
             logger.error("Got empty result from feeder, an internal error must have occurred.")
             continue
         try:
@@ -82,10 +82,8 @@ def create_sheet_task(self, sheet_json: str):
             logger.warning(f"cached result detected: {e}")
             stats["archived"] += 1
         except Exception as e:
-            logger.error(type(e))
-            logger.error(e)
-            logger.error(traceback.format_exc())
-            redis_publish_exception(e, self.name)
+            log_error(e, extra=f"{self.name}: {sheet_json}")
+            redis_publish_exception(e, self.name, traceback.format_exc())
             stats["failed"] += 1
             stats["errors"].append(str(e))
 
@@ -96,11 +94,11 @@ def create_sheet_task(self, sheet_json: str):
 @task_failure.connect(sender=create_sheet_task)
 @task_failure.connect(sender=create_archive_task)
 def task_failure_notifier(sender, **kwargs):
-    logger.warning("😅 From task_failure_notifier ==> Task failed successfully! ")
-    logger.error(kwargs['exception'])
-    logger.error(kwargs['traceback'])
-    logger.error("\n".join(traceback.format_list(traceback.extract_tb(kwargs['traceback']))))
-    redis_publish_exception(kwargs['exception'], sender.name)
+    traceback_msg = "\n".join(traceback.format_list(traceback.extract_tb(kwargs['traceback'])))
+    logger.warning("😅 From task_failure_notifier ==> Task failed successfully!")
+    log_error(kwargs['exception'], traceback_msg, f"task_failure: {sender.name}")
+    redis_publish_exception(kwargs['exception'], sender.name, traceback_msg)
+
 
 def choose_orchestrator(group, email):
     global ORCHESTRATORS
@@ -187,10 +185,11 @@ def get_all_urls(result: Metadata) -> List[models.ArchiveUrl]:
             if isinstance(prop, list):
                 for i, prop_media in enumerate(prop):
                     if prop_media := convert_if_media(prop_media):
-                        for j, url in enumerate(prop_media.urls): 
+                        for j, url in enumerate(prop_media.urls):
                             db_urls.append(models.ArchiveUrl(url=url, key=prop_media.get("id", f"{k}{prop_media.key}_{i}.{j}")))
     return db_urls
 
+
 def convert_if_media(media):
     if isinstance(media, Media): return media
     elif isinstance(media, dict):
@@ -199,13 +198,13 @@ def convert_if_media(media):
             logger.debug(f"error parsing {media} : {e}")
     return False
 
-def redis_publish_exception(exception, task_name):
+
+def redis_publish_exception(exception, task_name, traceback: str = ""):
+    REDIS_EXCEPTIONS_CHANNEL = settings.REDIS_EXCEPTIONS_CHANNEL
     try:
-        Rdis.publish(REDIS_EXCEPTIONS_CHANNEL, json.dumps({"exception": exception, "task": task_name}, default=str))
+        Rdis.publish(REDIS_EXCEPTIONS_CHANNEL, json.dumps({"exception": exception, "task": task_name, "traceback": traceback}, default=str))
     except Exception as e:
-        logger.error(e)
-        logger.error(traceback.format_exc())
-        logger.error(f"Could not publish to {REDIS_EXCEPTIONS_CHANNEL}")
+        log_error(e, f"[CRITICAL] Could not publish to {REDIS_EXCEPTIONS_CHANNEL}")
 
 
 @worker_init.connect

From ec93df98fa34420551ca8b46376c8998373e8055 Mon Sep 17 00:00:00 2001
From: msramalho <19508417+msramalho@users.noreply.github.com>
Date: Tue, 22 Oct 2024 13:01:24 +0100
Subject: [PATCH 28/33] refactoring worker file location

---
 docker-compose.yml                | 2 +-
 src/endpoints/interoperability.py | 2 +-
 src/endpoints/sheet.py            | 2 +-
 src/endpoints/task.py             | 2 +-
 src/endpoints/url.py              | 2 +-
 src/tests/endpoints/test_sheet.py | 4 ++--
 src/tests/endpoints/test_url.py   | 2 +-
 src/web/main.py                   | 2 +-
 src/worker/__init__.py            | 0
 src/{worker.py => worker/main.py} | 0
 10 files changed, 9 insertions(+), 9 deletions(-)
 create mode 100644 src/worker/__init__.py
 rename src/{worker.py => worker/main.py} (100%)

diff --git a/docker-compose.yml b/docker-compose.yml
index 5650f74..2274dca 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -31,7 +31,7 @@ services:
 
   worker:
     <<: *base-setup
-    command: celery --app=worker.celery worker --loglevel=info --logfile=logs/celery.log
+    command: celery --app=worker.main.celery worker --loglevel=info --logfile=logs/celery.log
     volumes:
       - ./src:/app
       - /var/run/docker.sock:/var/run/docker.sock
diff --git a/src/endpoints/interoperability.py b/src/endpoints/interoperability.py
index b784e03..5758dc1 100644
--- a/src/endpoints/interoperability.py
+++ b/src/endpoints/interoperability.py
@@ -6,7 +6,7 @@ import sqlalchemy
 
 from web.security import token_api_key_auth
 from db import models, schemas
-from worker import insert_result_into_db
+from worker.main import insert_result_into_db
 from core.logging import log_error
 
 
diff --git a/src/endpoints/sheet.py b/src/endpoints/sheet.py
index ecde4f4..5a75c89 100644
--- a/src/endpoints/sheet.py
+++ b/src/endpoints/sheet.py
@@ -7,7 +7,7 @@ from loguru import logger
 from core.config import ALLOW_ANY_EMAIL
 from web.security import get_token_or_user_auth
 from db import schemas
-from worker import create_sheet_task
+from worker.main import create_sheet_task
 
 sheet_router = APIRouter(prefix="/sheet", tags=["Google Spreadsheet operations"])
 
diff --git a/src/endpoints/task.py b/src/endpoints/task.py
index 532bad6..1a0c6ac 100644
--- a/src/endpoints/task.py
+++ b/src/endpoints/task.py
@@ -8,7 +8,7 @@ from web.security import get_token_or_user_auth
 
 from db import schemas
 from core.logging import log_error
-from worker import celery
+from worker.main import celery
 
 
 task_router = APIRouter(prefix="/task", tags=["Async task operations"])
diff --git a/src/endpoints/url.py b/src/endpoints/url.py
index 9a59b77..3932f73 100644
--- a/src/endpoints/url.py
+++ b/src/endpoints/url.py
@@ -10,7 +10,7 @@ from sqlalchemy.orm import Session
 from db import crud, schemas
 from db.database import get_db_dependency
 
-from worker import create_archive_task
+from worker.main import create_archive_task
 
 url_router = APIRouter(prefix="/url", tags=["Single URL operations"])
 
diff --git a/src/tests/endpoints/test_sheet.py b/src/tests/endpoints/test_sheet.py
index 1c0838f..f3e2559 100644
--- a/src/tests/endpoints/test_sheet.py
+++ b/src/tests/endpoints/test_sheet.py
@@ -8,7 +8,7 @@ def test_sheet_no_auth(client, test_no_auth):
     test_no_auth(client.post, "/sheet/archive")
 
 
-@patch("worker.create_sheet_task.delay", return_value=TaskResult(id="123-456-789", status="PENDING", result=""))
+@patch("worker.main.create_sheet_task.delay", return_value=TaskResult(id="123-456-789", status="PENDING", result=""))
 def test_sheet_rick(m1, client_with_auth):
 
     response = client_with_auth.post("/sheet/archive", json={"sheet_id": "123-sheet-id"})
@@ -26,7 +26,7 @@ def test_sheet_missing_sheet_data(client_with_auth):
     assert r.json() == {"detail": "sheet name or id is required"}
 
 
-@patch("worker.create_sheet_task.delay", return_value=TaskResult(id="123-API-789", status="PENDING", result=""))
+@patch("worker.main.create_sheet_task.delay", return_value=TaskResult(id="123-API-789", status="PENDING", result=""))
 def test_sheet_api(m1, client):
 
     response = client.post("/sheet/archive", json={"sheet_name": "456-sheet_name-id"}, headers={"Authorization": "Bearer this_is_the_test_api_token"})
diff --git a/src/tests/endpoints/test_url.py b/src/tests/endpoints/test_url.py
index b128493..01b6a24 100644
--- a/src/tests/endpoints/test_url.py
+++ b/src/tests/endpoints/test_url.py
@@ -10,7 +10,7 @@ def test_archive_url_unauthenticated(client, test_no_auth):
     test_no_auth(client.get, "/url/archive")
 
 
-@patch("worker.create_archive_task.delay", return_value=TaskResult(id="123-456-789", status="PENDING", result=""))
+@patch("worker.main.create_archive_task.delay", return_value=TaskResult(id="123-456-789", status="PENDING", result=""))
 def test_archive_url(m1, client_with_auth):
     response = client_with_auth.post("/url/archive", json={"url": "bad"})
     assert response.status_code == 422
diff --git a/src/web/main.py b/src/web/main.py
index addc3a9..adc837d 100644
--- a/src/web/main.py
+++ b/src/web/main.py
@@ -12,7 +12,7 @@ from sqlalchemy.orm import Session
 from loguru import logger
 
 from core.logging import logging_middleware, log_error
-from worker import create_archive_task, create_sheet_task, celery, insert_result_into_db
+from worker.main import create_archive_task, create_sheet_task, celery, insert_result_into_db
 
 from db import crud, models, schemas
 from web.security import get_user_auth, token_api_key_auth, get_token_or_user_auth
diff --git a/src/worker/__init__.py b/src/worker/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/src/worker.py b/src/worker/main.py
similarity index 100%
rename from src/worker.py
rename to src/worker/main.py

From e64f328d6638c4d2ecf4c7234255858b885c412c Mon Sep 17 00:00:00 2001
From: msramalho <19508417+msramalho@users.noreply.github.com>
Date: Wed, 23 Oct 2024 00:05:24 +0100
Subject: [PATCH 29/33] final round of tests

---
 src/.env.test                           |   3 +-
 src/core/events.py                      |   2 +-
 src/shared/settings.py                  |   2 +
 src/tests/orchestration-sheet.test.yaml |  25 +++
 src/tests/worker/test_worker_main.py    | 196 ++++++++++++++++++++++++
 src/worker/main.py                      |  24 +--
 6 files changed, 240 insertions(+), 12 deletions(-)
 create mode 100644 src/tests/orchestration-sheet.test.yaml
 create mode 100644 src/tests/worker/test_worker_main.py

diff --git a/src/.env.test b/src/.env.test
index bef1d88..5ff71bf 100644
--- a/src/.env.test
+++ b/src/.env.test
@@ -5,4 +5,5 @@ BLOCKED_EMAILS='["blocked@example.com"]'
 
 DATABASE_PATH="sqlite:///auto-archiver.test.db"
 API_BEARER_TOKEN=this_is_the_test_api_token
-USER_GROUPS_FILENAME=tests/user-groups.test.yaml
\ No newline at end of file
+USER_GROUPS_FILENAME=tests/user-groups.test.yaml
+SHEET_ORCHESTRATION_YAML=tests/orchestration-sheet.test.yaml
\ No newline at end of file
diff --git a/src/core/events.py b/src/core/events.py
index 445d027..c5cde2c 100644
--- a/src/core/events.py
+++ b/src/core/events.py
@@ -42,4 +42,4 @@ async def refresh_user_groups():
 
 @repeat_every(seconds=get_settings().REPEAT_COUNT_METRICS_SECONDS)
 async def repeat_measure_regular_metrics():
-    measure_regular_metrics(get_settings().DATABASE_PATH, get_settings().REPEAT_COUNT_METRICS_SECONDS)
+    await measure_regular_metrics(get_settings().DATABASE_PATH, get_settings().REPEAT_COUNT_METRICS_SECONDS)
diff --git a/src/shared/settings.py b/src/shared/settings.py
index 39e2598..56b9557 100644
--- a/src/shared/settings.py
+++ b/src/shared/settings.py
@@ -12,10 +12,12 @@ class Settings(BaseSettings):
 	# general
     SERVE_LOCAL_ARCHIVE: str = ""
     USER_GROUPS_FILENAME: str = "user-groups.yaml"
+    SHEET_ORCHESTRATION_YAML : str = "secrets/orchestration-sheet.yaml"
     
 	# database
     DATABASE_PATH: str
     DATABASE_QUERY_LIMIT: int = 100
+
     # redis
     CELERY_BROKER_URL: str = "redis://localhost:6379"
     CELERY_RESULT_BACKEND: str = "redis://localhost:6379"
diff --git a/src/tests/orchestration-sheet.test.yaml b/src/tests/orchestration-sheet.test.yaml
new file mode 100644
index 0000000..996b41a
--- /dev/null
+++ b/src/tests/orchestration-sheet.test.yaml
@@ -0,0 +1,25 @@
+steps:
+  feeder: gsheet_feeder # default -> only expects URL from CLI
+  archivers: # order matters
+    - youtubedl_archiver
+  enrichers:
+    - hash_enricher
+
+  formatter: html_formatter # defaults to mute_formatter
+  storages:
+    - local_storage
+  databases:
+    - console_db
+
+configurations:
+  gsheet_feeder:
+    sheet: test
+    header: 1 # defaults to 1 in GSheetsFeeder
+    service_account: "secrets/service_account.json"
+  hash_enricher:
+    algorithm: "SHA-256"
+  local_storage:
+    save_to: "./local_archive"
+    save_absolute: true
+    filename_generator: static
+    path_generator: flat
diff --git a/src/tests/worker/test_worker_main.py b/src/tests/worker/test_worker_main.py
new file mode 100644
index 0000000..5818443
--- /dev/null
+++ b/src/tests/worker/test_worker_main.py
@@ -0,0 +1,196 @@
+from unittest import mock
+
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from db import models, schemas
+from auto_archiver import Metadata
+from auto_archiver.core import Media
+
+
+@pytest.fixture()
+def worker_init():
+    from worker.main import at_start
+    at_start(None)
+
+
+class Test_create_archive_task():
+    URL = "https://example-live.com"
+    archive = schemas.ArchiveCreate(url=URL, tags=[], public=True, group_id=None, author_id="rick@example.com")
+
+    @patch("worker.main.insert_result_into_db")
+    @patch("worker.main.is_group_invalid_for_user", return_value=None)
+    @patch("worker.main.choose_orchestrator")
+    @patch("celery.app.task.Task.request")
+    def test_success(self, m_req, m_choose, m_is_group, m_insert, worker_init, db_session):
+        from worker.main import create_archive_task
+
+        m_req.id = "this-just-in"
+        mock_orchestrator = self.mock_orchestrator_choice(m_choose)
+
+        task = create_archive_task(self.archive.model_dump_json())
+
+        m_choose.assert_called_once()
+        mock_orchestrator.feed_item.assert_called_once()
+
+        assert task["status"] == "success"
+        assert task["metadata"]["url"] == self.URL
+        assert len(task["media"]) == 0
+
+    @patch("worker.main.is_group_invalid_for_user", return_value=True)
+    def test_raise_invalid(self, m_is_group, worker_init):
+        from worker.main import create_archive_task
+        with pytest.raises(Exception):
+            create_archive_task(self.archive.model_dump_json())
+
+    @patch("worker.main.insert_result_into_db", side_effect=Exception)
+    @patch("worker.main.is_group_invalid_for_user", return_value=False)
+    @patch("worker.main.choose_orchestrator")
+    def test_raise_db_error(self, m_choose, m_is_group, m_insert, worker_init):
+        from worker.main import create_archive_task
+        mock_orchestrator = self.mock_orchestrator_choice(m_choose)
+
+        with pytest.raises(Exception):
+            create_archive_task(self.archive.model_dump_json())
+        mock_orchestrator.feed_item.assert_called_once()
+
+    def mock_orchestrator_choice(self, m_choose):
+        mock_orchestrator = mock.MagicMock()
+        mock_orchestrator.configure_mock(feed_item=mock.MagicMock(return_value=Metadata().set_url(self.URL).success()))
+        m_choose.return_value = mock_orchestrator
+        return mock_orchestrator
+
+
+class Test_create_sheet_task():
+    URL = "https://example-live.com"
+    sheet = schemas.SubmitSheet(sheet_name="Sheet", sheet_id="123", author_id="rick@example.com", group_id=None)
+
+    # @patch("worker.main.insert_result_into_db")
+    @patch("worker.main.models.generate_uuid", return_value="constant-uuid")
+    @patch("worker.main.is_group_invalid_for_user", return_value=False)
+    @patch("worker.main.ArchivingOrchestrator")
+    def test_success(self, m_orch_generator, m_is_group, m_uuid, worker_init, db_session):
+        from worker.main import create_sheet_task
+
+        assert db_session.query(models.Archive).filter(models.Archive.url == self.URL).count() == 0
+
+        mock_metadata = Metadata().set_url(self.URL).success()
+        mock_metadata.add_media(Media("fn1.txt", urls=["outcome1.com"]))
+        m_orch = MagicMock()
+        m_orch.feed.return_value = iter([False, mock_metadata, mock_metadata])
+        m_orch_generator.return_value = m_orch
+
+        res = create_sheet_task(self.sheet.model_dump_json())
+        print(res)
+        assert res["archived"] == 1
+        assert res["failed"] == 0
+        assert len(res["errors"]) == 0
+        assert res["sheet"] == "Sheet"
+        assert res["sheet_id"] == "123"
+        assert res["success"] == True
+        assert len(res["time"]) > 0
+
+        # query created archive entry
+        inserted = db_session.query(models.Archive).filter(models.Archive.url == self.URL).one()
+        assert inserted is not None
+        assert inserted.url == self.URL
+        assert inserted.tags[0].id == "gsheet"
+
+    @patch("worker.main.insert_result_into_db", side_effect=Exception("some-error"))
+    @patch("worker.main.models.generate_uuid", return_value="constant-uuid")
+    @patch("worker.main.is_group_invalid_for_user", return_value=False)
+    @patch("worker.main.ArchivingOrchestrator")
+    def test_has_exception(self, m_orch_generator, m_is_group, m_uuid, worker_init, db_session):
+        from worker.main import create_sheet_task
+
+        assert db_session.query(models.Archive).filter(models.Archive.url == self.URL).count() == 0
+
+        mock_metadata = Metadata().set_url(self.URL).success()
+        mock_metadata.add_media(Media("fn1.txt", urls=["outcome1.com"]))
+        m_orch = MagicMock()
+        m_orch.feed.return_value = iter([mock_metadata])
+        m_orch_generator.return_value = m_orch
+
+        res = create_sheet_task(self.sheet.model_dump_json())
+        print(res)
+        assert res["archived"] == 0
+        assert res["failed"] == 1
+        assert res["errors"] == ["some-error"]
+        assert res["sheet_id"] == "123"
+        assert res["success"] == True
+
+        assert db_session.query(models.Archive).filter(models.Archive.url == self.URL).count() == 0
+
+    @patch("worker.main.is_group_invalid_for_user", return_value="Access denied")
+    def test_error_access(self, m_insert, worker_init, db_session):
+        from worker.main import create_sheet_task
+
+        res = create_sheet_task(self.sheet.model_dump_json())
+        assert "error" in res
+        assert res["error"] == "Access denied"
+
+
+def test_choose_orchestrator(worker_init):
+    from worker.main import choose_orchestrator
+
+    assert choose_orchestrator(None, "rick@example.com").__class__.__name__ == "ArchivingOrchestrator"
+
+
+@patch("worker.main.get_user_first_group", return_value="does-not-exist")
+def test_choose_orchestrator_assertion(worker_init):
+    from worker.main import choose_orchestrator
+
+    with pytest.raises(Exception):
+        choose_orchestrator(None, "rick@example.com")
+
+
+@patch("worker.main.read_user_groups")
+def test_get_user_first_group(m_read_user_groups, worker_init):
+    from worker.main import get_user_first_group
+
+    m_read_user_groups.return_value = {"users": {}}
+    assert get_user_first_group("email1") == "default"
+    m_read_user_groups.return_value = {"users": {"email1": []}}
+    assert get_user_first_group("email1") == "default"
+    m_read_user_groups.return_value = {"users": {"email1": ["group1", "group2"]}}
+    assert get_user_first_group("email1") == "group1"
+
+
+def test_is_group_invalid_for_user(worker_init, db_session):
+    from worker.main import is_group_invalid_for_user
+    from db.crud import upsert_user_groups
+
+    upsert_user_groups(db_session)
+
+    assert is_group_invalid_for_user(True, "", "") == False
+    assert is_group_invalid_for_user(False, "", "") == False
+
+    assert is_group_invalid_for_user(False, "default", "") == "User  is not part of default, no permission"
+    assert is_group_invalid_for_user(False, "spaceship", "jerry@example.com") == "User jerry@example.com is not part of spaceship, no permission"
+
+    assert is_group_invalid_for_user(False, "spaceship", "rick@example.com") == False
+
+
+def test_get_all_urls(worker_init, db_session):
+    from worker.main import get_all_urls
+    from auto_archiver import Metadata
+
+    meta = Metadata().set_url("https://example.com")
+    m1 = meta.add_media(Media("fn1.txt", urls=["outcome1.com"]))
+    m2 = meta.add_media(Media("fn2.txt", urls=["outcome2.com"]))
+    m3 = meta.add_media(Media("fn3.txt", urls=["outcome3.com"]))
+    m1.set("screenshot", Media("screenshot.png", urls=["screenshot.com"]))
+    m2.set("thumbnails", [Media("thumb1.png", urls=["thumb1.com"]), Media("thumb2.png", urls=["thumb2.com"])])
+    m3.set("ssl_data", Media("ssl_data.txt", urls=["ssl_data.com"]).to_dict())
+    m3.set("bad_data", {"bad": "dict is ignored"})
+
+    urls = [u.url for u in get_all_urls(meta)]
+    assert len(urls) == 7
+    assert "outcome1.com" in urls
+    assert "outcome2.com" in urls
+    assert "outcome3.com" in urls
+    assert "screenshot.com" in urls
+    assert "thumb1.com" in urls
+    assert "thumb2.com" in urls
+    assert "ssl_data.com" in urls
diff --git a/src/worker/main.py b/src/worker/main.py
index 9b6c417..bde1073 100644
--- a/src/worker/main.py
+++ b/src/worker/main.py
@@ -37,6 +37,7 @@ def create_archive_task(self, archive_json: str):
     url = archive.url
     logger.info(f"{url=} {archive=}")
 
+    # TODO: re-evaluate if this logic is to be used
     if not archive.rearchive:
         with get_db() as session:
             archives = crud.search_archives_by_url(session, url, archive.author_id, absolute_search=True)
@@ -63,11 +64,12 @@ def create_sheet_task(self, sheet_json: str):
     sheet.tags.add("gsheet")
     logger.info(f"SHEET START {sheet=}")
 
-    if (em := is_group_invalid_for_user(sheet.public, sheet.group_id, sheet.author_id)): return {"error": em}
+    if (em := is_group_invalid_for_user(sheet.public, sheet.group_id, sheet.author_id)):
+        return {"error": em}
 
     config = Config()
     # TODO: use choose_orchestrator and overwrite the feeder
-    config.parse(use_cli=False, yaml_config_filename="secrets/orchestration-sheet.yaml", overwrite_configs={"configurations": {"gsheet_feeder": {"sheet": sheet.sheet_name, "sheet_id": sheet.sheet_id, "header": sheet.header}}})
+    config.parse(use_cli=False, yaml_config_filename=get_settings().SHEET_ORCHESTRATION_YAML, overwrite_configs={"configurations": {"gsheet_feeder": {"sheet": sheet.sheet_name, "sheet_id": sheet.sheet_id, "header": sheet.header}}})
     orchestrator = ArchivingOrchestrator(config)
 
     stats = {"archived": 0, "failed": 0, "errors": []}
@@ -80,7 +82,6 @@ def create_sheet_task(self, sheet_json: str):
             stats["archived"] += 1
         except exc.IntegrityError as e:
             logger.warning(f"cached result detected: {e}")
-            stats["archived"] += 1
         except Exception as e:
             log_error(e, extra=f"{self.name}: {sheet_json}")
             redis_publish_exception(e, self.name, traceback.format_exc())
@@ -121,7 +122,8 @@ def read_user_groups():
 def get_user_first_group(email):
     user_groups_yaml = read_user_groups()
     groups = user_groups_yaml.get("users", {}).get(email, [])
-    if groups != None and len(groups): return groups[0]
+    if groups != None and len(groups):
+        return groups[0]
     return "default"
 
 
@@ -151,12 +153,14 @@ def is_group_invalid_for_user(public: bool, group_id: str, author_id: str):
     if public is true the requirement is not needed
     returns an error message if invalid, or False if all is good.
     """
-    if not public and group_id and len(group_id) > 0:
-        # ensure group is valid for user
-        with get_db() as session:
-            if not crud.is_user_in_group(session, group_id, author_id):
-                logger.error(em := f"User {author_id} is not part of {group_id}, no permission")
-                return em
+    if public: return False
+    if not group_id or len(group_id) == 0: return False
+    
+    # otherwise group must match
+    with get_db() as session:
+        if not crud.is_user_in_group(session, group_id, author_id):
+            logger.error(em := f"User {author_id} is not part of {group_id}, no permission")
+            return em
     return False
 
 

From 8e38c92def4190333de55ccc98ea793b0a3d8cfa Mon Sep 17 00:00:00 2001
From: msramalho <19508417+msramalho@users.noreply.github.com>
Date: Wed, 23 Oct 2024 09:55:15 +0100
Subject: [PATCH 30/33] adds missing files for tests

---
 src/tests/orchestration.test.yaml | 24 ++++++++++++++++++++++++
 src/tests/user-groups.test.yaml   |  6 +++---
 2 files changed, 27 insertions(+), 3 deletions(-)
 create mode 100644 src/tests/orchestration.test.yaml

diff --git a/src/tests/orchestration.test.yaml b/src/tests/orchestration.test.yaml
new file mode 100644
index 0000000..cb79ea9
--- /dev/null
+++ b/src/tests/orchestration.test.yaml
@@ -0,0 +1,24 @@
+steps:
+  feeder: cli_feeder
+  archivers: # order matters
+    - youtubedl_archiver
+  enrichers:
+    - hash_enricher
+
+  formatter: html_formatter # defaults to mute_formatter
+  storages:
+    - local_storage
+  databases:
+    - console_db
+
+configurations:
+  cli_feeder:
+    urls: 
+      - "url1"
+  hash_enricher:
+    algorithm: "SHA-256"
+  local_storage:
+    save_to: "./local_archive"
+    save_absolute: true
+    filename_generator: static
+    path_generator: flat
diff --git a/src/tests/user-groups.test.yaml b/src/tests/user-groups.test.yaml
index 7c5cfcc..545b38c 100644
--- a/src/tests/user-groups.test.yaml
+++ b/src/tests/user-groups.test.yaml
@@ -14,6 +14,6 @@ domains:
     - animated-characters
 
 orchestrators:
-  spaceship: secrets/orchestration-dev.yaml
-  interdimensional: secrets/orchestration-dev.yaml
-  default: secrets/orchestration-dev.yaml
+  spaceship: tests/orchestration.test.yaml
+  interdimensional: tests/orchestration.test.yaml
+  default: tests/orchestration.test.yaml

From bf91bb9e66efd219f61c7da051787a64fac9c9c4 Mon Sep 17 00:00:00 2001
From: msramalho <19508417+msramalho@users.noreply.github.com>
Date: Wed, 23 Oct 2024 10:11:37 +0100
Subject: [PATCH 31/33] simplifying tests

---
 src/.env.test                           |  2 +-
 src/tests/orchestration-sheet.test.yaml | 25 -------------------------
 2 files changed, 1 insertion(+), 26 deletions(-)
 delete mode 100644 src/tests/orchestration-sheet.test.yaml

diff --git a/src/.env.test b/src/.env.test
index 5ff71bf..5e5ea24 100644
--- a/src/.env.test
+++ b/src/.env.test
@@ -6,4 +6,4 @@ BLOCKED_EMAILS='["blocked@example.com"]'
 DATABASE_PATH="sqlite:///auto-archiver.test.db"
 API_BEARER_TOKEN=this_is_the_test_api_token
 USER_GROUPS_FILENAME=tests/user-groups.test.yaml
-SHEET_ORCHESTRATION_YAML=tests/orchestration-sheet.test.yaml
\ No newline at end of file
+SHEET_ORCHESTRATION_YAML=tests/orchestration.test.yaml
\ No newline at end of file
diff --git a/src/tests/orchestration-sheet.test.yaml b/src/tests/orchestration-sheet.test.yaml
deleted file mode 100644
index 996b41a..0000000
--- a/src/tests/orchestration-sheet.test.yaml
+++ /dev/null
@@ -1,25 +0,0 @@
-steps:
-  feeder: gsheet_feeder # default -> only expects URL from CLI
-  archivers: # order matters
-    - youtubedl_archiver
-  enrichers:
-    - hash_enricher
-
-  formatter: html_formatter # defaults to mute_formatter
-  storages:
-    - local_storage
-  databases:
-    - console_db
-
-configurations:
-  gsheet_feeder:
-    sheet: test
-    header: 1 # defaults to 1 in GSheetsFeeder
-    service_account: "secrets/service_account.json"
-  hash_enricher:
-    algorithm: "SHA-256"
-  local_storage:
-    save_to: "./local_archive"
-    save_absolute: true
-    filename_generator: static
-    path_generator: flat

From b17090319ad62d89b325e7f5ff14625acfe86621 Mon Sep 17 00:00:00 2001
From: msramalho <19508417+msramalho@users.noreply.github.com>
Date: Wed, 23 Oct 2024 10:43:35 +0100
Subject: [PATCH 32/33] closes #14 closes #21

---
 README.md | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/README.md b/README.md
index 236aba6..102fd2d 100644
--- a/README.md
+++ b/README.md
@@ -75,17 +75,14 @@ curl -XPOST -H "Authorization: Bearer GOOGLE_OAUTH_TOKEN" -H "Content-type: appl
 ```bash
 # can be done from top level but let's do it from the src folder for consistency with CI etc
 cd src
-# run tests 
-PYTHONPATH=. pipenv run pytest -v --color=yes tests/
+# run tests and generate coverage
+PYTHONPATH=. PIPENV_DOTENV_LOCATION=.env.test pipenv run coverage run -m pytest -vv --disable-warnings --color=yes tests/  && pipenv run coverage html
 
-# TO GET COVERAGE
-# run tests with coverage instead 
-PYTHONPATH=. pipenv run coverage run -m pytest -v --color=yes tests/
-
-# get coverage
+# get coverage report in command line
 pipenv run coverage report
-# get coverage HTML
 
+# get coverage HTML
 pipenv run coverage html
+
 # > open/run server on htmlcov/index.html to navigate through line coverage
 ```
\ No newline at end of file

From 6814e6dfe9369921cac77ce98a3e8e03470a6994 Mon Sep 17 00:00:00 2001
From: msramalho <19508417+msramalho@users.noreply.github.com>
Date: Wed, 23 Oct 2024 10:46:38 +0100
Subject: [PATCH 33/33] removes unintended pipfile

---
 Pipfile | 11 -----------
 1 file changed, 11 deletions(-)
 delete mode 100644 Pipfile

diff --git a/Pipfile b/Pipfile
deleted file mode 100644
index c398b0d..0000000
--- a/Pipfile
+++ /dev/null
@@ -1,11 +0,0 @@
-[[source]]
-url = "https://pypi.org/simple"
-verify_ssl = true
-name = "pypi"
-
-[packages]
-
-[dev-packages]
-
-[requires]
-python_version = "3.10"