major refactor of structure for worker V web: docker/app/secrets/envs/...

2026-06-11 13:08:34 +03:00 · 2025-02-10 00:41:50 +00:00
parent a1b730bad4
commit f8c45e2d92
74 changed files with 567 additions and 525 deletions
--- a/src/.env.alembic
+++ b/src/.env.alembic
@@ -1,5 +1,5 @@
 CHROME_APP_IDS='["1234567890"]'
 ALLOWED_ORIGINS='["allowed"]'
 BLOCKED_EMAILS='[]'
-DATABASE_PATH="sqlite:///./auto-archiver.db"
+DATABASE_PATH="sqlite:///./database/auto-archiver.db"
 API_BEARER_TOKEN=THIS_API_TOKEN_SHOULD_NEVER_BE_USED
--- a/src/.env.test
+++ b/src/.env.test
@@ -5,5 +5,5 @@ BLOCKED_EMAILS='["blocked@example.com"]'

 DATABASE_PATH="sqlite:///auto-archiver.test.db"
 API_BEARER_TOKEN=this_is_the_test_api_token
-USER_GROUPS_FILENAME=tests/user-groups.test.yaml
-SHEET_ORCHESTRATION_YAML=tests/orchestration.test.yaml
+USER_GROUPS_FILENAME=app/tests/user-groups.test.yaml
+SHEET_ORCHESTRATION_YAML=app/tests/orchestration.test.yaml
--- a/.example.env
+++ b/.example.env
@@ -1 +1,9 @@
-REDIS_PASSWORD=TODO
+REDIS_PASSWORD=TODO
+
+DATABASE_PATH="sqlite:///./database/auto-archiver.db"
+USER_GROUPS_FILENAME=app/user-groups.yaml
+CHROME_APP_IDS=000000000000000000000000000000000000000000000.apps.googleusercontent.com,000000000000000000000000000000000000000000001.apps.googleusercontent.com
+#ALLOWED_ORIGINS="http://localhost:8004" # dev only
+
+
+API_BEARER_TOKEN=TODO
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -35,7 +35,7 @@ jobs:
    - name: Install dependencies
      run: pipenv install --dev
      working-directory: src
-
+#TODO: fix working-directories here
    - name: Run tests with coverage
      run: PYTHONPATH=. PIPENV_DOTENV_LOCATION=.env.test pipenv run coverage run -m pytest -v --color=yes tests/
      working-directory: src
--- a/.gitignore
+++ b/.gitignore
@@ -2,26 +2,25 @@ orchestration.yaml
 my-archives
 *.pyc
 .DS_Store
-secrets
+secrets/*
 *.log
-__pycache
-.pytest_cach
+__pycache__
+.pytest_cache
 .env
 .env.dev
 .env.prod
 *.db
 redis/data/*
 .ipynb_checkpoints*
-src/user-groups.yaml
-src/user-groups.dev.yaml
+app/user-groups.yaml
+app/user-groups.dev.yaml
 wit*
-src/crawls
+app/crawls
 .coverage
-.pytest_cache/*
+.pytest_cache/
 htmlcov
 local_archive
 local_archive_test
 *db-wal
 *db-shm
-copy-files.sh
-.pytest_cache
+copy-files.sh
--- a/5
+++ b/5
@@ -6,6 +6,11 @@ dev:
 	docker compose -f docker-compose.yml -f docker-compose.dev.yml build
 	docker compose -f docker-compose.yml -f docker-compose.dev.yml up --remove-orphans

+
+dev-redis-only:
+	docker compose -f docker-compose.yml -f docker-compose.dev.yml build redis
+	docker compose -f docker-compose.yml -f docker-compose.dev.yml up --remove-orphans redis
+
 stop-dev:
 	docker compose -f docker-compose.yml -f docker-compose.dev.yml down --volumes

--- a/src/Pipfile
+++ b/src/Pipfile
@@ -5,7 +5,6 @@ name = "pypi"

 [packages]
 oscrypto = {git = "https://github.com/wbond/oscrypto.git", ref = "d5f3437ed24257895ae1edd9e503cfb352e635a8"}
-aiofiles = "==0.6.0"
 celery = ">=5.0"
 fastapi = "*"
 jinja2 = "*"
@@ -13,7 +12,6 @@ redis = "==3.5.3"
 requests = ">=2.25.1"
 uvicorn = ">=0.13.4"
 aiosqlite = "*"
-python-dotenv = "*"
 loguru = "*"
 sqlalchemy = "*"
 alembic = "*"
--- a/src/Pipfile.lock
+++ b/src/Pipfile.lock
@@ -1,7 +1,7 @@
 {
    "_meta": {
        "hash": {
-            "sha256": "14a7ead66a74419eebfa72478bbb7e3efe378df9f41e401738faa2871f5c4344"
+            "sha256": "f03b75b94f11f10065e9fd4b4f107a78c37f880f4537b4b19fd0aaad7afa9ab7"
        },
        "pipfile-spec": 6,
        "requires": {
@@ -16,103 +16,100 @@
        ]
    },
    "default": {
-        "aiofiles": {
-            "hashes": [
-                "sha256:bd3019af67f83b739f8e4053c6c0512a7f545b9a8d91aaeab55e6e0f9d123c27",
-                "sha256:e0281b157d3d5d59d803e3f4557dcc9a3dff28a4dd4829a9ff478adae50ca092"
-            ],
-            "index": "pypi",
-            "version": "==0.6.0"
-        },
        "aiohappyeyeballs": {
            "hashes": [
-                "sha256:5fdd7d87889c63183afc18ce9271f9b0a7d32c2303e394468dd45d514a757745",
-                "sha256:a980909d50efcd44795c4afeca523296716d50cd756ddca6af8c65b996e27de8"
+                "sha256:147ec992cf873d74f5062644332c539fcd42956dc69453fe5204195e560517e1",
+                "sha256:9b05052f9042985d32ecbe4b59a77ae19c006a78f1344d7fdad69d28ded3d0b0"
            ],
-            "markers": "python_version >= '3.8'",
-            "version": "==2.4.4"
+            "markers": "python_version >= '3.9'",
+            "version": "==2.4.6"
        },
        "aiohttp": {
            "hashes": [
-                "sha256:0882c2820fd0132240edbb4a51eb8ceb6eef8181db9ad5291ab3332e0d71df5f",
-                "sha256:0a6d3fbf2232e3a08c41eca81ae4f1dff3d8f1a30bae415ebe0af2d2458b8a33",
-                "sha256:0b7fb429ab1aafa1f48578eb315ca45bd46e9c37de11fe45c7f5f4138091e2f1",
-                "sha256:0eb98d90b6690827dcc84c246811feeb4e1eea683c0eac6caed7549be9c84665",
-                "sha256:0fd82b8e9c383af11d2b26f27a478640b6b83d669440c0a71481f7c865a51da9",
-                "sha256:10b4ff0ad793d98605958089fabfa350e8e62bd5d40aa65cdc69d6785859f94e",
-                "sha256:1642eceeaa5ab6c9b6dfeaaa626ae314d808188ab23ae196a34c9d97efb68350",
-                "sha256:1dac54e8ce2ed83b1f6b1a54005c87dfed139cf3f777fdc8afc76e7841101226",
-                "sha256:1e69966ea6ef0c14ee53ef7a3d68b564cc408121ea56c0caa2dc918c1b2f553d",
-                "sha256:1f21bb8d0235fc10c09ce1d11ffbd40fc50d3f08a89e4cf3a0c503dc2562247a",
-                "sha256:2170816e34e10f2fd120f603e951630f8a112e1be3b60963a1f159f5699059a6",
-                "sha256:21fef42317cf02e05d3b09c028712e1d73a9606f02467fd803f7c1f39cc59add",
-                "sha256:249cc6912405917344192b9f9ea5cd5b139d49e0d2f5c7f70bdfaf6b4dbf3a2e",
-                "sha256:3499c7ffbfd9c6a3d8d6a2b01c26639da7e43d47c7b4f788016226b1e711caa8",
-                "sha256:3af41686ccec6a0f2bdc66686dc0f403c41ac2089f80e2214a0f82d001052c03",
-                "sha256:3e23419d832d969f659c208557de4a123e30a10d26e1e14b73431d3c13444c2e",
-                "sha256:3ea1b59dc06396b0b424740a10a0a63974c725b1c64736ff788a3689d36c02d2",
-                "sha256:44167fc6a763d534a6908bdb2592269b4bf30a03239bcb1654781adf5e49caf1",
-                "sha256:479b8c6ebd12aedfe64563b85920525d05d394b85f166b7873c8bde6da612f9c",
-                "sha256:4af57160800b7a815f3fe0eba9b46bf28aafc195555f1824555fa2cfab6c1538",
-                "sha256:4b4fa1cb5f270fb3eab079536b764ad740bb749ce69a94d4ec30ceee1b5940d5",
-                "sha256:4eed954b161e6b9b65f6be446ed448ed3921763cc432053ceb606f89d793927e",
-                "sha256:541d823548ab69d13d23730a06f97460f4238ad2e5ed966aaf850d7c369782d9",
-                "sha256:568c1236b2fde93b7720f95a890741854c1200fba4a3471ff48b2934d2d93fd3",
-                "sha256:5854be2f3e5a729800bac57a8d76af464e160f19676ab6aea74bde18ad19d438",
-                "sha256:620598717fce1b3bd14dd09947ea53e1ad510317c85dda2c9c65b622edc96b12",
-                "sha256:6526e5fb4e14f4bbf30411216780c9967c20c5a55f2f51d3abd6de68320cc2f3",
-                "sha256:6fba278063559acc730abf49845d0e9a9e1ba74f85f0ee6efd5803f08b285853",
-                "sha256:70d1f9dde0e5dd9e292a6d4d00058737052b01f3532f69c0c65818dac26dc287",
-                "sha256:731468f555656767cda219ab42e033355fe48c85fbe3ba83a349631541715ba2",
-                "sha256:81b8fe282183e4a3c7a1b72f5ade1094ed1c6345a8f153506d114af5bf8accd9",
-                "sha256:84a585799c58b795573c7fa9b84c455adf3e1d72f19a2bf498b54a95ae0d194c",
-                "sha256:85992ee30a31835fc482468637b3e5bd085fa8fe9392ba0bdcbdc1ef5e9e3c55",
-                "sha256:8811f3f098a78ffa16e0ea36dffd577eb031aea797cbdba81be039a4169e242c",
-                "sha256:88a12ad8ccf325a8a5ed80e6d7c3bdc247d66175afedbe104ee2aaca72960d8e",
-                "sha256:8be8508d110d93061197fd2d6a74f7401f73b6d12f8822bbcd6d74f2b55d71b1",
-                "sha256:8e2bf8029dbf0810c7bfbc3e594b51c4cc9101fbffb583a3923aea184724203c",
-                "sha256:929f3ed33743a49ab127c58c3e0a827de0664bfcda566108989a14068f820194",
-                "sha256:92cde43018a2e17d48bb09c79e4d4cb0e236de5063ce897a5e40ac7cb4878773",
-                "sha256:92fc484e34b733704ad77210c7957679c5c3877bd1e6b6d74b185e9320cc716e",
-                "sha256:943a8b052e54dfd6439fd7989f67fc6a7f2138d0a2cf0a7de5f18aa4fe7eb3b1",
-                "sha256:9d73ee3725b7a737ad86c2eac5c57a4a97793d9f442599bea5ec67ac9f4bdc3d",
-                "sha256:9f5b3c1ed63c8fa937a920b6c1bec78b74ee09593b3f5b979ab2ae5ef60d7600",
-                "sha256:9fd46ce0845cfe28f108888b3ab17abff84ff695e01e73657eec3f96d72eef34",
-                "sha256:a344d5dc18074e3872777b62f5f7d584ae4344cd6006c17ba12103759d407af3",
-                "sha256:a60804bff28662cbcf340a4d61598891f12eea3a66af48ecfdc975ceec21e3c8",
-                "sha256:a8f5f7515f3552d899c61202d99dcb17d6e3b0de777900405611cd747cecd1b8",
-                "sha256:a9b7371665d4f00deb8f32208c7c5e652059b0fda41cf6dbcac6114a041f1cc2",
-                "sha256:aa54f8ef31d23c506910c21163f22b124facb573bff73930735cf9fe38bf7dff",
-                "sha256:aba807f9569455cba566882c8938f1a549f205ee43c27b126e5450dc9f83cc62",
-                "sha256:ae545f31489548c87b0cced5755cfe5a5308d00407000e72c4fa30b19c3220ac",
-                "sha256:af01e42ad87ae24932138f154105e88da13ce7d202a6de93fafdafb2883a00ef",
-                "sha256:b540bd67cfb54e6f0865ceccd9979687210d7ed1a1cc8c01f8e67e2f1e883d28",
-                "sha256:b6212a60e5c482ef90f2d788835387070a88d52cf6241d3916733c9176d39eab",
-                "sha256:b63de12e44935d5aca7ed7ed98a255a11e5cb47f83a9fded7a5e41c40277d104",
-                "sha256:ba74ec819177af1ef7f59063c6d35a214a8fde6f987f7661f4f0eecc468a8f76",
-                "sha256:bb49c7f1e6ebf3821a42d81d494f538107610c3a705987f53068546b0e90303e",
-                "sha256:bd176afcf8f5d2aed50c3647d4925d0db0579d96f75a31e77cbaf67d8a87742d",
-                "sha256:bd7227b87a355ce1f4bf83bfae4399b1f5bb42e0259cb9405824bd03d2f4336a",
-                "sha256:bf8d9bfee991d8acc72d060d53860f356e07a50f0e0d09a8dfedea1c554dd0d5",
-                "sha256:bfde76a8f430cf5c5584553adf9926534352251d379dcb266ad2b93c54a29745",
-                "sha256:c341c7d868750e31961d6d8e60ff040fb9d3d3a46d77fd85e1ab8e76c3e9a5c4",
-                "sha256:c7a06301c2fb096bdb0bd25fe2011531c1453b9f2c163c8031600ec73af1cc99",
-                "sha256:cb23d8bb86282b342481cad4370ea0853a39e4a32a0042bb52ca6bdde132df43",
-                "sha256:d119fafe7b634dbfa25a8c597718e69a930e4847f0b88e172744be24515140da",
-                "sha256:d40f9da8cabbf295d3a9dae1295c69975b86d941bc20f0a087f0477fa0a66231",
-                "sha256:d6c9af134da4bc9b3bd3e6a70072509f295d10ee60c697826225b60b9959acdd",
-                "sha256:dd7659baae9ccf94ae5fe8bfaa2c7bc2e94d24611528395ce88d009107e00c6d",
-                "sha256:de8d38f1c2810fa2a4f1d995a2e9c70bb8737b18da04ac2afbf3971f65781d87",
-                "sha256:e595c591a48bbc295ebf47cb91aebf9bd32f3ff76749ecf282ea7f9f6bb73886",
-                "sha256:ec2aa89305006fba9ffb98970db6c8221541be7bee4c1d027421d6f6df7d1ce2",
-                "sha256:ec82bf1fda6cecce7f7b915f9196601a1bd1a3079796b76d16ae4cce6d0ef89b",
-                "sha256:ed9ee95614a71e87f1a70bc81603f6c6760128b140bc4030abe6abaa988f1c3d",
-                "sha256:f047569d655f81cb70ea5be942ee5d4421b6219c3f05d131f64088c73bb0917f",
-                "sha256:ffa336210cf9cd8ed117011085817d00abe4c08f99968deef0013ea283547204",
-                "sha256:ffb3dc385f6bb1568aa974fe65da84723210e5d9707e360e9ecb51f59406cd2e"
+                "sha256:0450ada317a65383b7cce9576096150fdb97396dcfe559109b403c7242faffef",
+                "sha256:0b5263dcede17b6b0c41ef0c3ccce847d82a7da98709e75cf7efde3e9e3b5cae",
+                "sha256:0d5176f310a7fe6f65608213cc74f4228e4f4ce9fd10bcb2bb6da8fc66991462",
+                "sha256:0ed49efcd0dc1611378beadbd97beb5d9ca8fe48579fc04a6ed0844072261b6a",
+                "sha256:145a73850926018ec1681e734cedcf2716d6a8697d90da11284043b745c286d5",
+                "sha256:1987770fb4887560363b0e1a9b75aa303e447433c41284d3af2840a2f226d6e0",
+                "sha256:246067ba0cf5560cf42e775069c5d80a8989d14a7ded21af529a4e10e3e0f0e6",
+                "sha256:2c311e2f63e42c1bf86361d11e2c4a59f25d9e7aabdbdf53dc38b885c5435cdb",
+                "sha256:2cee3b117a8d13ab98b38d5b6bdcd040cfb4181068d05ce0c474ec9db5f3c5bb",
+                "sha256:2de1378f72def7dfb5dbd73d86c19eda0ea7b0a6873910cc37d57e80f10d64e1",
+                "sha256:30f546358dfa0953db92ba620101fefc81574f87b2346556b90b5f3ef16e55ce",
+                "sha256:34245498eeb9ae54c687a07ad7f160053911b5745e186afe2d0c0f2898a1ab8a",
+                "sha256:392432a2dde22b86f70dd4a0e9671a349446c93965f261dbaecfaf28813e5c42",
+                "sha256:3c0600bcc1adfaaac321422d615939ef300df81e165f6522ad096b73439c0f58",
+                "sha256:4016e383f91f2814e48ed61e6bda7d24c4d7f2402c75dd28f7e1027ae44ea204",
+                "sha256:40cd36749a1035c34ba8d8aaf221b91ca3d111532e5ccb5fa8c3703ab1b967ed",
+                "sha256:413ad794dccb19453e2b97c2375f2ca3cdf34dc50d18cc2693bd5aed7d16f4b9",
+                "sha256:4a93d28ed4b4b39e6f46fd240896c29b686b75e39cc6992692e3922ff6982b4c",
+                "sha256:4ee84c2a22a809c4f868153b178fe59e71423e1f3d6a8cd416134bb231fbf6d3",
+                "sha256:50c5c7b8aa5443304c55c262c5693b108c35a3b61ef961f1e782dd52a2f559c7",
+                "sha256:525410e0790aab036492eeea913858989c4cb070ff373ec3bc322d700bdf47c1",
+                "sha256:526c900397f3bbc2db9cb360ce9c35134c908961cdd0ac25b1ae6ffcaa2507ff",
+                "sha256:54775858c7f2f214476773ce785a19ee81d1294a6bedc5cc17225355aab74802",
+                "sha256:584096938a001378484aa4ee54e05dc79c7b9dd933e271c744a97b3b6f644957",
+                "sha256:6130459189e61baac5a88c10019b21e1f0c6d00ebc770e9ce269475650ff7f73",
+                "sha256:67453e603cea8e85ed566b2700efa1f6916aefbc0c9fcb2e86aaffc08ec38e78",
+                "sha256:68d54234c8d76d8ef74744f9f9fc6324f1508129e23da8883771cdbb5818cbef",
+                "sha256:6dfe7f984f28a8ae94ff3a7953cd9678550dbd2a1f9bda5dd9c5ae627744c78e",
+                "sha256:74bd573dde27e58c760d9ca8615c41a57e719bff315c9adb6f2a4281a28e8798",
+                "sha256:7603ca26d75b1b86160ce1bbe2787a0b706e592af5b2504e12caa88a217767b0",
+                "sha256:76719dd521c20a58a6c256d058547b3a9595d1d885b830013366e27011ffe804",
+                "sha256:7c3623053b85b4296cd3925eeb725e386644fd5bc67250b3bb08b0f144803e7b",
+                "sha256:7e44eba534381dd2687be50cbd5f2daded21575242ecfdaf86bbeecbc38dae8e",
+                "sha256:7fe3d65279bfbee8de0fb4f8c17fc4e893eed2dba21b2f680e930cc2b09075c5",
+                "sha256:8340def6737118f5429a5df4e88f440746b791f8f1c4ce4ad8a595f42c980bd5",
+                "sha256:84ede78acde96ca57f6cf8ccb8a13fbaf569f6011b9a52f870c662d4dc8cd854",
+                "sha256:850ff6155371fd802a280f8d369d4e15d69434651b844bde566ce97ee2277420",
+                "sha256:87a2e00bf17da098d90d4145375f1d985a81605267e7f9377ff94e55c5d769eb",
+                "sha256:88d385b8e7f3a870146bf5ea31786ef7463e99eb59e31db56e2315535d811f55",
+                "sha256:8a2fb742ef378284a50766e985804bd6adb5adb5aa781100b09befdbfa757b65",
+                "sha256:8dc0fba9a74b471c45ca1a3cb6e6913ebfae416678d90529d188886278e7f3f6",
+                "sha256:8fa1510b96c08aaad49303ab11f8803787c99222288f310a62f493faf883ede1",
+                "sha256:8fd12d0f989c6099e7b0f30dc6e0d1e05499f3337461f0b2b0dadea6c64b89df",
+                "sha256:9060addfa4ff753b09392efe41e6af06ea5dd257829199747b9f15bfad819460",
+                "sha256:930ffa1925393381e1e0a9b82137fa7b34c92a019b521cf9f41263976666a0d6",
+                "sha256:936d8a4f0f7081327014742cd51d320296b56aa6d324461a13724ab05f4b2933",
+                "sha256:97fe431f2ed646a3b56142fc81d238abcbaff08548d6912acb0b19a0cadc146b",
+                "sha256:9bd8695be2c80b665ae3f05cb584093a1e59c35ecb7d794d1edd96e8cc9201d7",
+                "sha256:9dec0000d2d8621d8015c293e24589d46fa218637d820894cb7356c77eca3259",
+                "sha256:a478aa11b328983c4444dacb947d4513cb371cd323f3845e53caeda6be5589d5",
+                "sha256:a481a574af914b6e84624412666cbfbe531a05667ca197804ecc19c97b8ab1b0",
+                "sha256:a4ac6a0f0f6402854adca4e3259a623f5c82ec3f0c049374133bcb243132baf9",
+                "sha256:a5e69046f83c0d3cb8f0d5bd9b8838271b1bc898e01562a04398e160953e8eb9",
+                "sha256:a7442662afebbf7b4c6d28cb7aab9e9ce3a5df055fc4116cc7228192ad6cb484",
+                "sha256:aa8a8caca81c0a3e765f19c6953416c58e2f4cc1b84829af01dd1c771bb2f91f",
+                "sha256:ab3247d58b393bda5b1c8f31c9edece7162fc13265334217785518dd770792b8",
+                "sha256:b10a47e5390c4b30a0d58ee12581003be52eedd506862ab7f97da7a66805befb",
+                "sha256:b34508f1cd928ce915ed09682d11307ba4b37d0708d1f28e5774c07a7674cac9",
+                "sha256:b8d3bb96c147b39c02d3db086899679f31958c5d81c494ef0fc9ef5bb1359b3d",
+                "sha256:b9d45dbb3aaec05cf01525ee1a7ac72de46a8c425cb75c003acd29f76b1ffe94",
+                "sha256:bf4480a5438f80e0f1539e15a7eb8b5f97a26fe087e9828e2c0ec2be119a9f72",
+                "sha256:c160a04283c8c6f55b5bf6d4cad59bb9c5b9c9cd08903841b25f1f7109ef1259",
+                "sha256:c96a43822f1f9f69cc5c3706af33239489a6294be486a0447fb71380070d4d5f",
+                "sha256:c9fd9dcf9c91affe71654ef77426f5cf8489305e1c66ed4816f5a21874b094b9",
+                "sha256:cddb31f8474695cd61fc9455c644fc1606c164b93bff2490390d90464b4655df",
+                "sha256:ce1bb21fc7d753b5f8a5d5a4bae99566386b15e716ebdb410154c16c91494d7f",
+                "sha256:d1c031a7572f62f66f1257db37ddab4cb98bfaf9b9434a3b4840bf3560f5e788",
+                "sha256:d589264dbba3b16e8951b6f145d1e6b883094075283dafcab4cdd564a9e353a0",
+                "sha256:dc065a4285307607df3f3686363e7f8bdd0d8ab35f12226362a847731516e42c",
+                "sha256:e10c440d142fa8b32cfdb194caf60ceeceb3e49807072e0dc3a8887ea80e8c16",
+                "sha256:e3552fe98e90fdf5918c04769f338a87fa4f00f3b28830ea9b78b1bdc6140e0d",
+                "sha256:e392804a38353900c3fd8b7cacbea5132888f7129f8e241915e90b85f00e3250",
+                "sha256:e4cecdb52aaa9994fbed6b81d4568427b6002f0a91c322697a4bfcc2b2363f5a",
+                "sha256:e5148ca8955affdfeb864aca158ecae11030e952b25b3ae15d4e2b5ba299bad2",
+                "sha256:e6b2732ef3bafc759f653a98881b5b9cdef0716d98f013d376ee8dfd7285abf1",
+                "sha256:ea756b5a7bac046d202a9a3889b9a92219f885481d78cd318db85b15cc0b7bcf",
+                "sha256:edb69b9589324bdc40961cdf0657815df674f1743a8d5ad9ab56a99e4833cfdd",
+                "sha256:f0203433121484b32646a5f5ea93ae86f3d9559d7243f07e8c0eab5ff8e3f70e",
+                "sha256:f6a19bcab7fbd8f8649d6595624856635159a6527861b9cdc3447af288a00c00",
+                "sha256:f752e80606b132140883bb262a457c475d219d7163d996dc9072434ffb0784c4",
+                "sha256:f7914ab70d2ee8ab91c13e5402122edbc77821c66d2758abb53aabe87f013287"
            ],
            "markers": "python_version >= '3.9'",
-            "version": "==3.11.11"
+            "version": "==3.11.12"
        },
        "aiosignal": {
            "hashes": [
@@ -132,12 +129,12 @@
        },
        "aiosqlite": {
            "hashes": [
-                "sha256:36a1deaca0cac40ebe32aac9977a6e2bbc7f5189f23f4a54d5908986729e5bd6",
-                "sha256:6d35c8c256637f4672f843c31021464090805bf925385ac39473fb16eaaca3d7"
+                "sha256:131bb8056daa3bc875608c631c678cda73922a2d4ba8aec373b19f18c17e7aa3",
+                "sha256:2549cf4057f95f53dcba16f2b64e8e2791d7e1adedb13197dd8ed77bb226d7d0"
            ],
            "index": "pypi",
-            "markers": "python_version >= '3.8'",
-            "version": "==0.20.0"
+            "markers": "python_version >= '3.9'",
+            "version": "==0.21.0"
        },
        "alembic": {
            "hashes": [
@@ -196,19 +193,19 @@
        },
        "attrs": {
            "hashes": [
-                "sha256:8f5c07333d543103541ba7be0e2ce16eeee8130cb0b3f9238ab904ce1e85baff",
-                "sha256:ac96cd038792094f438ad1f6ff80837353805ac950cd2aa0e0625ef19850c308"
+                "sha256:1c97078a80c814273a76b2a298a932eb681c87415c11dee0a6921de7f1b02c3e",
+                "sha256:c75a69e28a550a7e93789579c22aa26b0f5b83b75dc4e08fe092980051e1090a"
            ],
            "markers": "python_version >= '3.8'",
-            "version": "==24.3.0"
+            "version": "==25.1.0"
        },
        "authlib": {
            "hashes": [
-                "sha256:1c1e6608b5ed3624aeeee136ca7f8c120d6f51f731aa152b153d54741840e1f2",
-                "sha256:4bb20b978c8b636222b549317c1815e1fe62234fc1c5efe8855d84aebf3a74e3"
+                "sha256:30ead9ea4993cdbab821dc6e01e818362f92da290c04c7f6a1940f86507a790d",
+                "sha256:edc29c3f6a3e72cd9e9f45fff67fc663a2c364022eb0371c003f22d5405915c1"
            ],
            "markers": "python_version >= '3.9'",
-            "version": "==1.4.0"
+            "version": "==1.4.1"
        },
        "auto-archiver": {
            "hashes": [
@@ -221,11 +218,11 @@
        },
        "beautifulsoup4": {
            "hashes": [
-                "sha256:74e3d1928edc070d21748185c46e3fb33490f22f52a3addee9aee0f4f7781051",
-                "sha256:b80878c9f40111313e55da8ba20bdba06d8fa3969fc68304167741bbf9e082ed"
+                "sha256:1bd32405dacc920b42b83ba01644747ed77456a65760e285fbc47633ceddaf8b",
+                "sha256:99045d7d3f08f91f0d656bc9b7efbae189426cd913d830294a15eefa0ea4df16"
            ],
-            "markers": "python_full_version >= '3.6.0'",
-            "version": "==4.12.3"
+            "markers": "python_full_version >= '3.7.0'",
+            "version": "==4.13.3"
        },
        "billiard": {
            "hashes": [
@@ -245,19 +242,19 @@
        },
        "boto3": {
            "hashes": [
-                "sha256:53a5307f6a3526ee2f8590e3c45efa504a3ea4532c1bfe4926c0c19bf188d141",
-                "sha256:f9843a5d06f501d66ada06f5a5417f671823af2cf319e36ceefa1bafaaaaa953"
+                "sha256:0cf92ca0538ab115447e1c58050d43e1273e88c58ddfea2b6f133fdc508b400a",
+                "sha256:b10583bf8bd35be1b4027ee7e26b7cdf2078c79eab18357fd602cecb6d39400b"
            ],
            "markers": "python_version >= '3.8'",
-            "version": "==1.36.3"
+            "version": "==1.36.16"
        },
        "botocore": {
            "hashes": [
-                "sha256:536ab828e6f90dbb000e3702ac45fd76642113ae2db1b7b1373ad24104e89255",
-                "sha256:775b835e979da5c96548ed1a0b798101a145aec3cd46541d62e27dda5a94d7f8"
+                "sha256:10c6aa386ba1a9a0faef6bb5dbfc58fc2563a3c6b95352e86a583cd5f14b11f3",
+                "sha256:aca0348ccd730332082489b6817fdf89e1526049adcf6e9c8c11c96dd9f42c03"
            ],
            "markers": "python_version >= '3.8'",
-            "version": "==1.36.3"
+            "version": "==1.36.16"
        },
        "brotli": {
            "hashes": [
@@ -415,11 +412,11 @@
        },
        "certifi": {
            "hashes": [
-                "sha256:1275f7a45be9464efc1173084eaa30f866fe2e47d389406136d332ed4967ec56",
-                "sha256:b650d30f370c2b724812bee08008be0c4163b163ddaec3f2546c1caf65f191db"
+                "sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651",
+                "sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe"
            ],
            "markers": "python_version >= '3.6'",
-            "version": "==2024.12.14"
+            "version": "==2025.1.31"
        },
        "certvalidator": {
            "hashes": [
@@ -676,11 +673,11 @@
        },
        "dateparser": {
            "hashes": [
-                "sha256:0b21ad96534e562920a0083e97fd45fa959882d4162acc358705144520a35830",
-                "sha256:7975b43a4222283e0ae15be7b4999d08c9a70e2d378ac87385b1ccf2cffbbb30"
+                "sha256:7e4919aeb48481dbfc01ac9683c8e20bfe95bb715a38c1e9f6af889f4f30ccc3",
+                "sha256:bdcac262a467e6260030040748ad7c10d6bacd4f3b9cdb4cfd2251939174508c"
            ],
-            "markers": "python_version >= '3.7'",
-            "version": "==1.2.0"
+            "markers": "python_version >= '3.8'",
+            "version": "==1.2.1"
        },
        "dnspython": {
            "hashes": [
@@ -708,12 +705,12 @@
        },
        "fastapi": {
            "hashes": [
-                "sha256:9ec46f7addc14ea472958a96aae5b5de65f39721a46aaf5705c480d9a8b76654",
-                "sha256:e9240b29e36fa8f4bb7290316988e90c381e5092e0cbe84e7818cc3713bcf305"
+                "sha256:0ce9111231720190473e222cdf0f07f7206ad7e53ea02beb1d2dc36e2f0741e9",
+                "sha256:753a96dd7e036b34eeef8babdfcfe3f28ff79648f86551eb36bfc1b0bf4a8cbf"
            ],
            "index": "pypi",
            "markers": "python_version >= '3.8'",
-            "version": "==0.115.6"
+            "version": "==0.115.8"
        },
        "fastapi-mail": {
            "hashes": [
@@ -864,27 +861,27 @@
        },
        "google-api-core": {
            "hashes": [
-                "sha256:10d82ac0fca69c82a25b3efdeefccf6f28e02ebb97925a8cce8edbfe379929d9",
-                "sha256:e255640547a597a4da010876d333208ddac417d60add22b6851a0c66a831fcaf"
+                "sha256:bc78d608f5a5bf853b80bd70a795f703294de656c096c0968320830a4bc280f1",
+                "sha256:f8b36f5456ab0dd99a1b693a40a31d1e7757beea380ad1b38faaf8941eae9d8a"
            ],
            "markers": "python_version >= '3.7'",
-            "version": "==2.24.0"
+            "version": "==2.24.1"
        },
        "google-api-python-client": {
            "hashes": [
-                "sha256:55197f430f25c907394b44fa078545ffef89d33fd4dca501b7db9f0d8e224bd6",
-                "sha256:baef0bb631a60a0bd7c0bf12a5499e3a40cd4388484de7ee55c1950bf820a0cf"
+                "sha256:63d61fb3e4cf3fb31a70a87f45567c22f6dfe87bbfa27252317e3e2c42900db4",
+                "sha256:a8ccafaecfa42d15d5b5c3134ced8de08380019717fc9fb1ed510ca58eca3b7e"
            ],
            "markers": "python_version >= '3.7'",
-            "version": "==2.159.0"
+            "version": "==2.160.0"
        },
        "google-auth": {
            "hashes": [
-                "sha256:0054623abf1f9c83492c63d3f47e77f0a544caa3d40b2d98e099a611c2dd5d00",
-                "sha256:42664f18290a6be591be5329a96fe30184be1a1badb7292a7f686a9659de9ca0"
+                "sha256:8285113607d3b80a3f1543b75962447ba8a09fe85783432a784fdeef6ac094c4",
+                "sha256:e7dae6694313f434a2727bf2906f27ad259bae090d7aa896590d86feec3d9d4a"
            ],
            "markers": "python_version >= '3.7'",
-            "version": "==2.37.0"
+            "version": "==2.38.0"
        },
        "google-auth-httplib2": {
            "hashes": [
@@ -1038,10 +1035,11 @@
        },
        "instaloader": {
            "hashes": [
-                "sha256:754425eb17af44ce4bb6056e4eacd044a518d13b5efc11b9d80eb229bb96c652"
+                "sha256:43356f696231621ea5a93354f9a4578124fe131940ee9aa1e83c20f57e18f26d",
+                "sha256:a41a7372a18fb096b3ed545469479884de9cf768e12020c0e0e67c488d9d599c"
            ],
            "markers": "python_version >= '3.9'",
-            "version": "==4.14"
+            "version": "==4.14.1"
        },
        "itsdangerous": {
            "hashes": [
@@ -1239,11 +1237,11 @@
        },
        "mako": {
            "hashes": [
-                "sha256:42f48953c7eb91332040ff567eb7eea69b22e7a4affbc5ba8e845e8f730f6627",
-                "sha256:577b97e414580d3e088d47c2dbbe9594aa7a5146ed2875d4dfa9075af2dd3cc8"
+                "sha256:95920acccb578427a9aa38e37a186b1e43156c87260d7ba18ca63aa4c7cbd3a1",
+                "sha256:b5d65ff3462870feec922dbccf38f6efb44e5714d7b593a656be86663d8600ac"
            ],
            "markers": "python_version >= '3.8'",
-            "version": "==1.3.8"
+            "version": "==1.3.9"
        },
        "markdown-it-py": {
            "hashes": [
@@ -1322,11 +1320,11 @@
        },
        "marshmallow": {
            "hashes": [
-                "sha256:ec5d00d873ce473b7f2ffcb7104286a376c354cab0c2fa12f5573dab03e87210",
-                "sha256:f4debda3bb11153d81ac34b0d582bf23053055ee11e791b54b4b35493468040a"
+                "sha256:3350409f20a70a7e4e11a27661187b77cdcaeb20abca41c1454fe33636bea09c",
+                "sha256:e6d8affb6cb61d39d26402096dc0aee12d5a26d490a121f118d2e81dc0719dc6"
            ],
            "markers": "python_version >= '3.9'",
-            "version": "==3.25.1"
+            "version": "==3.26.1"
        },
        "mdurl": {
            "hashes": [
@@ -1785,11 +1783,11 @@
        },
        "proto-plus": {
            "hashes": [
-                "sha256:c91fc4a65074ade8e458e95ef8bac34d4008daa7cce4a12d6707066fca648961",
-                "sha256:fbb17f57f7bd05a68b7707e745e26528b0b3c34e378db91eef93912c54982d91"
+                "sha256:6e93d5f5ca267b54300880fff156b6a3386b3fa3f43b1da62e680fc0c586ef22",
+                "sha256:bf2dfaa3da281fc3187d12d224c707cb57214fb2c22ba854eb0c105a3fb2d4d7"
            ],
            "markers": "python_version >= '3.7'",
-            "version": "==1.25.0"
+            "version": "==1.26.0"
        },
        "protobuf": {
            "hashes": [
@@ -2075,7 +2073,6 @@
                "sha256:e324ee90a023d808f1959c46bcbc04446a10ced277783dc6ee09987c37ec10ca",
                "sha256:f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a"
            ],
-            "index": "pypi",
            "markers": "python_version >= '3.8'",
            "version": "==1.0.1"
        },
@@ -2097,10 +2094,10 @@
        },
        "pytz": {
            "hashes": [
-                "sha256:2aa355083c50a0f93fa581709deac0c9ad65cca8a9e9beac660adcbd493c798a",
-                "sha256:31c7c1817eb7fae7ca4b8c7ee50c72f93aa2dd863de768e1ef4245d426aa0725"
+                "sha256:89dd22dca55b46eac6eda23b2d72721bf1bdfef212645d81513ef5d03038de57",
+                "sha256:c2db42be2a2518b28e65f9207c4d05e6ff547d1efa4086469ef855e4ab70178e"
            ],
-            "version": "==2024.2"
+            "version": "==2025.1"
        },
        "pyyaml": {
            "hashes": [
@@ -2323,19 +2320,19 @@
        },
        "s3transfer": {
            "hashes": [
-                "sha256:3f25c900a367c8b7f7d8f9c34edc87e300bde424f779dc9f0a8ae4f9df9264f6",
-                "sha256:8fa0aa48177be1f3425176dfe1ab85dcd3d962df603c3dbfc585e6bf857ef0ff"
+                "sha256:3b39185cb72f5acc77db1a58b6e25b977f28d20496b6e58d6813d75f464d632f",
+                "sha256:be6ecb39fadd986ef1701097771f87e4d2f821f27f6071c872143884d2950fbc"
            ],
            "markers": "python_version >= '3.8'",
-            "version": "==0.11.1"
+            "version": "==0.11.2"
        },
        "selenium": {
            "hashes": [
-                "sha256:3d6a2e8e1b850a1078884ea19f4e011ecdc12263434d87a0b78769836fb82dd8",
-                "sha256:a9fae6eef48d470a1b0c6e45185d96f0dafb025e8da4b346cc41e4da3ac54fa0"
+                "sha256:0072d08670d7ec32db901bd0107695a330cecac9f196e3afb3fa8163026e022a",
+                "sha256:4238847e45e24e4472cfcf3554427512c7aab9443396435b1623ef406fff1cc1"
            ],
            "markers": "python_version >= '3.9'",
-            "version": "==4.28.0"
+            "version": "==4.28.1"
        },
        "six": {
            "hashes": [
@@ -2378,67 +2375,67 @@
        },
        "sqlalchemy": {
            "hashes": [
-                "sha256:03f0528c53ca0b67094c4764523c1451ea15959bbf0a8a8a3096900014db0278",
-                "sha256:12b0f1ec623cccf058cf21cb544f0e74656618165b083d78145cafde156ea7b6",
-                "sha256:12b28d99a9c14eaf4055810df1001557176716de0167b91026e648e65229bffb",
-                "sha256:1b2690456528a87234a75d1a1644cdb330a6926f455403c8e4f6cad6921f9098",
-                "sha256:1cdba1f73b64530c47b27118b7053b8447e6d6f3c8104e3ac59f3d40c33aa9fd",
-                "sha256:293f9ade06b2e68dd03cfb14d49202fac47b7bb94bffcff174568c951fbc7af2",
-                "sha256:2952748ecd67ed3b56773c185e85fc084f6bdcdec10e5032a7c25a6bc7d682ef",
-                "sha256:2f95fc8e3f34b5f6b3effb49d10ac97c569ec8e32f985612d9b25dd12d0d2e94",
-                "sha256:2fa2c0913f02341d25fb858e4fb2031e6b0813494cca1ba07d417674128ce11b",
-                "sha256:3151822aa1db0eb5afd65ccfafebe0ef5cda3a7701a279c8d0bf17781a793bb4",
-                "sha256:35bd2df269de082065d4b23ae08502a47255832cc3f17619a5cea92ce478b02b",
-                "sha256:41296bbcaa55ef5fdd32389a35c710133b097f7b2609d8218c0eabded43a1d84",
-                "sha256:44f569d0b1eb82301b92b72085583277316e7367e038d97c3a1a899d9a05e342",
-                "sha256:46954173612617a99a64aee103bcd3f078901b9a8dcfc6ae80cbf34ba23df989",
-                "sha256:4b12885dc85a2ab2b7d00995bac6d967bffa8594123b02ed21e8eb2205a7584b",
-                "sha256:4f581d365af9373a738c49e0c51e8b18e08d8a6b1b15cc556773bcd8a192fa8b",
-                "sha256:51bc9cfef83e0ac84f86bf2b10eaccb27c5a3e66a1212bef676f5bee6ef33ebb",
-                "sha256:521ef85c04c33009166777c77e76c8a676e2d8528dc83a57836b63ca9c69dcd1",
-                "sha256:5bc3339db84c5fb9130ac0e2f20347ee77b5dd2596ba327ce0d399752f4fce39",
-                "sha256:635d8a21577341dfe4f7fa59ec394b346da12420b86624a69e466d446de16aff",
-                "sha256:648ec5acf95ad59255452ef759054f2176849662af4521db6cb245263ae4aa33",
-                "sha256:650dcb70739957a492ad8acff65d099a9586b9b8920e3507ca61ec3ce650bb72",
-                "sha256:6b788f14c5bb91db7f468dcf76f8b64423660a05e57fe277d3f4fad7b9dcb7ce",
-                "sha256:6c67415258f9f3c69867ec02fea1bf6508153709ecbd731a982442a590f2b7e4",
-                "sha256:74bbd1d0a9bacf34266a7907d43260c8d65d31d691bb2356f41b17c2dca5b1d0",
-                "sha256:75311559f5c9881a9808eadbeb20ed8d8ba3f7225bef3afed2000c2a9f4d49b9",
-                "sha256:78361be6dc9073ed17ab380985d1e45e48a642313ab68ab6afa2457354ff692c",
-                "sha256:7b7e772dc4bc507fdec4ee20182f15bd60d2a84f1e087a8accf5b5b7a0dcf2ba",
-                "sha256:82df02816c14f8dc9f4d74aea4cb84a92f4b0620235daa76dde002409a3fbb5a",
-                "sha256:84b9f23b0fa98a6a4b99d73989350a94e4a4ec476b9a7dfe9b79ba5939f5e80b",
-                "sha256:8c4096727193762e72ce9437e2a86a110cf081241919ce3fab8e89c02f6b6658",
-                "sha256:8e47f1af09444f87c67b4f1bb6231e12ba6d4d9f03050d7fc88df6d075231a49",
-                "sha256:93d1543cd8359040c02b6614421c8e10cd7a788c40047dbc507ed46c29ae5636",
-                "sha256:94b564e38b344d3e67d2e224f0aec6ba09a77e4582ced41e7bfd0f757d926ec9",
-                "sha256:955a2a765aa1bd81aafa69ffda179d4fe3e2a3ad462a736ae5b6f387f78bfeb8",
-                "sha256:9d087663b7e1feabea8c578d6887d59bb00388158e8bff3a76be11aa3f748ca2",
-                "sha256:9df21b8d9e5c136ea6cde1c50d2b1c29a2b5ff2b1d610165c23ff250e0704087",
-                "sha256:a8998bf9f8658bd3839cbc44ddbe982955641863da0c1efe5b00c1ab4f5c16b1",
-                "sha256:b2eae3423e538c10d93ae3e87788c6a84658c3ed6db62e6a61bb9495b0ad16bb",
-                "sha256:b661b49d0cb0ab311a189b31e25576b7ac3e20783beb1e1817d72d9d02508bf5",
-                "sha256:bedee60385c1c0411378cbd4dc486362f5ee88deceea50002772912d798bb00f",
-                "sha256:c505edd429abdfe3643fa3b2e83efb3445a34a9dc49d5f692dd087be966020e0",
-                "sha256:cce918ada64c956b62ca2c2af59b125767097ec1dca89650a6221e887521bfd7",
-                "sha256:cf5ae8a9dcf657fd72144a7fd01f243236ea39e7344e579a121c4205aedf07bb",
-                "sha256:cf95a60b36997dad99692314c4713f141b61c5b0b4cc5c3426faad570b31ca01",
-                "sha256:d57bafbab289e147d064ffbd5cca2d7b1394b63417c0636cea1f2e93d16eb9e8",
-                "sha256:d70f53a0646cc418ca4853da57cf3ddddbccb8c98406791f24426f2dd77fd0e2",
-                "sha256:d75ead7dd4d255068ea0f21492ee67937bd7c90964c8f3c2bea83c7b7f81b95f",
-                "sha256:da36c3b0e891808a7542c5c89f224520b9a16c7f5e4d6a1156955605e54aef0e",
-                "sha256:db18ff6b8c0f1917f8b20f8eca35c28bbccb9f83afa94743e03d40203ed83de9",
-                "sha256:dfff7be361048244c3aa0f60b5e63221c5e0f0e509f4e47b8910e22b57d10ae7",
-                "sha256:e4fb5ac86d8fe8151966814f6720996430462e633d225497566b3996966b9bdb",
-                "sha256:e56a139bfe136a22c438478a86f8204c1eb5eed36f4e15c4224e4b9db01cb3e4",
-                "sha256:e6f5d254a22394847245f411a2956976401e84da4288aa70cbcd5190744062c1",
-                "sha256:e7402ff96e2b073a98ef6d6142796426d705addd27b9d26c3b32dbaa06d7d069",
-                "sha256:ea308cec940905ba008291d93619d92edaf83232ec85fbd514dcb329f3192761",
-                "sha256:eaa8039b6d20137a4e02603aba37d12cd2dde7887500b8855356682fc33933f4"
+                "sha256:0398361acebb42975deb747a824b5188817d32b5c8f8aba767d51ad0cc7bb08d",
+                "sha256:0561832b04c6071bac3aad45b0d3bb6d2c4f46a8409f0a7a9c9fa6673b41bc03",
+                "sha256:07258341402a718f166618470cde0c34e4cec85a39767dce4e24f61ba5e667ea",
+                "sha256:0a826f21848632add58bef4f755a33d45105d25656a0c849f2dc2df1c71f6f50",
+                "sha256:1052723e6cd95312f6a6eff9a279fd41bbae67633415373fdac3c430eca3425d",
+                "sha256:12d5b06a1f3aeccf295a5843c86835033797fea292c60e72b07bcb5d820e6dd3",
+                "sha256:12f5c9ed53334c3ce719155424dc5407aaa4f6cadeb09c5b627e06abb93933a1",
+                "sha256:2a0ef3f98175d77180ffdc623d38e9f1736e8d86b6ba70bff182a7e68bed7727",
+                "sha256:2f2951dc4b4f990a4b394d6b382accb33141d4d3bd3ef4e2b27287135d6bdd68",
+                "sha256:3868acb639c136d98107c9096303d2d8e5da2880f7706f9f8c06a7f961961149",
+                "sha256:386b7d136919bb66ced64d2228b92d66140de5fefb3c7df6bd79069a269a7b06",
+                "sha256:3d3043375dd5bbcb2282894cbb12e6c559654c67b5fffb462fda815a55bf93f7",
+                "sha256:3e35d5565b35b66905b79ca4ae85840a8d40d31e0b3e2990f2e7692071b179ca",
+                "sha256:402c2316d95ed90d3d3c25ad0390afa52f4d2c56b348f212aa9c8d072a40eee5",
+                "sha256:40310db77a55512a18827488e592965d3dec6a3f1e3d8af3f8243134029daca3",
+                "sha256:40e9cdbd18c1f84631312b64993f7d755d85a3930252f6276a77432a2b25a2f3",
+                "sha256:49aa2cdd1e88adb1617c672a09bf4ebf2f05c9448c6dbeba096a3aeeb9d4d443",
+                "sha256:57dd41ba32430cbcc812041d4de8d2ca4651aeefad2626921ae2a23deb8cd6ff",
+                "sha256:5dba1cdb8f319084f5b00d41207b2079822aa8d6a4667c0f369fce85e34b0c86",
+                "sha256:5e1d9e429028ce04f187a9f522818386c8b076723cdbe9345708384f49ebcec6",
+                "sha256:63178c675d4c80def39f1febd625a6333f44c0ba269edd8a468b156394b27753",
+                "sha256:6493bc0eacdbb2c0f0d260d8988e943fee06089cd239bd7f3d0c45d1657a70e2",
+                "sha256:64aa8934200e222f72fcfd82ee71c0130a9c07d5725af6fe6e919017d095b297",
+                "sha256:665255e7aae5f38237b3a6eae49d2358d83a59f39ac21036413fab5d1e810578",
+                "sha256:6db316d6e340f862ec059dc12e395d71f39746a20503b124edc255973977b728",
+                "sha256:70065dfabf023b155a9c2a18f573e47e6ca709b9e8619b2e04c54d5bcf193178",
+                "sha256:8455aa60da49cb112df62b4721bd8ad3654a3a02b9452c783e651637a1f21fa2",
+                "sha256:8b0ac78898c50e2574e9f938d2e5caa8fe187d7a5b69b65faa1ea4648925b096",
+                "sha256:8bf312ed8ac096d674c6aa9131b249093c1b37c35db6a967daa4c84746bc1bc9",
+                "sha256:92f99f2623ff16bd4aaf786ccde759c1f676d39c7bf2855eb0b540e1ac4530c8",
+                "sha256:9c8bcad7fc12f0cc5896d8e10fdf703c45bd487294a986903fe032c72201596b",
+                "sha256:9cd136184dd5f58892f24001cdce986f5d7e96059d004118d5410671579834a4",
+                "sha256:9eb4fa13c8c7a2404b6a8e3772c17a55b1ba18bc711e25e4d6c0c9f5f541b02a",
+                "sha256:a2bc4e49e8329f3283d99840c136ff2cd1a29e49b5624a46a290f04dff48e079",
+                "sha256:a5645cd45f56895cfe3ca3459aed9ff2d3f9aaa29ff7edf557fa7a23515a3725",
+                "sha256:a9afbc3909d0274d6ac8ec891e30210563b2c8bdd52ebbda14146354e7a69373",
+                "sha256:aa498d1392216fae47eaf10c593e06c34476ced9549657fca713d0d1ba5f7248",
+                "sha256:afd776cf1ebfc7f9aa42a09cf19feadb40a26366802d86c1fba080d8e5e74bdd",
+                "sha256:b335a7c958bc945e10c522c069cd6e5804f4ff20f9a744dd38e748eb602cbbda",
+                "sha256:b3c4817dff8cef5697f5afe5fec6bc1783994d55a68391be24cb7d80d2dbc3a6",
+                "sha256:b79ee64d01d05a5476d5cceb3c27b5535e6bb84ee0f872ba60d9a8cd4d0e6579",
+                "sha256:b87a90f14c68c925817423b0424381f0e16d80fc9a1a1046ef202ab25b19a444",
+                "sha256:bf89e0e4a30714b357f5d46b6f20e0099d38b30d45fa68ea48589faf5f12f62d",
+                "sha256:c058b84c3b24812c859300f3b5abf300daa34df20d4d4f42e9652a4d1c48c8a4",
+                "sha256:c09a6ea87658695e527104cf857c70f79f14e9484605e205217aae0ec27b45fc",
+                "sha256:c57b8e0841f3fce7b703530ed70c7c36269c6d180ea2e02e36b34cb7288c50c7",
+                "sha256:c9cea5b756173bb86e2235f2f871b406a9b9d722417ae31e5391ccaef5348f2c",
+                "sha256:cb39ed598aaf102251483f3e4675c5dd6b289c8142210ef76ba24aae0a8f8aba",
+                "sha256:e036549ad14f2b414c725349cce0772ea34a7ab008e9cd67f9084e4f371d1f32",
+                "sha256:e185ea07a99ce8b8edfc788c586c538c4b1351007e614ceb708fd01b095ef33e",
+                "sha256:e5a4d82bdb4bf1ac1285a68eab02d253ab73355d9f0fe725a97e1e0fa689decb",
+                "sha256:eae27ad7580529a427cfdd52c87abb2dfb15ce2b7a3e0fc29fbb63e2ed6f8120",
+                "sha256:ecef029b69843b82048c5b347d8e6049356aa24ed644006c9a9d7098c3bd3bfd",
+                "sha256:ee3bee874cb1fadee2ff2b79fc9fc808aa638670f28b2145074538d4a6a5028e",
+                "sha256:f0d3de936b192980209d7b5149e3c98977c3810d401482d05fb6d668d53c1c63",
+                "sha256:f53c0d6a859b2db58332e0e6a921582a02c1677cc93d4cbb36fdf49709b327b2",
+                "sha256:f9d57f1b3061b3e21476b0ad5f0397b112b94ace21d1f439f2db472e568178ae"
            ],
            "index": "pypi",
            "markers": "python_version >= '3.7'",
-            "version": "==2.0.37"
+            "version": "==2.0.38"
        },
        "starlette": {
            "hashes": [
@@ -2785,11 +2782,11 @@
        },
        "yt-dlp": {
            "hashes": [
-                "sha256:b8666b88e23c3fa5ee1e80920f4a9dfac7c405504a447214c0cf3d0c386edcfc",
-                "sha256:e8ec515d49bb62704915d13a22ee6fe03a5658d651e4e64574e3a17ee01f6e3b"
+                "sha256:1c9738266921ad43c568ad01ac3362fb7c7af549276fbec92bd72f140da16240",
+                "sha256:3e76bd896b9f96601021ca192ca0fbdd195e3c3dcc28302a3a34c9bc4979da7b"
            ],
            "markers": "python_version >= '3.9'",
-            "version": "==2025.1.15"
+            "version": "==2025.1.26"
        }
    },
    "develop": {
@@ -2803,80 +2800,75 @@
        },
        "certifi": {
            "hashes": [
-                "sha256:1275f7a45be9464efc1173084eaa30f866fe2e47d389406136d332ed4967ec56",
-                "sha256:b650d30f370c2b724812bee08008be0c4163b163ddaec3f2546c1caf65f191db"
+                "sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651",
+                "sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe"
            ],
            "markers": "python_version >= '3.6'",
-            "version": "==2024.12.14"
+            "version": "==2025.1.31"
        },
        "coverage": {
            "hashes": [
-                "sha256:05fca8ba6a87aabdd2d30d0b6c838b50510b56cdcfc604d40760dae7153b73d9",
-                "sha256:0aa9692b4fdd83a4647eeb7db46410ea1322b5ed94cd1715ef09d1d5922ba87f",
-                "sha256:0c807ca74d5a5e64427c8805de15b9ca140bba13572d6d74e262f46f50b13273",
-                "sha256:0d7a2bf79378d8fb8afaa994f91bfd8215134f8631d27eba3e0e2c13546ce994",
-                "sha256:0f460286cb94036455e703c66988851d970fdfd8acc2a1122ab7f4f904e4029e",
-                "sha256:204a8238afe787323a8b47d8be4df89772d5c1e4651b9ffa808552bdf20e1d50",
-                "sha256:2396e8116db77789f819d2bc8a7e200232b7a282c66e0ae2d2cd84581a89757e",
-                "sha256:254f1a3b1eef5f7ed23ef265eaa89c65c8c5b6b257327c149db1ca9d4a35f25e",
-                "sha256:26bcf5c4df41cad1b19c84af71c22cbc9ea9a547fc973f1f2cc9a290002c8b3c",
-                "sha256:27c6e64726b307782fa5cbe531e7647aee385a29b2107cd87ba7c0105a5d3853",
-                "sha256:299e91b274c5c9cdb64cbdf1b3e4a8fe538a7a86acdd08fae52301b28ba297f8",
-                "sha256:2bcfa46d7709b5a7ffe089075799b902020b62e7ee56ebaed2f4bdac04c508d8",
-                "sha256:2ccf240eb719789cedbb9fd1338055de2761088202a9a0b73032857e53f612fe",
-                "sha256:32ee6d8491fcfc82652a37109f69dee9a830e9379166cb73c16d8dc5c2915165",
-                "sha256:3f7b444c42bbc533aaae6b5a2166fd1a797cdb5eb58ee51a92bee1eb94a1e1cb",
-                "sha256:457574f4599d2b00f7f637a0700a6422243b3565509457b2dbd3f50703e11f59",
-                "sha256:489a01f94aa581dbd961f306e37d75d4ba16104bbfa2b0edb21d29b73be83609",
-                "sha256:4bcc276261505d82f0ad426870c3b12cb177752834a633e737ec5ee79bbdff18",
-                "sha256:4e0de1e902669dccbf80b0415fb6b43d27edca2fbd48c74da378923b05316098",
-                "sha256:4e4630c26b6084c9b3cb53b15bd488f30ceb50b73c35c5ad7871b869cb7365fd",
-                "sha256:4eea95ef275de7abaef630c9b2c002ffbc01918b726a39f5a4353916ec72d2f3",
-                "sha256:507a20fc863cae1d5720797761b42d2d87a04b3e5aeb682ef3b7332e90598f43",
-                "sha256:54a5f0f43950a36312155dae55c505a76cd7f2b12d26abeebbe7a0b36dbc868d",
-                "sha256:55b201b97286cf61f5e76063f9e2a1d8d2972fc2fcfd2c1272530172fd28c359",
-                "sha256:59af35558ba08b758aec4d56182b222976330ef8d2feacbb93964f576a7e7a90",
-                "sha256:5c912978f7fbf47ef99cec50c4401340436d200d41d714c7a4766f377c5b7b78",
-                "sha256:656c82b8a0ead8bba147de9a89bda95064874c91a3ed43a00e687f23cc19d53a",
-                "sha256:6713ba4b4ebc330f3def51df1d5d38fad60b66720948112f114968feb52d3f99",
-                "sha256:675cefc4c06e3b4c876b85bfb7c59c5e2218167bbd4da5075cbe3b5790a28988",
-                "sha256:6f93531882a5f68c28090f901b1d135de61b56331bba82028489bc51bdd818d2",
-                "sha256:714f942b9c15c3a7a5fe6876ce30af831c2ad4ce902410b7466b662358c852c0",
-                "sha256:79109c70cc0882e4d2d002fe69a24aa504dec0cc17169b3c7f41a1d341a73694",
-                "sha256:7bbd8c8f1b115b892e34ba66a097b915d3871db7ce0e6b9901f462ff3a975377",
-                "sha256:7ed2f37cfce1ce101e6dffdfd1c99e729dd2ffc291d02d3e2d0af8b53d13840d",
-                "sha256:7fb105327c8f8f0682e29843e2ff96af9dcbe5bab8eeb4b398c6a33a16d80a23",
-                "sha256:89d76815a26197c858f53c7f6a656686ec392b25991f9e409bcef020cd532312",
-                "sha256:9a7cfb50515f87f7ed30bc882f68812fd98bc2852957df69f3003d22a2aa0abf",
-                "sha256:9e1747bab246d6ff2c4f28b4d186b205adced9f7bd9dc362051cc37c4a0c7bd6",
-                "sha256:9e80eba8801c386f72e0712a0453431259c45c3249f0009aff537a517b52942b",
-                "sha256:a01ec4af7dfeb96ff0078ad9a48810bb0cc8abcb0115180c6013a6b26237626c",
-                "sha256:a372c89c939d57abe09e08c0578c1d212e7a678135d53aa16eec4430adc5e690",
-                "sha256:a3b204c11e2b2d883946fe1d97f89403aa1811df28ce0447439178cc7463448a",
-                "sha256:a534738b47b0de1995f85f582d983d94031dffb48ab86c95bdf88dc62212142f",
-                "sha256:a5e37dc41d57ceba70956fa2fc5b63c26dba863c946ace9705f8eca99daecdc4",
-                "sha256:aa744da1820678b475e4ba3dfd994c321c5b13381d1041fe9c608620e6676e25",
-                "sha256:ab32947f481f7e8c763fa2c92fd9f44eeb143e7610c4ca9ecd6a36adab4081bd",
-                "sha256:abb02e2f5a3187b2ac4cd46b8ced85a0858230b577ccb2c62c81482ca7d18852",
-                "sha256:b330368cb99ef72fcd2dc3ed260adf67b31499584dc8a20225e85bfe6f6cfed0",
-                "sha256:bc67deb76bc3717f22e765ab3e07ee9c7a5e26b9019ca19a3b063d9f4b874244",
-                "sha256:c0b1818063dc9e9d838c09e3a473c1422f517889436dd980f5d721899e66f315",
-                "sha256:c56e097019e72c373bae32d946ecf9858fda841e48d82df7e81c63ac25554078",
-                "sha256:c7827a5bc7bdb197b9e066cdf650b2887597ad124dd99777332776f7b7c7d0d0",
-                "sha256:ccc2b70a7ed475c68ceb548bf69cec1e27305c1c2606a5eb7c3afff56a1b3b27",
-                "sha256:d37a84878285b903c0fe21ac8794c6dab58150e9359f1aaebbeddd6412d53132",
-                "sha256:e2f0280519e42b0a17550072861e0bc8a80a0870de260f9796157d3fca2733c5",
-                "sha256:e4ae5ac5e0d1e4edfc9b4b57b4cbecd5bc266a6915c500f358817a8496739247",
-                "sha256:e67926f51821b8e9deb6426ff3164870976fe414d033ad90ea75e7ed0c2e5022",
-                "sha256:e78b270eadb5702938c3dbe9367f878249b5ef9a2fcc5360ac7bff694310d17b",
-                "sha256:ea3c8f04b3e4af80e17bab607c386a830ffc2fb88a5484e1df756478cf70d1d3",
-                "sha256:ec22b5e7fe7a0fa8509181c4aac1db48f3dd4d3a566131b313d1efc102892c18",
-                "sha256:f4f620668dbc6f5e909a0946a877310fb3d57aea8198bde792aae369ee1c23b5",
-                "sha256:fd34e7b3405f0cc7ab03d54a334c17a9e802897580d964bd8c2001f4b9fd488f"
+                "sha256:050172741de03525290e67f0161ae5f7f387c88fca50d47fceb4724ceaa591d2",
+                "sha256:08e5fb93576a6b054d3d326242af5ef93daaac9bb52bc25f12ccbc3fa94227cd",
+                "sha256:09d03f48d9025b8a6a116cddcb6c7b8ce80e4fb4c31dd2e124a7c377036ad58e",
+                "sha256:0d03c9452d9d1ccfe5d3a5df0427705022a49b356ac212d529762eaea5ef97b4",
+                "sha256:13100f98497086b359bf56fc035a762c674de8ef526daa389ac8932cb9bff1e0",
+                "sha256:25575cd5a7d2acc46b42711e8aff826027c0e4f80fb38028a74f31ac22aae69d",
+                "sha256:27700d859be68e4fb2e7bf774cf49933dcac6f81a9bc4c13bd41735b8d26a53b",
+                "sha256:2c81e53782043b323bd34c7de711ed9b4673414eb517eaf35af92185b873839c",
+                "sha256:397489c611b76302dfa1d9ea079e138dddc4af80fc6819d5f5119ec8ca6c0e47",
+                "sha256:476f29a258b9cd153f2be5bf5f119d670d2806363595263917bddc167d6e5cce",
+                "sha256:4bda710139ea646890d1c000feb533caff86904a0e0638f85e967c28cb8eec50",
+                "sha256:4cf96beb05d004e4c51cd846fcdf9eee9eb2681518524b66b2e7610507944c2f",
+                "sha256:4f21e3617f48d683f30cf2a6c8b739c838e600cb1454fe6b2eb486ac2bce8fbd",
+                "sha256:5128f3ba694c0a1bde55fc480090392c336236c3e1a10dad40dc1ab17c7675ff",
+                "sha256:532fe139691af134aa8b54ed60dd3c806aa81312d93693bd2883c7b61592c840",
+                "sha256:5a3f7cbbcb4ad95067a6525f83a6fc78d9cbc1e70f8abaeeaeaa72ef34f48fc3",
+                "sha256:5b48db06f53d1864fea6dbd855e6d51d41c0f06c212c3004511c0bdc6847b297",
+                "sha256:5e7ac966ab110bd94ee844f2643f196d78fde1cd2450399116d3efdd706e19f5",
+                "sha256:5edc16712187139ab635a2e644cc41fc239bc6d245b16124045743130455c652",
+                "sha256:60d4ad09dfc8c36c4910685faafcb8044c84e4dae302e86c585b3e2e7778726c",
+                "sha256:61c834cbb80946d6ebfddd9b393a4c46bec92fcc0fa069321fcb8049117f76ea",
+                "sha256:6ba27a0375c5ef4d2a7712f829265102decd5ff78b96d342ac2fa555742c4f4f",
+                "sha256:6c96a142057d83ee993eaf71629ca3fb952cda8afa9a70af4132950c2bd3deb9",
+                "sha256:6d60577673ba48d8ae8e362e61fd4ad1a640293ffe8991d11c86f195479100b7",
+                "sha256:7eb0504bb307401fd08bc5163a351df301438b3beb88a4fa044681295bbefc67",
+                "sha256:8e433b6e3a834a43dae2889adc125f3fa4c66668df420d8e49bc4ee817dd7a70",
+                "sha256:8fa4fffd90ee92f62ff7404b4801b59e8ea8502e19c9bf2d3241ce745b52926c",
+                "sha256:90de4e9ca4489e823138bd13098af9ac8028cc029f33f60098b5c08c675c7bda",
+                "sha256:a165b09e7d5f685bf659063334a9a7b1a2d57b531753d3e04bd442b3cfe5845b",
+                "sha256:a46d56e99a31d858d6912d31ffa4ede6a325c86af13139539beefca10a1234ce",
+                "sha256:ac476e6d0128fb7919b3fae726de72b28b5c9644cb4b579e4a523d693187c551",
+                "sha256:ac5d92e2cc121a13270697e4cb37e1eb4511ac01d23fe1b6c097facc3b46489e",
+                "sha256:adc2d941c0381edfcf3897f94b9f41b1e504902fab78a04b1677f2f72afead4b",
+                "sha256:b6ff5be3b1853e0862da9d349fe87f869f68e63a25f7c37ce1130b321140f963",
+                "sha256:bb35ae9f134fbd9cf7302a9654d5a1e597c974202678082dcc569eb39a8cde03",
+                "sha256:be05bde21d5e6eefbc3a6de6b9bee2b47894b8945342e8663192809c4d1f08ce",
+                "sha256:c27df03730059118b8a923cfc8b84b7e9976742560af528242f201880879c1da",
+                "sha256:c7719a5e1dc93883a6b319bc0374ecd46fb6091ed659f3fbe281ab991634b9b0",
+                "sha256:c86f4c7a6d1a54a24d804d9684d96e36a62d3ef7c0d7745ae2ea39e3e0293251",
+                "sha256:ca95d40900cf614e07f00cee8c2fad0371df03ca4d7a80161d84be2ec132b7a4",
+                "sha256:cd4839813b09ab1dd1be1bbc74f9a7787615f931f83952b6a9af1b2d3f708bf7",
+                "sha256:db4b1a69976b1b02acda15937538a1d3fe10b185f9d99920b17a740a0a102e06",
+                "sha256:dbb1a822fd858d9853333a7c95d4e70dde9a79e65893138ce32c2ec6457d7a36",
+                "sha256:de6b079b39246a7da9a40cfa62d5766bd52b4b7a88cf5a82ec4c45bf6e152306",
+                "sha256:df6ff122a0a10a30121d9f0cb3fbd03a6fe05861e4ec47adb9f25e9245aabc19",
+                "sha256:e0b0f272901a5172090c0802053fbc503cdc3fa2612720d2669a98a7384a7bec",
+                "sha256:e2778be4f574b39ec9dcd9e5e13644f770351ee0990a0ecd27e364aba95af89b",
+                "sha256:e3b746fa0ffc5b6b8856529de487da8b9aeb4fb394bb58de6502ef45f3434f12",
+                "sha256:e642e6a46a04e992ebfdabed79e46f478ec60e2c528e1e1a074d63800eda4286",
+                "sha256:eafea49da254a8289bed3fab960f808b322eda5577cb17a3733014928bbfbebd",
+                "sha256:f0f334ae844675420164175bf32b04e18a81fe57ad8eb7e0cfd4689d681ffed7",
+                "sha256:f382004fa4c93c01016d9226b9d696a08c53f6818b7ad59b4e96cb67e863353a",
+                "sha256:f4679fcc9eb9004fdd1b00231ef1ec7167168071bebc4d66327e28c1979b4449",
+                "sha256:fd2fffc8ce8692ce540103dff26279d2af22d424516ddebe2d7e4d6dbb3816b2",
+                "sha256:ff136607689c1c87f43d24203b6d2055b42030f352d5176f9c8b204d4235ef27",
+                "sha256:ff52b4e2ac0080c96e506819586c4b16cdbf46724bda90d308a7330a73cc8521",
+                "sha256:ff562952f15eff27247a4c4b03e45ce8a82e3fb197de6a7c54080f9d4ba07845"
            ],
            "index": "pypi",
            "markers": "python_version >= '3.9'",
-            "version": "==7.6.10"
+            "version": "==7.6.11"
        },
        "exceptiongroup": {
            "hashes": [
@@ -2953,12 +2945,12 @@
        },
        "pytest-asyncio": {
            "hashes": [
-                "sha256:0d0bb693f7b99da304a0634afc0a4b19e49d5e0de2d670f38dc4bfa5727c5075",
-                "sha256:3f8ef9a98f45948ea91a0ed3dc4268b5326c0e7bce73892acc654df4262ad45f"
+                "sha256:9e89518e0f9bd08928f97a3482fdc4e244df17529460bc038291ccaf8f85c7c3",
+                "sha256:fc1da2cf9f125ada7e710b4ddad05518d4cee187ae9412e9ac9271003497f07a"
            ],
            "index": "pypi",
            "markers": "python_version >= '3.9'",
-            "version": "==0.25.2"
+            "version": "==0.25.3"
        },
        "sniffio": {
            "hashes": [
--- a/src/alembic.ini
+++ b/src/alembic.ini
@@ -2,7 +2,7 @@

 [alembic]
 # path to migration scripts
-script_location = migrations
+script_location = app/migrations

 # template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
 # Uncomment the line below if you want the files to be prepended with date and time
--- a/app/example.user-groups.yaml
+++ b/app/example.user-groups.yaml
--- a/app/logs/.gitkeep
+++ b/app/logs/.gitkeep
--- a/app/migrations/env.py
+++ b/app/migrations/env.py
@@ -1,11 +1,10 @@
 from logging.config import fileConfig
-import os
 from sqlalchemy import engine_from_config
 from sqlalchemy import pool

 from alembic import context

-from shared.settings import get_settings
+from app.shared.settings import get_settings

 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
--- a/app/migrations/script.py.mako
+++ b/app/migrations/script.py.mako
--- a/app/migrations/versions/.gitkeep
+++ b/app/migrations/versions/.gitkeep
--- a/app/migrations/versions/02b2f6d17ed0_create_archives_store_until_column.py
+++ b/app/migrations/versions/02b2f6d17ed0_create_archives_store_until_column.py
--- a/app/migrations/versions/1636724ec4b1_rename_sheets_last_archived_col.py
+++ b/app/migrations/versions/1636724ec4b1_rename_sheets_last_archived_col.py
--- a/app/migrations/versions/89121d2c96d8_add_sheet_id_to_archive_table.py
+++ b/app/migrations/versions/89121d2c96d8_add_sheet_id_to_archive_table.py
--- a/app/migrations/versions/9369a264945b_modify_archive_url_to_have_uuid_id_.py
+++ b/app/migrations/versions/9369a264945b_modify_archive_url_to_have_uuid_id_.py
--- a/app/migrations/versions/93a611e4c066_vacuum_database_if_there_s_enough_space.py
+++ b/app/migrations/versions/93a611e4c066_vacuum_database_if_there_s_enough_space.py
--- a/app/migrations/versions/a23aaf3ae930_drop_active_column.py
+++ b/app/migrations/versions/a23aaf3ae930_drop_active_column.py
--- a/app/migrations/versions/fa012ec405b8_add_columns_to_groups_table.py
+++ b/app/migrations/versions/fa012ec405b8_add_columns_to_groups_table.py
--- a/app/shared/init.py
+++ b/app/shared/init.py
--- a/app/shared/aa_utils.py
+++ b/app/shared/aa_utils.py
@@ -0,0 +1,33 @@
+# TODO: code in this file should eventually be moved to the auto-archiver code base
+
+from typing import List
+from loguru import logger
+from auto_archiver import Metadata
+from auto_archiver.core import Media
+
+from app.shared.db import models
+
+def get_all_urls(result: Metadata) -> List[models.ArchiveUrl]:
+    db_urls = []
+    for m in result.media:
+        for i, url in enumerate(m.urls): db_urls.append(models.ArchiveUrl(url=url, key=m.get("id", f"media_{i}")))
+        for k, prop in m.properties.items():
+            if prop_converted := convert_if_media(prop):
+                for i, url in enumerate(prop_converted.urls): db_urls.append(models.ArchiveUrl(url=url, key=prop_converted.get("id", f"{k}_{i}")))
+            if isinstance(prop, list):
+                for i, prop_media in enumerate(prop):
+                    if prop_media := convert_if_media(prop_media):
+                        for j, url in enumerate(prop_media.urls):
+                            db_urls.append(models.ArchiveUrl(url=url, key=prop_media.get("id", f"{k}{prop_media.key}_{i}.{j}")))
+    return db_urls
+
+
+
+def convert_if_media(media):
+    if isinstance(media, Media): return media
+    elif isinstance(media, dict):
+        try: return Media.from_dict(media)
+        except Exception as e:
+            logger.debug(f"error parsing {media} : {e}")
+    return False
+
--- a/app/shared/business_logic.py
+++ b/app/shared/business_logic.py
@@ -0,0 +1,15 @@
+# TODO: temporary file for this code, maybe other code belongs here, maybe not. do decide
+
+
+import datetime
+from sqlalchemy.orm import Session
+
+from app.shared.db import crud
+
+
+def get_store_archive_until(db: Session, group_id: str) -> datetime.datetime:
+    group = crud.get_group(db, group_id)
+    max_lifespan = group.permissions.get("max_archive_lifespan_months", -1)
+    if max_lifespan == -1: return None
+
+    return datetime.datetime.now() + datetime.timedelta(days=30 * max_lifespan)
--- a/app/shared/config.py
+++ b/app/shared/config.py
--- a/app/shared/db/init.py
+++ b/app/shared/db/init.py
--- a/app/shared/db/crud.py
+++ b/app/shared/db/crud.py
@@ -4,15 +4,16 @@ from sqlalchemy.orm import Session, load_only
 from sqlalchemy import Column, or_, func, select
 from loguru import logger
 from datetime import datetime, timedelta
-
-from core.config import ALLOW_ANY_EMAIL
-from db.database import get_db
-from shared.settings import get_settings
-from shared.user_groups import UserGroups
-from utils.misc import fnv1a_hash_mod
-from . import models, schemas
 from sqlalchemy.ext.asyncio import AsyncSession

+from app.shared.config import ALLOW_ANY_EMAIL
+from app.shared.db.database import get_db
+from app.shared.db import models
+from app.shared import schemas
+from app.shared.settings import get_settings
+from app.shared.user_groups import UserGroups
+from app.shared.utils.misc import fnv1a_hash_mod
+
 DATABASE_QUERY_LIMIT = get_settings().DATABASE_QUERY_LIMIT


@@ -304,7 +305,7 @@ def delete_sheet(db: Session, sheet_id: str, email: str) -> bool:
 #--- Celery worker tasks


-def insert_result_into_db(db: Session, archive: schemas.ArchiveCreate) -> models.Archive:
+def store_archived_url(db: Session, archive: schemas.ArchiveCreate) -> models.Archive:
    # create and load user, tags, if needed
    create_or_get_user(db, archive.author_id)
    db_tags = [create_tag(db, tag) for tag in archive.tags]
--- a/app/shared/db/database.py
+++ b/app/shared/db/database.py
@@ -1,10 +1,11 @@
 from functools import lru_cache
 from sqlalchemy import Engine, create_engine, event, text
 from sqlalchemy.orm import sessionmaker
-from shared.settings import get_settings
 from contextlib import asynccontextmanager, contextmanager
 from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession, AsyncEngine, async_sessionmaker

+from app.shared.settings import get_settings
+

@lru_cache
 def make_engine(database_url: str):
--- a/app/shared/db/models.py
+++ b/app/shared/db/models.py
--- a/app/shared/db/user_state.py
+++ b/app/shared/db/user_state.py
@@ -3,11 +3,11 @@ from typing import Dict, Set
 import sqlalchemy
 from sqlalchemy.orm import Session
 from sqlalchemy import func
-from db import crud, models
 from datetime import datetime
-from shared.user_groups import GroupInfo, GroupPermissions
-from db.schemas import Usage, UsageResponse

+from app.shared.db import crud, models
+from app.shared.user_groups import GroupInfo, GroupPermissions
+from app.shared.schemas import Usage, UsageResponse

 class UserState:
    """
--- a/app/shared/log.py
+++ b/app/shared/log.py
@@ -0,0 +1,13 @@
+import traceback
+from loguru import logger
+
+
+# logging configurations
+logger.add("logs/api_logs.log", retention="30 days", rotation="3 days")
+logger.add("logs/error_logs.log", retention="30 days", level="ERROR")
+
+    
+def log_error(e: Exception, traceback_str: str = None, extra:str = ""):
+    if not traceback_str: traceback_str = traceback.format_exc()
+    if extra: extra = f"{extra}\n"
+    logger.error(f"{extra}{e.__class__.__name__}: {e}\n{traceback_str}")
--- a/app/shared/schemas.py
+++ b/app/shared/schemas.py
--- a/app/shared/settings.py
+++ b/app/shared/settings.py
@@ -31,8 +31,8 @@ class Settings(BaseSettings):
        return self.DATABASE_PATH.replace("sqlite://", "sqlite+aiosqlite://")

    # redis
+    REDIS_PASSWORD: str = ""
    CELERY_BROKER_URL: str = "redis://localhost:6379"
-    CELERY_RESULT_BACKEND: str = "redis://localhost:6379"
    REDIS_EXCEPTIONS_CHANNEL: str = "exceptions-channel"
    
 	# observability
@@ -40,8 +40,8 @@ class Settings(BaseSettings):

 	# security
    API_BEARER_TOKEN: Annotated[str, Len(min_length=20)]
-    ALLOWED_ORIGINS: Annotated[set[str], Len(min_length=1)]
-    CHROME_APP_IDS: Annotated[set[Annotated[str, Len(min_length=10)]], Len(min_length=1)]
+    ALLOWED_ORIGINS: Annotated[Set[str], Len(min_length=1)]
+    CHROME_APP_IDS: Annotated[Set[Annotated[str, Len(min_length=10)]], Len(min_length=1)]
    #TODO: deprecate blocklist?
    BLOCKED_EMAILS: Annotated[Set[str], Len(min_length=0)] = set()

--- a/app/shared/task_messaging.py
+++ b/app/shared/task_messaging.py
@@ -3,14 +3,14 @@ from functools import lru_cache
 from celery import Celery
 import redis

-from shared.settings import get_settings
+from app.shared.settings import get_settings

@lru_cache
 def get_celery(name:str="") -> Celery:
    return Celery(
        name,
        broker_url=get_settings().CELERY_BROKER_URL,
-        result_backend=get_settings().CELERY_RESULT_BACKEND,
+        result_backend=get_settings().CELERY_BROKER_URL,
    )


--- a/app/shared/user_groups.py
+++ b/app/shared/user_groups.py
@@ -1,3 +1,4 @@
+import os
 import yaml
 from loguru import logger
 from pydantic import BaseModel, field_validator, Field, model_validator
@@ -65,12 +66,19 @@ class GroupPermissions(BaseModel):
            raise ValueError("priority must be either 'low' or 'high'.")
        return v

+
 class GroupModel(BaseModel):
    description: str
    orchestrator: str
    orchestrator_sheet: str
    permissions: GroupPermissions

+    @field_validator('orchestrator', 'orchestrator_sheet', mode='before')
+    def validate_priority(cls, v):
+        if not os.path.exists(v):
+            raise ValueError(f"Orchestrator file not found with this path: {v}")
+        return v
+

 class UserGroupModel(BaseModel):
    users: Dict[str, List[str]] = Field(default_factory=dict)
@@ -125,6 +133,8 @@ class UserGroupModel(BaseModel):
        return self

 # for the API return values
+
+
 class GroupInfo(GroupPermissions):
    description: str = ""
-    service_account_emails: list[str] = []
+    service_account_emails: list[str] = []
--- a/app/shared/utils/misc.py
+++ b/app/shared/utils/misc.py
@@ -1,10 +1,3 @@
-import base64
-from fastapi.encoders import jsonable_encoder
-
-def custom_jsonable_encoder(obj):
-    if isinstance(obj, bytes):
-        return base64.b64encode(obj).decode('utf-8')
-    return jsonable_encoder(obj)

 def fnv1a_hash_mod(s: str, modulo:int) -> int:
    # receives a string and returns a number in [0:modulo-1], ensures an even distribution over the modulo range
--- a/app/tests/conftest.py
+++ b/app/tests/conftest.py
@@ -2,7 +2,7 @@ import os
 from fastapi.testclient import TestClient
 import pytest
 from unittest.mock import patch
-from core.config import ALLOW_ANY_EMAIL
+from app.shared.config import ALLOW_ANY_EMAIL
 from db.user_state import UserState
 from shared.settings import Settings

--- a/app/tests/db/test_crud.py
+++ b/app/tests/db/test_crud.py
@@ -63,7 +63,7 @@ def test_data(db_session):

 def test_get_archive(test_data, db_session):
    from db import crud
-    from core.config import ALLOW_ANY_EMAIL
+    from app.shared.config import ALLOW_ANY_EMAIL

    print(db_session.query(models.Group).all())

@@ -94,7 +94,7 @@ def test_get_archive(test_data, db_session):

 def test_search_archives_by_url(test_data, db_session):
    from db import crud
-    from core.config import ALLOW_ANY_EMAIL
+    from app.shared.config import ALLOW_ANY_EMAIL

    # rick's archives are private
    assert len(crud.search_archives_by_url(db_session, "https://example-0.com", "rick@example.com")) == 34
@@ -140,7 +140,7 @@ def test_search_archives_by_url(test_data, db_session):


 def test_search_archives_by_email(test_data, db_session):
-    from core.config import ALLOW_ANY_EMAIL
+    from app.shared.config import ALLOW_ANY_EMAIL
    from db import crud

    # lower/upper case
@@ -163,7 +163,7 @@ def test_search_archives_by_email(test_data, db_session):
@patch("db.crud.DATABASE_QUERY_LIMIT", new=25)
 def test_max_query_limit(test_data, db_session):
    from db import crud
-    from core.config import ALLOW_ANY_EMAIL
+    from app.shared.config import ALLOW_ANY_EMAIL

    assert len(crud.search_archives_by_url(db_session, "https://example", ALLOW_ANY_EMAIL)) == 25
    assert len(crud.search_archives_by_url(db_session, "https://example", ALLOW_ANY_EMAIL, limit=1000)) == 25
@@ -304,7 +304,7 @@ def test_create_tag(db_session):

 def test_is_user_in_group(test_data, db_session):
    from db import crud
-    from core.config import ALLOW_ANY_EMAIL
+    from app.shared.config import ALLOW_ANY_EMAIL

    # see user-groups.test.yaml
    test_pairs = [
--- a/app/tests/db/test_models.py
+++ b/app/tests/db/test_models.py
--- a/app/tests/endpoints/test_default.py
+++ b/app/tests/endpoints/test_default.py
@@ -1,7 +1,7 @@
 from unittest.mock import AsyncMock, MagicMock, patch
 from fastapi.testclient import TestClient
 import pytest
-from core.config import VERSION
+from app.shared.config import VERSION
 from tests.db.test_crud import test_data


@@ -103,7 +103,7 @@ async def test_prometheus_metrics(test_data, client_with_token, get_settings):
    assert 'disk_utilization{type="used"}' not in r.text

    # after metrics calculation
-    from utils.metrics import measure_regular_metrics
+    from web.utils.metrics import measure_regular_metrics
    await measure_regular_metrics(get_settings.DATABASE_PATH, 60 * 60 * 24 * 31 * 12 * 100)
    r2 = client_with_token.get("/metrics")
    assert 'disk_utilization{type="used"}' in r2.text
@@ -117,7 +117,7 @@ async def test_prometheus_metrics(test_data, client_with_token, get_settings):
    assert 'database_metrics_counter_total{query="count_by_user",user="jerry@example.com"} 33.0' in r2.text

    # 30s window, should not change the gauges nor the total in the counters
-    from utils.metrics import measure_regular_metrics
+    from web.utils.metrics import measure_regular_metrics
    await measure_regular_metrics(get_settings.DATABASE_PATH, 30)
    r3 = client_with_token.get("/metrics")
    assert 'database_metrics{query="count_archives"} 100.0' in r3.text
--- a/app/tests/endpoints/test_interoperability.py
+++ b/app/tests/endpoints/test_interoperability.py
@@ -2,7 +2,7 @@ from datetime import datetime
 import json
 from unittest.mock import patch

-from core.config import ALLOW_ANY_EMAIL
+from app.shared.config import ALLOW_ANY_EMAIL
 from db import crud


--- a/app/tests/endpoints/test_sheet.py
+++ b/app/tests/endpoints/test_sheet.py
@@ -4,7 +4,7 @@ from unittest.mock import MagicMock, patch

 from fastapi.testclient import TestClient

-from db.schemas import TaskResult
+from app.shared.schemas import TaskResult


 def test_endpoints_no_auth(client, test_no_auth):
--- a/app/tests/endpoints/test_task.py
+++ b/app/tests/endpoints/test_task.py
--- a/app/tests/endpoints/test_url.py
+++ b/app/tests/endpoints/test_url.py
@@ -1,7 +1,7 @@
 import json
 from unittest.mock import MagicMock, patch

-from db.schemas import ArchiveCreate, TaskResult
+from app.shared.schemas import ArchiveCreate, TaskResult


 def test_archive_url_unauthenticated(client, test_no_auth):
--- a/app/tests/orchestration.test.yaml
+++ b/app/tests/orchestration.test.yaml
--- a/app/tests/user-groups.test.broken.yaml
+++ b/app/tests/user-groups.test.broken.yaml
--- a/app/tests/user-groups.test.yaml
+++ b/app/tests/user-groups.test.yaml
--- a/app/tests/web/test_main.py
+++ b/app/tests/web/test_main.py
@@ -19,7 +19,7 @@ def test_alembic(db_session):

@patch("endpoints.default.crud.soft_delete_task", side_effect=Exception('mocked error'))
 def test_logging_middleware(m1, client_with_auth):
-    from utils.metrics import EXCEPTION_COUNTER
+    from web.utils.metrics import EXCEPTION_COUNTER
    assert len(EXCEPTION_COUNTER.collect()[0].samples) == 0
    with pytest.raises(Exception, match="mocked error"):
        client_with_auth.delete("/url/123")
--- a/app/tests/web/test_security.py
+++ b/app/tests/web/test_security.py
@@ -4,7 +4,7 @@ from fastapi import HTTPException
 from fastapi.security import HTTPAuthorizationCredentials
 import pytest

-from core.config import ALLOW_ANY_EMAIL
+from app.shared.config import ALLOW_ANY_EMAIL


 def test_secure_compare():
--- a/app/tests/worker/test_worker_main.py
+++ b/app/tests/worker/test_worker_main.py
--- a/app/web/init.py
+++ b/app/web/init.py
@@ -0,0 +1,3 @@
+from app.web.main import app_factory
+
+app = app_factory
--- a/app/web/endpoints/init.py
+++ b/app/web/endpoints/init.py
--- a/app/web/endpoints/default.py
+++ b/app/web/endpoints/default.py
@@ -2,15 +2,14 @@
 from typing import Dict
 from fastapi import APIRouter, Depends, Request, HTTPException
 from fastapi.responses import FileResponse, JSONResponse
-from sqlalchemy.orm import Session

-from core.config import VERSION, BREAKING_CHANGES
-from core.logging import log_error
-from db import crud
-from db.schemas import ActiveUser, UsageResponse
-from db.user_state import UserState
-from web.security import get_user_auth, bearer_security, get_user_state
-from shared.user_groups import GroupInfo
+from app.shared.config import VERSION, BREAKING_CHANGES
+from app.shared.log import log_error
+from app.shared.db import crud
+from app.shared.schemas import ActiveUser, UsageResponse
+from app.shared.db.user_state import UserState
+from app.web.security import get_user_auth, bearer_security, get_user_state
+from app.shared.user_groups import GroupInfo

 default_router = APIRouter()

@@ -57,4 +56,4 @@ def get_user_usage(

@default_router.get('/favicon.ico', include_in_schema=False)
 async def favicon() -> FileResponse:
-    return FileResponse("static/favicon.ico")
+    return FileResponse("web/static/favicon.ico")
--- a/app/web/endpoints/interoperability.py
+++ b/app/web/endpoints/interoperability.py
@@ -1,14 +1,19 @@
 import json
 from fastapi import APIRouter, Depends, HTTPException
 from fastapi.responses import JSONResponse
-from auto_archiver import Metadata
+from loguru import logger
 import sqlalchemy
+from auto_archiver import Metadata
+from sqlalchemy.orm import Session

-from core.config import ALLOW_ANY_EMAIL
-from web.security import token_api_key_auth
-from db import models, schemas
-from worker.main import insert_result_into_db, get_all_urls, get_store_until
-from core.logging import log_error
+from app.shared.aa_utils import get_all_urls
+from app.shared.config import ALLOW_ANY_EMAIL
+from app.shared import business_logic, schemas
+from app.shared.db import crud
+from app.shared.db.database import get_db_dependency
+from app.web.security import token_api_key_auth
+from app.shared.db import models
+from app.shared.log import log_error


 interoperability_router = APIRouter(prefix="/interop", tags=["Interoperability endpoints."])
@@ -18,7 +23,8 @@ interoperability_router = APIRouter(prefix="/interop", tags=["Interoperability e
@interoperability_router.post("/submit-archive", status_code=201, summary="Submit a manual archive entry, for data that was archived elsewhere.")
 def submit_manual_archive(
    manual: schemas.SubmitManualArchive,
-    auth=Depends(token_api_key_auth)
+    auth=Depends(token_api_key_auth),
+    db: Session = Depends(get_db_dependency)
 ):
    result: Metadata = Metadata.from_json(manual.result)
    manual.author_id = manual.author_id or ALLOW_ANY_EMAIL
@@ -34,10 +40,12 @@ def submit_manual_archive(
            id=models.generate_uuid(),
            result=json.loads(result.to_json()),
            urls=get_all_urls(result),
-            store_until=get_store_until(manual.group_id),
+            store_until=business_logic.get_store_archive_until(db, manual.group_id),
        )
-        archive_id = insert_result_into_db(archive)
+
+        db_archive = crud.store_archived_url(db, archive)
+        logger.debug(f"[MANUAL ARCHIVE STORED] {db_archive.author_id} {db_archive.url}")
+        return JSONResponse({"id": db_archive.id}, status_code=201)
    except sqlalchemy.exc.IntegrityError as e:
        log_error(e)
        raise HTTPException(status_code=422, detail=f"Cannot insert into DB due to integrity error, likely duplicate urls.")
-    return JSONResponse({"id": archive_id}, status_code=201)
--- a/app/web/endpoints/sheet.py
+++ b/app/web/endpoints/sheet.py
@@ -5,11 +5,12 @@ from fastapi.responses import JSONResponse
 from sqlalchemy import exc
 from sqlalchemy.orm import Session

-from db.user_state import UserState
-from shared.task_messaging import get_celery
-from web.security import get_user_state
-from db import schemas, crud
-from db.database import get_db_dependency
+from app.shared.db.user_state import UserState
+from app.shared import schemas
+from app.shared.task_messaging import get_celery
+from app.web.security import get_user_state
+from app.shared.db import crud
+from app.shared.db.database import get_db_dependency

 sheet_router = APIRouter(prefix="/sheet", tags=["Google Spreadsheet operations"])

--- a/app/web/endpoints/task.py
+++ b/app/web/endpoints/task.py
@@ -3,13 +3,11 @@ from fastapi import APIRouter, Depends
 from fastapi.encoders import jsonable_encoder
 from fastapi.responses import JSONResponse

-from loguru import logger
-from shared.task_messaging import get_celery
-from web.security import get_token_or_user_auth
-
-from db import schemas
-from core.logging import log_error
-from utils.misc import custom_jsonable_encoder
+from app.shared.task_messaging import get_celery
+from app.web.security import get_token_or_user_auth
+from app.shared import schemas
+from app.shared.log import log_error
+from app.web.utils.misc import custom_jsonable_encoder


 task_router = APIRouter(prefix="/task", tags=["Async task operations"])
--- a/app/web/endpoints/url.py
+++ b/app/web/endpoints/url.py
@@ -2,16 +2,16 @@
 from fastapi import APIRouter, Depends, HTTPException
 from fastapi.responses import JSONResponse
 from datetime import datetime
-
 from loguru import logger
-from core.config import ALLOW_ANY_EMAIL
-from db.user_state import UserState
-from shared.task_messaging import get_celery
-from web.security import get_token_or_user_auth, get_user_state
 from sqlalchemy.orm import Session

-from db import crud, schemas
-from db.database import get_db_dependency
+from app.shared.config import ALLOW_ANY_EMAIL
+from app.shared import schemas
+from app.shared.task_messaging import get_celery
+from app.web.security import get_token_or_user_auth, get_user_state
+from app.shared.db import crud
+from app.shared.db.user_state import UserState
+from app.shared.db.database import get_db_dependency

 from urllib.parse import urlparse

--- a/src/core/events.py
+++ b/src/core/events.py
@@ -7,14 +7,15 @@ from fastapi import FastAPI
 from contextlib import asynccontextmanager
 from fastapi_utils.tasks import repeat_every
 from loguru import logger
-
-from db import crud, models, schemas
-from db.database import get_db, get_db_async, make_engine, wal_checkpoint
-from shared.settings import get_settings
-from shared.task_messaging import get_celery
-from utils.metrics import measure_regular_metrics, redis_subscribe_worker_exceptions
 from fastapi_mail import FastMail, MessageSchema, MessageType

+from app.shared.db import crud, models
+from app.shared.db.database import get_db, get_db_async, make_engine, wal_checkpoint
+from app.shared import schemas
+from app.shared.settings import get_settings
+from app.shared.task_messaging import get_celery
+from app.web.utils.metrics import measure_regular_metrics, redis_subscribe_worker_exceptions
+
 celery = get_celery()


@@ -23,9 +24,12 @@ async def lifespan(app: FastAPI):
    # see https://fastapi.tiangolo.com/advanced/events/#lifespan

    # STARTUP
+    logger.debug("HERE 00")
    engine = make_engine(get_settings().DATABASE_PATH)
    models.Base.metadata.create_all(bind=engine)
-    alembic.config.main(argv=['--raiseerr', 'upgrade', 'head'])
+    logger.debug("HERE 01")
+    alembic.config.main(prog="alembic", argv=['--raiseerr', 'upgrade', 'head'])
+    logger.debug("HERE 02")
    logging.getLogger("uvicorn.access").disabled = True  # loguru
    asyncio.create_task(redis_subscribe_worker_exceptions(get_settings().REDIS_EXCEPTIONS_CHANNEL))
    asyncio.create_task(repeat_measure_regular_metrics())
@@ -88,6 +92,7 @@ async def archive_sheets_cronjob(frequency: str, interval: int, current_time_uni
 # TODO: on exception should logerror but also prometheus counter
 DELETE_WINDOW = get_settings().DELETE_SCHEDULED_ARCHIVES_NOTIFY_DAYS * 24 * 60 * 60

+
@repeat_every(seconds=DELETE_WINDOW, wait_first=180, on_exception=logger.error)
 async def notify_about_expired_archives():
    notify_from = datetime.datetime.now() + datetime.timedelta(days=get_settings().DELETE_SCHEDULED_ARCHIVES_NOTIFY_DAYS)
@@ -103,7 +108,7 @@ async def notify_about_expired_archives():
        # notify users
        for email in user_archives:
            list_of_archives = "\n".join([f'{a.url},{a.id}<br/>' for a in user_archives[email]])
-            #TODO: how can users download them in bulk?
+            # TODO: how can users download them in bulk?
            message = MessageSchema(
                subject="Auto Archiver: Archives Scheduled for Deletion",
                recipients=[email],
@@ -139,7 +144,6 @@ async def delete_expired_archives():
            logger.info(f"[CRON] Deleted {count_deleted} archives.")


-
@repeat_every(seconds=86400, wait_first=150, on_exception=logger.error)
 async def delete_stale_sheets():
    STALE_DAYS = get_settings().DELETE_STALE_SHEETS_DAYS
--- a/app/web/main.py
+++ b/app/web/main.py
@@ -7,28 +7,27 @@ from fastapi.staticfiles import StaticFiles
 from fastapi.middleware.cors import CORSMiddleware
 from prometheus_fastapi_instrumentator import Instrumentator
 from datetime import datetime
-import sqlalchemy
 from sqlalchemy.orm import Session
 from loguru import logger

-from core.logging import logging_middleware, log_error
-from shared.task_messaging import get_celery
-from worker.main import insert_result_into_db
+from app.shared.log import log_error
+from app.web.middleware import logging_middleware
+from app.shared import schemas
+from app.shared.task_messaging import get_celery

-from db import crud, models, schemas
-from web.security import get_user_auth, token_api_key_auth, get_token_or_user_auth
-from core.config import VERSION, API_DESCRIPTION
-from db.database import get_db_dependency
-from core.events import lifespan
-from shared.settings import get_settings
+from app.shared.db import crud
+from app.web.security import get_user_auth, token_api_key_auth, get_token_or_user_auth
+from app.shared.config import VERSION, API_DESCRIPTION
+from app.shared.db.database import get_db_dependency
+from app.web.events import lifespan
+from app.shared.settings import get_settings

-from auto_archiver import Metadata

-from endpoints.default import default_router
-from endpoints.url import url_router
-from endpoints.sheet import sheet_router
-from endpoints.task import task_router
-from endpoints.interoperability import interoperability_router
+from app.web.endpoints.default import default_router
+from app.web.endpoints.url import url_router
+from app.web.endpoints.sheet import sheet_router
+from app.web.endpoints.task import task_router
+from app.web.endpoints.interoperability import interoperability_router

 celery = get_celery()

@@ -161,14 +160,15 @@ def app_factory(settings = get_settings()):

    @app.post("/submit-archive", status_code=201, deprecated=True)  # DEPRECATED
    def submit_manual_archive(manual: schemas.SubmitManual, auth=Depends(token_api_key_auth)):
-        result = Metadata.from_json(manual.result)
-        logger.info(f"MANUAL SUBMIT {result.get_url()} {manual.author_id}")
-        manual.tags.add("manual")
-        try:
-            archive_id = insert_result_into_db(result, manual.tags, manual.public, manual.group_id, manual.author_id, models.generate_uuid())
-        except sqlalchemy.exc.IntegrityError as e:
-            log_error(e)
-            raise HTTPException(status_code=422, detail=f"Cannot insert into DB due to integrity error")
-        return JSONResponse({"id": archive_id})
+        raise HTTPException(status_code=410, detail="This endpoint is deprecated. Use /interop/submit-archive instead.")
+        # result = Metadata.from_json(manual.result)
+        # logger.info(f"MANUAL SUBMIT {result.get_url()} {manual.author_id}")
+        # manual.tags.add("manual")
+        # try:
+        #     # archive_id = insert_result_into_db(result, manual.tags, manual.public, manual.group_id, manual.author_id, models.generate_uuid())
+        # except sqlalchemy.exc.IntegrityError as e:
+        #     log_error(e)
+        #     raise HTTPException(status_code=422, detail=f"Cannot insert into DB due to integrity error")
+        # return JSONResponse({"id": archive_id})

    return app
--- a/app/web/middleware.py
+++ b/app/web/middleware.py
@@ -1,26 +1,17 @@
-import traceback
+
 from loguru import logger
 from fastapi import Request
+from app.shared.log import log_error


-# logging configurations
-logger.add("logs/api_logs.log", retention="30 days", rotation="3 days")
-logger.add("logs/error_logs.log", retention="30 days", level="ERROR")
-
-    
-def log_error(e: Exception, traceback_str: str = None, extra:str = ""):
-    if not traceback_str: traceback_str = traceback.format_exc()
-    if extra: extra = f"{extra}\n"
-    logger.error(f"{extra}{e.__class__.__name__}: {e}\n{traceback_str}")
-
 async def logging_middleware(request: Request, call_next):
    try:
        response = await call_next(request)
        logger.info(f"{request.client.host}:{request.client.port} {request.method} {request.url._url} - HTTP {response.status_code}")
        return response
    except Exception as e:
-        from utils.metrics import EXCEPTION_COUNTER
+        from web.utils.metrics import EXCEPTION_COUNTER
        EXCEPTION_COUNTER.labels(type=e.__class__.__name__).inc()
        logger.info(f"{request.client.host}:{request.client.port} {request.method} {request.url._url} - {e.__class__.__name__} {e}")
        log_error(e)
-        raise e
+        raise e
--- a/app/web/security.py
+++ b/app/web/security.py
@@ -2,11 +2,11 @@ from loguru import logger
 import requests, secrets
 from fastapi import HTTPException, status, Depends
 from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
-from core.config import ALLOW_ANY_EMAIL
-from shared.settings import get_settings
-from db.database import get_db
-from db import crud
-from db.user_state import UserState
+
+from app.shared.config import ALLOW_ANY_EMAIL
+from app.shared.settings import get_settings
+from app.shared.db.database import get_db
+from app.shared.db.user_state import UserState

 settings = get_settings()
 bearer_security = HTTPBearer()
--- a/app/web/static/.gitkeep
+++ b/app/web/static/.gitkeep
--- a/app/web/static/favicon.ico
+++ b/app/web/static/favicon.ico
--- a/app/web/utils/init.py
+++ b/app/web/utils/init.py
--- a/app/web/utils/metrics.py
+++ b/app/web/utils/metrics.py
@@ -3,12 +3,11 @@ import json
 import os
 import shutil
 from prometheus_client import Counter, Gauge
-import redis

-from db import crud
-from db.database import get_db
-from core.logging import log_error
-from shared.task_messaging import get_redis
+from app.shared.db import crud
+from app.shared.db.database import get_db
+from app.shared.log import log_error
+from app.shared.task_messaging import get_redis


 # Custom metrics
@@ -39,7 +38,7 @@ DATABASE_METRICS_COUNTER = Counter(
 )


-async def redis_subscribe_worker_exceptions(REDIS_EXCEPTIONS_CHANNEL):
+async def redis_subscribe_worker_exceptions(REDIS_EXCEPTIONS_CHANNEL: str):
    # Subscribe to Redis channel and increment the counter for each exception with info on the exception and task
    Redis = get_redis()
    PubSubExceptions = Redis.pubsub()
--- a/app/web/utils/misc.py
+++ b/app/web/utils/misc.py
@@ -0,0 +1,7 @@
+import base64
+from fastapi.encoders import jsonable_encoder
+
+def custom_jsonable_encoder(obj):
+    if isinstance(obj, bytes):
+        return base64.b64encode(obj).decode('utf-8')
+    return jsonable_encoder(obj)
--- a/app/worker/init.py
+++ b/app/worker/init.py
--- a/app/worker/main.py
+++ b/app/worker/main.py
@@ -1,19 +1,19 @@
+import json

 import traceback, datetime
-from typing import List
-
 from celery.signals import task_failure
-from auto_archiver import Config, ArchivingOrchestrator, Metadata
-from auto_archiver.core import Media
 from loguru import logger
-
-from db import crud, schemas, models
-from db.database import get_db
-from shared.task_messaging import get_celery, get_redis
-from shared.settings import get_settings
-import json
 from sqlalchemy import exc
-from core.logging import log_error
+
+from auto_archiver import Config, ArchivingOrchestrator, Metadata
+
+from app.shared.db import crud, models
+from app.shared.db.database import get_db
+from app.shared import business_logic, schemas
+from app.shared.task_messaging import get_celery, get_redis
+from app.shared.settings import get_settings
+from app.shared.log import log_error
+from app.shared.aa_utils import get_all_urls


 settings = get_settings()
@@ -24,8 +24,6 @@ Redis = get_redis()
 USER_GROUPS_FILENAME = settings.USER_GROUPS_FILENAME

 # TODO: after release, as it requires updating past entries with sheet_id where tag is used, drop tags
-
-
@celery.task(name="create_archive_task", bind=True, autoretry_for=(Exception,), retry_backoff=True, retry_kwargs={'max_retries': 0})
 def create_archive_task(self, archive_json: str):
    logger.info(archive_json)
@@ -96,6 +94,7 @@ def load_orchestrator(group_id: str, orchestrator_for_sheet: bool = False, overw
        else:
            orchestrator_fn = crud.get_group(session, group_id).orchestrator
        assert orchestrator_fn, f"no orchestrator found for {group_id}"
+        

    config = Config()
    config.parse(use_cli=False, yaml_config_filename=orchestrator_fn, overwrite_configs=overwrite_configs)
@@ -104,44 +103,13 @@ def load_orchestrator(group_id: str, orchestrator_for_sheet: bool = False, overw

 def insert_result_into_db(archive: schemas.ArchiveCreate) -> str:
    with get_db() as session:
-        db_task = crud.insert_result_into_db(session, archive)
+        db_task = crud.store_archived_url(session, archive)
        logger.debug(f"[ARCHIVE STORED] {db_task.author_id} {db_task.url}")
        return db_task.id

-
-# TODO: this should live within the auto-archiver
-def get_all_urls(result: Metadata) -> List[models.ArchiveUrl]:
-    db_urls = []
-    for m in result.media:
-        for i, url in enumerate(m.urls): db_urls.append(models.ArchiveUrl(url=url, key=m.get("id", f"media_{i}")))
-        for k, prop in m.properties.items():
-            if prop_converted := convert_if_media(prop):
-                for i, url in enumerate(prop_converted.urls): db_urls.append(models.ArchiveUrl(url=url, key=prop_converted.get("id", f"{k}_{i}")))
-            if isinstance(prop, list):
-                for i, prop_media in enumerate(prop):
-                    if prop_media := convert_if_media(prop_media):
-                        for j, url in enumerate(prop_media.urls):
-                            db_urls.append(models.ArchiveUrl(url=url, key=prop_media.get("id", f"{k}{prop_media.key}_{i}.{j}")))
-    return db_urls
-
-
 def get_store_until(group_id: str) -> datetime.datetime:
    with get_db() as session:
-        group = crud.get_group(session, group_id)
-        max_lifespan = group.permissions.get("max_archive_lifespan_months", -1)
-        if max_lifespan == -1: return None
-
-        return datetime.datetime.now() + datetime.timedelta(days=30 * max_lifespan)
-
-# TODO: this should live within the auto-archiver??
-def convert_if_media(media):
-    if isinstance(media, Media): return media
-    elif isinstance(media, dict):
-        try: return Media.from_dict(media)
-        except Exception as e:
-            logger.debug(f"error parsing {media} : {e}")
-    return False
-
+        return business_logic.get_store_archive_until(session, group_id)

 def redis_publish_exception(exception, task_name, traceback: str = ""):
    REDIS_EXCEPTIONS_CHANNEL = settings.REDIS_EXCEPTIONS_CHANNEL
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -1,20 +1,25 @@
 services:
  web:
-    command: uvicorn web:app --factory --host 0.0.0.0 --reload
+    command: uvicorn app.web:app --factory --host 0.0.0.0 --reload
    restart: "no"
-    env_file: src/.env.dev
+    env_file: .env.dev
+    volumes:
+      - ./app:/aa-api/app # for --reload to work
    environment:
-      - SERVE_LOCAL_ARCHIVE=/app/local_archive # See orchestration.yaml local_storage.save_to
-      - ALLOWED_ORIGINS=http://localhost:8004,chrome-extension://ojcimmjndnlmmlgnjaeojoebaceokpdp
-      - USER_GROUPS_FILENAME=user-groups.dev.yaml
-      - DATABASE_PATH=sqlite:////app/auto-archiver.db
+      - SERVE_LOCAL_ARCHIVE=/aa-api/app/local_archive # See orchestration.yaml local_storage.save_to
+      - ALLOWED_ORIGINS=["http://localhost:8000","http://localhost:8004","http://localhost:8081","chrome-extension://ojcimmjndnlmmlgnjaeojoebaceokpdp"]
+      - USER_GROUPS_FILENAME=/aa-api/app/user-groups.dev.yaml
+      - DATABASE_PATH=sqlite:////aa-api/app/database/auto-archiver.db
+    

  worker:
+  #TODO: add watchmedo
    restart: "no"
-    env_file: src/.env.dev
+    env_file: .env.dev

  redis:
+    command: redis-server /conf/redis.conf --requirepass ${REDIS_PASSWORD}
    restart: "no"
-    env_file: src/.env.dev
+    env_file: .env.dev
    ports:
      - 6379:6379
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,13 +1,3 @@
-# reusable YAML variables
-x-broker-url: &broker-url "redis://:${REDIS_PASSWORD}@redis:6379/0"
-
-x-base-setup: &base-setup
-  build: ./src
-  restart: always
-  env_file: src/.env.prod
-  environment:
-    CELERY_BROKER_URL: *broker-url
-    CELERY_RESULT_BACKEND: *broker-url

 volumes:
  crawls:
@@ -15,13 +5,21 @@ volumes:
 name: "auto-archiver-api"
 services:
  web:
-    <<: *base-setup
+    build: 
+      context: .
+      dockerfile: worker.Dockerfile
+    restart: always
+    env_file: .env.prod
+    environment:
+      CELERY_BROKER_URL: redis://:${REDIS_PASSWORD}@redis:6379/0
    ports:
      - "127.0.0.1:8004:8000"
    #TODO: should prod have the --reload flag?
-    command: uvicorn web:app --factory --host 0.0.0.0 --reload
+    command: uvicorn app.web:app --factory --host 0.0.0.0
    volumes:
-      - ./src:/app
+      # - ./app:/app
+      - ./app/logs:/aa-api/app/logs
+      - ./app/database:/aa-api/app/database
    depends_on:
      - redis
    healthcheck:
@@ -31,16 +29,19 @@ services:
      retries: 3

  worker:
-    <<: *base-setup
-    command: celery --app=worker.main.celery worker --loglevel=info --logfile=logs/celery.log
+    build: 
+      context: .
+      dockerfile: worker.Dockerfile
+    restart: always
+    env_file: .env.prod
+    command: celery --app=app.worker.main.celery worker --loglevel=info --logfile=/aa-api/app/logs/celery.log
    volumes:
-      - ./src:/app
+      - ./app/logs:/aa-api/app/logs
+      - ./app/database:/aa-api/app/database
      - /var/run/docker.sock:/var/run/docker.sock
      - crawls:/crawls # BROWSERTRIX_HOME_HOST:BROWSERTRIX_HOME_CONTAINER, do not change /crawls
    environment:
-      # celery broker-url needs to be duplicated here, do not remove
-      CELERY_BROKER_URL: *broker-url
-      CELERY_RESULT_BACKEND: *broker-url
+      CELERY_BROKER_URL: redis://:${REDIS_PASSWORD}@redis:6379/0
      WACZ_ENABLE_DOCKER: 1 # Enable calling docker from this container
      BROWSERTRIX_HOME_HOST: auto-archiver-api_crawls
      BROWSERTRIX_HOME_CONTAINER: /crawls
@@ -58,8 +59,8 @@ services:
    restart: always
    command: redis-server /conf/redis.conf --requirepass ${REDIS_PASSWORD}
    volumes:
-      - "./redis/data:/data"
-      - "./redis/config:/conf"
+      - ./redis/data:/data
+      - ./redis/config:/conf
    healthcheck:
      test: ["CMD", "redis-cli", "-a", "${REDIS_PASSWORD}", "ping"]
      interval: 30s
--- a/src/.example.env
+++ b/src/.example.env
@@ -1,7 +0,0 @@
-DATABASE_PATH="sqlite:///./auto-archiver.db"
-USER_GROUPS_FILENAME=user-groups.yaml
-CHROME_APP_IDS=000000000000000000000000000000000000000000000.apps.googleusercontent.com,000000000000000000000000000000000000000000001.apps.googleusercontent.com
-#ALLOWED_ORIGINS="http://localhost:8004" # dev only
-
-
-API_BEARER_TOKEN=TODO
--- a/src/db/init.py
+++ b/src/db/init.py
@@ -1 +0,0 @@
-# https://fastapi.tiangolo.com/tutorial/sql-databases/#review-all-the-files
--- a/src/web/init.py
+++ b/src/web/init.py
@@ -1,4 +0,0 @@
-from web.main import app_factory
-
-
-app = app_factory
--- a/worker.Dockerfile
+++ b/worker.Dockerfile
@@ -2,7 +2,7 @@
 FROM bellingcat/auto-archiver

 # set work directory
-WORKDIR /app
+WORKDIR /aa-api

 RUN curl -fsSL https://get.docker.com -o get-docker.sh && \
    sh get-docker.sh
@@ -13,10 +13,13 @@ ENV PYTHONDONTWRITEBYTECODE=1
 # install dependencies
 RUN pip install --upgrade pip && \
 	apt-get update 
-COPY Pipfile* ./
+COPY ./Pipfile* ./
 RUN pipenv install

-# copy src code over
-COPY . .
+# copy source code and .env files over
+COPY alembic.ini ./
+COPY .env* ./app/
+COPY ./secrets/ ./secrets/
+COPY ./app/ ./app/

 ENTRYPOINT ["pipenv", "run"]
				`@@ -1 +0,0 @@`
				`# https://fastapi.tiangolo.com/tutorial/sql-databases/#review-all-the-files`