bellingcat/auto-archiver-api
1
0
Fork 0
mirror of https://github.com/bellingcat/auto-archiver-api.git synced 2026-06-12 13:38:33 +03:00
Code Issues Packages Projects Releases Wiki Activity

new api mvp

Browse Source
This commit is contained in:
msramalho
2023-04-26 23:40:42 +01:00
parent 6c8283f517
commit 4119387feb
9 changed files with 370 additions and 209 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
README.md
View File

@@ -13,6 +13,7 @@ cd /src
* console 1 - `docker compose up redis` optionally add `dashboard` for flower dashboard and `web` if not running uvicorn locally * console 1 - `docker compose up redis` optionally add `dashboard` for flower dashboard and `web` if not running uvicorn locally
* console 2 - `pipenv shell` + `celery worker --app=worker.celery --loglevel=info --logfile=logs/celery_dev.log` * console 2 - `pipenv shell` + `celery worker --app=worker.celery --loglevel=info --logfile=logs/celery_dev.log`
* `celery --app=worker.celery worker --loglevel=info --logfile=logs/celery_dev.log` celery 5 * `celery --app=worker.celery worker --loglevel=info --logfile=logs/celery_dev.log` celery 5
* or with watchdog for dev auto-reload `watchmedo auto-restart -d ./ -- celery --app=worker.celery worker --loglevel=info --logfile=logs/celery_dev.log`
* console 3 - `pipenv shell` + `uvicorn main:app --host 0.0.0.0 --reload` * console 3 - `pipenv shell` + `uvicorn main:app --host 0.0.0.0 --reload`
orchestration must be from the console(?) orchestration must be from the console(?)
* turn off VPNs if connection to docker is not working * turn off VPNs if connection to docker is not working

1
src/Pipfile
View File

@@ -22,6 +22,7 @@ alembic = "*"
fastapi-utils = "*" fastapi-utils = "*"
[dev-packages] [dev-packages]
watchdog = "*"
[requires] [requires]
python_version = "3.10" python_version = "3.10"

218
src/Pipfile.lock generated
View File

@@ -1,7 +1,7 @@
{ {
"_meta": { "_meta": {
"hash": { "hash": {
"sha256": "8fbf19b1fc2cc928f83c9e52f4b6522b02ddedee2fc863b0434ee3567d928c8f" "sha256": "85ba632faeb7f00b42dfefbbe50a9a8348610a1297fdf70017991c9c2cb7980d"
}, },
"pipfile-spec": 6, "pipfile-spec": 6,
"requires": { "requires": {
@@ -26,11 +26,11 @@
}, },
"aiosqlite": { "aiosqlite": {
"hashes": [ "hashes": [
"sha256:c3511b841e3a2c5614900ba1d179f366826857586f78abd75e7cbeb88e75a557", "sha256:95ee77b91c8d2808bd08a59fbebf66270e9090c3d92ffbf260dc0db0b979577d",
"sha256:faa843ef5fb08bafe9a9b3859012d3d9d6f77ce3637899de20606b7fc39aa213" "sha256:edba222e03453e094a3ce605db1b970c4b3376264e56f32e2a4959f948d66a96"
], ],
"index": "pypi", "index": "pypi",
"version": "==0.18.0" "version": "==0.19.0"
}, },
"alembic": { "alembic": {
"hashes": [ "hashes": [
@@ -73,11 +73,11 @@
}, },
"attrs": { "attrs": {
"hashes": [ "hashes": [
"sha256:29e95c7f6778868dbd49170f98f8818f78f3dc5e0e37c0b1f474e3561b240836", "sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04",
"sha256:c9227bfc2f01993c03f68db37d1d15c9690188323c067c641f1a35ca58185f99" "sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015"
], ],
"markers": "python_version >= '3.6'", "markers": "python_version >= '3.7'",
"version": "==22.2.0" "version": "==23.1.0"
}, },
"authlib": { "authlib": {
"hashes": [ "hashes": [
@@ -88,11 +88,11 @@
}, },
"auto-archiver": { "auto-archiver": {
"hashes": [ "hashes": [
"sha256:033451db9789296cebbd00bca5446541b0a4e86bc9ae0d65a8bedf8b53015044", "sha256:850361f29e57719fa4661fb1184993f1b03da165a15b9144729665f00a832e31",
"sha256:b1a0fa870b0d7a0b2a0b259e6be92a3493aa2793cb04eee071263a7debcb7074" "sha256:df0e0034b974e0c51549616831b70f1e0a035e154bd39457a43701a6ee3b3fd7"
], ],
"index": "pypi", "index": "pypi",
"version": "==0.5.5" "version": "==0.5.7"
}, },
"beautifulsoup4": { "beautifulsoup4": {
"hashes": [ "hashes": [
@@ -111,19 +111,19 @@
}, },
"boto3": { "boto3": {
"hashes": [ "hashes": [
"sha256:03c2e1ddd29d993a6ab9b8a8fe184027957fc32bd405c496ad0c30311445925f", "sha256:2272a060005bf8299f7342cbf1344304eb44b7060cddba6784f676e3bc737bb8",
"sha256:4ea3319bba2e8ff7cd9560259ae64f073c7fb6312158aa375777687231cabe69" "sha256:deb53ad15ff0e75ae0be6d7115a2d34e4bafb0541484485f0feb61dabdfb5513"
], ],
"markers": "python_full_version >= '3.7.0'", "markers": "python_version >= '3.7'",
"version": "==1.26.112" "version": "==1.26.115"
}, },
"botocore": { "botocore": {
"hashes": [ "hashes": [
"sha256:1f52d9371d7b5ee30a53dcef7954c3cf22e04b131cfab5268035f3299ccde9e1", "sha256:58eee8cf8f4f3e515df29f6dc535dd86ed3f4cea40999c5bc74640ff40bdc71f",
"sha256:2cbaddb09b46dcb0a05490724d51acb224d3a8df433c347f995b4d78bfb02c8a" "sha256:dff327977d7c9f98f2dc54b51b8f70326952dd50ae23b885fdfa8bfeec014b76"
], ],
"markers": "python_full_version >= '3.7.0'", "markers": "python_version >= '3.7'",
"version": "==1.29.112" "version": "==1.29.115"
}, },
"brotli": { "brotli": {
"hashes": [ "hashes": [
@@ -410,7 +410,7 @@
"sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e", "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e",
"sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48" "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"
], ],
"markers": "python_full_version >= '3.7.0'", "markers": "python_version >= '3.7'",
"version": "==8.1.3" "version": "==8.1.3"
}, },
"cloudscraper": { "cloudscraper": {
@@ -465,7 +465,7 @@
"sha256:070b29b5bbf4b1ec2cd51c96ea040dc68a614de703910a91ad1abba18f9f379f", "sha256:070b29b5bbf4b1ec2cd51c96ea040dc68a614de703910a91ad1abba18f9f379f",
"sha256:86b8b7517efcc558f085a142cdb7620f0921543fcabdb538c8a4c4001d8178e3" "sha256:86b8b7517efcc558f085a142cdb7620f0921543fcabdb538c8a4c4001d8178e3"
], ],
"markers": "python_full_version >= '3.7.0'", "markers": "python_version >= '3.7'",
"version": "==1.1.8" "version": "==1.1.8"
}, },
"exceptiongroup": { "exceptiongroup": {
@@ -478,11 +478,11 @@
}, },
"fastapi": { "fastapi": {
"hashes": [ "hashes": [
"sha256:99d4fdb10e9dd9a24027ac1d0bd4b56702652056ca17a6c8721eec4ad2f14e18", "sha256:9569f0a381f8a457ec479d90fa01005cfddaae07546eb1f3fa035bc4797ae7d5",
"sha256:daf73bbe844180200be7966f68e8ec9fd8be57079dff1bacb366db32729e6eb5" "sha256:a870d443e5405982e1667dfe372663abf10754f246866056336d7f01c21dab07"
], ],
"index": "pypi", "index": "pypi",
"version": "==0.95.0" "version": "==0.95.1"
}, },
"fastapi-utils": { "fastapi-utils": {
"hashes": [ "hashes": [
@@ -501,18 +501,18 @@
}, },
"filelock": { "filelock": {
"hashes": [ "hashes": [
"sha256:3618c0da67adcc0506b015fd11ef7faf1b493f0b40d87728e19986b536890c37", "sha256:ad98852315c2ab702aeb628412cbf7e95b7ce8c3bf9565670b4eaecf1db370a9",
"sha256:f08a52314748335c6460fc8fe40cd5638b85001225db78c2aa01c8c0db83b318" "sha256:fc03ae43288c013d2ea83c8597001b1129db351aad9c57fe2409327916b8e718"
], ],
"markers": "python_full_version >= '3.7.0'", "markers": "python_version >= '3.7'",
"version": "==3.11.0" "version": "==3.12.0"
}, },
"flask": { "flask": {
"hashes": [ "hashes": [
"sha256:7eb373984bf1c770023fce9db164ed0c3353cd0b53f130f4693da0ca756a2e6d", "sha256:7eb373984bf1c770023fce9db164ed0c3353cd0b53f130f4693da0ca756a2e6d",
"sha256:c0bec9477df1cb867e5a67c9e1ab758de9cb4a3e52dd70681f59fa40a62b3f2d" "sha256:c0bec9477df1cb867e5a67c9e1ab758de9cb4a3e52dd70681f59fa40a62b3f2d"
], ],
"markers": "python_full_version >= '3.7.0'", "markers": "python_version >= '3.7'",
"version": "==2.2.3" "version": "==2.2.3"
}, },
"flower": { "flower": {
@@ -535,16 +535,16 @@
"sha256:4b9bb5d5a380a0befa0573b302651b8a9a89262c1730e37bf423cec511804c22", "sha256:4b9bb5d5a380a0befa0573b302651b8a9a89262c1730e37bf423cec511804c22",
"sha256:ce222e27b0de0d7bc63eb043b956996d6dccab14cc3b690aaea91c9cc99dc16e" "sha256:ce222e27b0de0d7bc63eb043b956996d6dccab14cc3b690aaea91c9cc99dc16e"
], ],
"markers": "python_full_version >= '3.7.0'", "markers": "python_version >= '3.7'",
"version": "==2.11.0" "version": "==2.11.0"
}, },
"google-api-python-client": { "google-api-python-client": {
"hashes": [ "hashes": [
"sha256:07b21ef21a542dd69cd7c09817a6079b2769cc2a791981402e8f0fcdb2d47f90", "sha256:0f320190ab9d5bd2fdb0cb894e8e53bb5e17d4888ee8dc4d26ba65ce378409e2",
"sha256:baf3c6f9b1679d89fcb88c29941a8b04b9a815d721880786baecc6a7f5bd376f" "sha256:3ca4e93821f4e9ac29b91ab0d9df168b42c8ad0fb8bff65b8c2ccb2d462b0464"
], ],
"markers": "python_full_version >= '3.7.0'", "markers": "python_version >= '3.7'",
"version": "==2.85.0" "version": "==2.86.0"
}, },
"google-auth": { "google-auth": {
"hashes": [ "hashes": [
@@ -574,7 +574,7 @@
"sha256:4168fcb568a826a52f23510412da405abd93f4d23ba544bb68d943b14ba3cb44", "sha256:4168fcb568a826a52f23510412da405abd93f4d23ba544bb68d943b14ba3cb44",
"sha256:b287dc48449d1d41af0c69f4ea26242b5ae4c3d7249a38b0984c86a4caffff1f" "sha256:b287dc48449d1d41af0c69f4ea26242b5ae4c3d7249a38b0984c86a4caffff1f"
], ],
"markers": "python_full_version >= '3.7.0'", "markers": "python_version >= '3.7'",
"version": "==1.59.0" "version": "==1.59.0"
}, },
"greenlet": { "greenlet": {
@@ -656,7 +656,7 @@
"sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d", "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d",
"sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761" "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"
], ],
"markers": "python_full_version >= '3.7.0'", "markers": "python_version >= '3.7'",
"version": "==0.14.0" "version": "==0.14.0"
}, },
"httplib2": { "httplib2": {
@@ -672,7 +672,7 @@
"sha256:401201aca462749773f02920139f302450cb548b70489b9b4b92be39fe3c3c50", "sha256:401201aca462749773f02920139f302450cb548b70489b9b4b92be39fe3c3c50",
"sha256:5f1f22bc65911eb1a6ffe7659bd6598e33dcfeeb904eb16ee1e705a09bf75916" "sha256:5f1f22bc65911eb1a6ffe7659bd6598e33dcfeeb904eb16ee1e705a09bf75916"
], ],
"markers": "python_full_version >= '3.7.0'", "markers": "python_version >= '3.7'",
"version": "==4.6.0" "version": "==4.6.0"
}, },
"idna": { "idna": {
@@ -688,7 +688,7 @@
"sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3", "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3",
"sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374" "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"
], ],
"markers": "python_full_version >= '3.7.0'", "markers": "python_version >= '3.7'",
"version": "==2.0.0" "version": "==2.0.0"
}, },
"instaloader": { "instaloader": {
@@ -703,7 +703,7 @@
"sha256:2c2349112351b88699d8d4b6b075022c0808887cb7ad10069318a8b0bc88db44", "sha256:2c2349112351b88699d8d4b6b075022c0808887cb7ad10069318a8b0bc88db44",
"sha256:5dbbc68b317e5e42f327f9021763545dc3fc3bfe22e6deb96aaf1fc38874156a" "sha256:5dbbc68b317e5e42f327f9021763545dc3fc3bfe22e6deb96aaf1fc38874156a"
], ],
"markers": "python_full_version >= '3.7.0'", "markers": "python_version >= '3.7'",
"version": "==2.1.2" "version": "==2.1.2"
}, },
"jinja2": { "jinja2": {
@@ -719,7 +719,7 @@
"sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980",
"sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe" "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"
], ],
"markers": "python_full_version >= '3.7.0'", "markers": "python_version >= '3.7'",
"version": "==1.0.1" "version": "==1.0.1"
}, },
"kombu": { "kombu": {
@@ -826,7 +826,7 @@
"sha256:c97c79c018b9165ac9922ae4f32da095ffd3c4e6872b45eded42926deea46818", "sha256:c97c79c018b9165ac9922ae4f32da095ffd3c4e6872b45eded42926deea46818",
"sha256:d60a3903dc3bb01a18ad6a89cdbe2e4eadc69c0bc8ef1e3773ba53d44c3f7a34" "sha256:d60a3903dc3bb01a18ad6a89cdbe2e4eadc69c0bc8ef1e3773ba53d44c3f7a34"
], ],
"markers": "python_full_version >= '3.7.0'", "markers": "python_version >= '3.7'",
"version": "==1.2.4" "version": "==1.2.4"
}, },
"markdown-it-py": { "markdown-it-py": {
@@ -834,7 +834,7 @@
"sha256:5a35f8d1870171d9acc47b99612dc146129b631baf04970128b568f190d0cc30", "sha256:5a35f8d1870171d9acc47b99612dc146129b631baf04970128b568f190d0cc30",
"sha256:7c9a5e412688bc771c67432cbfebcdd686c93ce6484913dccf06cb5a0bea35a1" "sha256:7c9a5e412688bc771c67432cbfebcdd686c93ce6484913dccf06cb5a0bea35a1"
], ],
"markers": "python_full_version >= '3.7.0'", "markers": "python_version >= '3.7'",
"version": "==2.2.0" "version": "==2.2.0"
}, },
"markupsafe": { "markupsafe": {
@@ -890,7 +890,7 @@
"sha256:f2bfb563d0211ce16b63c7cb9395d2c682a23187f54c3d79bfec33e6705473c6", "sha256:f2bfb563d0211ce16b63c7cb9395d2c682a23187f54c3d79bfec33e6705473c6",
"sha256:f8ffb705ffcf5ddd0e80b65ddf7bed7ee4f5a441ea7d3419e861a12eaf41af58" "sha256:f8ffb705ffcf5ddd0e80b65ddf7bed7ee4f5a441ea7d3419e861a12eaf41af58"
], ],
"markers": "python_full_version >= '3.7.0'", "markers": "python_version >= '3.7'",
"version": "==2.1.2" "version": "==2.1.2"
}, },
"marshmallow": { "marshmallow": {
@@ -898,7 +898,7 @@
"sha256:90032c0fd650ce94b6ec6dc8dfeb0e3ff50c144586462c389b81a07205bedb78", "sha256:90032c0fd650ce94b6ec6dc8dfeb0e3ff50c144586462c389b81a07205bedb78",
"sha256:93f0958568da045b0021ec6aeb7ac37c81bfcccbb9a0e7ed8559885070b3a19b" "sha256:93f0958568da045b0021ec6aeb7ac37c81bfcccbb9a0e7ed8559885070b3a19b"
], ],
"markers": "python_full_version >= '3.7.0'", "markers": "python_version >= '3.7'",
"version": "==3.19.0" "version": "==3.19.0"
}, },
"marshmallow-enum": { "marshmallow-enum": {
@@ -913,7 +913,7 @@
"sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8",
"sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba" "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"
], ],
"markers": "python_full_version >= '3.7.0'", "markers": "python_version >= '3.7'",
"version": "==0.1.2" "version": "==0.1.2"
}, },
"mutagen": { "mutagen": {
@@ -921,7 +921,7 @@
"sha256:6e5f8ba84836b99fe60be5fb27f84be4ad919bbb6b49caa6ae81e70584b55e58", "sha256:6e5f8ba84836b99fe60be5fb27f84be4ad919bbb6b49caa6ae81e70584b55e58",
"sha256:8af0728aa2d5c3ee5a727e28d0627966641fddfe804c23eabb5926a4d770aed5" "sha256:8af0728aa2d5c3ee5a727e28d0627966641fddfe804c23eabb5926a4d770aed5"
], ],
"markers": "python_full_version >= '3.7.0'", "markers": "python_version >= '3.7'",
"version": "==1.46.0" "version": "==1.46.0"
}, },
"mypy-extensions": { "mypy-extensions": {
@@ -952,7 +952,7 @@
"sha256:6f82bd3de45da303cf1f771ecafa1633750a358436a8bb60e06a1ceb745d2672", "sha256:6f82bd3de45da303cf1f771ecafa1633750a358436a8bb60e06a1ceb745d2672",
"sha256:c4ab89a56575d6d38a05aa16daeaa333109c1f96167aba8901ab18b6b5e0f7f5" "sha256:c4ab89a56575d6d38a05aa16daeaa333109c1f96167aba8901ab18b6b5e0f7f5"
], ],
"markers": "python_full_version >= '3.7.0'", "markers": "python_version >= '3.7'",
"version": "==1.2.0" "version": "==1.2.0"
}, },
"packaging": { "packaging": {
@@ -960,7 +960,7 @@
"sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61", "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61",
"sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f" "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"
], ],
"markers": "python_full_version >= '3.7.0'", "markers": "python_version >= '3.7'",
"version": "==23.1" "version": "==23.1"
}, },
"pluggy": { "pluggy": {
@@ -994,7 +994,7 @@
"sha256:f08aa300b67f1c012100d8eb62d47129e53d1150f4469fd78a29fa3cb68c66f2", "sha256:f08aa300b67f1c012100d8eb62d47129e53d1150f4469fd78a29fa3cb68c66f2",
"sha256:f2f4710543abec186aee332d6852ef5ae7ce2e9e807a3da570f36de5a732d88e" "sha256:f2f4710543abec186aee332d6852ef5ae7ce2e9e807a3da570f36de5a732d88e"
], ],
"markers": "python_full_version >= '3.7.0'", "markers": "python_version >= '3.7'",
"version": "==4.22.3" "version": "==4.22.3"
}, },
"py": { "py": {
@@ -1132,16 +1132,16 @@
"sha256:ecbbc51391248116c0a055899e6c3e7ffbb11fb5e2a4cd6f2d0b93272118a209", "sha256:ecbbc51391248116c0a055899e6c3e7ffbb11fb5e2a4cd6f2d0b93272118a209",
"sha256:f4a2b50e2b03d5776e7f21af73e2070e1b5c0d0df255a827e7c632962f8315af" "sha256:f4a2b50e2b03d5776e7f21af73e2070e1b5c0d0df255a827e7c632962f8315af"
], ],
"markers": "python_full_version >= '3.7.0'", "markers": "python_version >= '3.7'",
"version": "==1.10.7" "version": "==1.10.7"
}, },
"pygments": { "pygments": {
"hashes": [ "hashes": [
"sha256:77a3299119af881904cd5ecd1ac6a66214b6e9bed1f2db16993b54adede64094", "sha256:8ace4d3c1dd481894b2005f560ead0f9f19ee64fe983366be1a21e171d12775c",
"sha256:f7e36cffc4c517fbc252861b9a6e4644ca0e5abadf9a113c72d1358ad09b9500" "sha256:db2db3deb4b4179f399a09054b023b6a586b76499d36965813c71aa8ed7b5fd1"
], ],
"markers": "python_full_version >= '3.7.0'", "markers": "python_version >= '3.7'",
"version": "==2.15.0" "version": "==2.15.1"
}, },
"pyparsing": { "pyparsing": {
"hashes": [ "hashes": [
@@ -1188,7 +1188,7 @@
"sha256:70ca6ea68fe63ecc8fa4fcf00ae651fc8a5d02d93dcd12ae6d4fc7ca46c4d395", "sha256:70ca6ea68fe63ecc8fa4fcf00ae651fc8a5d02d93dcd12ae6d4fc7ca46c4d395",
"sha256:ce0d46ddb668b3be82f4ed5e503dbc33dd815d83e2eb6824211310d3fb172a27" "sha256:ce0d46ddb668b3be82f4ed5e503dbc33dd815d83e2eb6824211310d3fb172a27"
], ],
"markers": "python_full_version >= '3.7.0'", "markers": "python_version >= '3.7'",
"version": "==8.0.1" "version": "==8.0.1"
}, },
"python-twitter-v2": { "python-twitter-v2": {
@@ -1196,7 +1196,7 @@
"sha256:2397d518c17bfbc16a3d414b1cf6d3c231fd8d322f21c755ac2215c9ee675537", "sha256:2397d518c17bfbc16a3d414b1cf6d3c231fd8d322f21c755ac2215c9ee675537",
"sha256:4e03a30b2570fa4f17fbc7293d850fb8276c66be106d55e460b9287de37e1dd2" "sha256:4e03a30b2570fa4f17fbc7293d850fb8276c66be106d55e460b9287de37e1dd2"
], ],
"markers": "python_version >= '3.6' and python_version < '4.0'", "markers": "python_version >= '3.6' and python_version < '4'",
"version": "==0.8.1" "version": "==0.8.1"
}, },
"pytz": { "pytz": {
@@ -1363,7 +1363,7 @@
"sha256:22b74cae0278fd5086ff44144d3813be1cedc9115bdfabbfefd86400cb88b20a", "sha256:22b74cae0278fd5086ff44144d3813be1cedc9115bdfabbfefd86400cb88b20a",
"sha256:b5d573e13605423ec80bdd0cd5f8541f7844a0e71a13f74cf454ccb2f490708b" "sha256:b5d573e13605423ec80bdd0cd5f8541f7844a0e71a13f74cf454ccb2f490708b"
], ],
"markers": "python_full_version >= '3.7.0'", "markers": "python_version >= '3.7'",
"version": "==13.3.4" "version": "==13.3.4"
}, },
"rsa": { "rsa": {
@@ -1371,7 +1371,7 @@
"sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7", "sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7",
"sha256:e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21" "sha256:e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21"
], ],
"markers": "python_version >= '3.6' and python_version < '4.0'", "markers": "python_version >= '3.6' and python_version < '4'",
"version": "==4.9" "version": "==4.9"
}, },
"s3transfer": { "s3transfer": {
@@ -1379,7 +1379,7 @@
"sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd", "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd",
"sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947" "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947"
], ],
"markers": "python_full_version >= '3.7.0'", "markers": "python_version >= '3.7'",
"version": "==0.6.0" "version": "==0.6.0"
}, },
"selenium": { "selenium": {
@@ -1387,7 +1387,7 @@
"sha256:28430ac54a54fa59ad1f5392a1b89b169fe3ab2c2ccd1a9a10b6fe74f36cd6da", "sha256:28430ac54a54fa59ad1f5392a1b89b169fe3ab2c2ccd1a9a10b6fe74f36cd6da",
"sha256:61cda3a304f82637162bc155cae7bf88fdb04c115fa2cb1c1c2e1358fcd19a9f" "sha256:61cda3a304f82637162bc155cae7bf88fdb04c115fa2cb1c1c2e1358fcd19a9f"
], ],
"markers": "python_full_version >= '3.7.0'", "markers": "python_version >= '3.7'",
"version": "==4.8.3" "version": "==4.8.3"
}, },
"six": { "six": {
@@ -1403,7 +1403,7 @@
"sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101", "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101",
"sha256:eecefdce1e5bbfb7ad2eeaabf7c1eeb404d7757c379bd1f7e5cce9d8bf425384" "sha256:eecefdce1e5bbfb7ad2eeaabf7c1eeb404d7757c379bd1f7e5cce9d8bf425384"
], ],
"markers": "python_full_version >= '3.7.0'", "markers": "python_version >= '3.7'",
"version": "==1.3.0" "version": "==1.3.0"
}, },
"snscrape": { "snscrape": {
@@ -1423,11 +1423,11 @@
}, },
"soupsieve": { "soupsieve": {
"hashes": [ "hashes": [
"sha256:49e5368c2cda80ee7e84da9dbe3e110b70a4575f196efb74e51b94549d921955", "sha256:1c1bfee6819544a3447586c889157365a27e10d88cde3ad3da0cf0ddf646feb8",
"sha256:e28dba9ca6c7c00173e34e4ba57448f0688bb681b7c5e8bf4971daafc093d69a" "sha256:89d12b2d5dfcd2c9e8c22326da9d9aa9cb3dfab0a83a024f05704076ee8d35ea"
], ],
"markers": "python_full_version >= '3.7.0'", "markers": "python_version >= '3.7'",
"version": "==2.4" "version": "==2.4.1"
}, },
"sqlalchemy": { "sqlalchemy": {
"hashes": [ "hashes": [
@@ -1481,16 +1481,16 @@
"sha256:41da799057ea8620e4667a3e69a5b1923ebd32b1819c8fa75634bbe8d8bea9bd", "sha256:41da799057ea8620e4667a3e69a5b1923ebd32b1819c8fa75634bbe8d8bea9bd",
"sha256:e87fce5d7cbdde34b76f0ac69013fd9d190d581d80681493016666e6f96c6d5e" "sha256:e87fce5d7cbdde34b76f0ac69013fd9d190d581d80681493016666e6f96c6d5e"
], ],
"markers": "python_full_version >= '3.7.0'", "markers": "python_version >= '3.7'",
"version": "==0.26.1" "version": "==0.26.1"
}, },
"telethon": { "telethon": {
"hashes": [ "hashes": [
"sha256:613bae42acb5f2eeb1a0b92614e323021c66f374db62adf9826ea0c2c9120bb1", "sha256:b3990ec22351a3f3e1af376729c985025bbdd3bdabdde8c156112c3d3dfe1941",
"sha256:893c10f133974fba4c53eb1736b6514d596d1cd94c83436a711f3345df945199" "sha256:edc42fd58b8e1569830d3ead564cafa60fd51d684f03ee2a1fdd5f77a5a10438"
], ],
"markers": "python_version >= '3.5'", "markers": "python_version >= '3.5'",
"version": "==1.28.2" "version": "==1.28.5"
}, },
"text-unidecode": { "text-unidecode": {
"hashes": [ "hashes": [
@@ -1515,27 +1515,27 @@
}, },
"tornado": { "tornado": {
"hashes": [ "hashes": [
"sha256:1d54d13ab8414ed44de07efecb97d4ef7c39f7438cf5e976ccd356bebb1b5fca", "sha256:4546003dc8b5733489139d3bff5fa6a0211be505faf819bd9970e7c2b32e8122",
"sha256:20f638fd8cc85f3cbae3c732326e96addff0a15e22d80f049e00121651e82e72", "sha256:4d349846931557b7ec92f224b5d598b160e2ba26ae1812480b42e9622c884bf7",
"sha256:5c87076709343557ef8032934ce5f637dbb552efa7b21d08e89ae7619ed0eb23", "sha256:6164571f5b9f73143d1334df4584cb9ac86d20c461e17b6c189a19ead8bb93c1",
"sha256:5f8c52d219d4995388119af7ccaa0bcec289535747620116a58d830e7c25d8a8", "sha256:6cfff1e9c15c79e106b8352269d201f8fc0815914a6260f3893ca18b724ea94b",
"sha256:6fdfabffd8dfcb6cf887428849d30cf19a3ea34c2c248461e1f7d718ad30b66b", "sha256:720f53e6367b38190ae7fa398c25c086c69d88b3c6535bd6021a126b727fb5cd",
"sha256:87dcafae3e884462f90c90ecc200defe5e580a7fbbb4365eda7c7c1eb809ebc9", "sha256:912df5712024564e362ecce43c8d5862e14c78c8dd3846c9d889d44fbd7f4951",
"sha256:9b630419bde84ec666bfd7ea0a4cb2a8a651c2d5cccdbdd1972a0c859dfc3c13", "sha256:c37b6a384d54ce6a31168d40ab21ad2591ddaf34973075cc0cad154402ecd9e8",
"sha256:b8150f721c101abdef99073bf66d3903e292d851bee51910839831caba341a75", "sha256:c659ab04d5aa477dbe44152c67d93f3ad3243b992d94f795ca1d5c73c37337ce",
"sha256:ba09ef14ca9893954244fd872798b4ccb2367c165946ce2dd7376aebdde8e3ac", "sha256:c9114a61a4588c09065b9996ae05462350d17160b92b9bf9a1e93689cc0424dc",
"sha256:d3a2f5999215a3a06a4fc218026cd84c61b8b2b40ac5296a6db1f1451ef04c1e", "sha256:d68f3192936ff2c4add04dc21a436a43b4408d466746b78bb2b9d0a53a18683f",
"sha256:e5f923aa6a47e133d1cf87d60700889d7eae68988704e20c75fb2d65677a8e4b" "sha256:d7b737e18f701de3e4a3b0824260b4d740e4d60607b8089bb80e80ffd464780e"
], ],
"markers": "python_full_version >= '3.5.2'", "markers": "python_full_version >= '3.5.2'",
"version": "==6.2" "version": "==6.3"
}, },
"tqdm": { "tqdm": {
"hashes": [ "hashes": [
"sha256:1871fb68a86b8fb3b59ca4cdd3dcccbc7e6d613eeed31f4c332531977b89beb5", "sha256:1871fb68a86b8fb3b59ca4cdd3dcccbc7e6d613eeed31f4c332531977b89beb5",
"sha256:c4f53a17fe37e132815abceec022631be8ffe1b9381c2e6e30aa70edc99e9671" "sha256:c4f53a17fe37e132815abceec022631be8ffe1b9381c2e6e30aa70edc99e9671"
], ],
"markers": "python_full_version >= '3.7.0'", "markers": "python_version >= '3.7'",
"version": "==4.65.0" "version": "==4.65.0"
}, },
"trio": { "trio": {
@@ -1543,7 +1543,7 @@
"sha256:ce68f1c5400a47b137c5a4de72c7c901bd4e7a24fbdebfe9b41de8c6c04eaacf", "sha256:ce68f1c5400a47b137c5a4de72c7c901bd4e7a24fbdebfe9b41de8c6c04eaacf",
"sha256:f1dd0780a89bfc880c7c7994519cb53f62aacb2c25ff487001c0052bd721cdf0" "sha256:f1dd0780a89bfc880c7c7994519cb53f62aacb2c25ff487001c0052bd721cdf0"
], ],
"markers": "python_full_version >= '3.7.0'", "markers": "python_version >= '3.7'",
"version": "==0.22.0" "version": "==0.22.0"
}, },
"trio-websocket": { "trio-websocket": {
@@ -1551,7 +1551,7 @@
"sha256:0908435e4eecc49d830ae1c4d6c47b978a75f00594a2be2104d58b61a04cdb53", "sha256:0908435e4eecc49d830ae1c4d6c47b978a75f00594a2be2104d58b61a04cdb53",
"sha256:af13e9393f9051111300287947ec595d601758ce3d165328e7d36325135a8d62" "sha256:af13e9393f9051111300287947ec595d601758ce3d165328e7d36325135a8d62"
], ],
"markers": "python_full_version >= '3.7.0'", "markers": "python_version >= '3.7'",
"version": "==0.10.2" "version": "==0.10.2"
}, },
"typing-extensions": { "typing-extensions": {
@@ -1559,7 +1559,7 @@
"sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb", "sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb",
"sha256:fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4" "sha256:fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4"
], ],
"markers": "python_full_version >= '3.7.0'", "markers": "python_version >= '3.7'",
"version": "==4.5.0" "version": "==4.5.0"
}, },
"typing-inspect": { "typing-inspect": {
@@ -1582,7 +1582,7 @@
"sha256:3f21d09e1b2aa9f2dacca12da240ca37de3ba5237a93addfd6d593afe9073355", "sha256:3f21d09e1b2aa9f2dacca12da240ca37de3ba5237a93addfd6d593afe9073355",
"sha256:b44c4388f3d34f25862cfbb387578a4d70fec417649da694a132f628a23367e2" "sha256:b44c4388f3d34f25862cfbb387578a4d70fec417649da694a132f628a23367e2"
], ],
"markers": "python_full_version >= '3.7.0'", "markers": "python_version >= '3.7'",
"version": "==4.3" "version": "==4.3"
}, },
"uritemplate": { "uritemplate": {
@@ -1629,7 +1629,7 @@
"sha256:5a32fb5419f7bb8bd35de8548948fe27a06f857a4d086c87e142bf07aabc3fd7", "sha256:5a32fb5419f7bb8bd35de8548948fe27a06f857a4d086c87e142bf07aabc3fd7",
"sha256:a87c5aa7c1570c3aa87031e78c2052105e3681f57503fd4cb56470c3ab6106d6" "sha256:a87c5aa7c1570c3aa87031e78c2052105e3681f57503fd4cb56470c3ab6106d6"
], ],
"markers": "python_full_version >= '3.7.0'", "markers": "python_version >= '3.7'",
"version": "==0.3.15" "version": "==0.3.15"
}, },
"websockets": { "websockets": {
@@ -1704,7 +1704,7 @@
"sha256:fe10ddc59b304cb19a1bdf5bd0a7719cbbc9fbdd57ac80ed436b709fcf889106", "sha256:fe10ddc59b304cb19a1bdf5bd0a7719cbbc9fbdd57ac80ed436b709fcf889106",
"sha256:ff64a1d38d156d429404aaa84b27305e957fd10c30e5880d1765c9480bea490f" "sha256:ff64a1d38d156d429404aaa84b27305e957fd10c30e5880d1765c9480bea490f"
], ],
"markers": "python_full_version >= '3.7.0'", "markers": "python_version >= '3.7'",
"version": "==10.4" "version": "==10.4"
}, },
"werkzeug": { "werkzeug": {
@@ -1712,7 +1712,7 @@
"sha256:2e1ccc9417d4da358b9de6f174e3ac094391ea1d4fbef2d667865d819dfd0afe", "sha256:2e1ccc9417d4da358b9de6f174e3ac094391ea1d4fbef2d667865d819dfd0afe",
"sha256:56433961bc1f12533306c624f3be5e744389ac61d722175d543e1751285da612" "sha256:56433961bc1f12533306c624f3be5e744389ac61d722175d543e1751285da612"
], ],
"markers": "python_full_version >= '3.7.0'", "markers": "python_version >= '3.7'",
"version": "==2.2.3" "version": "==2.2.3"
}, },
"wsproto": { "wsproto": {
@@ -1720,7 +1720,7 @@
"sha256:ad565f26ecb92588a3e43bc3d96164de84cd9902482b130d0ddbaa9664a85065", "sha256:ad565f26ecb92588a3e43bc3d96164de84cd9902482b130d0ddbaa9664a85065",
"sha256:b9acddd652b585d75b20477888c56642fdade28bdfd3579aa24a4d2c037dd736" "sha256:b9acddd652b585d75b20477888c56642fdade28bdfd3579aa24a4d2c037dd736"
], ],
"markers": "python_full_version >= '3.7.0'", "markers": "python_version >= '3.7'",
"version": "==1.2.0" "version": "==1.2.0"
}, },
"yt-dlp": { "yt-dlp": {
@@ -1728,9 +1728,43 @@
"sha256:3b2df037c80922f0f83f63ee2f9253496b4a8668c0fe8d2a836ba9040f853b07", "sha256:3b2df037c80922f0f83f63ee2f9253496b4a8668c0fe8d2a836ba9040f853b07",
"sha256:9af92de5effc193bdb51216d9ebf28874d96180d202fae752b0d9f2a63380f3a" "sha256:9af92de5effc193bdb51216d9ebf28874d96180d202fae752b0d9f2a63380f3a"
], ],
"markers": "python_full_version >= '3.7.0'", "markers": "python_version >= '3.7'",
"version": "==2023.2.17" "version": "==2023.2.17"
} }
}, },
"develop": {} "develop": {
"watchdog": {
"hashes": [
"sha256:0e06ab8858a76e1219e68c7573dfeba9dd1c0219476c5a44d5333b01d7e1743a",
"sha256:13bbbb462ee42ec3c5723e1205be8ced776f05b100e4737518c67c8325cf6100",
"sha256:233b5817932685d39a7896b1090353fc8efc1ef99c9c054e46c8002561252fb8",
"sha256:25f70b4aa53bd743729c7475d7ec41093a580528b100e9a8c5b5efe8899592fc",
"sha256:2b57a1e730af3156d13b7fdddfc23dea6487fceca29fc75c5a868beed29177ae",
"sha256:336adfc6f5cc4e037d52db31194f7581ff744b67382eb6021c868322e32eef41",
"sha256:3aa7f6a12e831ddfe78cdd4f8996af9cf334fd6346531b16cec61c3b3c0d8da0",
"sha256:3ed7c71a9dccfe838c2f0b6314ed0d9b22e77d268c67e015450a29036a81f60f",
"sha256:4c9956d27be0bb08fc5f30d9d0179a855436e655f046d288e2bcc11adfae893c",
"sha256:4d98a320595da7a7c5a18fc48cb633c2e73cda78f93cac2ef42d42bf609a33f9",
"sha256:4f94069eb16657d2c6faada4624c39464f65c05606af50bb7902e036e3219be3",
"sha256:5113334cf8cf0ac8cd45e1f8309a603291b614191c9add34d33075727a967709",
"sha256:51f90f73b4697bac9c9a78394c3acbbd331ccd3655c11be1a15ae6fe289a8c83",
"sha256:5d9f3a10e02d7371cd929b5d8f11e87d4bad890212ed3901f9b4d68767bee759",
"sha256:7ade88d0d778b1b222adebcc0927428f883db07017618a5e684fd03b83342bd9",
"sha256:7c5f84b5194c24dd573fa6472685b2a27cc5a17fe5f7b6fd40345378ca6812e3",
"sha256:7e447d172af52ad204d19982739aa2346245cc5ba6f579d16dac4bfec226d2e7",
"sha256:8ae9cda41fa114e28faf86cb137d751a17ffd0316d1c34ccf2235e8a84365c7f",
"sha256:8f3ceecd20d71067c7fd4c9e832d4e22584318983cabc013dbf3f70ea95de346",
"sha256:9fac43a7466eb73e64a9940ac9ed6369baa39b3bf221ae23493a9ec4d0022674",
"sha256:a70a8dcde91be523c35b2bf96196edc5730edb347e374c7de7cd20c43ed95397",
"sha256:adfdeab2da79ea2f76f87eb42a3ab1966a5313e5a69a0213a3cc06ef692b0e96",
"sha256:ba07e92756c97e3aca0912b5cbc4e5ad802f4557212788e72a72a47ff376950d",
"sha256:c07253088265c363d1ddf4b3cdb808d59a0468ecd017770ed716991620b8f77a",
"sha256:c9d8c8ec7efb887333cf71e328e39cffbf771d8f8f95d308ea4125bf5f90ba64",
"sha256:d00e6be486affb5781468457b21a6cbe848c33ef43f9ea4a73b4882e5f188a44",
"sha256:d429c2430c93b7903914e4db9a966c7f2b068dd2ebdd2fa9b9ce094c7d459f33"
],
"index": "pypi",
"version": "==3.0.0"
}
}
} }

44
src/db/crud.py
View File

@@ -1,9 +1,11 @@
from functools import cache from functools import cache
from sqlalchemy.orm import Session, load_only from sqlalchemy.orm import Session, load_only
from sqlalchemy import Column
from loguru import logger from loguru import logger
from . import models, schemas from . import models, schemas
import yaml import yaml
## --------------- TASK = Archive
def get_task(db: Session, task_id: str): def get_task(db: Session, task_id: str):
return base_query(db).filter(models.Archive.id == task_id).first() return base_query(db).filter(models.Archive.id == task_id).first()
@@ -17,22 +19,19 @@ def search_tasks_by_url(db: Session, url:str, skip: int = 0, limit: int = 100):
def search_tasks_by_email(db: Session, email:str, skip: int = 0, limit: int = 100): def search_tasks_by_email(db: Session, email:str, skip: int = 0, limit: int = 100):
return base_query(db).filter(models.Archive.author.has(email=email)).offset(skip).limit(limit).all() return base_query(db).filter(models.Archive.author.has(email=email)).offset(skip).limit(limit).all()
def create_task(db: Session, task: schemas.TaskCreate): def create_task(db: Session, task: schemas.ArchiveCreate, tags:list[models.Tag],urls:list[models.ArchiveUrl]):
db_task = models.Archive(id=task.id, url=task.url, author=task.author, result=task.result) db_task = models.Archive(id=task.id, url=task.url, author_id=task.author_id, result=task.result, group_id=task.group_id)
logger.debug(tags)
db_task.tags = tags # will this work? TODO: test if I don't call create tag before
db_task.urls = urls # will this work to create ArchiveUrl? TODO: test
db.add(db_task) db.add(db_task)
db.commit() db.commit()
db.refresh(db_task) db.refresh(db_task)
return db_task return db_task
# def delete_task(db: Session, task_id: str, email:str)->bool:
# db_task = db.query(models.Task).filter(models.Task.id == task_id, models.Task.author==email).first()
# if db_task:
# db.delete(db_task)
# db.commit()
# return db_task is not None
def soft_delete_task(db: Session, task_id: str, email:str)->bool: def soft_delete_task(db: Session, task_id: str, email:str)->bool:
db_task = db.query(models.Archive).filter(models.Archive.id == task_id, models.Archive.author==email, models.Archive.deleted==False).first() # TODO: implement hard-delete with cronjob that deletes from S3
db_task = db.query(models.Archive).filter(models.Archive.id == task_id, models.Archive.author_id==email, models.Archive.deleted==False).first()
if db_task: if db_task:
db_task.deleted = True db_task.deleted = True
db.commit() db.commit()
@@ -44,6 +43,31 @@ def base_query(db:Session):
.options(load_only(models.Archive.id, models.Archive.created_at, models.Archive.url, models.Archive.result))\ .options(load_only(models.Archive.id, models.Archive.created_at, models.Archive.url, models.Archive.result))\
.filter(models.Archive.deleted == False) .filter(models.Archive.deleted == False)
## --------------- TAG
def create_tag(db: Session, tag: str):
db_tag = db.query(models.Tag).filter(models.Tag.id==tag).first()
if not db_tag:
db_tag = models.Tag(id=tag)
db.add(db_tag)
db.commit()
db.refresh(db_tag)
return db_tag
def search_tags(db: Session, tag:str, skip: int = 0, limit: int = 100):
return db.query(models.Tag).filter(models.Tag.url.like(f'%{tag}%')).offset(skip).limit(limit).all()
def get_group_for_user(db:Session, group_name:str, email:str)->models.Group:
return db.query(models.association_table_user_groups).filter_by(user_id=email, group_id=group_name).first()
def get_user_groups(db: Session, email:str):
groups = db.query(models.association_table_user_groups).filter_by(user_id=email).with_entities(Column("group_id")).all()
return [g[0] for g in groups]
## --------------- INIT User-Groups
@cache @cache
def get_group(db:Session, group_name:str)->models.Group: def get_group(db:Session, group_name:str)->models.Group:
db_group = db.query(models.Group).filter(models.Group.id==group_name).first() db_group = db.query(models.Group).filter(models.Group.id==group_name).first()

37
src/db/models.py
View File

@@ -4,25 +4,8 @@ from sqlalchemy.orm import relationship
import uuid import uuid
from .database import Base from .database import Base
# class Task(Base):
# __tablename__ = "tasks"
# id = Column(String, primary_key=True, index=True)
# url = Column(String, index=True)
# author = Column(String, index=True)
# result = Column(JSON, default=None)
# created_at = Column(DateTime(timezone=True), server_default=func.now())
# # updated_at = Column(DateTime(timezone=True), onupdate=func.now())
# deleted = Column(Boolean, default=False)
# # items = relationship("Item", back_populates="owner")
# # tags = relationship("Tag", back_populates="owner")
def generate_uuid(): def generate_uuid():
return str(uuid.uuid4()) return str(uuid.uuid4())
### new data model below
# many to many association tables # many to many association tables
association_table_archive_tags = Table( association_table_archive_tags = Table(
@@ -45,23 +28,33 @@ class Archive(Base):
id = Column(String, primary_key=True, index=True) id = Column(String, primary_key=True, index=True)
url = Column(String, index=True) url = Column(String, index=True)
result = Column(JSON, default=None) result = Column(JSON, default=None)
public = Column(Boolean, default=True) # if public=false, access to group and author
deleted = Column(Boolean, default=False)
created_at = Column(DateTime(timezone=True), server_default=func.now()) created_at = Column(DateTime(timezone=True), server_default=func.now())
updated_at = Column(DateTime(timezone=True), onupdate=func.now()) updated_at = Column(DateTime(timezone=True), onupdate=func.now())
deleted = Column(Boolean, default=False)
public = Column(Boolean, default=True) # if public=false, access to group and author
group_id = Column(String, ForeignKey("groups.id"), default=None) group_id = Column(String, ForeignKey("groups.id"), default=None)
author_id = Column(String, ForeignKey("users.email")) author_id = Column(String, ForeignKey("users.email"))
group = relationship("Group", back_populates="archives")
tags = relationship("Tag", back_populates="archives", secondary=association_table_archive_tags) tags = relationship("Tag", back_populates="archives", secondary=association_table_archive_tags)
group = relationship("Group", back_populates="archives")
author = relationship("User", back_populates="archives") author = relationship("User", back_populates="archives")
urls = relationship("ArchiveUrl", back_populates="archive")
class ArchiveUrl(Base):
__tablename__ = "archive_urls"
url = Column(String, primary_key=True, index=True)
key = Column(String, default=None)
archive_id = Column(String, ForeignKey("archives.id"))
archive = relationship("Archive", back_populates="urls")
class Tag(Base): class Tag(Base):
__tablename__ = "tags" __tablename__ = "tags"
id = Column(String, primary_key=True, index=True, default=generate_uuid) id = Column(String, primary_key=True, index=True)
name = Column(String, unique=True, index=True)
created_at = Column(DateTime(timezone=True), server_default=func.now()) created_at = Column(DateTime(timezone=True), server_default=func.now())
archives = relationship("Archive", back_populates="tags", secondary=association_table_archive_tags) archives = relationship("Archive", back_populates="tags", secondary=association_table_archive_tags)

25
src/db/schemas.py
View File

@@ -1,16 +1,31 @@
from pydantic import BaseModel from pydantic import BaseModel
from datetime import datetime from datetime import datetime
class TaskCreate(BaseModel): class ArchiveCreate(BaseModel):
id: str id: str | None = None
url: str url: str
author: str result: dict | None = None
result: dict public: bool = True
author_id: str | None = None
group_id: str | None = None
tags: list = []
# urls: list = []
class Task(TaskCreate):
class Archive(ArchiveCreate):
created_at: datetime created_at: datetime
updated_at: datetime | None
deleted: bool deleted: bool
class Config: class Config:
orm_mode = True orm_mode = True
# class TagCreate(BaseModel):
# id: str
# class Tag(TagCreate):
# created_at: datetime
# # class Config:
# # orm_mode = True

57
src/main.py
View File

@@ -15,14 +15,15 @@ from worker import create_archive_task, celery
from db import crud, models, schemas from db import crud, models, schemas
from db.database import engine, SessionLocal from db.database import engine, SessionLocal
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
from security import get_bearer_auth, get_basic_auth from security import get_bearer_auth, get_basic_auth, bearer_security
load_dotenv() load_dotenv()
# Configuration # Configuration
ALLOWED_ORIGINS = os.environ.get("ALLOWED_ORIGINS", "chrome-extension://ondkcheoicfckabcnkdgbepofpjmjcmb,chrome-extension://ojcimmjndnlmmlgnjaeojoebaceokpdp").split(",") ALLOWED_ORIGINS = os.environ.get("ALLOWED_ORIGINS", "chrome-extension://ondkcheoicfckabcnkdgbepofpjmjcmb,chrome-extension://ojcimmjndnlmmlgnjaeojoebaceokpdp").split(",")
VERSION = "0.2.0" VERSION = "0.3.1"
# min-version refers to the version of auto-archiver-extension on the webstore
BREAKING_CHANGES = {"minVersion": "0.3.0", "message": "The latest update has breaking changes, please update the extension to the most recent version."}
app = FastAPI() app = FastAPI()
app.add_middleware( app.add_middleware(
@@ -41,8 +42,16 @@ def get_db():
@app.get("/") @app.get("/")
def home(): return JSONResponse({"version": VERSION}) async def home(request: Request):
status = {"version": VERSION, "breakingChanges": BREAKING_CHANGES}
try:
# if authenticated will load available groups
email = await get_bearer_auth(await bearer_security(request))
db: Session = next(get_db())
status["groups"] = crud.get_user_groups(db, email)
except HTTPException: pass
except Exception as e: logger.error(e)
return JSONResponse(status)
# logging configurations # logging configurations
logger.add("logs/api_logs.log", retention="30 days", rotation="3 days") logger.add("logs/api_logs.log", retention="30 days", rotation="3 days")
@@ -55,36 +64,59 @@ async def logging_middleware(request: Request, call_next):
# Bearer protected below # Bearer protected below
@app.get("/tasks/search-url", response_model=list[schemas.Task]) @app.get("/groups", response_model=list[str])
def search(url:str, skip: int = 0, limit: int = 100, db: Session = Depends(get_db), email = Depends(get_bearer_auth)): def get_user_groups(db: Session = Depends(get_db), email = Depends(get_bearer_auth)):
return crud.get_user_groups(db, email)
@app.get("/tasks/search-url", response_model=list[schemas.Archive])
def search(url:str, skip: int = 0, limit: int = 100, db: Session = Depends(get_db), _email = Depends(get_bearer_auth)):
return crud.search_tasks_by_url(db, url, skip=skip, limit=limit) return crud.search_tasks_by_url(db, url, skip=skip, limit=limit)
# @app.get("/tasks/search", response_model=list[schemas.Task]) # @app.get("/tasks/search", response_model=list[schemas.Task])
# def search(skip: int = 0, limit: int = 100, db: Session = Depends(get_db), email = Depends(get_bearer_auth)): # def search(skip: int = 0, limit: int = 100, db: Session = Depends(get_db), email = Depends(get_bearer_auth)):
# return crud.get_tasks(db, skip=skip, limit=limit) # return crud.get_tasks(db, skip=skip, limit=limit)
@app.get("/tasks/sync", response_model=list[schemas.Task]) @app.get("/tasks/sync", response_model=list[schemas.Archive])
def search(skip: int = 0, limit: int = 100, db: Session = Depends(get_db), email = Depends(get_bearer_auth)): def search(skip: int = 0, limit: int = 100, db: Session = Depends(get_db), email = Depends(get_bearer_auth)):
return crud.search_tasks_by_email(db, email, skip=skip, limit=limit) return crud.search_tasks_by_email(db, email, skip=skip, limit=limit)
@app.post("/tasks", status_code=201) @app.post("/tasks", status_code=201)
def run_task(payload = Body(...), email = Depends(get_bearer_auth)): def run_task(archive:schemas.ArchiveCreate, email = Depends(get_bearer_auth)):
url = payload.get('url') archive.author_id = email
logger.info(f"new task for user {email}: {url}") url = archive.url
logger.warning(archive)
logger.info(f"new {archive.public=} task for {email=} and {archive.group_id=}: {url}")
if type(url)!=str or len(url)<=5: if type(url)!=str or len(url)<=5:
raise HTTPException(status_code=422, detail=f"Invalid URL received: {url}") raise HTTPException(status_code=422, detail=f"Invalid URL received: {url}")
task = create_archive_task.delay(url=payload.get('url'), email=email) logger.info("creating task")
task = create_archive_task.delay(archive.json())
return JSONResponse({"id": task.id}) return JSONResponse({"id": task.id})
# @app.post("/tasks", status_code=201)
# def run_task(payload = Body(...), email = Depends(get_bearer_auth)):
# url = payload.get('url')
# public = payload.get('public', True)
# group = payload.get('group', None)
# logger.info(f"new {public=} task for {email=} and {group=}: {url}")
# if type(url)!=str or len(url)<=5:
# raise HTTPException(status_code=422, detail=f"Invalid URL received: {url}")
# task = create_archive_task.delay(url=payload.get('url'), email=email, public=public, group=group)
# return JSONResponse({"id": task.id})
@app.get("/tasks/{task_id}") @app.get("/tasks/{task_id}")
def get_status(task_id, email = Depends(get_bearer_auth)): def get_status(task_id, email = Depends(get_bearer_auth)):
logger.info(f"status check for user {email}") logger.info(f"status check for user {email}")
task_result = AsyncResult(task_id, app=celery) task_result = AsyncResult(task_id, app=celery)
logger.info(task_result)
result = { result = {
"id": task_id, "id": task_id,
"status": task_result.status, "status": task_result.status,
"result": task_result.result "result": task_result.result
} }
try:
if task_result.result and "error" in task_result.result:
result["status"] = "FAILURE"
except Exception as e: logger.error(traceback.format_exc())
try: try:
json_result = jsonable_encoder(result, exclude_unset=True) json_result = jsonable_encoder(result, exclude_unset=True)
return JSONResponse(json_result) return JSONResponse(json_result)
@@ -94,6 +126,7 @@ def get_status(task_id, email = Depends(get_bearer_auth)):
return JSONResponse({ return JSONResponse({
"id": task_id, "id": task_id,
"status": "FAILURE", "status": "FAILURE",
"result": {"error": e}
}) })

79
src/requirements.txt
View File

@@ -7,19 +7,19 @@
# -i https://pypi.org/simple # -i https://pypi.org/simple
aiofiles==0.6.0 aiofiles==0.6.0
aiosqlite==0.18.0 aiosqlite==0.19.0
alembic==1.9.4 alembic==1.10.3
amqp==2.6.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' amqp==2.6.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
anyio==3.6.2; python_full_version >= '3.6.2' anyio==3.6.2; python_full_version >= '3.6.2'
argparse==1.4.0 argparse==1.4.0
async-generator==1.10; python_version >= '3.5' async-generator==1.10; python_version >= '3.5'
attrs==22.2.0; python_version >= '3.6' attrs==23.1.0; python_version >= '3.7'
authlib==0.15.6 authlib==0.15.6
auto-archiver==0.4.3 auto-archiver==0.5.6
beautifulsoup4==4.11.2; python_version >= '3.6' beautifulsoup4==4.12.2; python_version >= '3.6'
billiard==3.6.4.0 billiard==3.6.4.0
boto3==1.26.79; python_version >= '3.7' boto3==1.26.115; python_version >= '3.7'
botocore==1.29.79; python_version >= '3.7' botocore==1.29.115; python_version >= '3.7'
brotli==1.0.9; platform_python_implementation == 'CPython' brotli==1.0.9; platform_python_implementation == 'CPython'
bs4==0.0.1 bs4==0.0.1
cachetools==5.3.0; python_version ~= '3.7' cachetools==5.3.0; python_version ~= '3.7'
@@ -31,24 +31,25 @@ click==8.1.3; python_version >= '3.7'
cloudscraper==1.2.69 cloudscraper==1.2.69
cryptography==38.0.4; python_version >= '3.6' cryptography==38.0.4; python_version >= '3.6'
dataclasses-json==0.5.7; python_version >= '3.6' dataclasses-json==0.5.7; python_version >= '3.6'
dateparser==1.1.7; python_version >= '3.7' dateparser==1.1.8; python_version >= '3.7'
exceptiongroup==1.1.0; python_version < '3.11' exceptiongroup==1.1.1; python_version < '3.11'
fastapi==0.92.0 fastapi-utils==0.2.1
fastapi==0.95.1
ffmpeg-python==0.2.0 ffmpeg-python==0.2.0
filelock==3.9.0; python_version >= '3.7' filelock==3.12.0; python_version >= '3.7'
flask==2.2.3; python_version >= '3.7' flask==2.2.3; python_version >= '3.7'
flower==0.9.7 flower==0.9.7
future==0.18.3; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' future==0.18.3; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
google-api-core==2.11.0; python_version >= '3.7' google-api-core==2.11.0; python_version >= '3.7'
google-api-python-client==2.79.0; python_version >= '3.7' google-api-python-client==2.86.0; python_version >= '3.7'
google-auth-httplib2==0.1.0 google-auth-httplib2==0.1.0
google-auth-oauthlib==1.0.0; python_version >= '3.6' google-auth-oauthlib==1.0.0; python_version >= '3.6'
google-auth==2.16.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' google-auth==2.17.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
googleapis-common-protos==1.58.0; python_version >= '3.7' googleapis-common-protos==1.59.0; python_version >= '3.7'
greenlet==2.0.2; platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32'))))) greenlet==2.0.2; python_version >= '3' and platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32')))))
gspread==5.7.2; python_version not in '3.0, 3.1, 3.2, 3.3' and python_version >= '3.6' gspread==5.8.0; python_version not in '3.0, 3.1, 3.2, 3.3' and python_version >= '3.6'
h11==0.14.0; python_version >= '3.7' h11==0.14.0; python_version >= '3.7'
httplib2==0.21.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' httplib2==0.22.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
humanize==4.6.0; python_version >= '3.7' humanize==4.6.0; python_version >= '3.7'
idna==3.4; python_version >= '3.5' idna==3.4; python_version >= '3.5'
iniconfig==2.0.0; python_version >= '3.7' iniconfig==2.0.0; python_version >= '3.7'
@@ -57,7 +58,7 @@ itsdangerous==2.1.2; python_version >= '3.7'
jinja2==3.1.2 jinja2==3.1.2
jmespath==1.0.1; python_version >= '3.7' jmespath==1.0.1; python_version >= '3.7'
kombu==4.6.11; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' kombu==4.6.11; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
loguru==0.6.0 loguru==0.7.0
lxml==4.9.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' lxml==4.9.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
mako==1.2.4; python_version >= '3.7' mako==1.2.4; python_version >= '3.7'
markdown-it-py==2.2.0; python_version >= '3.7' markdown-it-py==2.2.0; python_version >= '3.7'
@@ -70,59 +71,59 @@ mypy-extensions==1.0.0; python_version >= '3.5'
oauth2client==4.1.3 oauth2client==4.1.3
oauthlib==3.2.2; python_version >= '3.6' oauthlib==3.2.2; python_version >= '3.6'
outcome==1.2.0; python_version >= '3.7' outcome==1.2.0; python_version >= '3.7'
packaging==23.0; python_version >= '3.7' packaging==23.1; python_version >= '3.7'
pluggy==0.13.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' pluggy==0.13.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
prometheus-client==0.8.0 prometheus-client==0.8.0
protobuf==4.22.0; python_version >= '3.7' protobuf==4.22.3; python_version >= '3.7'
py==1.11.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' py==1.11.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
pyaes==1.6.1 pyaes==1.6.1
pyasn1-modules==0.2.8 pyasn1-modules==0.2.8
pyasn1==0.4.8 pyasn1==0.4.8
pycparser==2.21 pycparser==2.21
pycryptodomex==3.17; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' pycryptodomex==3.17; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
pydantic==1.10.5; python_version >= '3.7' pydantic==1.10.7; python_version >= '3.7'
pygments==2.14.0; python_version >= '3.6' pygments==2.15.0; python_version >= '3.7'
pyparsing==3.0.9; python_version >= '3.1' pyparsing==3.0.9; python_version >= '3.1'
pysocks==1.7.1 pysocks==1.7.1
pytest==6.2.4 pytest==6.2.4
python-dateutil==2.8.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' python-dateutil==2.8.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
python-dotenv==1.0.0 python-dotenv==1.0.0
python-slugify==8.0.1; python_version >= '3.7' python-slugify==8.0.1; python_version >= '3.7'
python-twitter-v2==0.8.1; python_version >= '3.6' and python_version < '4.0' python-twitter-v2==0.8.1; python_version >= '3.6' and python_version < '4'
pytz-deprecation-shim==0.1.0.post0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' pytz-deprecation-shim==0.1.0.post0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
pytz==2022.7.1 pytz==2023.3
pyyaml==6.0; python_version >= '3.6' pyyaml==6.0; python_version >= '3.6'
redis==3.5.3 redis==3.5.3
regex==2022.10.31; python_version >= '3.6' regex==2023.3.23; python_version >= '3.8'
requests-oauthlib==1.3.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' requests-oauthlib==1.3.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
requests-toolbelt==0.10.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' requests-toolbelt==0.10.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
requests==2.28.2 requests==2.28.2
rich==13.3.1; python_version >= '3.7' rich==13.3.4; python_version >= '3.7'
rsa==4.9; python_version >= '3.6' and python_version < '4.0' rsa==4.9; python_version >= '3.6' and python_version < '4'
s3transfer==0.6.0; python_version >= '3.7' s3transfer==0.6.0; python_version >= '3.7'
selenium==4.8.2; python_version >= '3.7' selenium==4.8.3; python_version >= '3.7'
six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
sniffio==1.3.0; python_version >= '3.7' sniffio==1.3.0; python_version >= '3.7'
snscrape==0.5.0.20230113; python_version ~= '3.8' snscrape==0.6.2.20230320; python_version ~= '3.8'
sortedcontainers==2.4.0 sortedcontainers==2.4.0
soupsieve==2.4; python_version >= '3.7' soupsieve==2.4.1; python_version >= '3.7'
sqlalchemy==2.0.4 sqlalchemy==1.4.47
starlette==0.25.0; python_version >= '3.7' starlette==0.26.1; python_version >= '3.7'
telethon==1.27.0; python_version >= '3.5' telethon==1.28.5; python_version >= '3.5'
text-unidecode==1.3 text-unidecode==1.3
tiktok-downloader==0.3.4 tiktok-downloader==0.3.4
toml==0.10.2; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' toml==0.10.2; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
tornado==6.2; python_full_version >= '3.5.2' tornado==6.3; python_full_version >= '3.5.2'
tqdm==4.64.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' tqdm==4.65.0; python_version >= '3.7'
trio-websocket==0.9.2; python_version >= '3.5' trio-websocket==0.10.2; python_version >= '3.7'
trio==0.22.0; python_version >= '3.7' trio==0.22.0; python_version >= '3.7'
typing-extensions==4.5.0; python_version >= '3.7' typing-extensions==4.5.0; python_version >= '3.7'
typing-inspect==0.8.0 typing-inspect==0.8.0
tzdata==2022.7; python_version >= '3.6' tzdata==2023.3; python_version >= '3.6'
tzlocal==4.2; python_version >= '3.6' tzlocal==4.3; python_version >= '3.7'
uritemplate==4.1.1; python_version >= '3.6' uritemplate==4.1.1; python_version >= '3.6'
urllib3==1.26.14; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' urllib3==1.26.14; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
uvicorn==0.20.0 uvicorn==0.21.1
vine==1.3.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' vine==1.3.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
vk-api==11.9.9 vk-api==11.9.9
vk-url-scraper==0.3.15; python_version >= '3.7' vk-url-scraper==0.3.15; python_version >= '3.7'

117
src/worker.py
View File

@@ -1,12 +1,14 @@
import os, re import os, re, traceback, yaml
from celery import Celery from celery import Celery, states
from celery.exceptions import Ignore
from celery.signals import task_failure
from auto_archiver import Config, ArchivingOrchestrator, Metadata from auto_archiver import Config, ArchivingOrchestrator, Metadata
# from auto_archiver.enrichers import ScreenshotEnricher # from auto_archiver.enrichers import ScreenshotEnricher
from loguru import logger from loguru import logger
from db import crud, schemas from db import crud, schemas, models
from db.database import engine, SessionLocal from db.database import engine, SessionLocal
from contextlib import contextmanager from contextlib import contextmanager
import json import json
@@ -14,6 +16,8 @@ import json
celery = Celery(__name__) celery = Celery(__name__)
celery.conf.broker_url = os.environ.get("CELERY_BROKER_URL", "redis://localhost:6379") celery.conf.broker_url = os.environ.get("CELERY_BROKER_URL", "redis://localhost:6379")
celery.conf.result_backend = os.environ.get("CELERY_RESULT_BACKEND", "redis://localhost:6379") celery.conf.result_backend = os.environ.get("CELERY_RESULT_BACKEND", "redis://localhost:6379")
USER_GROUPS_FILENAME=os.environ.get("USER_GROUPS_FILENAME", "user-groups.yaml")
@contextmanager @contextmanager
def get_db(): def get_db():
@@ -21,34 +25,89 @@ def get_db():
try: yield session try: yield session
finally: session.close() finally: session.close()
config_default = Config()
config_default.parse(use_cli=False, yaml_config_filename=os.environ.get("ORCHESTRATION_CONFIG_DEFAULT", "secrets/orchestration.yaml"))
config_bcat = None
if (config_bcat_file := os.environ.get("ORCHESTRATION_CONFIG_BELLINGCAT")):
config_bcat = Config()
config_bcat.parse(use_cli=False, yaml_config_filename=config_bcat_file)
orchestrators = {"bellingcat": None, "default": None}
@celery.task(name="create_archive_task", bind=True, autoretry_for=(Exception,), retry_backoff=True, retry_kwargs={'max_retries': 5}) @celery.task(name="create_archive_task", bind=True, autoretry_for=(Exception,), retry_backoff=True, retry_kwargs={'max_retries': 5})
def create_archive_task(self, url: str, email:str=""): def create_archive_task(self, archive_json: str):
orchestrator = choose_orchestrator(email)
result = orchestrator.feed_item(Metadata().set_url(url)).to_json() archive = schemas.ArchiveCreate.parse_raw(archive_json)
if not archive.public and archive.group_id and len(archive.group_id) > 0:
# ensure group is valid for user
with get_db() as session:
db_group = crud.get_group_for_user(session, archive.group_id, archive.author_id)
if not db_group:
logger.error(em := f"User {archive.author_id} is not part of {archive.group_id}, no permission")
return {"error": em}
url = archive.url
logger.info(f"{url=}")
logger.info(f"{archive=}")
orchestrator = choose_orchestrator(archive.group_id, archive.author_id)
result = orchestrator.feed_item(Metadata().set_url(url))
if not result:
logger.error(f"UNABLE TO archive: {url}")
return {"error": "unable to archive"}
result_json = result.to_json()
with get_db() as session: with get_db() as session:
db_task = crud.create_task(session, task=schemas.TaskCreate(id=self.request.id, url=url, author=email, result=json.loads(result))) # create DB URLs
db_urls = [models.ArchiveUrl(url=url, key=m.get("id", f"media_{i}")) for i, m in enumerate(result.media) for url in m.urls]
# create DB TAGs if needed
db_tags = [crud.create_tag(session, tag) for tag in archive.tags]
# insert archive
db_task = crud.create_task(session, task=schemas.ArchiveCreate(id=self.request.id, url=url, result=json.loads(result_json), public=archive.public, author_id=archive.author_id, group_id=archive.group_id), tags=db_tags, urls=db_urls)
logger.debug(f"Added {db_task.id=} to database on {db_task.created_at}") logger.debug(f"Added {db_task.id=} to database on {db_task.created_at}")
return result return result_json
@task_failure.connect(sender=create_archive_task)
def task_failure_notifier(sender=None, **kwargs):
logger.warning("😅 From task_failure_notifier ==> Task failed successfully! ")
logger.error(kwargs['exception'])
logger.error(kwargs['traceback'])
logger.error("\n".join(traceback.format_list(traceback.extract_tb(kwargs['traceback']))))
def choose_orchestrator(group, email):
global ORCHESTRATORS
if group not in ORCHESTRATORS: group = get_user_first_group(email)
assert group in ORCHESTRATORS, f"{group=} not in configurations"
logger.info(f"CHOOSE Orchestrator for {group=}, {email=}")
return ArchivingOrchestrator(ORCHESTRATORS.get(group))
def read_user_groups():
# read yaml safely
with open(USER_GROUPS_FILENAME) as inf:
try:
return yaml.safe_load(inf)
except yaml.YAMLError as e:
logger.error(f"could not open user groups filename {USER_GROUPS_FILENAME}: {e}")
raise e
def get_user_first_group(email):
user_groups_yaml = read_user_groups()
groups = user_groups_yaml.get("users", {}).get(email, [])
if len(groups): return groups[0]
return "default"
def choose_orchestrator(email): def load_orchestrators():
global orchestrators, config_bcat global ORCHESTRATORS
if re.match(r'^[\w.]+@bellingcat\.com$', email) and config_bcat: ORCHESTRATORS = {}
logger.debug("Using bellingcat config for orchestration") """
if not orchestrators["bellingcat"]: reads the orchestrators key in the config file to load different orchestrators for different groups
orchestrators["bellingcat"] = ArchivingOrchestrator(config_bcat) """
return orchestrators["bellingcat"] user_groups_yaml = read_user_groups()
logger.debug("Using default config for orchestration")
if not orchestrators["default"]: orchestrators_config = user_groups_yaml.get("orchestrators", {})
orchestrators["default"] = ArchivingOrchestrator(config_default) assert len(orchestrators_config), f"No orchestrators key found in {USER_GROUPS_FILENAME}. please see the example file"
return orchestrators["default"] assert "default" in orchestrators_config, "please include a 'default' orchestrator to be used when the user has no group"
logger.debug(f"Found {len(orchestrators_config)} group orchestrators.")
for group, config_filename in orchestrators_config.items():
config = Config()
config.parse(use_cli=False, yaml_config_filename=config_filename)
ORCHESTRATORS[group] = config
return ORCHESTRATORS
## INIT
ORCHESTRATORS = {}
load_orchestrators()