From c6b39db295b5c1b5791d7d1d89cd426b09691161 Mon Sep 17 00:00:00 2001 From: msramalho <19508417+msramalho@users.noreply.github.com> Date: Tue, 21 Feb 2023 14:11:32 +0000 Subject: [PATCH] MVP celery worker for auto-archiver --- .gitignore | 13 ++ LICENSE | 21 +++ README.md | 10 ++ docker-compose.yml | 47 ++++++ src/Dockerfile | 17 ++ src/Pipfile | 21 +++ src/Pipfile.lock | 373 +++++++++++++++++++++++++++++++++++++++++++ src/main.py | 49 ++++++ src/requirements.txt | 9 ++ src/worker.py | 32 ++++ 10 files changed, 592 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README.md create mode 100644 docker-compose.yml create mode 100644 src/Dockerfile create mode 100644 src/Pipfile create mode 100644 src/Pipfile.lock create mode 100644 src/main.py create mode 100644 src/requirements.txt create mode 100644 src/worker.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..75e3a7f --- /dev/null +++ b/.gitignore @@ -0,0 +1,13 @@ +orchestration.yaml +my-archives +*.pyc +.DS_Store +secrets +*.log +__pycache +.pytest_cach +logs +#temp +static +templates +tests \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..f593c11 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 Stichting Bellingcat + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..24628bb --- /dev/null +++ b/README.md @@ -0,0 +1,10 @@ +## development +http://localhost:8004 + +cd /src +* `pipenv install --editable ../../auto-archiver` +* console 1 - `docker compose up web redis dashboard ` +* console 2 - `pipenv shell` + `celery worker --app=worker.celery --loglevel=info --logfile=logs/celery_dev.log` +* console 3 - `pipenv shell` + `uvicorn main:app --host 0.0.0.0 --reload` +orchestration must be from the console(?) + diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..a188308 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,47 @@ +version: '3.8' + +services: + + web: + build: ./project + ports: + - 8004:8000 + command: uvicorn main:app --host 0.0.0.0 --reload + volumes: + - ./project:/usr/src/app + environment: + - CELERY_BROKER_URL=redis://redis:6379/0 + - CELERY_RESULT_BACKEND=redis://redis:6379/0 + depends_on: + - redis + + worker: + build: ./project + command: celery worker --app=worker.celery --loglevel=info --logfile=logs/celery.log + volumes: + - ./project:/usr/src/app + environment: + - CELERY_BROKER_URL=redis://redis:6379/0 + - CELERY_RESULT_BACKEND=redis://redis:6379/0 + depends_on: + - web + - redis + + redis: + image: redis:6-alpine + # DEV ONLY + ports: + - 6379:6379 + + dashboard: + build: ./project + command: flower --app=worker.celery --port=5555 --broker=redis://redis:6379/0 + ports: + - 5556:5555 + environment: + - CELERY_BROKER_URL=redis://redis:6379/0 + - CELERY_RESULT_BACKEND=redis://redis:6379/0 + depends_on: + - web + - redis + # - worker diff --git a/src/Dockerfile b/src/Dockerfile new file mode 100644 index 0000000..e8c267c --- /dev/null +++ b/src/Dockerfile @@ -0,0 +1,17 @@ +# pull official base image 3.9 +FROM python:3.9.5-slim-buster + +# set work directory +WORKDIR /usr/src/app + +# set environment variables +ENV PYTHONUNBUFFERED 1 +ENV PYTHONDONTWRITEBYTECODE 1 + +# install dependencies +RUN pip install --upgrade pip +COPY ./requirements.txt . +RUN pip install -r requirements.txt + +# copy project code over +COPY . . diff --git a/src/Pipfile b/src/Pipfile new file mode 100644 index 0000000..a1e54e9 --- /dev/null +++ b/src/Pipfile @@ -0,0 +1,21 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +aiofiles = "==0.6.0" +celery = "==4.4.7" +fastapi = "==0.64.0" +flower = "==0.9.7" +jinja2 = "==3.0.3" +pytest = "==6.2.4" +redis = "==3.5.3" +requests = "==2.25.1" +uvicorn = "==0.13.4" +auto-archiver = {editable = true, path = "./../../auto-archiver"} + +[dev-packages] + +[requires] +python_version = "3.10" diff --git a/src/Pipfile.lock b/src/Pipfile.lock new file mode 100644 index 0000000..a538683 --- /dev/null +++ b/src/Pipfile.lock @@ -0,0 +1,373 @@ +{ + "_meta": { + "hash": { + "sha256": "77804d59b6d31a417d2ad33eef4b0da6bbebf4b81b58bfe34404cea0fc6dd520" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.10" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "aiofiles": { + "hashes": [ + "sha256:bd3019af67f83b739f8e4053c6c0512a7f545b9a8d91aaeab55e6e0f9d123c27", + "sha256:e0281b157d3d5d59d803e3f4557dcc9a3dff28a4dd4829a9ff478adae50ca092" + ], + "index": "pypi", + "version": "==0.6.0" + }, + "amqp": { + "hashes": [ + "sha256:70cdb10628468ff14e57ec2f751c7aa9e48e7e3651cfd62d431213c0c4e58f21", + "sha256:aa7f313fb887c91f15474c1229907a04dac0b8135822d6603437803424c0aa59" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==2.6.1" + }, + "attrs": { + "hashes": [ + "sha256:29e95c7f6778868dbd49170f98f8818f78f3dc5e0e37c0b1f474e3561b240836", + "sha256:c9227bfc2f01993c03f68db37d1d15c9690188323c067c641f1a35ca58185f99" + ], + "markers": "python_version >= '3.6'", + "version": "==22.2.0" + }, + "billiard": { + "hashes": [ + "sha256:299de5a8da28a783d51b197d496bef4f1595dd023a93a4f59dde1886ae905547", + "sha256:87103ea78fa6ab4d5c751c4909bcff74617d985de7fa8b672cf8618afd5a875b" + ], + "version": "==3.6.4.0" + }, + "celery": { + "hashes": [ + "sha256:a92e1d56e650781fb747032a3997d16236d037c8199eacd5217d1a72893bca45", + "sha256:d220b13a8ed57c78149acf82c006785356071844afe0b27012a4991d44026f9f" + ], + "index": "pypi", + "version": "==4.4.7" + }, + "certifi": { + "hashes": [ + "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3", + "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18" + ], + "markers": "python_version >= '3.6'", + "version": "==2022.12.7" + }, + "chardet": { + "hashes": [ + "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa", + "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==4.0.0" + }, + "click": { + "hashes": [ + "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a", + "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==7.1.2" + }, + "fastapi": { + "hashes": [ + "sha256:62a438d0ff466640939414436339ce4e303964f3f823b7288e300baa869162e3", + "sha256:9bbd7b7b9291bbc3bbd72cbc82f5d456369802dab0d142a85350b06c5c7e6379" + ], + "index": "pypi", + "version": "==0.64.0" + }, + "flower": { + "hashes": [ + "sha256:8d6d6ac03e60b3a4227d156da489eb435e2442d82e89922d413df9054b9221eb", + "sha256:cf27a254268bb06fd4972408d0518237fcd847f7da4b4cd8055e228150ace8f3" + ], + "index": "pypi", + "version": "==0.9.7" + }, + "h11": { + "hashes": [ + "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d", + "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761" + ], + "markers": "python_version >= '3.7'", + "version": "==0.14.0" + }, + "humanize": { + "hashes": [ + "sha256:401201aca462749773f02920139f302450cb548b70489b9b4b92be39fe3c3c50", + "sha256:5f1f22bc65911eb1a6ffe7659bd6598e33dcfeeb904eb16ee1e705a09bf75916" + ], + "markers": "python_version >= '3.7'", + "version": "==4.6.0" + }, + "idna": { + "hashes": [ + "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6", + "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.10" + }, + "iniconfig": { + "hashes": [ + "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3", + "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374" + ], + "markers": "python_version >= '3.7'", + "version": "==2.0.0" + }, + "jinja2": { + "hashes": [ + "sha256:077ce6014f7b40d03b47d1f1ca4b0fc8328a692bd284016f806ed0eaca390ad8", + "sha256:611bb273cd68f3b993fabdc4064fc858c5b47a973cb5aa7999ec1ba405c87cd7" + ], + "index": "pypi", + "version": "==3.0.3" + }, + "kombu": { + "hashes": [ + "sha256:be48cdffb54a2194d93ad6533d73f69408486483d189fe9f5990ee24255b0e0a", + "sha256:ca1b45faac8c0b18493d02a8571792f3c40291cf2bcf1f55afed3d8f3aa7ba74" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==4.6.11" + }, + "markupsafe": { + "hashes": [ + "sha256:0576fe974b40a400449768941d5d0858cc624e3249dfd1e0c33674e5c7ca7aed", + "sha256:085fd3201e7b12809f9e6e9bc1e5c96a368c8523fad5afb02afe3c051ae4afcc", + "sha256:090376d812fb6ac5f171e5938e82e7f2d7adc2b629101cec0db8b267815c85e2", + "sha256:0b462104ba25f1ac006fdab8b6a01ebbfbce9ed37fd37fd4acd70c67c973e460", + "sha256:137678c63c977754abe9086a3ec011e8fd985ab90631145dfb9294ad09c102a7", + "sha256:1bea30e9bf331f3fef67e0a3877b2288593c98a21ccb2cf29b74c581a4eb3af0", + "sha256:22152d00bf4a9c7c83960521fc558f55a1adbc0631fbb00a9471e097b19d72e1", + "sha256:22731d79ed2eb25059ae3df1dfc9cb1546691cc41f4e3130fe6bfbc3ecbbecfa", + "sha256:2298c859cfc5463f1b64bd55cb3e602528db6fa0f3cfd568d3605c50678f8f03", + "sha256:28057e985dace2f478e042eaa15606c7efccb700797660629da387eb289b9323", + "sha256:2e7821bffe00aa6bd07a23913b7f4e01328c3d5cc0b40b36c0bd81d362faeb65", + "sha256:2ec4f2d48ae59bbb9d1f9d7efb9236ab81429a764dedca114f5fdabbc3788013", + "sha256:340bea174e9761308703ae988e982005aedf427de816d1afe98147668cc03036", + "sha256:40627dcf047dadb22cd25ea7ecfe9cbf3bbbad0482ee5920b582f3809c97654f", + "sha256:40dfd3fefbef579ee058f139733ac336312663c6706d1163b82b3003fb1925c4", + "sha256:4cf06cdc1dda95223e9d2d3c58d3b178aa5dacb35ee7e3bbac10e4e1faacb419", + "sha256:50c42830a633fa0cf9e7d27664637532791bfc31c731a87b202d2d8ac40c3ea2", + "sha256:55f44b440d491028addb3b88f72207d71eeebfb7b5dbf0643f7c023ae1fba619", + "sha256:608e7073dfa9e38a85d38474c082d4281f4ce276ac0010224eaba11e929dd53a", + "sha256:63ba06c9941e46fa389d389644e2d8225e0e3e5ebcc4ff1ea8506dce646f8c8a", + "sha256:65608c35bfb8a76763f37036547f7adfd09270fbdbf96608be2bead319728fcd", + "sha256:665a36ae6f8f20a4676b53224e33d456a6f5a72657d9c83c2aa00765072f31f7", + "sha256:6d6607f98fcf17e534162f0709aaad3ab7a96032723d8ac8750ffe17ae5a0666", + "sha256:7313ce6a199651c4ed9d7e4cfb4aa56fe923b1adf9af3b420ee14e6d9a73df65", + "sha256:7668b52e102d0ed87cb082380a7e2e1e78737ddecdde129acadb0eccc5423859", + "sha256:7df70907e00c970c60b9ef2938d894a9381f38e6b9db73c5be35e59d92e06625", + "sha256:7e007132af78ea9df29495dbf7b5824cb71648d7133cf7848a2a5dd00d36f9ff", + "sha256:835fb5e38fd89328e9c81067fd642b3593c33e1e17e2fdbf77f5676abb14a156", + "sha256:8bca7e26c1dd751236cfb0c6c72d4ad61d986e9a41bbf76cb445f69488b2a2bd", + "sha256:8db032bf0ce9022a8e41a22598eefc802314e81b879ae093f36ce9ddf39ab1ba", + "sha256:99625a92da8229df6d44335e6fcc558a5037dd0a760e11d84be2260e6f37002f", + "sha256:9cad97ab29dfc3f0249b483412c85c8ef4766d96cdf9dcf5a1e3caa3f3661cf1", + "sha256:a4abaec6ca3ad8660690236d11bfe28dfd707778e2442b45addd2f086d6ef094", + "sha256:a6e40afa7f45939ca356f348c8e23048e02cb109ced1eb8420961b2f40fb373a", + "sha256:a6f2fcca746e8d5910e18782f976489939d54a91f9411c32051b4aab2bd7c513", + "sha256:a806db027852538d2ad7555b203300173dd1b77ba116de92da9afbc3a3be3eed", + "sha256:abcabc8c2b26036d62d4c746381a6f7cf60aafcc653198ad678306986b09450d", + "sha256:b8526c6d437855442cdd3d87eede9c425c4445ea011ca38d937db299382e6fa3", + "sha256:bb06feb762bade6bf3c8b844462274db0c76acc95c52abe8dbed28ae3d44a147", + "sha256:c0a33bc9f02c2b17c3ea382f91b4db0e6cde90b63b296422a939886a7a80de1c", + "sha256:c4a549890a45f57f1ebf99c067a4ad0cb423a05544accaf2b065246827ed9603", + "sha256:ca244fa73f50a800cf8c3ebf7fd93149ec37f5cb9596aa8873ae2c1d23498601", + "sha256:cf877ab4ed6e302ec1d04952ca358b381a882fbd9d1b07cccbfd61783561f98a", + "sha256:d9d971ec1e79906046aa3ca266de79eac42f1dbf3612a05dc9368125952bd1a1", + "sha256:da25303d91526aac3672ee6d49a2f3db2d9502a4a60b55519feb1a4c7714e07d", + "sha256:e55e40ff0cc8cc5c07996915ad367fa47da6b3fc091fdadca7f5403239c5fec3", + "sha256:f03a532d7dee1bed20bc4884194a16160a2de9ffc6354b3878ec9682bb623c54", + "sha256:f1cd098434e83e656abf198f103a8207a8187c0fc110306691a2e94a78d0abb2", + "sha256:f2bfb563d0211ce16b63c7cb9395d2c682a23187f54c3d79bfec33e6705473c6", + "sha256:f8ffb705ffcf5ddd0e80b65ddf7bed7ee4f5a441ea7d3419e861a12eaf41af58" + ], + "markers": "python_version >= '3.7'", + "version": "==2.1.2" + }, + "packaging": { + "hashes": [ + "sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2", + "sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97" + ], + "markers": "python_version >= '3.7'", + "version": "==23.0" + }, + "pluggy": { + "hashes": [ + "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0", + "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.13.1" + }, + "prometheus-client": { + "hashes": [ + "sha256:983c7ac4b47478720db338f1491ef67a100b474e3bc7dafcbaefb7d0b8f9b01c", + "sha256:c6e6b706833a6bd1fd51711299edee907857be10ece535126a158f911ee80915" + ], + "version": "==0.8.0" + }, + "py": { + "hashes": [ + "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719", + "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==1.11.0" + }, + "pydantic": { + "hashes": [ + "sha256:1fd326aff5d6c36f05735c7c9b3d5b0e933b4ca52ad0b6e4b38038d82703d35b", + "sha256:2185a3b3d98ab4506a3f6707569802d2d92c3a7ba3a9a35683a7709ea6c2aaa2", + "sha256:261f357f0aecda005934e413dfd7aa4077004a174dafe414a8325e6098a8e419", + "sha256:305d0376c516b0dfa1dbefeae8c21042b57b496892d721905a6ec6b79494a66d", + "sha256:3257bd714de9db2102b742570a56bf7978e90441193acac109b1f500290f5718", + "sha256:3353072625ea2a9a6c81ad01b91e5c07fa70deb06368c71307529abf70d23325", + "sha256:36e44a4de37b8aecffa81c081dbfe42c4d2bf9f6dff34d03dce157ec65eb0f15", + "sha256:3bb99cf9655b377db1a9e47fa4479e3330ea96f4123c6c8200e482704bf1eda2", + "sha256:3f9d9b2be177c3cb6027cd67fbf323586417868c06c3c85d0d101703136e6b31", + "sha256:45edea10b75d3da43cfda12f3792833a3fa70b6eee4db1ed6aed528cef17c74e", + "sha256:51782fd81f09edcf265823c3bf43ff36d00db246eca39ee765ef58dc8421a642", + "sha256:532e97c35719f137ee5405bd3eeddc5c06eb91a032bc755a44e34a712420daf3", + "sha256:58e41dd1e977531ac6073b11baac8c013f3cd8706a01d3dc74e86955be8b2c0c", + "sha256:5920824fe1e21cbb3e38cf0f3dd24857c8959801d1031ce1fac1d50857a03bfb", + "sha256:5f3bc8f103b56a8c88021d481410874b1f13edf6e838da607dcb57ecff9b4594", + "sha256:63200cd8af1af2c07964546b7bc8f217e8bda9d0a2ef0ee0c797b36353914984", + "sha256:663d2dd78596c5fa3eb996bc3f34b8c2a592648ad10008f98d1348be7ae212fb", + "sha256:6a4b0aab29061262065bbdede617ef99cc5914d1bf0ddc8bcd8e3d7928d85bd6", + "sha256:6bb0452d7b8516178c969d305d9630a3c9b8cf16fcf4713261c9ebd465af0d73", + "sha256:72ef3783be8cbdef6bca034606a5de3862be6b72415dc5cb1fb8ddbac110049a", + "sha256:76c930ad0746c70f0368c4596020b736ab65b473c1f9b3872310a835d852eb19", + "sha256:7c5b94d598c90f2f46b3a983ffb46ab806a67099d118ae0da7ef21a2a4033b28", + "sha256:7ce1612e98c6326f10888df951a26ec1a577d8df49ddcaea87773bfbe23ba5cc", + "sha256:8481dca324e1c7b715ce091a698b181054d22072e848b6fc7895cd86f79b4449", + "sha256:87f831e81ea0589cd18257f84386bf30154c5f4bed373b7b75e5cb0b5d53ea87", + "sha256:9a9d9155e2a9f38b2eb9374c88f02fd4d6851ae17b65ee786a87d032f87008f8", + "sha256:9e337ac83686645a46db0e825acceea8e02fca4062483f40e9ae178e8bd1103a", + "sha256:b429f7c457aebb7fbe7cd69c418d1cd7c6fdc4d3c8697f45af78b8d5a7955760", + "sha256:b473d00ccd5c2061fd896ac127b7755baad233f8d996ea288af14ae09f8e0d1e", + "sha256:bd46a0e6296346c477e59a954da57beaf9c538da37b9df482e50f836e4a7d4bb", + "sha256:c428c0f64a86661fb4873495c4fac430ec7a7cef2b8c1c28f3d1a7277f9ea5ab", + "sha256:c9e5b778b6842f135902e2d82624008c6a79710207e28e86966cd136c621bfee", + "sha256:ca9075ab3de9e48b75fa8ccb897c34ccc1519177ad8841d99f7fd74cf43be5bf", + "sha256:f582cac9d11c227c652d3ce8ee223d94eb06f4228b52a8adaafa9fa62e73d5c9", + "sha256:f5bee6c523d13944a1fdc6f0525bc86dbbd94372f17b83fa6331aabacc8fd08e", + "sha256:f836444b4c5ece128b23ec36a446c9ab7f9b0f7981d0d27e13a7c366ee163f8a" + ], + "markers": "python_version >= '3.7'", + "version": "==1.10.5" + }, + "pytest": { + "hashes": [ + "sha256:50bcad0a0b9c5a72c8e4e7c9855a3ad496ca6a881a3641b4260605450772c54b", + "sha256:91ef2131a9bd6be8f76f1f08eac5c5317221d6ad1e143ae03894b862e8976890" + ], + "index": "pypi", + "version": "==6.2.4" + }, + "pytz": { + "hashes": [ + "sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0", + "sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a" + ], + "version": "==2022.7.1" + }, + "redis": { + "hashes": [ + "sha256:0e7e0cfca8660dea8b7d5cd8c4f6c5e29e11f31158c0b0ae91a397f00e5a05a2", + "sha256:432b788c4530cfe16d8d943a09d40ca6c16149727e4afe8c2c9d5580c59d9f24" + ], + "index": "pypi", + "version": "==3.5.3" + }, + "requests": { + "hashes": [ + "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804", + "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e" + ], + "index": "pypi", + "version": "==2.25.1" + }, + "starlette": { + "hashes": [ + "sha256:bd2ffe5e37fb75d014728511f8e68ebf2c80b0fa3d04ca1479f4dc752ae31ac9", + "sha256:ebe8ee08d9be96a3c9f31b2cb2a24dbdf845247b745664bd8a3f9bd0c977fdbc" + ], + "markers": "python_version >= '3.6'", + "version": "==0.13.6" + }, + "toml": { + "hashes": [ + "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", + "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.10.2" + }, + "tornado": { + "hashes": [ + "sha256:1d54d13ab8414ed44de07efecb97d4ef7c39f7438cf5e976ccd356bebb1b5fca", + "sha256:20f638fd8cc85f3cbae3c732326e96addff0a15e22d80f049e00121651e82e72", + "sha256:5c87076709343557ef8032934ce5f637dbb552efa7b21d08e89ae7619ed0eb23", + "sha256:5f8c52d219d4995388119af7ccaa0bcec289535747620116a58d830e7c25d8a8", + "sha256:6fdfabffd8dfcb6cf887428849d30cf19a3ea34c2c248461e1f7d718ad30b66b", + "sha256:87dcafae3e884462f90c90ecc200defe5e580a7fbbb4365eda7c7c1eb809ebc9", + "sha256:9b630419bde84ec666bfd7ea0a4cb2a8a651c2d5cccdbdd1972a0c859dfc3c13", + "sha256:b8150f721c101abdef99073bf66d3903e292d851bee51910839831caba341a75", + "sha256:ba09ef14ca9893954244fd872798b4ccb2367c165946ce2dd7376aebdde8e3ac", + "sha256:d3a2f5999215a3a06a4fc218026cd84c61b8b2b40ac5296a6db1f1451ef04c1e", + "sha256:e5f923aa6a47e133d1cf87d60700889d7eae68988704e20c75fb2d65677a8e4b" + ], + "markers": "python_full_version >= '3.5.2'", + "version": "==6.2" + }, + "typing-extensions": { + "hashes": [ + "sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb", + "sha256:fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4" + ], + "markers": "python_version >= '3.7'", + "version": "==4.5.0" + }, + "urllib3": { + "hashes": [ + "sha256:076907bf8fd355cde77728471316625a4d2f7e713c125f51953bb5b3eecf4f72", + "sha256:75edcdc2f7d85b137124a6c3c9fc3933cdeaa12ecb9a6a959f22797a0feca7e1" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", + "version": "==1.26.14" + }, + "uvicorn": { + "hashes": [ + "sha256:3292251b3c7978e8e4a7868f4baf7f7f7bb7e40c759ecc125c37e99cdea34202", + "sha256:7587f7b08bd1efd2b9bad809a3d333e972f1d11af8a5e52a9371ee3a5de71524" + ], + "index": "pypi", + "version": "==0.13.4" + }, + "vine": { + "hashes": [ + "sha256:133ee6d7a9016f177ddeaf191c1f58421a1dcc6ee9a42c58b34bed40e1d2cd87", + "sha256:ea4947cc56d1fd6f2095c8d543ee25dad966f78692528e68b4fada11ba3f98af" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.3.0" + } + }, + "develop": {} +} diff --git a/src/main.py b/src/main.py new file mode 100644 index 0000000..e1d3d7c --- /dev/null +++ b/src/main.py @@ -0,0 +1,49 @@ +from celery.result import AsyncResult +from fastapi import Body, FastAPI, Form, Request +from fastapi.encoders import jsonable_encoder +from fastapi.responses import JSONResponse +from fastapi.staticfiles import StaticFiles +from fastapi.templating import Jinja2Templates +from pydantic.json import pydantic_encoder + + +from worker import create_task, create_archive_task, celery + + +app = FastAPI() +app.mount("/static", StaticFiles(directory="static"), name="static") +templates = Jinja2Templates(directory="templates") + + + +@app.get("/") +def home(request: Request): + return templates.TemplateResponse("home.html", context={"request": request}) + + +@app.post("/tasks", status_code=201) +def run_task(payload = Body(...)): + # task_type = payload["type"] + # task = create_task.delay(int(task_type)) + task = create_archive_task.delay(payload["url"]) + return JSONResponse({"task_id": task.id}) + + +@app.get("/tasks/{task_id}") +def get_status(task_id): + task_result = AsyncResult(task_id, app=celery) + result = { + "task_id": task_id, + "task_status": task_result.status, + "task_result": task_result.result + } + try: + json_result = jsonable_encoder(result, custom_encoder=pydantic_encoder) + return JSONResponse(json_result)#content=json_result) + except Exception as e: + print(e) + print(task_result.result) + return JSONResponse({ + "task_id": task_id, + "task_status": "FAILURE", + }) diff --git a/src/requirements.txt b/src/requirements.txt new file mode 100644 index 0000000..1638b8e --- /dev/null +++ b/src/requirements.txt @@ -0,0 +1,9 @@ +aiofiles==0.6.0 +celery==4.4.7 +fastapi==0.64.0 +flower==0.9.7 +Jinja2==3.0.3 +pytest==6.2.4 +redis==3.5.3 +requests==2.25.1 +uvicorn==0.13.4 diff --git a/src/worker.py b/src/worker.py new file mode 100644 index 0000000..902a7e8 --- /dev/null +++ b/src/worker.py @@ -0,0 +1,32 @@ +import os +import time + +from celery import Celery +from dataclasses import asdict +from auto_archiver import Config, ArchivingOrchestrator, Metadata + + +celery = Celery(__name__) +celery.conf.broker_url = os.environ.get("CELERY_BROKER_URL", "redis://localhost:6379") +celery.conf.result_backend = os.environ.get("CELERY_RESULT_BACKEND", "redis://localhost:6379") + + +@celery.task(name="create_task") +def create_task(task_type): + print("DEV MODE") + time.sleep(int(task_type) * 10) + return True + + +# from configs.v2config import ConfigV2 +# from auto_archiver import ArchivingOrchestrator +config = Config() +config.parse(use_cli=False, yaml_config_filename="orchestration.yaml") +orchestrator = None + +@celery.task(name="create_archive_task") +def create_archive_task(url: str = ""): + global orchestrator + if not orchestrator: orchestrator = ArchivingOrchestrator(config) + return orchestrator.feed_item(Metadata().set_url(url)).to_json() + return True