diff --git a/docker-compose.yml b/docker-compose.yml index 16ab4e6..475c6f8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,13 +33,13 @@ services: redis: image: redis:6-alpine - # command: redis-server /conf/redis.conf # local dev + # command: redis-server /conf/redis.conf # DEV ONLY command: redis-server /conf/redis.conf --requirepass ${REDIS_PASSWORD} volumes: - "./redis/data:/data" - "./redis/config:/conf" restart: always - # TODO: DEV ONLY + # DEV ONLY # ports: # - 6379:6379 diff --git a/src/.example.env b/src/.example.env index 02df5c9..9228b0f 100644 --- a/src/.example.env +++ b/src/.example.env @@ -1,4 +1,6 @@ GOOGLE_CHROME_APP_ID=0000000000000000000000000000000000.apps.googleusercontent.com +# for validating non-email limited JWT +GOOGLE_CHROME_APP_ID_PUBLIC=0000000000000000000000000000000000.apps.googleusercontent.com ALLOWED_EMAILS=email1,email2 ORCHESTRATION_CONFIG_DEFAULT=secrets/orchestration.yaml diff --git a/src/Pipfile b/src/Pipfile index dd8d218..d31f2d0 100644 --- a/src/Pipfile +++ b/src/Pipfile @@ -17,7 +17,7 @@ aiosqlite = "*" python-dotenv = "*" loguru = "*" sqlalchemy = "*" -auto-archiver = ">=0.5.12" +auto-archiver = "*" alembic = "*" fastapi-utils = "*" diff --git a/src/Pipfile.lock b/src/Pipfile.lock index 04f7dcb..36ce21b 100644 --- a/src/Pipfile.lock +++ b/src/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "85ba632faeb7f00b42dfefbbe50a9a8348610a1297fdf70017991c9c2cb7980d" + "sha256": "2db7ae3621bf6c9c8a256e8d34ae667eafaf4e4203c450ca1aac415f5a7cac2f" }, "pipfile-spec": 6, "requires": { @@ -197,18 +197,18 @@ }, "auto-archiver": { "hashes": [ - "sha256:2d28139eaff36b05b99f611ace518b59ae06f439d3a8e104b9251c9f3eb27c50", - "sha256:a7ab3c339295fc2862a0c363638fdbcbf32285ae9c3f0ccc592c3f4312acf165" + "sha256:1b337b9c9dfb7757fd861b3a06de1c4ad3f2e38c17f702ba2ae80d2c19ce6293", + "sha256:2f8695be28e8c8c84c3e83903b6ab90fbd670c3403ed5a59e581ace8ff7c7a98" ], "index": "pypi", - "version": "==0.5.10" + "version": "==0.5.15" }, "beautifulsoup4": { "hashes": [ "sha256:492bbc69dca35d12daac71c4db1bfff0c876c00ef4a2ffacce226d4638eb72da", "sha256:bd2520ca0d9d7d12694a53d44ac482d181b4ec1888909b035a3dbf40d0f57d4a" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==4.12.2" }, "billiard": { @@ -228,19 +228,19 @@ }, "boto3": { "hashes": [ - "sha256:23523d5d6aa51bba2461d67f6eb458d83b6a52d18e3d953b1ce71209b66462ec", - "sha256:ba7ca9215a1026620741273da10d0d3cceb9f7649f7c101e616a287071826f9d" + "sha256:110263d6be92383b9d5de61f4b36f732f0c98080dcfaf0aeb2f53f93b45b030a", + "sha256:82d92428599c107d8cf7c716e4d805193f07585968d261d9751a0e0777d8b178" ], "markers": "python_version >= '3.7'", - "version": "==1.26.135" + "version": "==1.26.136" }, "botocore": { "hashes": [ - "sha256:06502a4473924ef60ac0de908385a5afab9caee6c5b49cf6a330fab0d76ddf5f", - "sha256:0c61d4e5e04fe5329fa65da6b31492ef9d0d5174d72fc2af69de2ed0f87804ca" + "sha256:134c9a84d3ce112fd7ed00b626d6a48f9d232742dffd9ee77f190b53c44bbcc8", + "sha256:1f36cfa1586c9ceeaf2835a886c5272dd81c9467cf1d515f2dc6a82a2a658da0" ], "markers": "python_version >= '3.7'", - "version": "==1.29.135" + "version": "==1.29.136" }, "brotli": { "hashes": [ @@ -507,7 +507,7 @@ "sha256:f8303414c7b03f794347ad062c0516cee0e15f7a612abd0ce1e25caf6ceb47df", "sha256:fca62a8301b605b954ad2e9c3666f9d97f63872aa4efcae5492baca2056b74ab" ], - "markers": "python_version >= '3.7'", + "markers": "python_full_version >= '3.7.0'", "version": "==3.1.0" }, "click": { @@ -825,7 +825,7 @@ "sha256:f82d4d717d8ef19188687aa32b8363e96062911e63ba22a0cff7802a8e58e5f1", "sha256:fc3a569657468b6f3fb60587e48356fe512c1754ca05a564f11366ac9e306526" ], - "markers": "platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32')))))", + "markers": "python_version >= '3' and (platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32'))))))", "version": "==2.0.2" }, "gspread": { @@ -862,11 +862,11 @@ }, "httpx": { "hashes": [ - "sha256:447556b50c1921c351ea54b4fe79d91b724ed2b027462ab9a329465d147d5a4e", - "sha256:507d676fc3e26110d41df7d35ebd8b3b8585052450f4097401c9be59d928c63e" + "sha256:06781eb9ac53cde990577af654bd990a4949de37a28bdb4a230d434f3a30b9bd", + "sha256:5853a43053df830c20f8110c5e69fe44d035d850b2dfe795e196f00fdb774bdd" ], "markers": "python_version >= '3.7'", - "version": "==0.24.0" + "version": "==0.24.1" }, "humanize": { "hashes": [ @@ -1317,42 +1317,41 @@ }, "pycryptodomex": { "hashes": [ - "sha256:0af93aad8d62e810247beedef0261c148790c52f3cd33643791cc6396dd217c1", - "sha256:12056c38e49d972f9c553a3d598425f8a1c1d35b2e4330f89d5ff1ffb70de041", - "sha256:23d83b610bd97704f0cd3acc48d99b76a15c8c1540d8665c94d514a49905bad7", - "sha256:2d4d395f109faba34067a08de36304e846c791808524614c731431ee048fe70a", - "sha256:32e764322e902bbfac49ca1446604d2839381bbbdd5a57920c9daaf2e0b778df", - "sha256:3c2516b42437ae6c7a29ef3ddc73c8d4714e7b6df995b76be4695bbe4b3b5cd2", - "sha256:40e8a11f578bd0851b02719c862d55d3ee18d906c8b68a9c09f8c564d6bb5b92", - "sha256:4b51e826f0a04d832eda0790bbd0665d9bfe73e5a4d8ea93b6a9b38beeebe935", - "sha256:4c4674f4b040321055c596aac926d12f7f6859dfe98cd12f4d9453b43ab6adc8", - "sha256:55eed98b4150a744920597c81b3965b632038781bab8a08a12ea1d004213c600", - "sha256:599bb4ae4bbd614ca05f49bd4e672b7a250b80b13ae1238f05fd0f09d87ed80a", - "sha256:5c23482860302d0d9883404eaaa54b0615eefa5274f70529703e2c43cc571827", - "sha256:64b876d57cb894b31056ad8dd6a6ae1099b117ae07a3d39707221133490e5715", - "sha256:67a3648025e4ddb72d43addab764336ba2e670c8377dba5dd752e42285440d31", - "sha256:6feedf4b0e36b395329b4186a805f60f900129cdf0170e120ecabbfcb763995d", - "sha256:78f0ddd4adc64baa39b416f3637aaf99f45acb0bcdc16706f0cc7ebfc6f10109", - "sha256:7a6651a07f67c28b6e978d63aa3a3fccea0feefed9a8453af3f7421a758461b7", - "sha256:7a8dc3ee7a99aae202a4db52de5a08aa4d01831eb403c4d21da04ec2f79810db", - "sha256:7cc28dd33f1f3662d6da28ead4f9891035f63f49d30267d3b41194c8778997c8", - "sha256:7fa0b52df90343fafe319257b31d909be1d2e8852277fb0376ba89d26d2921db", - "sha256:88b0d5bb87eaf2a31e8a759302b89cf30c97f2f8ca7d83b8c9208abe8acb447a", - "sha256:a4fa037078e92c7cc49f6789a8bac3de06856740bb2038d05f2d9a2e4b165d59", - "sha256:a57e3257bacd719769110f1f70dd901c5b6955e9596ad403af11a3e6e7e3311c", - "sha256:ab33c2d9f275e05e235dbca1063753b5346af4a5cac34a51fa0da0d4edfb21d7", - "sha256:c84689c73358dfc23f9fdcff2cb9e7856e65e2ce3b5ed8ff630d4c9bdeb1867b", - "sha256:c92537b596bd5bffb82f8964cabb9fef1bca8a28a9e0a69ffd3ec92a4a7ad41b", - "sha256:caa937ff29d07a665dfcfd7a84f0d4207b2ebf483362fa9054041d67fdfacc20", - "sha256:d38ab9e53b1c09608ba2d9b8b888f1e75d6f66e2787e437adb1fecbffec6b112", - "sha256:d4cf0128da167562c49b0e034f09e9cedd733997354f2314837c2fa461c87bb1", - "sha256:db23d7341e21b273d2440ec6faf6c8b1ca95c8894da612e165be0b89a8688340", - "sha256:ee8bf4fdcad7d66beb744957db8717afc12d176e3fd9c5d106835133881a049b", - "sha256:f854c8476512cebe6a8681cc4789e4fcff6019c17baa0fd72b459155dc605ab4", - "sha256:fd29d35ac80755e5c0a99d96b44fb9abbd7e871849581ea6a4cb826d24267537" + "sha256:160a39a708c36fa0b168ab79386dede588e62aec06eb505add870739329aecc6", + "sha256:192306cf881fe3467dda0e174a4f47bb3a8bb24b90c9cdfbdc248eec5fc0578c", + "sha256:1949e09ea49b09c36d11a951b16ff2a05a0ffe969dda1846e4686ee342fe8646", + "sha256:215be2980a6b70704c10796dd7003eb4390e7be138ac6fb8344bf47e71a8d470", + "sha256:27072a494ce621cc7a9096bbf60ed66826bb94db24b49b7359509e7951033e74", + "sha256:2dc4eab20f4f04a2d00220fdc9258717b82d31913552e766d5f00282c031b70a", + "sha256:302a8f37c224e7b5d72017d462a2be058e28f7be627bdd854066e16722d0fc0c", + "sha256:3d9314ac785a5b75d5aaf924c5f21d6ca7e8df442e5cf4f0fefad4f6e284d422", + "sha256:3e3ecb5fe979e7c1bb0027e518340acf7ee60415d79295e5251d13c68dde576e", + "sha256:4d9379c684efea80fdab02a3eb0169372bca7db13f9332cb67483b8dc8b67c37", + "sha256:50308fcdbf8345e5ec224a5502b4215178bdb5e95456ead8ab1a69ffd94779cb", + "sha256:5594a125dae30d60e94f37797fc67ce3c744522de7992c7c360d02fdb34918f8", + "sha256:58fc0aceb9c961b9897facec9da24c6a94c5db04597ec832060f53d4d6a07196", + "sha256:6421d23d6a648e83ba2670a352bcd978542dad86829209f59d17a3f087f4afef", + "sha256:6875eb8666f68ddbd39097867325bd22771f595b4e2b0149739b5623c8bf899b", + "sha256:6ed3606832987018615f68e8ed716a7065c09a0fe94afd7c9ca1b6777f0ac6eb", + "sha256:71687eed47df7e965f6e0bf3cadef98f368d5221f0fb89d2132effe1a3e6a194", + "sha256:73d64b32d84cf48d9ec62106aa277dbe99ab5fbfd38c5100bc7bddd3beb569f7", + "sha256:75672205148bdea34669173366df005dbd52be05115e919551ee97171083423d", + "sha256:76f0a46bee539dae4b3dfe37216f678769349576b0080fdbe431d19a02da42ff", + "sha256:8ff129a5a0eb5ff16e45ca4fa70a6051da7f3de303c33b259063c19be0c43d35", + "sha256:ac614363a86cc53d8ba44b6c469831d1555947e69ab3276ae8d6edc219f570f7", + "sha256:ba95abd563b0d1b88401658665a260852a8e6c647026ee6a0a65589287681df8", + "sha256:bbdcce0a226d9205560a5936b05208c709b01d493ed8307792075dedfaaffa5f", + "sha256:bec6c80994d4e7a38312072f89458903b65ec99bed2d65aa4de96d997a53ea7a", + "sha256:c2953afebf282a444c51bf4effe751706b4d0d63d7ca2cc51db21f902aa5b84e", + "sha256:d35a8ffdc8b05e4b353ba281217c8437f02c57d7233363824e9d794cf753c419", + "sha256:d56c9ec41258fd3734db9f5e4d2faeabe48644ba9ca23b18e1839b3bdf093222", + "sha256:d84e105787f5e5d36ec6a581ff37a1048d12e638688074b2a00bcf402f9aa1c2", + "sha256:e00a4bacb83a2627e8210cb353a2e31f04befc1155db2976e5e239dd66482278", + "sha256:f237278836dda412a325e9340ba2e6a84cb0f56b9244781e5b61f10b3905de88", + "sha256:f9ab5ef0718f6a8716695dea16d83b671b22c45e9c0c78fd807c32c0192e54b5" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==3.17" + "version": "==3.18.0" }, "pydantic": { "hashes": [ @@ -1644,7 +1643,7 @@ "sha256:2d11b9b8dd03868f09b4fffadc84a6a8cda574e40dc90821bd845720ebb8e89c", "sha256:69cdf53799e63f38b95b9bf9c875f8c90e78dd62b2f00c13a911c7a3b9fa4704" ], - "markers": "python_version >= '3.7'", + "markers": "python_full_version >= '3.7.0'", "version": "==13.3.5" }, "rsa": { @@ -1652,7 +1651,7 @@ "sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7", "sha256:e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21" ], - "markers": "python_version >= '3.6' and python_version < '4.0'", + "markers": "python_version >= '3.6' and python_version < '4'", "version": "==4.9" }, "s3transfer": { @@ -1791,7 +1790,7 @@ "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'", "version": "==0.10.2" }, "tornado": { @@ -1808,7 +1807,7 @@ "sha256:c2de14066c4a38b4ecbbcd55c5cc4b5340eb04f1c5e81da7451ef555859c833f", "sha256:c367ab6c0393d71171123ca5515c61ff62fe09024fa6bf299cd1339dc9456829" ], - "markers": "python_full_version >= '3.5.2'", + "markers": "python_version >= '3.5.2'", "version": "==6.3.2" }, "tqdm": { @@ -1882,6 +1881,12 @@ "index": "pypi", "version": "==0.22.0" }, + "uwsgi": { + "hashes": [ + "sha256:35a30d83791329429bc04fe44183ce4ab512fcf6968070a7bfba42fc5a0552a9" + ], + "version": "==2.0.21" + }, "vine": { "hashes": [ "sha256:133ee6d7a9016f177ddeaf191c1f58421a1dcc6ee9a42c58b34bed40e1d2cd87", @@ -1994,7 +1999,7 @@ "sha256:ad565f26ecb92588a3e43bc3d96164de84cd9902482b130d0ddbaa9664a85065", "sha256:b9acddd652b585d75b20477888c56642fdade28bdfd3579aa24a4d2c037dd736" ], - "markers": "python_version >= '3.7'", + "markers": "python_full_version >= '3.7.0'", "version": "==1.2.0" }, "yarl": { diff --git a/src/db/schemas.py b/src/db/schemas.py index 157af02..4a9bdc0 100644 --- a/src/db/schemas.py +++ b/src/db/schemas.py @@ -31,6 +31,7 @@ class Archive(ArchiveCreate): # # orm_mode = True class SubmitSheet(BaseModel): - sheet_name: str + sheet_name: str | None= None + sheet_id: str | None = None header: int = 1 \ No newline at end of file diff --git a/src/main.py b/src/main.py index c07f8d9..db16852 100644 --- a/src/main.py +++ b/src/main.py @@ -15,7 +15,7 @@ from worker import create_archive_task, create_sheet_task, celery from db import crud, models, schemas from db.database import engine, SessionLocal from sqlalchemy.orm import Session -from security import get_bearer_auth, get_basic_auth, bearer_security +from security import get_bearer_auth, get_basic_auth, bearer_security, get_bearer_auth_public load_dotenv() @@ -131,7 +131,7 @@ def get_status(task_id, email = Depends(get_bearer_auth)): @app.delete("/tasks/{task_id}") -def get_status(task_id, db: Session = Depends(get_db), email = Depends(get_bearer_auth)): +def get_status(task_id, db: Session = Depends(get_db), email = Depends(get_bearer_auth_public)): logger.info(f"deleting task {task_id} request by {email}") return JSONResponse({ "id": task_id, @@ -139,8 +139,10 @@ def get_status(task_id, db: Session = Depends(get_db), email = Depends(get_beare }) @app.post("/sheet", status_code=201) -def run_task(sheet:schemas.SubmitSheet, basic_auth = Depends(get_basic_auth)): - logger.info("LAUNCHING SHEET TASK") +def run_task(sheet:schemas.SubmitSheet, email = Depends(get_bearer_auth)): + logger.info(f"LAUNCHING SHEET TASK for {email=}") + if not sheet.sheet_name and not sheet.sheet_id: + raise HTTPException(status_code=422, detail=f"sheet name or id is required") task = create_sheet_task.delay(sheet.json()) return JSONResponse({"id": task.id}) diff --git a/src/requirements.txt b/src/requirements.txt index f283eeb..cff2614 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -1,133 +1,134 @@ -# -# These requirements were autogenerated by pipenv -# To regenerate from the project's Pipfile, run: -# -# pipenv lock --requirements -# - # -i https://pypi.org/simple aiofiles==0.6.0 +aiohttp==3.8.4 ; python_version >= '3.6' +aiosignal==1.3.1 ; python_version >= '3.7' aiosqlite==0.19.0 -alembic==1.10.3 -amqp==2.6.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' -anyio==3.6.2; python_full_version >= '3.6.2' +alembic==1.11.1 +amqp==2.6.1 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' +anyio==3.6.2 ; python_full_version >= '3.6.2' argparse==1.4.0 -async-generator==1.10; python_version >= '3.5' -attrs==23.1.0; python_version >= '3.7' +async-generator==1.10 ; python_version >= '3.5' +async-timeout==4.0.2 ; python_version >= '3.6' +attrs==23.1.0 ; python_version >= '3.7' authlib==0.15.6 -auto-archiver==0.5.6 -beautifulsoup4==4.12.2; python_version >= '3.6' +auto-archiver==0.5.15 +beautifulsoup4==4.12.2 ; python_full_version >= '3.6.0' billiard==3.6.4.0 -boto3==1.26.115; python_version >= '3.7' -botocore==1.29.115; python_version >= '3.7' -brotli==1.0.9; platform_python_implementation == 'CPython' +blinker==1.6.2 ; python_version >= '3.7' +boto3==1.26.136 ; python_version >= '3.7' +botocore==1.29.136 ; python_version >= '3.7' +brotli==1.0.9 ; platform_python_implementation == 'CPython' bs4==0.0.1 -cachetools==5.3.0; python_version ~= '3.7' +cachetools==5.3.0 ; python_version ~= '3.7' celery==4.4.7 -certifi==2022.12.7; python_version >= '3.6' +certifi==2023.5.7 ; python_version >= '3.6' cffi==1.15.1 -charset-normalizer==3.0.1 -click==8.1.3; python_version >= '3.7' -cloudscraper==1.2.69 -cryptography==38.0.4; python_version >= '3.6' -dataclasses-json==0.5.7; python_version >= '3.6' -dateparser==1.1.8; python_version >= '3.7' -exceptiongroup==1.1.1; python_version < '3.11' +charset-normalizer==3.1.0 ; python_full_version >= '3.7.0' +click==8.1.3 ; python_version >= '3.7' +cloudscraper==1.2.71 +cryptography==38.0.4 ; python_version >= '3.6' +dataclasses-json==0.5.7 ; python_version >= '3.6' +dateparser==1.1.8 ; python_version >= '3.7' +exceptiongroup==1.1.1 ; python_version < '3.11' +fastapi==0.95.2 fastapi-utils==0.2.1 -fastapi==0.95.1 ffmpeg-python==0.2.0 -filelock==3.12.0; python_version >= '3.7' -flask==2.2.3; python_version >= '3.7' +filelock==3.12.0 ; python_version >= '3.7' +flask==2.3.2 ; python_version >= '3.8' flower==0.9.7 -future==0.18.3; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' -google-api-core==2.11.0; python_version >= '3.7' -google-api-python-client==2.86.0; python_version >= '3.7' +frozenlist==1.3.3 ; python_version >= '3.7' +future==0.18.3 ; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' +google-api-core==2.11.0 ; python_version >= '3.7' +google-api-python-client==2.86.0 ; python_version >= '3.7' +google-auth==2.18.1 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' google-auth-httplib2==0.1.0 -google-auth-oauthlib==1.0.0; python_version >= '3.6' -google-auth==2.17.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' -googleapis-common-protos==1.59.0; python_version >= '3.7' -greenlet==2.0.2; python_version >= '3' and platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32'))))) -gspread==5.8.0; python_version not in '3.0, 3.1, 3.2, 3.3' and python_version >= '3.6' -h11==0.14.0; python_version >= '3.7' -httplib2==0.22.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' -humanize==4.6.0; python_version >= '3.7' -idna==3.4; python_version >= '3.5' -iniconfig==2.0.0; python_version >= '3.7' -instaloader==4.9.6; python_version >= '3.8' -itsdangerous==2.1.2; python_version >= '3.7' +google-auth-oauthlib==1.0.0 ; python_version >= '3.6' +googleapis-common-protos==1.59.0 ; python_version >= '3.7' +greenlet==2.0.2 ; python_version >= '3' and (platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32')))))) +gspread==5.9.0 ; python_version not in '3.0, 3.1, 3.2, 3.3' and python_version >= '3.6' +h11==0.14.0 ; python_version >= '3.7' +httpcore==0.17.1 ; python_version >= '3.7' +httplib2==0.22.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' +httpx==0.24.1 ; python_version >= '3.7' +humanize==4.6.0 ; python_version >= '3.7' +idna==3.4 ; python_version >= '3.5' +iniconfig==2.0.0 ; python_version >= '3.7' +instaloader==4.9.6 ; python_version >= '3.8' +itsdangerous==2.1.2 ; python_version >= '3.7' jinja2==3.1.2 -jmespath==1.0.1; python_version >= '3.7' -kombu==4.6.11; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' +jmespath==1.0.1 ; python_version >= '3.7' +kombu==4.6.11 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' loguru==0.7.0 -lxml==4.9.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' -mako==1.2.4; python_version >= '3.7' -markdown-it-py==2.2.0; python_version >= '3.7' -markupsafe==2.1.2; python_version >= '3.7' +lxml==4.9.2 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' +mako==1.2.4 ; python_version >= '3.7' +markdown-it-py==2.2.0 ; python_version >= '3.7' +markupsafe==2.1.2 ; python_version >= '3.7' +marshmallow==3.19.0 ; python_version >= '3.7' marshmallow-enum==1.5.1 -marshmallow==3.19.0; python_version >= '3.7' -mdurl==0.1.2; python_version >= '3.7' -mutagen==1.46.0; python_version >= '3.7' -mypy-extensions==1.0.0; python_version >= '3.5' +mdurl==0.1.2 ; python_version >= '3.7' +multidict==6.0.4 ; python_version >= '3.7' +mutagen==1.46.0 ; python_version >= '3.7' +mypy-extensions==1.0.0 ; python_version >= '3.5' oauth2client==4.1.3 -oauthlib==3.2.2; python_version >= '3.6' -outcome==1.2.0; python_version >= '3.7' -packaging==23.1; python_version >= '3.7' -pluggy==0.13.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' +oauthlib==3.2.2 ; python_version >= '3.6' +outcome==1.2.0 ; python_version >= '3.7' +packaging==23.1 ; python_version >= '3.7' +pluggy==0.13.1 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' prometheus-client==0.8.0 -protobuf==4.22.3; python_version >= '3.7' -py==1.11.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' +protobuf==4.23.1 ; python_version >= '3.7' +py==1.11.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' pyaes==1.6.1 -pyasn1-modules==0.2.8 -pyasn1==0.4.8 +pyasn1==0.5.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' +pyasn1-modules==0.3.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' pycparser==2.21 -pycryptodomex==3.17; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' -pydantic==1.10.7; python_version >= '3.7' -pygments==2.15.0; python_version >= '3.7' -pyparsing==3.0.9; python_version >= '3.1' +pycryptodomex==3.18.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' +pydantic==1.10.7 ; python_version >= '3.7' +pygments==2.15.1 ; python_version >= '3.7' +pyparsing==3.0.9 ; python_version >= '3.1' pysocks==1.7.1 pytest==6.2.4 -python-dateutil==2.8.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' +python-dateutil==2.8.2 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' python-dotenv==1.0.0 -python-slugify==8.0.1; python_version >= '3.7' -python-twitter-v2==0.8.1; python_version >= '3.6' and python_version < '4' -pytz-deprecation-shim==0.1.0.post0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' +python-slugify==8.0.1 ; python_version >= '3.7' +python-twitter-v2==0.8.1 ; python_version >= '3.6' and python_version < '4.0' pytz==2023.3 -pyyaml==6.0; python_version >= '3.6' +pyyaml==6.0 ; python_version >= '3.6' redis==3.5.3 -regex==2023.3.23; python_version >= '3.8' -requests-oauthlib==1.3.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' -requests-toolbelt==0.10.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' -requests==2.28.2 -rich==13.3.4; python_version >= '3.7' -rsa==4.9; python_version >= '3.6' and python_version < '4' -s3transfer==0.6.0; python_version >= '3.7' -selenium==4.8.3; python_version >= '3.7' -six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' -sniffio==1.3.0; python_version >= '3.7' -snscrape==0.6.2.20230320; python_version ~= '3.8' +regex==2023.5.5 ; python_version >= '3.6' +requests==2.30.0 +requests-oauthlib==1.3.1 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' +requests-toolbelt==1.0.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' +rich==13.3.5 ; python_full_version >= '3.7.0' +rsa==4.9 ; python_version >= '3.6' and python_version < '4' +s3transfer==0.6.1 ; python_version >= '3.7' +selenium==4.9.1 ; python_version >= '3.7' +six==1.16.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' +sniffio==1.3.0 ; python_version >= '3.7' +snscrape==0.6.2.20230320 ; python_version ~= '3.8' sortedcontainers==2.4.0 -soupsieve==2.4.1; python_version >= '3.7' -sqlalchemy==1.4.47 -starlette==0.26.1; python_version >= '3.7' -telethon==1.28.5; python_version >= '3.5' +soupsieve==2.4.1 ; python_version >= '3.7' +sqlalchemy==1.4.48 +starlette==0.27.0 ; python_version >= '3.7' +telethon==1.28.5 ; python_version >= '3.5' text-unidecode==1.3 -tiktok-downloader==0.3.4 -toml==0.10.2; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' -tornado==6.3; python_full_version >= '3.5.2' -tqdm==4.65.0; python_version >= '3.7' -trio-websocket==0.10.2; python_version >= '3.7' -trio==0.22.0; python_version >= '3.7' -typing-extensions==4.5.0; python_version >= '3.7' +tiktok-downloader==0.3.5 +toml==0.10.2 ; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2' +tornado==6.3.2 ; python_version >= '3.5.2' +tqdm==4.65.0 ; python_version >= '3.7' +trio==0.22.0 ; python_version >= '3.7' +trio-websocket==0.10.2 ; python_version >= '3.7' +typing-extensions==4.5.0 ; python_version >= '3.7' typing-inspect==0.8.0 -tzdata==2023.3; python_version >= '3.6' -tzlocal==4.3; python_version >= '3.7' -uritemplate==4.1.1; python_version >= '3.6' -urllib3==1.26.14; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' -uvicorn==0.21.1 -vine==1.3.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' +tzlocal==5.0.1 ; python_version >= '3.7' +uritemplate==4.1.1 ; python_version >= '3.6' +urllib3==1.26.15 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' +uvicorn==0.22.0 +uwsgi==2.0.21 +vine==1.3.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' vk-api==11.9.9 -vk-url-scraper==0.3.15; python_version >= '3.7' -websockets==10.4; python_version >= '3.7' -werkzeug==2.2.3; python_version >= '3.7' -wsproto==1.2.0; python_version >= '3.7' -yt-dlp==2023.2.17; python_version >= '3.7' +vk-url-scraper==0.3.24 ; python_version >= '3.7' +websockets==11.0.3 ; python_version >= '3.7' +werkzeug==2.3.4 ; python_version >= '3.8' +wsproto==1.2.0 ; python_full_version >= '3.7.0' +yarl==1.9.2 ; python_version >= '3.7' +yt-dlp==2023.3.4 ; python_version >= '3.7' diff --git a/src/security.py b/src/security.py index 0d3fbee..97c0c0f 100644 --- a/src/security.py +++ b/src/security.py @@ -8,8 +8,10 @@ from fastapi.security import HTTPBasic, HTTPBasicCredentials, HTTPBearer, HTTPAu # Configuration GOOGLE_CHROME_APP_ID = os.environ.get("GOOGLE_CHROME_APP_ID") assert len(GOOGLE_CHROME_APP_ID)>10, "GOOGLE_CHROME_APP_ID env variable not set" +GOOGLE_CHROME_APP_ID_PUBLIC = os.environ.get("GOOGLE_CHROME_APP_ID_PUBLIC") +assert len(GOOGLE_CHROME_APP_ID_PUBLIC)>10, "GOOGLE_CHROME_APP_ID_PUBLIC env variable not set" ALLOWED_EMAILS = set([e.strip().lower() for e in os.environ.get("ALLOWED_EMAILS", "").split(",")]) -assert len(GOOGLE_CHROME_APP_ID)>=1, "at least one ALLOWED_EMAILS is required from the env variable" +assert len(ALLOWED_EMAILS)>=1, "at least one ALLOWED_EMAILS is required from the env variable" logger.info(f"{len(ALLOWED_EMAILS)=}") basic_security = HTTPBasic() @@ -54,6 +56,38 @@ def custom_is_email_allowed(email, any_bellingcat_email=False): return email.lower() in ALLOWED_EMAILS or (any_bellingcat_email and re.match(r'^[\w.]+@bellingcat\.com$', email)) +#--------------------- Bearer Auth ANY EMAIL + +async def get_bearer_auth_public(credentials: HTTPAuthorizationCredentials = Depends(bearer_security)): + # validates the Bearer token in the case that it requires it + access_token = credentials.credentials + valid_user, info = authenticate_user_public(access_token) + if valid_user: return info + logger.debug(f"TOKEN FAILURE: {valid_user=} {info=}") + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail=info, + headers={"WWW-Authenticate": "Bearer"}, + ) + +def authenticate_user_public(access_token): + # https://cloud.google.com/docs/authentication/token-types#access + if type(access_token)!=str or len(access_token)<10: return False, "invalid access_token" + r = requests.get("https://oauth2.googleapis.com/tokeninfo", {"access_token":access_token}) + if r.status_code!=200: return False, "error occurred" + try: + j = r.json() + if j.get("azp") != GOOGLE_CHROME_APP_ID_PUBLIC and j.get("aud")!=GOOGLE_CHROME_APP_ID_PUBLIC: + return False, f"token does not belong to correct APP_ID" + if j.get("email_verified") != "true": + return False, f"email '{j.get('email')}' not verified" + if int(j.get("expires_in", -1)) <= 0: + return False, "Token expired" + return True, j.get('email') + except Exception as e: + logger.warning(f"EXCEPTION occurred: {e}") + return False, f"EXCEPTION occurred" + #--------------------- Basic Auth SFP = os.environ.get("STATIC_FILE_PASSWORD", "") # min length is 20 chars async def get_basic_auth(credentials: HTTPBasicCredentials = Depends(basic_security)): diff --git a/src/worker.py b/src/worker.py index d1f2ba9..7980da0 100644 --- a/src/worker.py +++ b/src/worker.py @@ -1,12 +1,11 @@ -import os, re, traceback, yaml +import os, re, traceback, yaml, datetime from celery import Celery, states from celery.exceptions import Ignore from celery.signals import task_failure from auto_archiver import Config, ArchivingOrchestrator, Metadata # from auto_archiver.enrichers import ScreenshotEnricher -from auto_archiver.feeders import GsheetsFeeder from loguru import logger from db import crud, schemas, models @@ -63,18 +62,13 @@ def create_archive_task(self, archive_json: str): def create_sheet_task(self, sheet_json: str): logger.info(f"STARTING {sheet_json}") sheet = schemas.SubmitSheet.parse_raw(sheet_json) - orchestrator = choose_orchestrator("bellingcat", "") - #TODO: modify archiver to accept sheetId too config = Config() - config.parse(use_cli=False, yaml_config_filename="secrets/orchestration-sheet.yaml") - config.feeder.sheet = sheet.sheet_name - config.feeder.header = sheet.header + config.parse(use_cli=False, yaml_config_filename="secrets/orchestration-sheet.yaml", overwrite_configs={"configurations": {"gsheet_feeder": {"sheet": sheet.sheet_name, "sheet_id": sheet.sheet_id, "header": sheet.header}}}) orchestrator = ArchivingOrchestrator(config) - #TODO: make auto-archiver config.parse receive an overriding dict - result = orchestrator.feed() # TODO: save into local DB + orchestrator.feed() - return result.to_json() + return {"success": True, "sheet": sheet.sheet_name, "sheet_id": sheet.sheet_id, "time": datetime.datetime.now().isoformat()} @task_failure.connect(sender=create_archive_task) def task_failure_notifier(sender=None, **kwargs):