mirror of
https://github.com/bellingcat/auto-archiver-api.git
synced 2026-06-11 13:08:34 +03:00
sheet processing implemented
This commit is contained in:
msramalho
@@ -33,13 +33,13 @@ services:
|
||||
|
||||
redis:
|
||||
image: redis:6-alpine
|
||||
# command: redis-server /conf/redis.conf # local dev
|
||||
# command: redis-server /conf/redis.conf # DEV ONLY
|
||||
command: redis-server /conf/redis.conf --requirepass ${REDIS_PASSWORD}
|
||||
volumes:
|
||||
- "./redis/data:/data"
|
||||
- "./redis/config:/conf"
|
||||
restart: always
|
||||
# TODO: DEV ONLY
|
||||
# DEV ONLY
|
||||
# ports:
|
||||
# - 6379:6379
|
||||
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
GOOGLE_CHROME_APP_ID=0000000000000000000000000000000000.apps.googleusercontent.com
|
||||
# for validating non-email limited JWT
|
||||
GOOGLE_CHROME_APP_ID_PUBLIC=0000000000000000000000000000000000.apps.googleusercontent.com
|
||||
|
||||
ALLOWED_EMAILS=email1,email2
|
||||
ORCHESTRATION_CONFIG_DEFAULT=secrets/orchestration.yaml
|
||||
|
||||
@@ -17,7 +17,7 @@ aiosqlite = "*"
|
||||
python-dotenv = "*"
|
||||
loguru = "*"
|
||||
sqlalchemy = "*"
|
||||
auto-archiver = ">=0.5.12"
|
||||
auto-archiver = "*"
|
||||
alembic = "*"
|
||||
fastapi-utils = "*"
|
||||
|
||||
|
||||
115
src/Pipfile.lock
generated
115
src/Pipfile.lock
generated
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"_meta": {
|
||||
"hash": {
|
||||
"sha256": "85ba632faeb7f00b42dfefbbe50a9a8348610a1297fdf70017991c9c2cb7980d"
|
||||
"sha256": "2db7ae3621bf6c9c8a256e8d34ae667eafaf4e4203c450ca1aac415f5a7cac2f"
|
||||
},
|
||||
"pipfile-spec": 6,
|
||||
"requires": {
|
||||
@@ -197,18 +197,18 @@
|
||||
},
|
||||
"auto-archiver": {
|
||||
"hashes": [
|
||||
"sha256:2d28139eaff36b05b99f611ace518b59ae06f439d3a8e104b9251c9f3eb27c50",
|
||||
"sha256:a7ab3c339295fc2862a0c363638fdbcbf32285ae9c3f0ccc592c3f4312acf165"
|
||||
"sha256:1b337b9c9dfb7757fd861b3a06de1c4ad3f2e38c17f702ba2ae80d2c19ce6293",
|
||||
"sha256:2f8695be28e8c8c84c3e83903b6ab90fbd670c3403ed5a59e581ace8ff7c7a98"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==0.5.10"
|
||||
"version": "==0.5.15"
|
||||
},
|
||||
"beautifulsoup4": {
|
||||
"hashes": [
|
||||
"sha256:492bbc69dca35d12daac71c4db1bfff0c876c00ef4a2ffacce226d4638eb72da",
|
||||
"sha256:bd2520ca0d9d7d12694a53d44ac482d181b4ec1888909b035a3dbf40d0f57d4a"
|
||||
],
|
||||
"markers": "python_version >= '3.6'",
|
||||
"markers": "python_full_version >= '3.6.0'",
|
||||
"version": "==4.12.2"
|
||||
},
|
||||
"billiard": {
|
||||
@@ -228,19 +228,19 @@
|
||||
},
|
||||
"boto3": {
|
||||
"hashes": [
|
||||
"sha256:23523d5d6aa51bba2461d67f6eb458d83b6a52d18e3d953b1ce71209b66462ec",
|
||||
"sha256:ba7ca9215a1026620741273da10d0d3cceb9f7649f7c101e616a287071826f9d"
|
||||
"sha256:110263d6be92383b9d5de61f4b36f732f0c98080dcfaf0aeb2f53f93b45b030a",
|
||||
"sha256:82d92428599c107d8cf7c716e4d805193f07585968d261d9751a0e0777d8b178"
|
||||
],
|
||||
"markers": "python_version >= '3.7'",
|
||||
"version": "==1.26.135"
|
||||
"version": "==1.26.136"
|
||||
},
|
||||
"botocore": {
|
||||
"hashes": [
|
||||
"sha256:06502a4473924ef60ac0de908385a5afab9caee6c5b49cf6a330fab0d76ddf5f",
|
||||
"sha256:0c61d4e5e04fe5329fa65da6b31492ef9d0d5174d72fc2af69de2ed0f87804ca"
|
||||
"sha256:134c9a84d3ce112fd7ed00b626d6a48f9d232742dffd9ee77f190b53c44bbcc8",
|
||||
"sha256:1f36cfa1586c9ceeaf2835a886c5272dd81c9467cf1d515f2dc6a82a2a658da0"
|
||||
],
|
||||
"markers": "python_version >= '3.7'",
|
||||
"version": "==1.29.135"
|
||||
"version": "==1.29.136"
|
||||
},
|
||||
"brotli": {
|
||||
"hashes": [
|
||||
@@ -507,7 +507,7 @@
|
||||
"sha256:f8303414c7b03f794347ad062c0516cee0e15f7a612abd0ce1e25caf6ceb47df",
|
||||
"sha256:fca62a8301b605b954ad2e9c3666f9d97f63872aa4efcae5492baca2056b74ab"
|
||||
],
|
||||
"markers": "python_version >= '3.7'",
|
||||
"markers": "python_full_version >= '3.7.0'",
|
||||
"version": "==3.1.0"
|
||||
},
|
||||
"click": {
|
||||
@@ -825,7 +825,7 @@
|
||||
"sha256:f82d4d717d8ef19188687aa32b8363e96062911e63ba22a0cff7802a8e58e5f1",
|
||||
"sha256:fc3a569657468b6f3fb60587e48356fe512c1754ca05a564f11366ac9e306526"
|
||||
],
|
||||
"markers": "platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32')))))",
|
||||
"markers": "python_version >= '3' and (platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32'))))))",
|
||||
"version": "==2.0.2"
|
||||
},
|
||||
"gspread": {
|
||||
@@ -862,11 +862,11 @@
|
||||
},
|
||||
"httpx": {
|
||||
"hashes": [
|
||||
"sha256:447556b50c1921c351ea54b4fe79d91b724ed2b027462ab9a329465d147d5a4e",
|
||||
"sha256:507d676fc3e26110d41df7d35ebd8b3b8585052450f4097401c9be59d928c63e"
|
||||
"sha256:06781eb9ac53cde990577af654bd990a4949de37a28bdb4a230d434f3a30b9bd",
|
||||
"sha256:5853a43053df830c20f8110c5e69fe44d035d850b2dfe795e196f00fdb774bdd"
|
||||
],
|
||||
"markers": "python_version >= '3.7'",
|
||||
"version": "==0.24.0"
|
||||
"version": "==0.24.1"
|
||||
},
|
||||
"humanize": {
|
||||
"hashes": [
|
||||
@@ -1317,42 +1317,41 @@
|
||||
},
|
||||
"pycryptodomex": {
|
||||
"hashes": [
|
||||
"sha256:0af93aad8d62e810247beedef0261c148790c52f3cd33643791cc6396dd217c1",
|
||||
"sha256:12056c38e49d972f9c553a3d598425f8a1c1d35b2e4330f89d5ff1ffb70de041",
|
||||
"sha256:23d83b610bd97704f0cd3acc48d99b76a15c8c1540d8665c94d514a49905bad7",
|
||||
"sha256:2d4d395f109faba34067a08de36304e846c791808524614c731431ee048fe70a",
|
||||
"sha256:32e764322e902bbfac49ca1446604d2839381bbbdd5a57920c9daaf2e0b778df",
|
||||
"sha256:3c2516b42437ae6c7a29ef3ddc73c8d4714e7b6df995b76be4695bbe4b3b5cd2",
|
||||
"sha256:40e8a11f578bd0851b02719c862d55d3ee18d906c8b68a9c09f8c564d6bb5b92",
|
||||
"sha256:4b51e826f0a04d832eda0790bbd0665d9bfe73e5a4d8ea93b6a9b38beeebe935",
|
||||
"sha256:4c4674f4b040321055c596aac926d12f7f6859dfe98cd12f4d9453b43ab6adc8",
|
||||
"sha256:55eed98b4150a744920597c81b3965b632038781bab8a08a12ea1d004213c600",
|
||||
"sha256:599bb4ae4bbd614ca05f49bd4e672b7a250b80b13ae1238f05fd0f09d87ed80a",
|
||||
"sha256:5c23482860302d0d9883404eaaa54b0615eefa5274f70529703e2c43cc571827",
|
||||
"sha256:64b876d57cb894b31056ad8dd6a6ae1099b117ae07a3d39707221133490e5715",
|
||||
"sha256:67a3648025e4ddb72d43addab764336ba2e670c8377dba5dd752e42285440d31",
|
||||
"sha256:6feedf4b0e36b395329b4186a805f60f900129cdf0170e120ecabbfcb763995d",
|
||||
"sha256:78f0ddd4adc64baa39b416f3637aaf99f45acb0bcdc16706f0cc7ebfc6f10109",
|
||||
"sha256:7a6651a07f67c28b6e978d63aa3a3fccea0feefed9a8453af3f7421a758461b7",
|
||||
"sha256:7a8dc3ee7a99aae202a4db52de5a08aa4d01831eb403c4d21da04ec2f79810db",
|
||||
"sha256:7cc28dd33f1f3662d6da28ead4f9891035f63f49d30267d3b41194c8778997c8",
|
||||
"sha256:7fa0b52df90343fafe319257b31d909be1d2e8852277fb0376ba89d26d2921db",
|
||||
"sha256:88b0d5bb87eaf2a31e8a759302b89cf30c97f2f8ca7d83b8c9208abe8acb447a",
|
||||
"sha256:a4fa037078e92c7cc49f6789a8bac3de06856740bb2038d05f2d9a2e4b165d59",
|
||||
"sha256:a57e3257bacd719769110f1f70dd901c5b6955e9596ad403af11a3e6e7e3311c",
|
||||
"sha256:ab33c2d9f275e05e235dbca1063753b5346af4a5cac34a51fa0da0d4edfb21d7",
|
||||
"sha256:c84689c73358dfc23f9fdcff2cb9e7856e65e2ce3b5ed8ff630d4c9bdeb1867b",
|
||||
"sha256:c92537b596bd5bffb82f8964cabb9fef1bca8a28a9e0a69ffd3ec92a4a7ad41b",
|
||||
"sha256:caa937ff29d07a665dfcfd7a84f0d4207b2ebf483362fa9054041d67fdfacc20",
|
||||
"sha256:d38ab9e53b1c09608ba2d9b8b888f1e75d6f66e2787e437adb1fecbffec6b112",
|
||||
"sha256:d4cf0128da167562c49b0e034f09e9cedd733997354f2314837c2fa461c87bb1",
|
||||
"sha256:db23d7341e21b273d2440ec6faf6c8b1ca95c8894da612e165be0b89a8688340",
|
||||
"sha256:ee8bf4fdcad7d66beb744957db8717afc12d176e3fd9c5d106835133881a049b",
|
||||
"sha256:f854c8476512cebe6a8681cc4789e4fcff6019c17baa0fd72b459155dc605ab4",
|
||||
"sha256:fd29d35ac80755e5c0a99d96b44fb9abbd7e871849581ea6a4cb826d24267537"
|
||||
"sha256:160a39a708c36fa0b168ab79386dede588e62aec06eb505add870739329aecc6",
|
||||
"sha256:192306cf881fe3467dda0e174a4f47bb3a8bb24b90c9cdfbdc248eec5fc0578c",
|
||||
"sha256:1949e09ea49b09c36d11a951b16ff2a05a0ffe969dda1846e4686ee342fe8646",
|
||||
"sha256:215be2980a6b70704c10796dd7003eb4390e7be138ac6fb8344bf47e71a8d470",
|
||||
"sha256:27072a494ce621cc7a9096bbf60ed66826bb94db24b49b7359509e7951033e74",
|
||||
"sha256:2dc4eab20f4f04a2d00220fdc9258717b82d31913552e766d5f00282c031b70a",
|
||||
"sha256:302a8f37c224e7b5d72017d462a2be058e28f7be627bdd854066e16722d0fc0c",
|
||||
"sha256:3d9314ac785a5b75d5aaf924c5f21d6ca7e8df442e5cf4f0fefad4f6e284d422",
|
||||
"sha256:3e3ecb5fe979e7c1bb0027e518340acf7ee60415d79295e5251d13c68dde576e",
|
||||
"sha256:4d9379c684efea80fdab02a3eb0169372bca7db13f9332cb67483b8dc8b67c37",
|
||||
"sha256:50308fcdbf8345e5ec224a5502b4215178bdb5e95456ead8ab1a69ffd94779cb",
|
||||
"sha256:5594a125dae30d60e94f37797fc67ce3c744522de7992c7c360d02fdb34918f8",
|
||||
"sha256:58fc0aceb9c961b9897facec9da24c6a94c5db04597ec832060f53d4d6a07196",
|
||||
"sha256:6421d23d6a648e83ba2670a352bcd978542dad86829209f59d17a3f087f4afef",
|
||||
"sha256:6875eb8666f68ddbd39097867325bd22771f595b4e2b0149739b5623c8bf899b",
|
||||
"sha256:6ed3606832987018615f68e8ed716a7065c09a0fe94afd7c9ca1b6777f0ac6eb",
|
||||
"sha256:71687eed47df7e965f6e0bf3cadef98f368d5221f0fb89d2132effe1a3e6a194",
|
||||
"sha256:73d64b32d84cf48d9ec62106aa277dbe99ab5fbfd38c5100bc7bddd3beb569f7",
|
||||
"sha256:75672205148bdea34669173366df005dbd52be05115e919551ee97171083423d",
|
||||
"sha256:76f0a46bee539dae4b3dfe37216f678769349576b0080fdbe431d19a02da42ff",
|
||||
"sha256:8ff129a5a0eb5ff16e45ca4fa70a6051da7f3de303c33b259063c19be0c43d35",
|
||||
"sha256:ac614363a86cc53d8ba44b6c469831d1555947e69ab3276ae8d6edc219f570f7",
|
||||
"sha256:ba95abd563b0d1b88401658665a260852a8e6c647026ee6a0a65589287681df8",
|
||||
"sha256:bbdcce0a226d9205560a5936b05208c709b01d493ed8307792075dedfaaffa5f",
|
||||
"sha256:bec6c80994d4e7a38312072f89458903b65ec99bed2d65aa4de96d997a53ea7a",
|
||||
"sha256:c2953afebf282a444c51bf4effe751706b4d0d63d7ca2cc51db21f902aa5b84e",
|
||||
"sha256:d35a8ffdc8b05e4b353ba281217c8437f02c57d7233363824e9d794cf753c419",
|
||||
"sha256:d56c9ec41258fd3734db9f5e4d2faeabe48644ba9ca23b18e1839b3bdf093222",
|
||||
"sha256:d84e105787f5e5d36ec6a581ff37a1048d12e638688074b2a00bcf402f9aa1c2",
|
||||
"sha256:e00a4bacb83a2627e8210cb353a2e31f04befc1155db2976e5e239dd66482278",
|
||||
"sha256:f237278836dda412a325e9340ba2e6a84cb0f56b9244781e5b61f10b3905de88",
|
||||
"sha256:f9ab5ef0718f6a8716695dea16d83b671b22c45e9c0c78fd807c32c0192e54b5"
|
||||
],
|
||||
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
|
||||
"version": "==3.17"
|
||||
"version": "==3.18.0"
|
||||
},
|
||||
"pydantic": {
|
||||
"hashes": [
|
||||
@@ -1644,7 +1643,7 @@
|
||||
"sha256:2d11b9b8dd03868f09b4fffadc84a6a8cda574e40dc90821bd845720ebb8e89c",
|
||||
"sha256:69cdf53799e63f38b95b9bf9c875f8c90e78dd62b2f00c13a911c7a3b9fa4704"
|
||||
],
|
||||
"markers": "python_version >= '3.7'",
|
||||
"markers": "python_full_version >= '3.7.0'",
|
||||
"version": "==13.3.5"
|
||||
},
|
||||
"rsa": {
|
||||
@@ -1652,7 +1651,7 @@
|
||||
"sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7",
|
||||
"sha256:e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21"
|
||||
],
|
||||
"markers": "python_version >= '3.6' and python_version < '4.0'",
|
||||
"markers": "python_version >= '3.6' and python_version < '4'",
|
||||
"version": "==4.9"
|
||||
},
|
||||
"s3transfer": {
|
||||
@@ -1791,7 +1790,7 @@
|
||||
"sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b",
|
||||
"sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"
|
||||
],
|
||||
"markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
|
||||
"markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'",
|
||||
"version": "==0.10.2"
|
||||
},
|
||||
"tornado": {
|
||||
@@ -1808,7 +1807,7 @@
|
||||
"sha256:c2de14066c4a38b4ecbbcd55c5cc4b5340eb04f1c5e81da7451ef555859c833f",
|
||||
"sha256:c367ab6c0393d71171123ca5515c61ff62fe09024fa6bf299cd1339dc9456829"
|
||||
],
|
||||
"markers": "python_full_version >= '3.5.2'",
|
||||
"markers": "python_version >= '3.5.2'",
|
||||
"version": "==6.3.2"
|
||||
},
|
||||
"tqdm": {
|
||||
@@ -1882,6 +1881,12 @@
|
||||
"index": "pypi",
|
||||
"version": "==0.22.0"
|
||||
},
|
||||
"uwsgi": {
|
||||
"hashes": [
|
||||
"sha256:35a30d83791329429bc04fe44183ce4ab512fcf6968070a7bfba42fc5a0552a9"
|
||||
],
|
||||
"version": "==2.0.21"
|
||||
},
|
||||
"vine": {
|
||||
"hashes": [
|
||||
"sha256:133ee6d7a9016f177ddeaf191c1f58421a1dcc6ee9a42c58b34bed40e1d2cd87",
|
||||
@@ -1994,7 +1999,7 @@
|
||||
"sha256:ad565f26ecb92588a3e43bc3d96164de84cd9902482b130d0ddbaa9664a85065",
|
||||
"sha256:b9acddd652b585d75b20477888c56642fdade28bdfd3579aa24a4d2c037dd736"
|
||||
],
|
||||
"markers": "python_version >= '3.7'",
|
||||
"markers": "python_full_version >= '3.7.0'",
|
||||
"version": "==1.2.0"
|
||||
},
|
||||
"yarl": {
|
||||
|
||||
@@ -31,6 +31,7 @@ class Archive(ArchiveCreate):
|
||||
# # orm_mode = True
|
||||
|
||||
class SubmitSheet(BaseModel):
|
||||
sheet_name: str
|
||||
sheet_name: str | None= None
|
||||
sheet_id: str | None = None
|
||||
header: int = 1
|
||||
|
||||
10
src/main.py
10
src/main.py
@@ -15,7 +15,7 @@ from worker import create_archive_task, create_sheet_task, celery
|
||||
from db import crud, models, schemas
|
||||
from db.database import engine, SessionLocal
|
||||
from sqlalchemy.orm import Session
|
||||
from security import get_bearer_auth, get_basic_auth, bearer_security
|
||||
from security import get_bearer_auth, get_basic_auth, bearer_security, get_bearer_auth_public
|
||||
|
||||
load_dotenv()
|
||||
|
||||
@@ -131,7 +131,7 @@ def get_status(task_id, email = Depends(get_bearer_auth)):
|
||||
|
||||
|
||||
@app.delete("/tasks/{task_id}")
|
||||
def get_status(task_id, db: Session = Depends(get_db), email = Depends(get_bearer_auth)):
|
||||
def get_status(task_id, db: Session = Depends(get_db), email = Depends(get_bearer_auth_public)):
|
||||
logger.info(f"deleting task {task_id} request by {email}")
|
||||
return JSONResponse({
|
||||
"id": task_id,
|
||||
@@ -139,8 +139,10 @@ def get_status(task_id, db: Session = Depends(get_db), email = Depends(get_beare
|
||||
})
|
||||
|
||||
@app.post("/sheet", status_code=201)
|
||||
def run_task(sheet:schemas.SubmitSheet, basic_auth = Depends(get_basic_auth)):
|
||||
logger.info("LAUNCHING SHEET TASK")
|
||||
def run_task(sheet:schemas.SubmitSheet, email = Depends(get_bearer_auth)):
|
||||
logger.info(f"LAUNCHING SHEET TASK for {email=}")
|
||||
if not sheet.sheet_name and not sheet.sheet_id:
|
||||
raise HTTPException(status_code=422, detail=f"sheet name or id is required")
|
||||
task = create_sheet_task.delay(sheet.json())
|
||||
return JSONResponse({"id": task.id})
|
||||
|
||||
|
||||
@@ -1,133 +1,134 @@
|
||||
#
|
||||
# These requirements were autogenerated by pipenv
|
||||
# To regenerate from the project's Pipfile, run:
|
||||
#
|
||||
# pipenv lock --requirements
|
||||
#
|
||||
|
||||
# -i https://pypi.org/simple
|
||||
aiofiles==0.6.0
|
||||
aiohttp==3.8.4 ; python_version >= '3.6'
|
||||
aiosignal==1.3.1 ; python_version >= '3.7'
|
||||
aiosqlite==0.19.0
|
||||
alembic==1.10.3
|
||||
amqp==2.6.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
|
||||
anyio==3.6.2; python_full_version >= '3.6.2'
|
||||
alembic==1.11.1
|
||||
amqp==2.6.1 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
|
||||
anyio==3.6.2 ; python_full_version >= '3.6.2'
|
||||
argparse==1.4.0
|
||||
async-generator==1.10; python_version >= '3.5'
|
||||
attrs==23.1.0; python_version >= '3.7'
|
||||
async-generator==1.10 ; python_version >= '3.5'
|
||||
async-timeout==4.0.2 ; python_version >= '3.6'
|
||||
attrs==23.1.0 ; python_version >= '3.7'
|
||||
authlib==0.15.6
|
||||
auto-archiver==0.5.6
|
||||
beautifulsoup4==4.12.2; python_version >= '3.6'
|
||||
auto-archiver==0.5.15
|
||||
beautifulsoup4==4.12.2 ; python_full_version >= '3.6.0'
|
||||
billiard==3.6.4.0
|
||||
boto3==1.26.115; python_version >= '3.7'
|
||||
botocore==1.29.115; python_version >= '3.7'
|
||||
brotli==1.0.9; platform_python_implementation == 'CPython'
|
||||
blinker==1.6.2 ; python_version >= '3.7'
|
||||
boto3==1.26.136 ; python_version >= '3.7'
|
||||
botocore==1.29.136 ; python_version >= '3.7'
|
||||
brotli==1.0.9 ; platform_python_implementation == 'CPython'
|
||||
bs4==0.0.1
|
||||
cachetools==5.3.0; python_version ~= '3.7'
|
||||
cachetools==5.3.0 ; python_version ~= '3.7'
|
||||
celery==4.4.7
|
||||
certifi==2022.12.7; python_version >= '3.6'
|
||||
certifi==2023.5.7 ; python_version >= '3.6'
|
||||
cffi==1.15.1
|
||||
charset-normalizer==3.0.1
|
||||
click==8.1.3; python_version >= '3.7'
|
||||
cloudscraper==1.2.69
|
||||
cryptography==38.0.4; python_version >= '3.6'
|
||||
dataclasses-json==0.5.7; python_version >= '3.6'
|
||||
dateparser==1.1.8; python_version >= '3.7'
|
||||
exceptiongroup==1.1.1; python_version < '3.11'
|
||||
charset-normalizer==3.1.0 ; python_full_version >= '3.7.0'
|
||||
click==8.1.3 ; python_version >= '3.7'
|
||||
cloudscraper==1.2.71
|
||||
cryptography==38.0.4 ; python_version >= '3.6'
|
||||
dataclasses-json==0.5.7 ; python_version >= '3.6'
|
||||
dateparser==1.1.8 ; python_version >= '3.7'
|
||||
exceptiongroup==1.1.1 ; python_version < '3.11'
|
||||
fastapi==0.95.2
|
||||
fastapi-utils==0.2.1
|
||||
fastapi==0.95.1
|
||||
ffmpeg-python==0.2.0
|
||||
filelock==3.12.0; python_version >= '3.7'
|
||||
flask==2.2.3; python_version >= '3.7'
|
||||
filelock==3.12.0 ; python_version >= '3.7'
|
||||
flask==2.3.2 ; python_version >= '3.8'
|
||||
flower==0.9.7
|
||||
future==0.18.3; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
google-api-core==2.11.0; python_version >= '3.7'
|
||||
google-api-python-client==2.86.0; python_version >= '3.7'
|
||||
frozenlist==1.3.3 ; python_version >= '3.7'
|
||||
future==0.18.3 ; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
google-api-core==2.11.0 ; python_version >= '3.7'
|
||||
google-api-python-client==2.86.0 ; python_version >= '3.7'
|
||||
google-auth==2.18.1 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
|
||||
google-auth-httplib2==0.1.0
|
||||
google-auth-oauthlib==1.0.0; python_version >= '3.6'
|
||||
google-auth==2.17.3; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
|
||||
googleapis-common-protos==1.59.0; python_version >= '3.7'
|
||||
greenlet==2.0.2; python_version >= '3' and platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32')))))
|
||||
gspread==5.8.0; python_version not in '3.0, 3.1, 3.2, 3.3' and python_version >= '3.6'
|
||||
h11==0.14.0; python_version >= '3.7'
|
||||
httplib2==0.22.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
humanize==4.6.0; python_version >= '3.7'
|
||||
idna==3.4; python_version >= '3.5'
|
||||
iniconfig==2.0.0; python_version >= '3.7'
|
||||
instaloader==4.9.6; python_version >= '3.8'
|
||||
itsdangerous==2.1.2; python_version >= '3.7'
|
||||
google-auth-oauthlib==1.0.0 ; python_version >= '3.6'
|
||||
googleapis-common-protos==1.59.0 ; python_version >= '3.7'
|
||||
greenlet==2.0.2 ; python_version >= '3' and (platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32'))))))
|
||||
gspread==5.9.0 ; python_version not in '3.0, 3.1, 3.2, 3.3' and python_version >= '3.6'
|
||||
h11==0.14.0 ; python_version >= '3.7'
|
||||
httpcore==0.17.1 ; python_version >= '3.7'
|
||||
httplib2==0.22.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
httpx==0.24.1 ; python_version >= '3.7'
|
||||
humanize==4.6.0 ; python_version >= '3.7'
|
||||
idna==3.4 ; python_version >= '3.5'
|
||||
iniconfig==2.0.0 ; python_version >= '3.7'
|
||||
instaloader==4.9.6 ; python_version >= '3.8'
|
||||
itsdangerous==2.1.2 ; python_version >= '3.7'
|
||||
jinja2==3.1.2
|
||||
jmespath==1.0.1; python_version >= '3.7'
|
||||
kombu==4.6.11; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
|
||||
jmespath==1.0.1 ; python_version >= '3.7'
|
||||
kombu==4.6.11 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
|
||||
loguru==0.7.0
|
||||
lxml==4.9.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
|
||||
mako==1.2.4; python_version >= '3.7'
|
||||
markdown-it-py==2.2.0; python_version >= '3.7'
|
||||
markupsafe==2.1.2; python_version >= '3.7'
|
||||
lxml==4.9.2 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
|
||||
mako==1.2.4 ; python_version >= '3.7'
|
||||
markdown-it-py==2.2.0 ; python_version >= '3.7'
|
||||
markupsafe==2.1.2 ; python_version >= '3.7'
|
||||
marshmallow==3.19.0 ; python_version >= '3.7'
|
||||
marshmallow-enum==1.5.1
|
||||
marshmallow==3.19.0; python_version >= '3.7'
|
||||
mdurl==0.1.2; python_version >= '3.7'
|
||||
mutagen==1.46.0; python_version >= '3.7'
|
||||
mypy-extensions==1.0.0; python_version >= '3.5'
|
||||
mdurl==0.1.2 ; python_version >= '3.7'
|
||||
multidict==6.0.4 ; python_version >= '3.7'
|
||||
mutagen==1.46.0 ; python_version >= '3.7'
|
||||
mypy-extensions==1.0.0 ; python_version >= '3.5'
|
||||
oauth2client==4.1.3
|
||||
oauthlib==3.2.2; python_version >= '3.6'
|
||||
outcome==1.2.0; python_version >= '3.7'
|
||||
packaging==23.1; python_version >= '3.7'
|
||||
pluggy==0.13.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
oauthlib==3.2.2 ; python_version >= '3.6'
|
||||
outcome==1.2.0 ; python_version >= '3.7'
|
||||
packaging==23.1 ; python_version >= '3.7'
|
||||
pluggy==0.13.1 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
prometheus-client==0.8.0
|
||||
protobuf==4.22.3; python_version >= '3.7'
|
||||
py==1.11.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
|
||||
protobuf==4.23.1 ; python_version >= '3.7'
|
||||
py==1.11.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
|
||||
pyaes==1.6.1
|
||||
pyasn1-modules==0.2.8
|
||||
pyasn1==0.4.8
|
||||
pyasn1==0.5.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
|
||||
pyasn1-modules==0.3.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
|
||||
pycparser==2.21
|
||||
pycryptodomex==3.17; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
|
||||
pydantic==1.10.7; python_version >= '3.7'
|
||||
pygments==2.15.0; python_version >= '3.7'
|
||||
pyparsing==3.0.9; python_version >= '3.1'
|
||||
pycryptodomex==3.18.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
|
||||
pydantic==1.10.7 ; python_version >= '3.7'
|
||||
pygments==2.15.1 ; python_version >= '3.7'
|
||||
pyparsing==3.0.9 ; python_version >= '3.1'
|
||||
pysocks==1.7.1
|
||||
pytest==6.2.4
|
||||
python-dateutil==2.8.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
python-dateutil==2.8.2 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
python-dotenv==1.0.0
|
||||
python-slugify==8.0.1; python_version >= '3.7'
|
||||
python-twitter-v2==0.8.1; python_version >= '3.6' and python_version < '4'
|
||||
pytz-deprecation-shim==0.1.0.post0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
|
||||
python-slugify==8.0.1 ; python_version >= '3.7'
|
||||
python-twitter-v2==0.8.1 ; python_version >= '3.6' and python_version < '4.0'
|
||||
pytz==2023.3
|
||||
pyyaml==6.0; python_version >= '3.6'
|
||||
pyyaml==6.0 ; python_version >= '3.6'
|
||||
redis==3.5.3
|
||||
regex==2023.3.23; python_version >= '3.8'
|
||||
requests-oauthlib==1.3.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
requests-toolbelt==0.10.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
requests==2.28.2
|
||||
rich==13.3.4; python_version >= '3.7'
|
||||
rsa==4.9; python_version >= '3.6' and python_version < '4'
|
||||
s3transfer==0.6.0; python_version >= '3.7'
|
||||
selenium==4.8.3; python_version >= '3.7'
|
||||
six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
sniffio==1.3.0; python_version >= '3.7'
|
||||
snscrape==0.6.2.20230320; python_version ~= '3.8'
|
||||
regex==2023.5.5 ; python_version >= '3.6'
|
||||
requests==2.30.0
|
||||
requests-oauthlib==1.3.1 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
requests-toolbelt==1.0.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
rich==13.3.5 ; python_full_version >= '3.7.0'
|
||||
rsa==4.9 ; python_version >= '3.6' and python_version < '4'
|
||||
s3transfer==0.6.1 ; python_version >= '3.7'
|
||||
selenium==4.9.1 ; python_version >= '3.7'
|
||||
six==1.16.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
sniffio==1.3.0 ; python_version >= '3.7'
|
||||
snscrape==0.6.2.20230320 ; python_version ~= '3.8'
|
||||
sortedcontainers==2.4.0
|
||||
soupsieve==2.4.1; python_version >= '3.7'
|
||||
sqlalchemy==1.4.47
|
||||
starlette==0.26.1; python_version >= '3.7'
|
||||
telethon==1.28.5; python_version >= '3.5'
|
||||
soupsieve==2.4.1 ; python_version >= '3.7'
|
||||
sqlalchemy==1.4.48
|
||||
starlette==0.27.0 ; python_version >= '3.7'
|
||||
telethon==1.28.5 ; python_version >= '3.5'
|
||||
text-unidecode==1.3
|
||||
tiktok-downloader==0.3.4
|
||||
toml==0.10.2; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
tornado==6.3; python_full_version >= '3.5.2'
|
||||
tqdm==4.65.0; python_version >= '3.7'
|
||||
trio-websocket==0.10.2; python_version >= '3.7'
|
||||
trio==0.22.0; python_version >= '3.7'
|
||||
typing-extensions==4.5.0; python_version >= '3.7'
|
||||
tiktok-downloader==0.3.5
|
||||
toml==0.10.2 ; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'
|
||||
tornado==6.3.2 ; python_version >= '3.5.2'
|
||||
tqdm==4.65.0 ; python_version >= '3.7'
|
||||
trio==0.22.0 ; python_version >= '3.7'
|
||||
trio-websocket==0.10.2 ; python_version >= '3.7'
|
||||
typing-extensions==4.5.0 ; python_version >= '3.7'
|
||||
typing-inspect==0.8.0
|
||||
tzdata==2023.3; python_version >= '3.6'
|
||||
tzlocal==4.3; python_version >= '3.7'
|
||||
uritemplate==4.1.1; python_version >= '3.6'
|
||||
urllib3==1.26.14; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
|
||||
uvicorn==0.21.1
|
||||
vine==1.3.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
tzlocal==5.0.1 ; python_version >= '3.7'
|
||||
uritemplate==4.1.1 ; python_version >= '3.6'
|
||||
urllib3==1.26.15 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
|
||||
uvicorn==0.22.0
|
||||
uwsgi==2.0.21
|
||||
vine==1.3.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
|
||||
vk-api==11.9.9
|
||||
vk-url-scraper==0.3.15; python_version >= '3.7'
|
||||
websockets==10.4; python_version >= '3.7'
|
||||
werkzeug==2.2.3; python_version >= '3.7'
|
||||
wsproto==1.2.0; python_version >= '3.7'
|
||||
yt-dlp==2023.2.17; python_version >= '3.7'
|
||||
vk-url-scraper==0.3.24 ; python_version >= '3.7'
|
||||
websockets==11.0.3 ; python_version >= '3.7'
|
||||
werkzeug==2.3.4 ; python_version >= '3.8'
|
||||
wsproto==1.2.0 ; python_full_version >= '3.7.0'
|
||||
yarl==1.9.2 ; python_version >= '3.7'
|
||||
yt-dlp==2023.3.4 ; python_version >= '3.7'
|
||||
|
||||
@@ -8,8 +8,10 @@ from fastapi.security import HTTPBasic, HTTPBasicCredentials, HTTPBearer, HTTPAu
|
||||
# Configuration
|
||||
GOOGLE_CHROME_APP_ID = os.environ.get("GOOGLE_CHROME_APP_ID")
|
||||
assert len(GOOGLE_CHROME_APP_ID)>10, "GOOGLE_CHROME_APP_ID env variable not set"
|
||||
GOOGLE_CHROME_APP_ID_PUBLIC = os.environ.get("GOOGLE_CHROME_APP_ID_PUBLIC")
|
||||
assert len(GOOGLE_CHROME_APP_ID_PUBLIC)>10, "GOOGLE_CHROME_APP_ID_PUBLIC env variable not set"
|
||||
ALLOWED_EMAILS = set([e.strip().lower() for e in os.environ.get("ALLOWED_EMAILS", "").split(",")])
|
||||
assert len(GOOGLE_CHROME_APP_ID)>=1, "at least one ALLOWED_EMAILS is required from the env variable"
|
||||
assert len(ALLOWED_EMAILS)>=1, "at least one ALLOWED_EMAILS is required from the env variable"
|
||||
logger.info(f"{len(ALLOWED_EMAILS)=}")
|
||||
|
||||
basic_security = HTTPBasic()
|
||||
@@ -54,6 +56,38 @@ def custom_is_email_allowed(email, any_bellingcat_email=False):
|
||||
return email.lower() in ALLOWED_EMAILS or (any_bellingcat_email and re.match(r'^[\w.]+@bellingcat\.com$', email))
|
||||
|
||||
|
||||
#--------------------- Bearer Auth ANY EMAIL
|
||||
|
||||
async def get_bearer_auth_public(credentials: HTTPAuthorizationCredentials = Depends(bearer_security)):
|
||||
# validates the Bearer token in the case that it requires it
|
||||
access_token = credentials.credentials
|
||||
valid_user, info = authenticate_user_public(access_token)
|
||||
if valid_user: return info
|
||||
logger.debug(f"TOKEN FAILURE: {valid_user=} {info=}")
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail=info,
|
||||
headers={"WWW-Authenticate": "Bearer"},
|
||||
)
|
||||
|
||||
def authenticate_user_public(access_token):
|
||||
# https://cloud.google.com/docs/authentication/token-types#access
|
||||
if type(access_token)!=str or len(access_token)<10: return False, "invalid access_token"
|
||||
r = requests.get("https://oauth2.googleapis.com/tokeninfo", {"access_token":access_token})
|
||||
if r.status_code!=200: return False, "error occurred"
|
||||
try:
|
||||
j = r.json()
|
||||
if j.get("azp") != GOOGLE_CHROME_APP_ID_PUBLIC and j.get("aud")!=GOOGLE_CHROME_APP_ID_PUBLIC:
|
||||
return False, f"token does not belong to correct APP_ID"
|
||||
if j.get("email_verified") != "true":
|
||||
return False, f"email '{j.get('email')}' not verified"
|
||||
if int(j.get("expires_in", -1)) <= 0:
|
||||
return False, "Token expired"
|
||||
return True, j.get('email')
|
||||
except Exception as e:
|
||||
logger.warning(f"EXCEPTION occurred: {e}")
|
||||
return False, f"EXCEPTION occurred"
|
||||
|
||||
#--------------------- Basic Auth
|
||||
SFP = os.environ.get("STATIC_FILE_PASSWORD", "") # min length is 20 chars
|
||||
async def get_basic_auth(credentials: HTTPBasicCredentials = Depends(basic_security)):
|
||||
|
||||
@@ -1,12 +1,11 @@
|
||||
|
||||
import os, re, traceback, yaml
|
||||
import os, re, traceback, yaml, datetime
|
||||
|
||||
from celery import Celery, states
|
||||
from celery.exceptions import Ignore
|
||||
from celery.signals import task_failure
|
||||
from auto_archiver import Config, ArchivingOrchestrator, Metadata
|
||||
# from auto_archiver.enrichers import ScreenshotEnricher
|
||||
from auto_archiver.feeders import GsheetsFeeder
|
||||
from loguru import logger
|
||||
|
||||
from db import crud, schemas, models
|
||||
@@ -63,18 +62,13 @@ def create_archive_task(self, archive_json: str):
|
||||
def create_sheet_task(self, sheet_json: str):
|
||||
logger.info(f"STARTING {sheet_json}")
|
||||
sheet = schemas.SubmitSheet.parse_raw(sheet_json)
|
||||
orchestrator = choose_orchestrator("bellingcat", "")
|
||||
#TODO: modify archiver to accept sheetId too
|
||||
config = Config()
|
||||
config.parse(use_cli=False, yaml_config_filename="secrets/orchestration-sheet.yaml")
|
||||
config.feeder.sheet = sheet.sheet_name
|
||||
config.feeder.header = sheet.header
|
||||
config.parse(use_cli=False, yaml_config_filename="secrets/orchestration-sheet.yaml", overwrite_configs={"configurations": {"gsheet_feeder": {"sheet": sheet.sheet_name, "sheet_id": sheet.sheet_id, "header": sheet.header}}})
|
||||
orchestrator = ArchivingOrchestrator(config)
|
||||
#TODO: make auto-archiver config.parse receive an overriding dict
|
||||
result = orchestrator.feed()
|
||||
# TODO: save into local DB
|
||||
orchestrator.feed()
|
||||
|
||||
return result.to_json()
|
||||
return {"success": True, "sheet": sheet.sheet_name, "sheet_id": sheet.sheet_id, "time": datetime.datetime.now().isoformat()}
|
||||
|
||||
@task_failure.connect(sender=create_archive_task)
|
||||
def task_failure_notifier(sender=None, **kwargs):
|
||||
|
||||
Reference in New Issue
Block a user