bellingcat/auto-archiver-api
1
0
Fork 0
mirror of https://github.com/bellingcat/auto-archiver-api.git synced 2026-06-12 21:48:35 +03:00
Code Issues Packages Projects Releases Wiki Activity

task retries

Browse Source
This commit is contained in:
msramalho
2023-03-10 11:00:02 +00:00
parent 67237b6fd1
commit a400443dc0
2 changed files with 6 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/main.py
View File

@@ -1,5 +1,5 @@
from celery.result import AsyncResult from celery.result import AsyncResult
from fastapi import Body, FastAPI, Depends, Request from fastapi import Body, FastAPI, Depends, Request, HTTPException
from fastapi.encoders import jsonable_encoder from fastapi.encoders import jsonable_encoder
from fastapi.responses import JSONResponse, FileResponse from fastapi.responses import JSONResponse, FileResponse
from fastapi.staticfiles import StaticFiles from fastapi.staticfiles import StaticFiles
@@ -68,7 +68,10 @@ def search(skip: int = 0, limit: int = 100, db: Session = Depends(get_db), email
@app.post("/tasks", status_code=201) @app.post("/tasks", status_code=201)
def run_task(payload = Body(...), email = Depends(get_bearer_auth)): def run_task(payload = Body(...), email = Depends(get_bearer_auth)):
logger.info(f"new task for user {email}: {payload.get('url')}") url = payload.get('url')
logger.info(f"new task for user {email}: {url}")
if type(url)!=str or len(url)<=5:
raise HTTPException(status_code=422, detail=f"Invalid URL received: {url}")
task = create_archive_task.delay(url=payload.get('url'), email=email) task = create_archive_task.delay(url=payload.get('url'), email=email)
return JSONResponse({"id": task.id}) return JSONResponse({"id": task.id})

3
src/worker.py
View File

@@ -31,9 +31,8 @@ if (config_bcat_file := os.environ.get("ORCHESTRATION_CONFIG_BELLINGCAT")):
orchestrators = {"bellingcat": None, "default": None} orchestrators = {"bellingcat": None, "default": None}
@celery.task(name="create_archive_task", bind=True) @celery.task(name="create_archive_task", bind=True, autoretry_for=(Exception,), retry_backoff=True, retry_kwargs={'max_retries': 5})
def create_archive_task(self, url: str, email:str=""): def create_archive_task(self, url: str, email:str=""):
assert type(url)==str and len(url)>5, f"Invalid URL received: {url}"
orchestrator = choose_orchestrator(email) orchestrator = choose_orchestrator(email)
result = orchestrator.feed_item(Metadata().set_url(url)).to_json() result = orchestrator.feed_item(Metadata().set_url(url)).to_json()
with get_db() as session: with get_db() as session: