task retries

src/main.py

@@ -1,5 +1,5 @@
 from celery.result import AsyncResult
-from fastapi import Body, FastAPI, Depends, Request
+from fastapi import Body, FastAPI, Depends, Request, HTTPException
 from fastapi.encoders import jsonable_encoder
 from fastapi.responses import JSONResponse, FileResponse
 from fastapi.staticfiles import StaticFiles
@@ -68,7 +68,10 @@ def search(skip: int = 0, limit: int = 100, db: Session = Depends(get_db), email
 
 @app.post("/tasks", status_code=201)
 def run_task(payload = Body(...), email = Depends(get_bearer_auth)):
-    logger.info(f"new task for user {email}: {payload.get('url')}")
+    url = payload.get('url')
+    logger.info(f"new task for user {email}: {url}")
+    if type(url)!=str or len(url)<=5:
+        raise HTTPException(status_code=422, detail=f"Invalid URL received: {url}")
     task = create_archive_task.delay(url=payload.get('url'), email=email)
     return JSONResponse({"id": task.id})
 

src/worker.py

@@ -31,9 +31,8 @@ if (config_bcat_file := os.environ.get("ORCHESTRATION_CONFIG_BELLINGCAT")):
 
 orchestrators = {"bellingcat": None, "default": None}
 
-@celery.task(name="create_archive_task", bind=True)
+@celery.task(name="create_archive_task", bind=True, autoretry_for=(Exception,), retry_backoff=True, retry_kwargs={'max_retries': 5})
 def create_archive_task(self, url: str, email:str=""):
-    assert type(url)==str and len(url)>5, f"Invalid URL received: {url}"
     orchestrator = choose_orchestrator(email)
     result = orchestrator.feed_item(Metadata().set_url(url)).to_json()
     with get_db() as session: