WIP decoupling worker/web, cleaning worker code

src/endpoints/__init__.py

@@ -1,5 +0,0 @@
-from endpoints.default import default_router
-from endpoints.url import url_router
-from endpoints.task import task_router
-from endpoints.interoperability import interoperability_router
-from endpoints.sheet import sheet_router

src/endpoints/sheet.py

@@ -6,13 +6,14 @@ from sqlalchemy import exc
 from sqlalchemy.orm import Session
 
 from db.user_state import UserState
+from shared.task_messaging import get_celery
 from web.security import token_api_key_auth, get_user_state
 from db import schemas, crud
 from db.database import get_db_dependency
-from worker.main import create_sheet_task
 
 sheet_router = APIRouter(prefix="/sheet", tags=["Google Spreadsheet operations"])
 
+celery = get_celery()
 
 @sheet_router.post("/create", status_code=201, summary="Store a new Google Sheet for regular archiving.")
 def create_sheet(
@@ -73,7 +74,7 @@ def archive_user_sheet(
     if not user.can_manually_trigger(sheet.group_id):
         raise HTTPException(status_code=429, detail="User cannot manually trigger sheet archiving in this group.")
 
-    task = create_sheet_task.delay(schemas.SubmitSheet(sheet_id=id, author_id=user.email, group=sheet.group_id).model_dump_json())
+    task = celery.signature("create_sheet_task", args=[schemas.SubmitSheet(sheet_id=id, author_id=user.email, group=sheet.group_id).model_dump_json()]).delay()
 
     return JSONResponse({"id": task.id}, status_code=201)
 
@@ -86,5 +87,5 @@ def archive_sheet(
     sheet.author_id = sheet.author_id or "api-endpoint"
     if not sheet.sheet_id:
         raise HTTPException(status_code=422, detail=f"sheet id is required")
-    task = create_sheet_task.delay(sheet.model_dump_json())
+    task = celery.signature("create_sheet_task", args=[sheet.model_dump_json()]).delay()
     return JSONResponse({"id": task.id}, status_code=201)

src/endpoints/task.py

@@ -4,16 +4,17 @@ from fastapi.encoders import jsonable_encoder
 from fastapi.responses import JSONResponse
 
 from loguru import logger
+from shared.task_messaging import get_celery
 from web.security import get_token_or_user_auth
 
 from db import schemas
 from core.logging import log_error
-from worker.main import celery
 from utils.misc import custom_jsonable_encoder
 
 
 task_router = APIRouter(prefix="/task", tags=["Async task operations"])
 
+celery = get_celery()
 
 @task_router.get("/{task_id}", summary="Check the status of an async task by its id, works for URLs and Sheet tasks.")
 def get_status(task_id, email=Depends(get_token_or_user_auth)) -> schemas.TaskResult:

src/endpoints/url.py

@@ -6,17 +6,18 @@ from datetime import datetime
 from loguru import logger
 from core.config import ALLOW_ANY_EMAIL
 from db.user_state import UserState
+from shared.task_messaging import get_celery
 from web.security import get_token_or_user_auth, get_user_state
 from sqlalchemy.orm import Session
 
 from db import crud, schemas
 from db.database import get_db_dependency
 
-from worker.main import create_archive_task
 from urllib.parse import urlparse
 
 url_router = APIRouter(prefix="/url", tags=["Single URL operations"])
 
+celery = get_celery()
 
 @url_router.post("/archive", status_code=201, summary="Submit a single URL archive request, starts an archiving task.", response_description="task_id for the archiving task, will match the archive id.")
 def archive_url(
@@ -24,6 +25,7 @@ def archive_url(
     email=Depends(get_token_or_user_auth),
     db: Session = Depends(get_db_dependency)
 ) -> schemas.Task:
+    archive.author_id = email
     logger.info(f"new {archive.public=} task for {email=} and {archive.group_id=}: {archive.url}")
 
     parsed_url = urlparse(archive.url)
@@ -39,15 +41,9 @@ def archive_url(
         if not user.has_quota_max_monthly_mbs(archive.group_id):
             raise HTTPException(status_code=429, detail="User has reached their monthly MB quota.")
     
-    # TODO: deprecate ArchiveCreate
-    backwards_compatible_archive = schemas.ArchiveCreate(
-        url=archive.url,
-        author_id=email,
-        group_id=archive.group_id,
-        public=archive.public,
-    )
+    archive_create = schemas.ArchiveCreate(**archive.model_dump())
 
-    task = create_archive_task.delay(backwards_compatible_archive.model_dump_json())
+    task = celery.signature("create_archive_task", args=[archive_create.model_dump_json()]).delay()
     task_response = schemas.Task(id=task.id)
     return JSONResponse(task_response.model_dump(), status_code=201)