major refactor of structure for worker V web: docker/app/secrets/envs/...

app/web/endpoints/default.py

@@ -0,0 +1,59 @@
+
+from typing import Dict
+from fastapi import APIRouter, Depends, Request, HTTPException
+from fastapi.responses import FileResponse, JSONResponse
+
+from app.shared.config import VERSION, BREAKING_CHANGES
+from app.shared.log import log_error
+from app.shared.db import crud
+from app.shared.schemas import ActiveUser, UsageResponse
+from app.shared.db.user_state import UserState
+from app.web.security import get_user_auth, bearer_security, get_user_state
+from app.shared.user_groups import GroupInfo
+
+default_router = APIRouter()
+
+
+@default_router.get("/")
+async def home(request: Request):
+    # TODO: maybe split into 2 routes: one non authenticated and one authenticated for the groups info only, necessary only for the extension
+    status = {"version": VERSION, "breakingChanges": BREAKING_CHANGES}
+    try:
+        email = await get_user_auth(await bearer_security(request))
+        status["groups"] = crud.get_user_groups(email)
+    except HTTPException: pass  # not authenticated is fine
+    except Exception as e: log_error(e)
+    return JSONResponse(status)
+
+
+@default_router.get("/health")
+async def health():
+    return JSONResponse({"status": "ok"})
+
+
+@default_router.get("/user/active", summary="Check if the user is active and can use the tool.")
+async def active(
+    user: UserState = Depends(get_user_state),
+) -> ActiveUser:
+    return {"active": user.active}
+
+
+@default_router.get("/user/permissions", summary="Get the user's global 'all' permissions and the permissions for each group they belong to.")
+def get_user_permissions(
+    user: UserState = Depends(get_user_state),
+) -> Dict[str, GroupInfo]:
+    return user.permissions
+
+@default_router.get("/user/usage", summary="Get the user's monthly URLs/MBs usage along with the total active sheets, breakdown by group.")
+def get_user_usage(
+    user: UserState = Depends(get_user_state),
+) -> UsageResponse:
+    if not user.active:
+        raise HTTPException(status_code=403, detail="User is not active.")
+    return user.usage()
+    
+
+
+@default_router.get('/favicon.ico', include_in_schema=False)
+async def favicon() -> FileResponse:
+    return FileResponse("web/static/favicon.ico")

app/web/endpoints/interoperability.py

@@ -0,0 +1,51 @@
+import json
+from fastapi import APIRouter, Depends, HTTPException
+from fastapi.responses import JSONResponse
+from loguru import logger
+import sqlalchemy
+from auto_archiver import Metadata
+from sqlalchemy.orm import Session
+
+from app.shared.aa_utils import get_all_urls
+from app.shared.config import ALLOW_ANY_EMAIL
+from app.shared import business_logic, schemas
+from app.shared.db import crud
+from app.shared.db.database import get_db_dependency
+from app.web.security import token_api_key_auth
+from app.shared.db import models
+from app.shared.log import log_error
+
+
+interoperability_router = APIRouter(prefix="/interop", tags=["Interoperability endpoints."])
+
+
+# ----- endpoint to submit data archived elsewhere
+@interoperability_router.post("/submit-archive", status_code=201, summary="Submit a manual archive entry, for data that was archived elsewhere.")
+def submit_manual_archive(
+    manual: schemas.SubmitManualArchive,
+    auth=Depends(token_api_key_auth),
+    db: Session = Depends(get_db_dependency)
+):
+    result: Metadata = Metadata.from_json(manual.result)
+    manual.author_id = manual.author_id or ALLOW_ANY_EMAIL
+    manual.tags.add("manual")
+
+    try:
+        archive = schemas.ArchiveCreate(
+            author_id=manual.author_id,
+            url=result.get_url(),
+            public=manual.public,
+            group_id=manual.group_id,
+            tags=manual.tags,
+            id=models.generate_uuid(),
+            result=json.loads(result.to_json()),
+            urls=get_all_urls(result),
+            store_until=business_logic.get_store_archive_until(db, manual.group_id),
+        )
+
+        db_archive = crud.store_archived_url(db, archive)
+        logger.debug(f"[MANUAL ARCHIVE STORED] {db_archive.author_id} {db_archive.url}")
+        return JSONResponse({"id": db_archive.id}, status_code=201)
+    except sqlalchemy.exc.IntegrityError as e:
+        log_error(e)
+        raise HTTPException(status_code=422, detail=f"Cannot insert into DB due to integrity error, likely duplicate urls.")

app/web/endpoints/sheet.py

@@ -0,0 +1,80 @@
+
+from fastapi import APIRouter, Depends, HTTPException
+from fastapi.responses import JSONResponse
+
+from sqlalchemy import exc
+from sqlalchemy.orm import Session
+
+from app.shared.db.user_state import UserState
+from app.shared import schemas
+from app.shared.task_messaging import get_celery
+from app.web.security import get_user_state
+from app.shared.db import crud
+from app.shared.db.database import get_db_dependency
+
+sheet_router = APIRouter(prefix="/sheet", tags=["Google Spreadsheet operations"])
+
+celery = get_celery()
+
+@sheet_router.post("/create", status_code=201, summary="Store a new Google Sheet for regular archiving.")
+def create_sheet(
+    sheet: schemas.SheetAdd,
+    user: UserState = Depends(get_user_state),
+    db: Session = Depends(get_db_dependency),
+) -> schemas.SheetResponse:
+
+    if not user.in_group(sheet.group_id):
+        raise HTTPException(status_code=403, detail="User does not have access to this group.")
+
+    if not user.has_quota_monthly_sheets(sheet.group_id):
+        raise HTTPException(status_code=429, detail="User has reached their sheet quota for this group.")
+
+    if not user.is_sheet_frequency_allowed(sheet.group_id, sheet.frequency):
+        raise HTTPException(status_code=422, detail="Invalid frequency selected for this group.")
+
+    try:
+        return crud.create_sheet(db, sheet.id, sheet.name, user.email, sheet.group_id, sheet.frequency)
+    except exc.IntegrityError as e:
+        raise HTTPException(status_code=400, detail="Sheet with this ID is already being archived.") from e
+
+
+@sheet_router.get("/mine", status_code=200, summary="Get the authenticated user's Google Sheets.")
+def get_user_sheets(
+    user: UserState = Depends(get_user_state),
+    db: Session = Depends(get_db_dependency)
+) -> list[schemas.SheetResponse]:
+    return crud.get_user_sheets(db, user.email)
+
+
+@sheet_router.delete("/{id}", summary="Delete a Google Sheet by ID.")
+def delete_sheet(
+    id: str,
+    user: UserState = Depends(get_user_state),
+    db: Session = Depends(get_db_dependency),
+) -> schemas.TaskDelete:
+    return JSONResponse({
+        "id": id,
+        "deleted": crud.delete_sheet(db, id, user.email)
+    })
+
+
+@sheet_router.post("/{id}/archive", status_code=201, summary="Trigger an archiving task for a GSheet you own.", response_description="task_id for the archiving task.")
+def archive_user_sheet(
+    id: str,
+    user: UserState = Depends(get_user_state),
+    db: Session = Depends(get_db_dependency),
+) -> schemas.Task:
+
+    sheet = crud.get_user_sheet(db, user.email, sheet_id=id)
+    if not sheet:
+        raise HTTPException(status_code=403, detail="No access to this sheet.")
+
+    if not user.in_group(sheet.group_id):
+        raise HTTPException(status_code=403, detail="User does not have access to this group.")
+
+    if not user.can_manually_trigger(sheet.group_id):
+        raise HTTPException(status_code=429, detail="User cannot manually trigger sheet archiving in this group.")
+
+    task = celery.signature("create_sheet_task", args=[schemas.SubmitSheet(sheet_id=id, author_id=user.email, group_id=sheet.group_id).model_dump_json()]).delay()
+
+    return JSONResponse({"id": task.id}, status_code=201)

app/web/endpoints/task.py

@@ -0,0 +1,40 @@
+from celery.result import AsyncResult
+from fastapi import APIRouter, Depends
+from fastapi.encoders import jsonable_encoder
+from fastapi.responses import JSONResponse
+
+from app.shared.task_messaging import get_celery
+from app.web.security import get_token_or_user_auth
+from app.shared import schemas
+from app.shared.log import log_error
+from app.web.utils.misc import custom_jsonable_encoder
+
+
+task_router = APIRouter(prefix="/task", tags=["Async task operations"])
+
+celery = get_celery()
+
+@task_router.get("/{task_id}", summary="Check the status of an async task by its id, works for URLs and Sheet tasks.")
+def get_status(task_id, email=Depends(get_token_or_user_auth)) -> schemas.TaskResult:
+    task = AsyncResult(task_id, app=celery)
+    try:
+        if task.status == "FAILURE":
+            # *FAILURE* The task raised an exception, or has exceeded the retry limit.
+            # The :attr:`result` attribute then contains the exception raised by the task.
+            # https://docs.celeryq.dev/en/stable/_modules/celery/result.html#AsyncResult
+            raise task.result
+
+        response = {
+            "id": task_id,
+            "status": task.status,
+            "result": task.result
+        }
+        return JSONResponse(jsonable_encoder(response, exclude_unset=True, custom_encoder={bytes: custom_jsonable_encoder}))
+
+    except Exception as e:
+        log_error(e)
+        return JSONResponse({
+            "id": task_id,
+            "status": "FAILURE",
+            "result": {"error": str(e)}
+        })

app/web/endpoints/url.py

@@ -0,0 +1,77 @@
+
+from fastapi import APIRouter, Depends, HTTPException
+from fastapi.responses import JSONResponse
+from datetime import datetime
+from loguru import logger
+from sqlalchemy.orm import Session
+
+from app.shared.config import ALLOW_ANY_EMAIL
+from app.shared import schemas
+from app.shared.task_messaging import get_celery
+from app.web.security import get_token_or_user_auth, get_user_state
+from app.shared.db import crud
+from app.shared.db.user_state import UserState
+from app.shared.db.database import get_db_dependency
+
+from urllib.parse import urlparse
+
+url_router = APIRouter(prefix="/url", tags=["Single URL operations"])
+
+celery = get_celery()
+
+@url_router.post("/archive", status_code=201, summary="Submit a single URL archive request, starts an archiving task.", response_description="task_id for the archiving task, will match the archive id.")
+def archive_url(
+    archive: schemas.ArchiveTrigger,
+    email=Depends(get_token_or_user_auth),
+    db: Session = Depends(get_db_dependency)
+) -> schemas.Task:
+    archive.author_id = email
+    logger.info(f"new {archive.public=} task for {email=} and {archive.group_id=}: {archive.url}")
+
+    parsed_url = urlparse(archive.url)
+    if not all([parsed_url.scheme, parsed_url.netloc]):
+        raise HTTPException(status_code=400, detail="Invalid URL received.")
+
+    if email != ALLOW_ANY_EMAIL:
+        user = UserState(db, email)
+        if archive.group_id and not user.in_group(archive.group_id):
+            raise HTTPException(status_code=403, detail="User does not have access to this group.")
+        if not user.has_quota_max_monthly_urls(archive.group_id):
+            raise HTTPException(status_code=429, detail="User has reached their monthly URL quota.")
+        if not user.has_quota_max_monthly_mbs(archive.group_id):
+            raise HTTPException(status_code=429, detail="User has reached their monthly MB quota.")
+    
+    archive_create = schemas.ArchiveCreate(**archive.model_dump())
+
+    task = celery.signature("create_archive_task", args=[archive_create.model_dump_json()]).delay()
+    task_response = schemas.Task(id=task.id)
+    return JSONResponse(task_response.model_dump(), status_code=201)
+
+
+@url_router.get("/search", summary="Search for archive entries by URL.")
+def search_by_url(
+        url: str, skip: int = 0, limit: int = 25,
+        archived_after: datetime = None, archived_before: datetime = None,
+        db: Session = Depends(get_db_dependency),
+        email: str = Depends(get_token_or_user_auth)
+) -> list[schemas.ArchiveResult]:
+
+    if email != ALLOW_ANY_EMAIL:
+        user = UserState(db, email)
+        if not user.read and not user.read_public:
+            raise HTTPException(status_code=403, detail="User does not have read access.")
+
+    return crud.search_archives_by_url(db, url.strip(), email, skip=skip, limit=limit, archived_after=archived_after, archived_before=archived_before)
+
+
+@url_router.delete("/{id}", summary="Delete a single URL archive by id.")
+def delete_task(
+    id:str, 
+    user: UserState = Depends(get_user_state), 
+    db: Session = Depends(get_db_dependency)
+) -> schemas.TaskDelete:
+    logger.info(f"deleting url archive task {id} request by {user.email}")
+    return JSONResponse({
+        "id": id,
+        "deleted": crud.soft_delete_task(db, id, user.email)
+    })