From 17d429e6d3defde0a1f96680863fd2ae0a08053d Mon Sep 17 00:00:00 2001
From: msramalho <19508417+msramalho@users.noreply.github.com>
Date: Mon, 30 Jun 2025 10:55:19 +0100
Subject: [PATCH] new logging strategy, combining API+Worker logs

---
 app/shared/log.py                     | 13 ++++++++++---
 app/shared/user_groups.py             |  3 ++-
 app/shared/utils/misc.py              |  2 +-
 app/tests/web/routers/test_default.py |  2 +-
 app/web/config.py                     |  2 +-
 app/web/db/crud.py                    |  2 +-
 app/web/events.py                     |  2 +-
 app/web/main.py                       |  2 +-
 app/web/middleware.py                 |  7 +++----
 app/web/routers/interoperability.py   |  3 +--
 app/web/routers/url.py                |  2 +-
 app/web/security.py                   |  2 +-
 app/worker/main.py                    |  3 +--
 app/worker/worker_log.py              | 12 +++++-------
 14 files changed, 30 insertions(+), 27 deletions(-)

diff --git a/app/shared/log.py b/app/shared/log.py
index 152bcc0..138a650 100644
--- a/app/shared/log.py
+++ b/app/shared/log.py
@@ -1,11 +1,18 @@
 import traceback
 
-from loguru import logger
+from auto_archiver.utils.custom_logger import logger
 
 
 # logging configurations
-logger.add("logs/api_logs.log", retention="30 days")
-logger.add("logs/error_logs.log", retention="30 days", level="ERROR")
+logger.add(
+    "logs/all_logs.log", retention="30 days", format="{extra[serialized]}"
+)
+logger.add(
+    "logs/all_error_logs.log",
+    retention="120 days",
+    level="ERROR",
+    format="{extra[serialized]}",
+)
 
 
 def log_error(e: Exception, traceback_str: str = None, extra: str = ""):
diff --git a/app/shared/user_groups.py b/app/shared/user_groups.py
index 444fd6b..039928d 100644
--- a/app/shared/user_groups.py
+++ b/app/shared/user_groups.py
@@ -3,7 +3,6 @@ import os
 from typing import Dict, List, Set
 
 import yaml
-from loguru import logger
 from pydantic import (
     BaseModel,
     Field,
@@ -13,6 +12,8 @@ from pydantic import (
 )
 from typing_extensions import Self
 
+from app.shared.log import logger
+
 
 class UserGroups:
     def __init__(self, filename):
diff --git a/app/shared/utils/misc.py b/app/shared/utils/misc.py
index 7fd250d..9549f36 100644
--- a/app/shared/utils/misc.py
+++ b/app/shared/utils/misc.py
@@ -1,9 +1,9 @@
 from typing import List
 
 from auto_archiver.core import Media, Metadata
-from loguru import logger
 
 from app.shared.db import models
+from app.shared.log import logger
 
 
 def fnv1a_hash_mod(s: str, modulo: int) -> int:
diff --git a/app/tests/web/routers/test_default.py b/app/tests/web/routers/test_default.py
index 970e705..158863d 100644
--- a/app/tests/web/routers/test_default.py
+++ b/app/tests/web/routers/test_default.py
@@ -3,8 +3,8 @@ from unittest.mock import MagicMock
 
 import pytest
 from fastapi.testclient import TestClient
-from loguru import logger
 
+from app.shared.log import logger
 from app.shared.schemas import Usage, UsageResponse
 from app.shared.user_groups import GroupInfo
 from app.web.config import VERSION
diff --git a/app/web/config.py b/app/web/config.py
index 5d8d5a0..e387e00 100644
--- a/app/web/config.py
+++ b/app/web/config.py
@@ -1,4 +1,4 @@
-VERSION = "0.12.0"
+VERSION = "0.12.1"
 
 API_DESCRIPTION = """
 #### API for the Auto-Archiver project, a tool to archive web pages and Google Sheets.
diff --git a/app/web/db/crud.py b/app/web/db/crud.py
index 308c526..a6743ff 100644
--- a/app/web/db/crud.py
+++ b/app/web/db/crud.py
@@ -4,7 +4,6 @@ from typing import Any, Type
 
 from cachetools import LRUCache, cached
 from cachetools.keys import hashkey
-from loguru import logger
 from sqlalchemy import (
     Column,
     ColumnElement,
@@ -21,6 +20,7 @@ from sqlalchemy.orm import Session, load_only
 
 from app.shared.db import models
 from app.shared.db.models import Archive, Group
+from app.shared.log import logger
 from app.shared.settings import get_settings
 from app.shared.user_groups import UserGroups
 from app.shared.utils.misc import fnv1a_hash_mod
diff --git a/app/web/events.py b/app/web/events.py
index e9af845..8e07c61 100644
--- a/app/web/events.py
+++ b/app/web/events.py
@@ -8,7 +8,6 @@ import alembic.config
 from fastapi import FastAPI
 from fastapi_mail import FastMail, MessageSchema, MessageType
 from fastapi_utils.tasks import repeat_every
-from loguru import logger
 
 from app.shared import schemas
 from app.shared.db import models
@@ -18,6 +17,7 @@ from app.shared.db.database import (
     make_engine,
     wal_checkpoint,
 )
+from app.shared.log import logger
 from app.shared.settings import get_settings
 from app.shared.task_messaging import get_celery
 from app.web.db import crud
diff --git a/app/web/main.py b/app/web/main.py
index df5ca73..8bc11f1 100644
--- a/app/web/main.py
+++ b/app/web/main.py
@@ -3,9 +3,9 @@ import os
 from fastapi import Depends, FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.staticfiles import StaticFiles
-from loguru import logger
 from prometheus_fastapi_instrumentator import Instrumentator
 
+from app.shared.log import logger
 from app.shared.settings import Settings, get_settings
 from app.shared.task_messaging import get_celery
 from app.web.config import API_DESCRIPTION, VERSION
diff --git a/app/web/middleware.py b/app/web/middleware.py
index b39ae01..cef1c9f 100644
--- a/app/web/middleware.py
+++ b/app/web/middleware.py
@@ -1,9 +1,8 @@
 import traceback
 
 from fastapi import Request
-from loguru import logger
 
-from app.shared.log import log_error
+from app.shared.log import log_error, logger
 from app.web.utils.metrics import EXCEPTION_COUNTER
 
 
@@ -13,14 +12,14 @@ async def logging_middleware(request: Request, call_next):
         # TODO: use Origin to have summary prometheus metrics on where
         #  requests come from
         # origin = request.headers.get("origin")
-        logger.info(
+        logger.debug(
             f"{request.client.host}:{request.client.port} {request.method} {request.url._url} - HTTP {response.status_code}"
         )
         return response
     except Exception as e:
         location = f"{request.method} {request.url._url}"
         await increase_exceptions_counter(e, location)
-        logger.info(
+        logger.error(
             f"{request.client.host}:{request.client.port} {location} - {e.__class__.__name__} {e}"
         )
         raise e
diff --git a/app/web/routers/interoperability.py b/app/web/routers/interoperability.py
index 008d248..516b9f3 100644
--- a/app/web/routers/interoperability.py
+++ b/app/web/routers/interoperability.py
@@ -5,13 +5,12 @@ import sqlalchemy
 from auto_archiver.core import Metadata
 from fastapi import APIRouter, Depends, HTTPException
 from fastapi.responses import JSONResponse
-from loguru import logger
 from sqlalchemy.orm import Session
 
 from app.shared import business_logic, schemas
 from app.shared.db import models, worker_crud
 from app.shared.db.database import get_db_dependency
-from app.shared.log import log_error
+from app.shared.log import log_error, logger
 from app.shared.utils.misc import get_all_urls
 from app.web.config import ALLOW_ANY_EMAIL
 from app.web.security import token_api_key_auth
diff --git a/app/web/routers/url.py b/app/web/routers/url.py
index dd3e7c5..b9a012a 100644
--- a/app/web/routers/url.py
+++ b/app/web/routers/url.py
@@ -4,11 +4,11 @@ from urllib.parse import urlparse
 
 from fastapi import APIRouter, Depends, HTTPException
 from fastapi.responses import JSONResponse
-from loguru import logger
 from sqlalchemy.orm import Session
 
 from app.shared import schemas
 from app.shared.db.database import get_db_dependency
+from app.shared.log import logger
 from app.shared.schemas import DeleteResponse
 from app.shared.task_messaging import get_celery
 from app.web.config import ALLOW_ANY_EMAIL
diff --git a/app/web/security.py b/app/web/security.py
index 5b73997..3682d75 100644
--- a/app/web/security.py
+++ b/app/web/security.py
@@ -6,10 +6,10 @@ import requests
 from fastapi import Depends, HTTPException, status
 from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
 from firebase_admin import auth, credentials, exceptions
-from loguru import logger
 from sqlalchemy.orm import Session
 
 from app.shared.db.database import get_db_dependency
+from app.shared.log import logger
 from app.shared.settings import get_settings
 from app.web.config import ALLOW_ANY_EMAIL
 from app.web.db.user_state import UserState
diff --git a/app/worker/main.py b/app/worker/main.py
index 20b7252..a7100fc 100644
--- a/app/worker/main.py
+++ b/app/worker/main.py
@@ -4,7 +4,6 @@ import traceback
 
 from auto_archiver.core.orchestrator import ArchivingOrchestrator
 from celery.signals import task_failure
-from loguru import logger
 from sqlalchemy import exc
 
 from app.shared import business_logic, constants, schemas
@@ -14,7 +13,7 @@ from app.shared.log import log_error
 from app.shared.settings import get_settings
 from app.shared.task_messaging import get_celery, get_redis
 from app.shared.utils.misc import get_all_urls
-from app.worker.worker_log import setup_celery_logger
+from app.worker.worker_log import logger, setup_celery_logger
 
 
 settings = get_settings()
diff --git a/app/worker/worker_log.py b/app/worker/worker_log.py
index c1c8dcc..34dba1c 100644
--- a/app/worker/worker_log.py
+++ b/app/worker/worker_log.py
@@ -1,7 +1,6 @@
 import sys
 
-from loguru import logger
-
+from app.shared.log import logger
 from app.shared.task_messaging import get_celery
 
 
@@ -14,16 +13,15 @@ def setup_celery_logger(c):
     for handler in celery_logger.handlers[:]:
         celery_logger.removeHandler(handler)
 
-    # Set up Loguru logging
-    logger.add("logs/celery_logs.log", retention="30 days", level="DEBUG")
-    logger.add("logs/celery_error_logs.log", retention="30 days", level="ERROR")
-
     # Redirect Celery logs to Loguru
     class InterceptHandler:
         @staticmethod
         def write(message):
             if message.strip():
-                logger.info(message.strip())
+                msg = message.strip()
+                # TODO: serialize to include extra information
+                with logger.contextualize(worker=True):
+                    logger.info(msg)
 
         # Required to prevent issues with buffered output
         @staticmethod