new logging strategy, combining API+Worker logs

app/shared/log.py

@@ -1,11 +1,18 @@
 import traceback
 
-from loguru import logger
+from auto_archiver.utils.custom_logger import logger
 
 
 # logging configurations
-logger.add("logs/api_logs.log", retention="30 days")
+logger.add(
-logger.add("logs/error_logs.log", retention="30 days", level="ERROR")
+    "logs/all_logs.log", retention="30 days", format="{extra[serialized]}"
+)
+logger.add(
+    "logs/all_error_logs.log",
+    retention="120 days",
+    level="ERROR",
+    format="{extra[serialized]}",
+)
 
 
 def log_error(e: Exception, traceback_str: str = None, extra: str = ""):

app/shared/user_groups.py

@@ -3,7 +3,6 @@ import os
 from typing import Dict, List, Set
 
 import yaml
-from loguru import logger
 from pydantic import (
     BaseModel,
     Field,
@@ -13,6 +12,8 @@ from pydantic import (
 )
 from typing_extensions import Self
 
+from app.shared.log import logger
+
 
 class UserGroups:
     def __init__(self, filename):

app/shared/utils/misc.py

@@ -1,9 +1,9 @@
 from typing import List
 
 from auto_archiver.core import Media, Metadata
-from loguru import logger
 
 from app.shared.db import models
+from app.shared.log import logger
 
 
 def fnv1a_hash_mod(s: str, modulo: int) -> int:

app/tests/web/routers/test_default.py

@@ -3,8 +3,8 @@ from unittest.mock import MagicMock
 
 import pytest
 from fastapi.testclient import TestClient
-from loguru import logger
 
+from app.shared.log import logger
 from app.shared.schemas import Usage, UsageResponse
 from app.shared.user_groups import GroupInfo
 from app.web.config import VERSION

app/web/config.py

@@ -1,4 +1,4 @@
-VERSION = "0.12.0"
+VERSION = "0.12.1"
 
 API_DESCRIPTION = """
 #### API for the Auto-Archiver project, a tool to archive web pages and Google Sheets.

app/web/db/crud.py

@@ -4,7 +4,6 @@ from typing import Any, Type
 
 from cachetools import LRUCache, cached
 from cachetools.keys import hashkey
-from loguru import logger
 from sqlalchemy import (
     Column,
     ColumnElement,
@@ -21,6 +20,7 @@ from sqlalchemy.orm import Session, load_only
 
 from app.shared.db import models
 from app.shared.db.models import Archive, Group
+from app.shared.log import logger
 from app.shared.settings import get_settings
 from app.shared.user_groups import UserGroups
 from app.shared.utils.misc import fnv1a_hash_mod

app/web/events.py

@@ -8,7 +8,6 @@ import alembic.config
 from fastapi import FastAPI
 from fastapi_mail import FastMail, MessageSchema, MessageType
 from fastapi_utils.tasks import repeat_every
-from loguru import logger
 
 from app.shared import schemas
 from app.shared.db import models
@@ -18,6 +17,7 @@ from app.shared.db.database import (
     make_engine,
     wal_checkpoint,
 )
+from app.shared.log import logger
 from app.shared.settings import get_settings
 from app.shared.task_messaging import get_celery
 from app.web.db import crud

app/web/main.py

@@ -3,9 +3,9 @@ import os
 from fastapi import Depends, FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.staticfiles import StaticFiles
-from loguru import logger
 from prometheus_fastapi_instrumentator import Instrumentator
 
+from app.shared.log import logger
 from app.shared.settings import Settings, get_settings
 from app.shared.task_messaging import get_celery
 from app.web.config import API_DESCRIPTION, VERSION

app/web/middleware.py

@@ -1,9 +1,8 @@
 import traceback
 
 from fastapi import Request
-from loguru import logger
 
-from app.shared.log import log_error
+from app.shared.log import log_error, logger
 from app.web.utils.metrics import EXCEPTION_COUNTER
 
 
@@ -13,14 +12,14 @@ async def logging_middleware(request: Request, call_next):
         # TODO: use Origin to have summary prometheus metrics on where
         #  requests come from
         # origin = request.headers.get("origin")
-        logger.info(
+        logger.debug(
             f"{request.client.host}:{request.client.port} {request.method} {request.url._url} - HTTP {response.status_code}"
         )
         return response
     except Exception as e:
         location = f"{request.method} {request.url._url}"
         await increase_exceptions_counter(e, location)
-        logger.info(
+        logger.error(
             f"{request.client.host}:{request.client.port} {location} - {e.__class__.__name__} {e}"
         )
         raise e

app/web/routers/interoperability.py

@@ -5,13 +5,12 @@ import sqlalchemy
 from auto_archiver.core import Metadata
 from fastapi import APIRouter, Depends, HTTPException
 from fastapi.responses import JSONResponse
-from loguru import logger
 from sqlalchemy.orm import Session
 
 from app.shared import business_logic, schemas
 from app.shared.db import models, worker_crud
 from app.shared.db.database import get_db_dependency
-from app.shared.log import log_error
+from app.shared.log import log_error, logger
 from app.shared.utils.misc import get_all_urls
 from app.web.config import ALLOW_ANY_EMAIL
 from app.web.security import token_api_key_auth

app/web/routers/url.py

@@ -4,11 +4,11 @@ from urllib.parse import urlparse
 
 from fastapi import APIRouter, Depends, HTTPException
 from fastapi.responses import JSONResponse
-from loguru import logger
 from sqlalchemy.orm import Session
 
 from app.shared import schemas
 from app.shared.db.database import get_db_dependency
+from app.shared.log import logger
 from app.shared.schemas import DeleteResponse
 from app.shared.task_messaging import get_celery
 from app.web.config import ALLOW_ANY_EMAIL

app/web/security.py

@@ -6,10 +6,10 @@ import requests
 from fastapi import Depends, HTTPException, status
 from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
 from firebase_admin import auth, credentials, exceptions
-from loguru import logger
 from sqlalchemy.orm import Session
 
 from app.shared.db.database import get_db_dependency
+from app.shared.log import logger
 from app.shared.settings import get_settings
 from app.web.config import ALLOW_ANY_EMAIL
 from app.web.db.user_state import UserState

app/worker/main.py

@@ -4,7 +4,6 @@ import traceback
 
 from auto_archiver.core.orchestrator import ArchivingOrchestrator
 from celery.signals import task_failure
-from loguru import logger
 from sqlalchemy import exc
 
 from app.shared import business_logic, constants, schemas
@@ -14,7 +13,7 @@ from app.shared.log import log_error
 from app.shared.settings import get_settings
 from app.shared.task_messaging import get_celery, get_redis
 from app.shared.utils.misc import get_all_urls
-from app.worker.worker_log import setup_celery_logger
+from app.worker.worker_log import logger, setup_celery_logger
 
 
 settings = get_settings()

app/worker/worker_log.py

@@ -1,7 +1,6 @@
 import sys
 
-from loguru import logger
+from app.shared.log import logger
-
 from app.shared.task_messaging import get_celery
 
 
@@ -14,16 +13,15 @@ def setup_celery_logger(c):
     for handler in celery_logger.handlers[:]:
         celery_logger.removeHandler(handler)
 
-    # Set up Loguru logging
-    logger.add("logs/celery_logs.log", retention="30 days", level="DEBUG")
-    logger.add("logs/celery_error_logs.log", retention="30 days", level="ERROR")
-
     # Redirect Celery logs to Loguru
     class InterceptHandler:
         @staticmethod
         def write(message):
             if message.strip():
-                logger.info(message.strip())
+                msg = message.strip()
+                # TODO: serialize to include extra information
+                with logger.contextualize(worker=True):
+                    logger.info(msg)
 
         # Required to prevent issues with buffered output
         @staticmethod