new logging strategy, combining API+Worker logs

app/web/config.py

@@ -1,4 +1,4 @@
-VERSION = "0.12.0"
+VERSION = "0.12.1"
 
 API_DESCRIPTION = """
 #### API for the Auto-Archiver project, a tool to archive web pages and Google Sheets.

app/web/db/crud.py

@@ -4,7 +4,6 @@ from typing import Any, Type
 
 from cachetools import LRUCache, cached
 from cachetools.keys import hashkey
-from loguru import logger
 from sqlalchemy import (
     Column,
     ColumnElement,
@@ -21,6 +20,7 @@ from sqlalchemy.orm import Session, load_only
 
 from app.shared.db import models
 from app.shared.db.models import Archive, Group
+from app.shared.log import logger
 from app.shared.settings import get_settings
 from app.shared.user_groups import UserGroups
 from app.shared.utils.misc import fnv1a_hash_mod

app/web/events.py

@@ -8,7 +8,6 @@ import alembic.config
 from fastapi import FastAPI
 from fastapi_mail import FastMail, MessageSchema, MessageType
 from fastapi_utils.tasks import repeat_every
-from loguru import logger
 
 from app.shared import schemas
 from app.shared.db import models
@@ -18,6 +17,7 @@ from app.shared.db.database import (
     make_engine,
     wal_checkpoint,
 )
+from app.shared.log import logger
 from app.shared.settings import get_settings
 from app.shared.task_messaging import get_celery
 from app.web.db import crud

app/web/main.py

@@ -3,9 +3,9 @@ import os
 from fastapi import Depends, FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.staticfiles import StaticFiles
-from loguru import logger
 from prometheus_fastapi_instrumentator import Instrumentator
 
+from app.shared.log import logger
 from app.shared.settings import Settings, get_settings
 from app.shared.task_messaging import get_celery
 from app.web.config import API_DESCRIPTION, VERSION

app/web/middleware.py

@@ -1,9 +1,8 @@
 import traceback
 
 from fastapi import Request
-from loguru import logger
 
-from app.shared.log import log_error
+from app.shared.log import log_error, logger
 from app.web.utils.metrics import EXCEPTION_COUNTER
 
 
@@ -13,14 +12,14 @@ async def logging_middleware(request: Request, call_next):
         # TODO: use Origin to have summary prometheus metrics on where
         #  requests come from
         # origin = request.headers.get("origin")
-        logger.info(
+        logger.debug(
             f"{request.client.host}:{request.client.port} {request.method} {request.url._url} - HTTP {response.status_code}"
         )
         return response
     except Exception as e:
         location = f"{request.method} {request.url._url}"
         await increase_exceptions_counter(e, location)
-        logger.info(
+        logger.error(
             f"{request.client.host}:{request.client.port} {location} - {e.__class__.__name__} {e}"
         )
         raise e

app/web/routers/interoperability.py

@@ -5,13 +5,12 @@ import sqlalchemy
 from auto_archiver.core import Metadata
 from fastapi import APIRouter, Depends, HTTPException
 from fastapi.responses import JSONResponse
-from loguru import logger
 from sqlalchemy.orm import Session
 
 from app.shared import business_logic, schemas
 from app.shared.db import models, worker_crud
 from app.shared.db.database import get_db_dependency
-from app.shared.log import log_error
+from app.shared.log import log_error, logger
 from app.shared.utils.misc import get_all_urls
 from app.web.config import ALLOW_ANY_EMAIL
 from app.web.security import token_api_key_auth

app/web/routers/url.py

@@ -4,11 +4,11 @@ from urllib.parse import urlparse
 
 from fastapi import APIRouter, Depends, HTTPException
 from fastapi.responses import JSONResponse
-from loguru import logger
 from sqlalchemy.orm import Session
 
 from app.shared import schemas
 from app.shared.db.database import get_db_dependency
+from app.shared.log import logger
 from app.shared.schemas import DeleteResponse
 from app.shared.task_messaging import get_celery
 from app.web.config import ALLOW_ANY_EMAIL

app/web/security.py

@@ -6,10 +6,10 @@ import requests
 from fastapi import Depends, HTTPException, status
 from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
 from firebase_admin import auth, credentials, exceptions
-from loguru import logger
 from sqlalchemy.orm import Session
 
 from app.shared.db.database import get_db_dependency
+from app.shared.log import logger
 from app.shared.settings import get_settings
 from app.web.config import ALLOW_ANY_EMAIL
 from app.web.db.user_state import UserState