new logging strategy, combining API+Worker logs

app/worker/main.py

@@ -4,7 +4,6 @@ import traceback
 
 from auto_archiver.core.orchestrator import ArchivingOrchestrator
 from celery.signals import task_failure
-from loguru import logger
 from sqlalchemy import exc
 
 from app.shared import business_logic, constants, schemas
@@ -14,7 +13,7 @@ from app.shared.log import log_error
 from app.shared.settings import get_settings
 from app.shared.task_messaging import get_celery, get_redis
 from app.shared.utils.misc import get_all_urls
-from app.worker.worker_log import setup_celery_logger
+from app.worker.worker_log import logger, setup_celery_logger
 
 
 settings = get_settings()

app/worker/worker_log.py

@@ -1,7 +1,6 @@
 import sys
 
-from loguru import logger
-
+from app.shared.log import logger
 from app.shared.task_messaging import get_celery
 
 
@@ -14,16 +13,15 @@ def setup_celery_logger(c):
     for handler in celery_logger.handlers[:]:
         celery_logger.removeHandler(handler)
 
-    # Set up Loguru logging
-    logger.add("logs/celery_logs.log", retention="30 days", level="DEBUG")
-    logger.add("logs/celery_error_logs.log", retention="30 days", level="ERROR")
-
     # Redirect Celery logs to Loguru
     class InterceptHandler:
         @staticmethod
         def write(message):
             if message.strip():
-                logger.info(message.strip())
+                msg = message.strip()
+                # TODO: serialize to include extra information
+                with logger.contextualize(worker=True):
+                    logger.info(msg)
 
         # Required to prevent issues with buffered output
         @staticmethod