optimizing compose and logging

.env.alembic

@@ -1,5 +1,5 @@
 CHROME_APP_IDS='["1234567890"]'
 ALLOWED_ORIGINS='["allowed"]'
 BLOCKED_EMAILS='[]'
-DATABASE_PATH="sqlite:///./app/database/auto-archiver.db"
+DATABASE_PATH="sqlite:///./database/auto-archiver.db"
 API_BEARER_TOKEN=THIS_API_TOKEN_SHOULD_NEVER_BE_USED

app/shared/log.py

@@ -3,8 +3,8 @@ from loguru import logger
 
 
 # logging configurations
-logger.add("app/logs/api_logs.log", retention="30 days", rotation="3 days")
-logger.add("app/logs/error_logs.log", retention="30 days", level="ERROR")
+logger.add("logs/api_logs.log", retention="30 days")
+logger.add("logs/error_logs.log", retention="30 days", level="ERROR")
 
     
 def log_error(e: Exception, traceback_str: str = None, extra:str = ""):

app/shared/task_messaging.py

@@ -11,6 +11,7 @@ def get_celery(name:str="") -> Celery:
         name,
         broker_url=get_settings().CELERY_BROKER_URL,
         result_backend=get_settings().CELERY_BROKER_URL,
+        broker_connection_retry_on_startup=False
     )
 
 

app/web/db/crud.py

@@ -181,7 +181,6 @@ def upsert_user_groups(db: Session):
     import json
     # upsert groups and save a map of groupid -> dbobject
     for group_id, g in ug.groups.items():
-        logger.debug(f"GROUP {group_id} => {g.service_account_email}")
         upsert_group(db, group_id, g.description, g.orchestrator, g.orchestrator_sheet, g.service_account_email, json.loads(g.permissions.model_dump_json()), list(group_domains.get(group_id, [])))
     db_groups: dict[str, models.Group] = {g.id: g for g in db.query(models.Group).all()}
 

docker-compose.dev.yml

@@ -10,11 +10,11 @@ services:
       - SERVE_LOCAL_ARCHIVE=/aa-api/app/local_archive # See orchestration.yaml local_storage.save_to
       - ALLOWED_ORIGINS=["http://localhost:8000","http://localhost:8004","http://localhost:8081","chrome-extension://ojcimmjndnlmmlgnjaeojoebaceokpdp"]
       - USER_GROUPS_FILENAME=/aa-api/app/user-groups.dev.yaml
-      - DATABASE_PATH=sqlite:////aa-api/app/database/auto-archiver.db
+      - DATABASE_PATH=sqlite:////aa-api/database/auto-archiver.db
     
 
   worker:
-    command: watchmedo auto-restart --patterns="*.py" --recursive --ignore-directories -- celery -- --app=app.worker.main.celery worker --loglevel=info --logfile=/aa-api/app/logs/celery.log
+    command: watchmedo auto-restart --patterns="*.py" --recursive --ignore-directories -- celery -- --app=app.worker.main.celery worker --loglevel=debug --logfile=/aa-api/logs/celery.log
     restart: "no"
     env_file: .env.dev
     volumes:

docker-compose.yml

@@ -15,12 +15,10 @@ services:
       REDIS_HOSTNAME: redis
     ports:
       - "127.0.0.1:8004:8000"
-    #TODO: should prod have the --reload flag?
     command: uvicorn app.web:app --factory --host 0.0.0.0
     volumes:
-      # - ./app:/app
-      - ./app/logs:/aa-api/app/logs
-      - ./app/database:/aa-api/app/database
+      - ./logs:/aa-api/logs
+      - ./app/database:/aa-api/database
     depends_on:
       - redis
     healthcheck:
@@ -35,10 +33,10 @@ services:
       dockerfile: worker.Dockerfile
     restart: always
     env_file: .env.prod
-    command: celery --app=app.worker.main.celery worker --loglevel=info --logfile=/aa-api/app/logs/celery.log
+    command: celery --app=app.worker.main.celery worker --loglevel=warning --logfile=/aa-api/logs/celery.log
     volumes:
-      - ./app/logs:/aa-api/app/logs
-      - ./app/database:/aa-api/app/database
+      - ./logs:/aa-api/logs
+      - ./app/database:/aa-api/database
       - /var/run/docker.sock:/var/run/docker.sock
       - crawls:/crawls # BROWSERTRIX_HOME_HOST:BROWSERTRIX_HOME_CONTAINER, do not change /crawls
     environment: