From 4f9d447ec77a1dcaaeea42082bd0c8624709102e Mon Sep 17 00:00:00 2001 From: msramalho <19508417+msramalho@users.noreply.github.com> Date: Tue, 11 Feb 2025 23:30:45 +0000 Subject: [PATCH] optimizing compose and logging --- .env.alembic | 2 +- app/shared/log.py | 4 ++-- app/shared/task_messaging.py | 1 + app/web/db/crud.py | 1 - docker-compose.dev.yml | 4 ++-- docker-compose.yml | 12 +++++------- {app/logs => logs}/.gitkeep | 0 7 files changed, 11 insertions(+), 13 deletions(-) rename {app/logs => logs}/.gitkeep (100%) diff --git a/.env.alembic b/.env.alembic index da29332..8691557 100644 --- a/.env.alembic +++ b/.env.alembic @@ -1,5 +1,5 @@ CHROME_APP_IDS='["1234567890"]' ALLOWED_ORIGINS='["allowed"]' BLOCKED_EMAILS='[]' -DATABASE_PATH="sqlite:///./app/database/auto-archiver.db" +DATABASE_PATH="sqlite:///./database/auto-archiver.db" API_BEARER_TOKEN=THIS_API_TOKEN_SHOULD_NEVER_BE_USED \ No newline at end of file diff --git a/app/shared/log.py b/app/shared/log.py index d11b7a3..68587e2 100644 --- a/app/shared/log.py +++ b/app/shared/log.py @@ -3,8 +3,8 @@ from loguru import logger # logging configurations -logger.add("app/logs/api_logs.log", retention="30 days", rotation="3 days") -logger.add("app/logs/error_logs.log", retention="30 days", level="ERROR") +logger.add("logs/api_logs.log", retention="30 days") +logger.add("logs/error_logs.log", retention="30 days", level="ERROR") def log_error(e: Exception, traceback_str: str = None, extra:str = ""): diff --git a/app/shared/task_messaging.py b/app/shared/task_messaging.py index 52dcba3..7f0f09e 100644 --- a/app/shared/task_messaging.py +++ b/app/shared/task_messaging.py @@ -11,6 +11,7 @@ def get_celery(name:str="") -> Celery: name, broker_url=get_settings().CELERY_BROKER_URL, result_backend=get_settings().CELERY_BROKER_URL, + broker_connection_retry_on_startup=False ) diff --git a/app/web/db/crud.py b/app/web/db/crud.py index 5e1f976..be2a915 100644 --- a/app/web/db/crud.py +++ b/app/web/db/crud.py @@ -181,7 +181,6 @@ def upsert_user_groups(db: Session): import json # upsert groups and save a map of groupid -> dbobject for group_id, g in ug.groups.items(): - logger.debug(f"GROUP {group_id} => {g.service_account_email}") upsert_group(db, group_id, g.description, g.orchestrator, g.orchestrator_sheet, g.service_account_email, json.loads(g.permissions.model_dump_json()), list(group_domains.get(group_id, []))) db_groups: dict[str, models.Group] = {g.id: g for g in db.query(models.Group).all()} diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 5a8bc83..3a7129d 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -10,11 +10,11 @@ services: - SERVE_LOCAL_ARCHIVE=/aa-api/app/local_archive # See orchestration.yaml local_storage.save_to - ALLOWED_ORIGINS=["http://localhost:8000","http://localhost:8004","http://localhost:8081","chrome-extension://ojcimmjndnlmmlgnjaeojoebaceokpdp"] - USER_GROUPS_FILENAME=/aa-api/app/user-groups.dev.yaml - - DATABASE_PATH=sqlite:////aa-api/app/database/auto-archiver.db + - DATABASE_PATH=sqlite:////aa-api/database/auto-archiver.db worker: - command: watchmedo auto-restart --patterns="*.py" --recursive --ignore-directories -- celery -- --app=app.worker.main.celery worker --loglevel=info --logfile=/aa-api/app/logs/celery.log + command: watchmedo auto-restart --patterns="*.py" --recursive --ignore-directories -- celery -- --app=app.worker.main.celery worker --loglevel=debug --logfile=/aa-api/logs/celery.log restart: "no" env_file: .env.dev volumes: diff --git a/docker-compose.yml b/docker-compose.yml index 420ef6d..074b311 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,12 +15,10 @@ services: REDIS_HOSTNAME: redis ports: - "127.0.0.1:8004:8000" - #TODO: should prod have the --reload flag? command: uvicorn app.web:app --factory --host 0.0.0.0 volumes: - # - ./app:/app - - ./app/logs:/aa-api/app/logs - - ./app/database:/aa-api/app/database + - ./logs:/aa-api/logs + - ./app/database:/aa-api/database depends_on: - redis healthcheck: @@ -35,10 +33,10 @@ services: dockerfile: worker.Dockerfile restart: always env_file: .env.prod - command: celery --app=app.worker.main.celery worker --loglevel=info --logfile=/aa-api/app/logs/celery.log + command: celery --app=app.worker.main.celery worker --loglevel=warning --logfile=/aa-api/logs/celery.log volumes: - - ./app/logs:/aa-api/app/logs - - ./app/database:/aa-api/app/database + - ./logs:/aa-api/logs + - ./app/database:/aa-api/database - /var/run/docker.sock:/var/run/docker.sock - crawls:/crawls # BROWSERTRIX_HOME_HOST:BROWSERTRIX_HOME_CONTAINER, do not change /crawls environment: diff --git a/app/logs/.gitkeep b/logs/.gitkeep similarity index 100% rename from app/logs/.gitkeep rename to logs/.gitkeep