From 273adb4ba02c5d1749908431d09b6fb893d04928 Mon Sep 17 00:00:00 2001 From: msramalho <19508417+msramalho@users.noreply.github.com> Date: Mon, 28 Oct 2024 15:01:11 +0000 Subject: [PATCH 1/9] fix docker health command for worker --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 2274dca..8764f02 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -47,7 +47,7 @@ services: - web - redis healthcheck: - test: ["CMD", "pipenv", "run", "celery", "-A", "worker.celery", "status"] + test: ["CMD", "pipenv", "run", "celery", "-A", "worker.main.celery", "status"] interval: 30s timeout: 10s retries: 3 From dbe3cbb3f5011092fe15db462096724178ff380a Mon Sep 17 00:00:00 2001 From: msramalho <19508417+msramalho@users.noreply.github.com> Date: Mon, 28 Oct 2024 15:01:17 +0000 Subject: [PATCH 2/9] version bump --- src/core/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/config.py b/src/core/config.py index 6a37fd0..7f6393e 100644 --- a/src/core/config.py +++ b/src/core/config.py @@ -1,4 +1,4 @@ -VERSION = "0.7.0" +VERSION = "0.7.1" API_DESCRIPTION = """ #### API for the Auto-Archiver project, a tool to archive web pages and Google Sheets. From 8dc91bbe0f3132e9da17d45dde63e851ba8ecb5e Mon Sep 17 00:00:00 2001 From: msramalho <19508417+msramalho@users.noreply.github.com> Date: Mon, 28 Oct 2024 15:02:17 +0000 Subject: [PATCH 3/9] fixes logic of error logging --- src/core/logging.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/core/logging.py b/src/core/logging.py index fbaed29..5ff03db 100644 --- a/src/core/logging.py +++ b/src/core/logging.py @@ -5,15 +5,14 @@ from fastapi import Request # logging configurations logger.add("logs/api_logs.log", retention="30 days", rotation="3 days") -error_logger = logger.add("logs/error_logs.log", retention="30 days") +logger.add("logs/error_logs.log", retention="30 days", level="ERROR") def log_error(e: Exception, traceback_str: str = None, extra:str = ""): # EXCEPTION_COUNTER.labels(type(e).__name__).inc() if not traceback_str: traceback_str = traceback.format_exc() if extra: extra = f"{extra}\n" - logger.error(f"{extra}{e.__class__.__name__}: {e}") - error_logger.error(f"{extra}{e.__class__.__name__}: {e}\n{traceback_str}") + logger.error(f"{extra}{e.__class__.__name__}: {e}\n{traceback_str}") async def logging_middleware(request: Request, call_next): try: From bd3fd9f48bebb5e14da1585a61bc473670ed4e76 Mon Sep 17 00:00:00 2001 From: msramalho <19508417+msramalho@users.noreply.github.com> Date: Mon, 28 Oct 2024 15:02:28 +0000 Subject: [PATCH 4/9] adds cache for db handling --- src/db/database.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/db/database.py b/src/db/database.py index d18333d..d42466a 100644 --- a/src/db/database.py +++ b/src/db/database.py @@ -1,9 +1,11 @@ +from functools import lru_cache from sqlalchemy import Engine, create_engine, event from sqlalchemy.orm import sessionmaker from shared.settings import get_settings from contextlib import contextmanager +@lru_cache def make_engine(database_url: str): engine = create_engine(database_url, connect_args={"check_same_thread": False}) From 66b070bfa2ef38361799b32f12332b75d7e132ec Mon Sep 17 00:00:00 2001 From: msramalho <19508417+msramalho@users.noreply.github.com> Date: Mon, 28 Oct 2024 15:02:36 +0000 Subject: [PATCH 5/9] removes empty space --- src/tests/endpoints/test_url.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/tests/endpoints/test_url.py b/src/tests/endpoints/test_url.py index 01b6a24..3d05434 100644 --- a/src/tests/endpoints/test_url.py +++ b/src/tests/endpoints/test_url.py @@ -1,5 +1,3 @@ - - import json from unittest.mock import patch From 8b3c4d32281fbb6ff05f1fb2aa22d63129fa4fd4 Mon Sep 17 00:00:00 2001 From: msramalho <19508417+msramalho@users.noreply.github.com> Date: Mon, 28 Oct 2024 15:04:18 +0000 Subject: [PATCH 6/9] exclude more endpoints from metrics --- src/web/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/web/main.py b/src/web/main.py index adc837d..826cc03 100644 --- a/src/web/main.py +++ b/src/web/main.py @@ -51,7 +51,7 @@ def app_factory(settings = get_settings()): app.include_router(interoperability_router) # prometheus exposed in /metrics with authentication - Instrumentator(should_group_status_codes=False, excluded_handlers=["/metrics", "/health"]).instrument(app).expose(app, dependencies=[Depends(token_api_key_auth)]) + Instrumentator(should_group_status_codes=False, excluded_handlers=["/metrics", "/health", "/openapi.json", "/favicon.ico"]).instrument(app).expose(app, dependencies=[Depends(token_api_key_auth)]) local_dir = settings.SERVE_LOCAL_ARCHIVE if not os.path.isdir(local_dir) and os.path.isdir(local_dir.replace("/app", ".")): From 5283b341c791e1279cd2008b725ae3427ae16326 Mon Sep 17 00:00:00 2001 From: msramalho <19508417+msramalho@users.noreply.github.com> Date: Mon, 28 Oct 2024 15:04:40 +0000 Subject: [PATCH 7/9] minor change to security length check --- src/web/security.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/web/security.py b/src/web/security.py index b310782..bfa678a 100644 --- a/src/web/security.py +++ b/src/web/security.py @@ -15,10 +15,9 @@ def secure_compare(token, api_key): # Factory method to create an authentication dependency for a specific key def api_key_auth(api_key): + assert len(api_key) >= 20, "Invalid API key, must be at least 20 chars" async def auth(bearer: HTTPAuthorizationCredentials = Depends(bearer_security), auto_error=True): - assert len(api_key) >= 20, "Invalid API key, must be at least 20 chars" - is_correct = secure_compare(bearer.credentials, api_key) if is_correct: return True From a454eb778d99859378b9d7871aa8e868124d8f7c Mon Sep 17 00:00:00 2001 From: msramalho <19508417+msramalho@users.noreply.github.com> Date: Mon, 28 Oct 2024 15:05:11 +0000 Subject: [PATCH 8/9] ignoring script file --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 16d4ed8..1df91d8 100644 --- a/.gitignore +++ b/.gitignore @@ -22,4 +22,5 @@ htmlcov local_archive local_archive_test *db-wal -*db-shm \ No newline at end of file +*db-shm +copy-files.sh \ No newline at end of file From d7fb55980bb5a11d4d43062a4e23ceae9dbc3e34 Mon Sep 17 00:00:00 2001 From: msramalho <19508417+msramalho@users.noreply.github.com> Date: Mon, 28 Oct 2024 15:17:30 +0000 Subject: [PATCH 9/9] fix cache issues in tests --- src/tests/conftest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/tests/conftest.py b/src/tests/conftest.py index 1b69916..97c18e8 100644 --- a/src/tests/conftest.py +++ b/src/tests/conftest.py @@ -26,7 +26,8 @@ def mock_settings(): def test_db(get_settings: Settings): from db.database import make_engine from db import models - + + make_engine.cache_clear() engine = make_engine(get_settings.DATABASE_PATH) fs = get_settings.DATABASE_PATH.replace("sqlite:///", "")