isolating api methods, introducing first tests

src/core/config.py

@@ -0,0 +1,27 @@
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
+
+VERSION = "0.7.0"
+API_DESCRIPTION = """
+#### API for the Auto-Archiver project, a tool to archive web pages and Google Sheets.
+
+**Usage notes:**
+- The API requires a Bearer token for most operations, which you can obtain by logging in with your Google account.
+- You can use this API to archive single URLs or entire Google Sheets. 
+- Once you submit a URL or Sheet for archiving, the API will return a task_id that you can use to check the status of the archiving process. It works asynchronously.
+"""
+
+ALLOWED_ORIGINS = os.environ.get("ALLOWED_ORIGINS", "chrome-extension://ondkcheoicfckabcnkdgbepofpjmjcmb,chrome-extension://ojcimmjndnlmmlgnjaeojoebaceokpdp").split(",")
+
+BREAKING_CHANGES = {"minVersion": "0.3.1", "message": "The latest update has breaking changes, please update the extension to the most recent version."}
+
+SERVE_LOCAL_ARCHIVE = os.environ.get("SERVE_LOCAL_ARCHIVE", "")
+
+SQLALCHEMY_DATABASE_URL = os.environ.get("DATABASE_PATH")
+
+REPEAT_COUNT_METRICS_SECONDS = 15
+
+CHROME_APP_IDS = set([app_id.strip() for app_id in os.environ.get("CHROME_APP_IDS", "").split(",")])
+BLOCKED_EMAILS = set([e.strip().lower() for e in os.environ.get("BLOCKED_EMAILS", "").split(",")])

src/core/events.py

@@ -0,0 +1,44 @@
+import asyncio
+import logging
+import alembic.config
+from fastapi import FastAPI
+from sqlalchemy.orm import Session
+from contextlib import asynccontextmanager
+from fastapi_utils.tasks import repeat_every
+from loguru import logger
+
+from db import crud, models
+from db.database import get_db, engine
+from utils.metrics import measure_regular_metrics, redis_subscribe_worker_exceptions
+from core.config import REPEAT_COUNT_METRICS_SECONDS
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    # see https://fastapi.tiangolo.com/advanced/events/#lifespan
+
+    # STARTUP
+    models.Base.metadata.create_all(bind=engine)
+    alembic.config.main(argv=['--raiseerr', 'upgrade', 'head'])
+    # disabling uvicorn logger since we use loguru in logging_middleware
+    logging.getLogger("uvicorn.access").disabled = True
+    asyncio.create_task(redis_subscribe_worker_exceptions())
+    asyncio.create_task(refresh_user_groups())
+    asyncio.create_task(measure_regular_metrics())
+
+    yield  # separates startup from shutdown instructions
+
+    # SHUTDOWN
+    logger.info("shutting down")
+
+
+# CRON JOBS
+
+@repeat_every(seconds=60 * 60)  # 1 hour
+async def refresh_user_groups():
+    db: Session = next(get_db())
+    crud.upsert_user_groups(db)
+
+@repeat_every(seconds=REPEAT_COUNT_METRICS_SECONDS)
+async def repeat_measure_regular_metrics():
+    measure_regular_metrics()

src/core/logging.py

@@ -0,0 +1,15 @@
+from loguru import logger
+from fastapi import Request
+from utils.metrics import EXCEPTION_COUNTER
+
+
+# logging configurations
+logger.add("logs/api_logs.log", retention="30 days", rotation="3 days")
+async def logging_middleware(request: Request, call_next):
+    try:
+        response = await call_next(request)
+        logger.info(f"{request.client.host}:{request.client.port} {request.method} {request.url._url} - HTTP {response.status_code}")
+        return response
+    except Exception as e:
+        EXCEPTION_COUNTER.labels(type(e).__name__).inc()
+        raise e