refactor to use pydantic settings and WAL sqlite mode

src/db/crud.py

@@ -5,12 +5,13 @@ from loguru import logger
 from datetime import datetime, timedelta
 
 from security import ALLOW_ANY_EMAIL
+from shared.settings import Settings
 from . import models, schemas
-import yaml, os
+import yaml
 
 DOMAIN_GROUPS = {}
 DOMAIN_GROUPS_LOADED = False
-MAX_LIMIT = 100
+DATABASE_QUERY_LIMIT = Settings().DATABASE_QUERY_LIMIT
 
 # --------------- TASK = Archive
 
@@ -39,12 +40,12 @@ def search_archives_by_url(db: Session, url: str, email: str, skip: int = 0, lim
         query = query.filter(models.Archive.created_at >= archived_after)
     if archived_before:
         query = query.filter(models.Archive.created_at <= archived_before)
-    return query.order_by(models.Archive.created_at.desc()).offset(skip).limit(min(limit, MAX_LIMIT)).all()
+    return query.order_by(models.Archive.created_at.desc()).offset(skip).limit(min(limit, DATABASE_QUERY_LIMIT)).all()
 
 
 def search_archives_by_email(db: Session, email: str, skip: int = 0, limit: int = 100):
     email = email.lower()
-    return base_query(db).filter(models.Archive.author.has(email=email)).offset(skip).limit(min(limit, MAX_LIMIT)).all()
+    return base_query(db).filter(models.Archive.author.has(email=email)).offset(skip).limit(min(limit, DATABASE_QUERY_LIMIT)).all()
 
 
 def create_task(db: Session, task: schemas.ArchiveCreate, tags: list[models.Tag], urls: list[models.ArchiveUrl]):
@@ -76,7 +77,7 @@ def count_by_user_since(db:Session, seconds_delta: int = 15):
     return db.query(models.Archive.author_id,func.count().label('total'))\
         .filter(models.Archive.created_at >= time_threshold)\
         .group_by(models.Archive.author_id)\
-        .order_by(func.count().desc()).limit(5 * MAX_LIMIT).all()
+        .order_by(func.count().desc()).limit(5 * DATABASE_QUERY_LIMIT).all()
 
 def base_query(db: Session):
     # allow only some fields to be returned, for example author should remain hidden
@@ -98,7 +99,7 @@ def create_tag(db: Session, tag: str):
 
 
 def search_tags(db: Session, tag: str, skip: int = 0, limit: int = 100):
-    return db.query(models.Tag).filter(models.Tag.url.like(f'%{tag}%')).offset(skip).limit(min(limit, MAX_LIMIT)).all()
+    return db.query(models.Tag).filter(models.Tag.url.like(f'%{tag}%')).offset(skip).limit(min(limit, DATABASE_QUERY_LIMIT)).all()
 
 
 def is_user_in_group(db: Session, group_name: str, email: str) -> models.Group:
@@ -148,7 +149,7 @@ def upsert_user_groups(db: Session):
     along with new participation of users in groups
     """
     logger.debug("Updating user-groups configuration.")
-    filename = os.environ.get("USER_GROUPS_FILENAME", "user-groups.yaml")
+    filename = Settings().USER_GROUPS_FILENAME
 
     # read yaml safely
     with open(filename) as inf:

src/db/database.py

@@ -1,17 +1,36 @@
-from sqlalchemy import create_engine
+from sqlalchemy import Engine, create_engine, event
 from sqlalchemy.orm import sessionmaker, declarative_base
-from core.config import SQLALCHEMY_DATABASE_URL
+from shared.settings import Settings
+from contextlib import contextmanager
 
 
-engine = create_engine(
-    SQLALCHEMY_DATABASE_URL, connect_args={"check_same_thread": False}
-)
-SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
+settings = Settings()
 
-Base = declarative_base()
+def make_engine(database_url: str):
+    engine = create_engine(database_url, connect_args={"check_same_thread": False})
+
+    @event.listens_for(engine, "connect")
+    def set_sqlite_pragma(conn, _) -> None:
+        cursor = conn.cursor()
+        cursor.execute("PRAGMA journal_mode=WAL")
+        cursor.close()
+
+    return engine
 
 
+def make_session_local(engine: Engine):
+    session_local = sessionmaker(autocommit=False, autoflush=False, bind=engine)
+    return session_local
+
+
+@contextmanager
 def get_db():
-    session = SessionLocal()
+    session = make_session_local(make_engine(settings.DATABASE_PATH))()
     try: yield session
     finally: session.close()
+
+
+def get_db_dependency():
+    # to use with Depends and ensure proper session closing
+    with get_db() as db:
+        yield db

src/db/models.py

@@ -1,8 +1,10 @@
 from sqlalchemy import Column, String, JSON, DateTime, Boolean, Table, ForeignKey
 from sqlalchemy.sql import func
-from sqlalchemy.orm import relationship
+from sqlalchemy.orm import relationship, declarative_base
 import uuid
-from .database import Base
+
+
+Base = declarative_base()
 
 def generate_uuid():
     return str(uuid.uuid4())
@@ -59,7 +61,6 @@ class Tag(Base):
 
     archives = relationship("Archive", back_populates="tags", secondary=association_table_archive_tags)
 
-
 class User(Base):
     __tablename__ = "users"
 

src/db/schemas.py

@@ -2,6 +2,13 @@ from pydantic import BaseModel
 from datetime import datetime
 
 
+class Tag(BaseModel):
+    id: str
+    created_at: datetime
+
+    model_config = { "from_attributes": True }
+    __hash__ = object.__hash__
+
 class ArchiveCreate(BaseModel):
     id: str | None = None
     url: str
@@ -9,7 +16,7 @@ class ArchiveCreate(BaseModel):
     public: bool = True
     author_id: str | None = None
     group_id: str | None = None
-    tags: set = set()
+    tags: set[Tag] | None = set()
     rearchive: bool = True
     # urls: list = []
 
@@ -28,7 +35,7 @@ class SubmitSheet(BaseModel):
     public: bool = False
     author_id: str | None = None
     group_id: str | None = None
-    tags: set | None = set()
+    tags: set[Tag] | None = set()
     columns: dict | None = {} # TODO: implement
 
 class SubmitManual(BaseModel):
@@ -36,7 +43,7 @@ class SubmitManual(BaseModel):
     public: bool = False
     author_id: str | None = None
     group_id: str | None = None
-    tags: set | None = set()
+    tags: set[Tag] | None = set()
 
 class Task(BaseModel):
     id: str