major refactor of structure for worker V web: docker/app/secrets/envs/...

app/shared/aa_utils.py

@@ -0,0 +1,33 @@
+# TODO: code in this file should eventually be moved to the auto-archiver code base
+
+from typing import List
+from loguru import logger
+from auto_archiver import Metadata
+from auto_archiver.core import Media
+
+from app.shared.db import models
+
+def get_all_urls(result: Metadata) -> List[models.ArchiveUrl]:
+    db_urls = []
+    for m in result.media:
+        for i, url in enumerate(m.urls): db_urls.append(models.ArchiveUrl(url=url, key=m.get("id", f"media_{i}")))
+        for k, prop in m.properties.items():
+            if prop_converted := convert_if_media(prop):
+                for i, url in enumerate(prop_converted.urls): db_urls.append(models.ArchiveUrl(url=url, key=prop_converted.get("id", f"{k}_{i}")))
+            if isinstance(prop, list):
+                for i, prop_media in enumerate(prop):
+                    if prop_media := convert_if_media(prop_media):
+                        for j, url in enumerate(prop_media.urls):
+                            db_urls.append(models.ArchiveUrl(url=url, key=prop_media.get("id", f"{k}{prop_media.key}_{i}.{j}")))
+    return db_urls
+
+
+
+def convert_if_media(media):
+    if isinstance(media, Media): return media
+    elif isinstance(media, dict):
+        try: return Media.from_dict(media)
+        except Exception as e:
+            logger.debug(f"error parsing {media} : {e}")
+    return False
+

app/shared/business_logic.py

@@ -0,0 +1,15 @@
+# TODO: temporary file for this code, maybe other code belongs here, maybe not. do decide
+
+
+import datetime
+from sqlalchemy.orm import Session
+
+from app.shared.db import crud
+
+
+def get_store_archive_until(db: Session, group_id: str) -> datetime.datetime:
+    group = crud.get_group(db, group_id)
+    max_lifespan = group.permissions.get("max_archive_lifespan_months", -1)
+    if max_lifespan == -1: return None
+
+    return datetime.datetime.now() + datetime.timedelta(days=30 * max_lifespan)

app/shared/config.py

@@ -0,0 +1,13 @@
+VERSION = "0.8.0"
+API_DESCRIPTION = """
+#### API for the Auto-Archiver project, a tool to archive web pages and Google Sheets.
+
+**Usage notes:**
+- The API requires a Bearer token for most operations, which you can obtain by logging in with your Google account.
+- You can use this API to archive single URLs or entire Google Sheets. 
+- Once you submit a URL or Sheet for archiving, the API will return a task_id that you can use to check the status of the archiving process. It works asynchronously.
+"""
+BREAKING_CHANGES = {"minVersion": "0.3.1", "message": "The latest update has breaking changes, please update the extension to the most recent version."}
+
+# changing this will corrupt the database logic
+ALLOW_ANY_EMAIL = "*"

app/shared/db/crud.py

@@ -0,0 +1,314 @@
+from collections import defaultdict
+from functools import lru_cache
+from sqlalchemy.orm import Session, load_only
+from sqlalchemy import Column, or_, func, select
+from loguru import logger
+from datetime import datetime, timedelta
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.shared.config import ALLOW_ANY_EMAIL
+from app.shared.db.database import get_db
+from app.shared.db import models
+from app.shared import schemas
+from app.shared.settings import get_settings
+from app.shared.user_groups import UserGroups
+from app.shared.utils.misc import fnv1a_hash_mod
+
+DATABASE_QUERY_LIMIT = get_settings().DATABASE_QUERY_LIMIT
+
+
+def get_limit(user_limit: int):
+    return max(1, min(user_limit, DATABASE_QUERY_LIMIT))
+
+# --------------- TASK = Archive
+
+def base_query(db: Session):
+    # NOTE: load_only is for optimization and not obfuscation, use .with_entities() if needed
+    return db.query(models.Archive)\
+        .filter(models.Archive.deleted == False)\
+        .options(load_only(models.Archive.id, models.Archive.created_at, models.Archive.url, models.Archive.result, models.Archive.store_until))
+
+def get_archive(db: Session, id: str, email: str):
+    query = base_query(db).filter(models.Archive.id == id)
+    if email != ALLOW_ANY_EMAIL:
+        groups = get_user_groups(email)
+        query = query.filter(or_(models.Archive.public == True, models.Archive.author_id == email, models.Archive.group_id.in_(groups)))
+    return query.first()
+
+def search_archives_by_url(db: Session, url: str, email: str, skip: int = 0, limit: int = 100, archived_after: datetime = None, archived_before: datetime = None, absolute_search: bool = False)-> list[models.Archive]:
+    # searches for partial URLs, if email is * no ownership filtering happens
+    query = base_query(db)
+    if email != ALLOW_ANY_EMAIL:
+        groups = get_user_groups(email)
+        query = query.filter(or_(models.Archive.public == True, models.Archive.author_id == email, models.Archive.group_id.in_(groups)))
+    if absolute_search:
+        query = query.filter(models.Archive.url == url)
+    else:
+        query = query.filter(models.Archive.url.like(f'%{url}%'))
+    if archived_after:
+        query = query.filter(models.Archive.created_at > archived_after)
+    if archived_before:
+        query = query.filter(models.Archive.created_at < archived_before)
+    return query.order_by(models.Archive.created_at.desc()).offset(skip).limit(get_limit(limit)).all()
+
+
+def search_archives_by_email(db: Session, email: str, skip: int = 0, limit: int = 100):
+    return base_query(db).filter(models.Archive.author_id == email).order_by(models.Archive.created_at.desc()).offset(skip).limit(get_limit(limit)).all()
+
+#TODO: rename task to archive
+def create_task(db: Session, task: schemas.ArchiveCreate, tags: list[models.Tag], urls: list[models.ArchiveUrl]) -> models.Archive:
+    db_task = models.Archive(id=task.id, url=task.url, result=task.result, public=task.public, author_id=task.author_id, group_id=task.group_id, sheet_id=task.sheet_id, store_until=task.store_until)
+    db_task.tags = tags
+    db_task.urls = urls
+    db.add(db_task)
+    db.commit()
+    db.refresh(db_task)
+    return db_task
+
+
+def soft_delete_task(db: Session, task_id: str, email: str) -> bool:
+    # TODO: implement hard-delete with cronjob that deletes from S3
+    db_task = db.query(models.Archive).filter(models.Archive.id == task_id, models.Archive.author_id == email, models.Archive.deleted == False).first()
+    if db_task:
+        db_task.deleted = True
+        db.commit()
+    return db_task is not None
+
+
+def count_archives(db: Session):
+    return db.query(func.count(models.Archive.id)).scalar()
+
+
+def count_archive_urls(db: Session):
+    return db.query(func.count(models.ArchiveUrl.url)).scalar()
+
+
+def count_users(db: Session):
+    return db.query(func.count(models.User.email)).scalar()
+
+
+def count_by_user_since(db: Session, seconds_delta: int = 15):
+    time_threshold = datetime.now() - timedelta(seconds=seconds_delta)
+    return db.query(models.Archive.author_id, func.count().label('total'))\
+        .filter(models.Archive.created_at >= time_threshold)\
+        .group_by(models.Archive.author_id)\
+        .order_by(func.count().desc())\
+        .limit(500).all()
+
+async def find_by_store_until(db: AsyncSession, store_until_is_before:datetime) -> dict:
+    res =  await db.execute(
+        select(models.Archive)
+        .filter(models.Archive.deleted ==False, models.Archive.store_until < store_until_is_before)
+    )
+    return res.scalars()
+
+async def soft_delete_expired_archives(db: AsyncSession) -> dict:
+    to_delete = await find_by_store_until(db, datetime.now())
+    counter = 0
+    for archive in to_delete:
+        archive.deleted = True
+        counter += 1
+    await db.commit()
+    return counter
+# --------------- TAG
+
+
+def create_tag(db: Session, tag: str) -> models.Tag:
+    db_tag = db.query(models.Tag).filter(models.Tag.id == tag).first()
+    if not db_tag:
+        db_tag = models.Tag(id=tag)
+        db.add(db_tag)
+        db.commit()
+        db.refresh(db_tag)
+    return db_tag
+
+
+def is_user_in_group(db: Session, email: str, group_name: str) -> models.Group:
+    if email == ALLOW_ANY_EMAIL: return True
+    return len(group_name) and len(email) and group_name in get_user_groups(email)
+
+
+@lru_cache
+def get_user_groups(email: str) -> list[str]:
+    """
+    given an email retrieves the user groups from the DB and then the email-domain groups from a global variable, the email does not need to belong to an existing user. 
+    """
+    if not email or not len(email) or "@" not in email: return []
+
+    with get_db() as db:
+        # get user groups
+        user_groups = db.query(models.association_table_user_groups).filter_by(user_id=email).with_entities(Column("group_id")).all()
+        user_level_groups_names = [g[0] for g in user_groups]
+
+        # get domain groups
+        domain = email.split('@')[1]
+        domain_level_groups = db.query(models.Group.id).filter(models.Group.domains.contains(domain)).with_entities(Column("id")).all()
+        domain_level_groups_names = [g[0] for g in domain_level_groups]
+
+        return list(set(user_level_groups_names + domain_level_groups_names))
+
+
+# --------------- INIT User-Groups
+
+def get_group(db: Session, group_name: str) -> models.Group:
+    return db.query(models.Group).filter(models.Group.id == group_name).first()
+
+
+def create_or_get_user(db: Session, author_id: str) -> models.User:
+    if type(author_id) == str: author_id = author_id.lower()
+    db_user = db.query(models.User).filter(models.User.email == author_id).first()
+    if not db_user:
+        db_user = models.User(email=author_id)
+        db.add(db_user)
+        db.commit()
+        db.refresh(db_user)
+    return db_user
+
+
+def upsert_group(db: Session, group_name: str, description: str, orchestrator: str, orchestrator_sheet: str, permissions: dict, domains: list) -> models.Group:
+    db_group = db.query(models.Group).filter(models.Group.id == group_name).first()
+    if db_group is None:
+        db_group = models.Group(id=group_name, description=description, orchestrator=orchestrator, orchestrator_sheet=orchestrator_sheet, permissions=permissions, domains=domains)
+        db.add(db_group)
+    else:
+        db_group.description = description
+        db_group.orchestrator = orchestrator
+        db_group.orchestrator_sheet = orchestrator_sheet
+        db_group.permissions = permissions
+        db_group.domains = domains
+    db.commit()
+    db.refresh(db_group)
+    return db_group
+
+
+def upsert_user(db: Session, email: str):
+    email = email.lower()
+    db_user = db.query(models.User).filter(models.User.email == email).first()
+    if db_user is None:
+        db_user = models.User(email=email)
+        db.add(db_user)
+        db.commit()
+    return db_user
+
+
+def upsert_user_groups(db: Session):
+    def display_email_pii(email: str):
+        return f"'{email[0:3]}...@{email.split('@')[1]}'"
+    """
+    reads the user_groups yaml file and inserts any new users, groups, 
+    along with new participation of users in groups
+    """
+    logger.debug("Updating user-groups configuration.")
+    filename = get_settings().USER_GROUPS_FILENAME
+
+    ug = UserGroups(filename)
+
+    # delete all user-groups relationships
+    db.query(models.association_table_user_groups).delete()
+
+    # create a map of group_id -> domains and another of domain -> groups
+    group_domains = defaultdict(set)
+    domain_groups = defaultdict(list)
+    for domain, explicit_groups in ug.domains.items():
+        domain_groups[domain] = list(set(explicit_groups))
+        for group in explicit_groups:
+            group_domains[group].add(domain)
+    import json
+    # upsert groups and save a map of groupid -> dbobject
+    for group_id, g in ug.groups.items():
+        upsert_group(db, group_id, g.description, g.orchestrator, g.orchestrator_sheet, json.loads(g.permissions.model_dump_json()), list(group_domains.get(group_id, [])))
+    db_groups: dict[str, models.Group] = {g.id: g for g in db.query(models.Group).all()}
+
+    # integrity checks
+    for group_in_domains in group_domains:
+        if group_in_domains not in db_groups:
+            logger.warning(f"[CONFIG] Group '{group_in_domains}' does not exist in the database: domains setting will not work.")
+
+    # reinsert users in their EXPLICITLY DEFINED groups
+    # domain groups are check live, as there may be new users that are not explicitly registered but belong to a domain
+    for email, explicit_groups in ug.users.items():
+        explicit_groups = explicit_groups or []
+        logger.info(f"EXPLICIT {display_email_pii(email)} => {explicit_groups}")
+
+        db_user = upsert_user(db, email)
+
+        # connect users to groups
+        for group_id in explicit_groups:
+            if group_id not in db_groups:
+                logger.warning(f"[CONFIG] Group {group_id} does not exist in config file, skipping for email={display_email_pii(email)}.")
+                continue
+            db_groups[group_id].users.append(db_user)
+
+    db.commit()
+    count_user_groups = db.query(models.association_table_user_groups).count()
+    count_groups = db.query(func.count(models.Group.id)).scalar()
+
+    logger.success(f"[CONFIG] DONE: [users={count_users(db)}, groups={count_groups}, explicit user groups={count_user_groups}].")
+
+
+# --------------- SHEET
+def create_sheet(db: Session, sheet_id: str, name: str, email: str, group_id: str, frequency: str):
+    db_sheet = models.Sheet(id=sheet_id, name=name, author_id=email, group_id=group_id, frequency=frequency)
+    db.add(db_sheet)
+    db.commit()
+    db.refresh(db_sheet)
+    return db_sheet
+
+
+def get_user_sheet(db: Session, email: str, sheet_id: str) -> models.Sheet:
+    return db.query(models.Sheet).filter(models.Sheet.author_id == email, models.Sheet.id == sheet_id).first()
+
+
+def get_user_sheets(db: Session, email: str) -> list[models.Sheet]:
+    return db.query(models.Sheet).filter(models.Sheet.author_id == email).order_by(models.Sheet.last_url_archived_at.desc()).all()
+
+
+async def get_sheets_by_id_hash(db: AsyncSession, frequency: str, modulo: str, id_hash: str) -> list[models.Sheet]:
+    result = await db.execute(
+        select(models.Sheet).filter(models.Sheet.frequency == frequency)
+    )
+    filtered = []
+    for sheet in result.scalars():
+        if fnv1a_hash_mod(sheet.id, modulo) == id_hash:
+            filtered.append(sheet)
+    return filtered
+
+async def delete_stale_sheets(db: AsyncSession, inactivity_days: int) -> dict:
+    time_threshold = datetime.now() - timedelta(days=inactivity_days)
+    result = await db.execute(
+        select(models.Sheet).filter(models.Sheet.last_url_archived_at < time_threshold)
+    )
+    deleted = defaultdict(list)
+    for sheet in result.scalars():
+        await db.delete(sheet)
+        deleted[sheet.author_id].append(sheet)
+    await db.commit()
+    return dict(deleted)
+
+def update_sheet_last_url_archived_at(db: Session, sheet_id: str):
+    db_sheet = db.query(models.Sheet).filter(models.Sheet.id == sheet_id).first()
+    if db_sheet:
+        db_sheet.last_url_archived_at = datetime.now()
+        db.commit()
+        return True
+    return False
+
+
+def delete_sheet(db: Session, sheet_id: str, email: str) -> bool:
+    db_sheet = db.query(models.Sheet).filter(models.Sheet.id == sheet_id, models.Sheet.author_id == email).first()
+    if db_sheet:
+        db.delete(db_sheet)
+        db.commit()
+    return db_sheet is not None
+
+
+#--- Celery worker tasks
+
+
+def store_archived_url(db: Session, archive: schemas.ArchiveCreate) -> models.Archive:
+    # create and load user, tags, if needed
+    create_or_get_user(db, archive.author_id)
+    db_tags = [create_tag(db, tag) for tag in archive.tags]
+    # insert everything
+    db_task = create_task(db, task=archive, tags=db_tags, urls=archive.urls)
+    return db_task

app/shared/db/database.py

@@ -0,0 +1,67 @@
+from functools import lru_cache
+from sqlalchemy import Engine, create_engine, event, text
+from sqlalchemy.orm import sessionmaker
+from contextlib import asynccontextmanager, contextmanager
+from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession, AsyncEngine, async_sessionmaker
+
+from app.shared.settings import get_settings
+
+
+@lru_cache
+def make_engine(database_url: str):
+    engine = create_engine(database_url, connect_args={"check_same_thread": False})
+
+    @event.listens_for(engine, "connect")
+    def set_sqlite_pragma(conn, _) -> None:
+        cursor = conn.cursor()
+        cursor.execute("PRAGMA journal_mode=WAL")
+        cursor.close()
+
+    return engine
+
+
+def make_session_local(engine: Engine):
+    session_local = sessionmaker(autocommit=False, autoflush=False, bind=engine)
+    return session_local
+
+
+@contextmanager
+def get_db():
+    session = make_session_local(make_engine(get_settings().DATABASE_PATH))()
+    try: yield session
+    finally: session.close()
+
+
+def get_db_dependency():
+    # to use with Depends and ensure proper session closing
+    with get_db() as db:
+        yield db
+
+def wal_checkpoint():
+    # WAL checkpointing, make sure the .sqlite file receives the latest changes
+    # to be called at startup as it halts writes
+    with get_db() as db:
+        db.execute(text("PRAGMA wal_checkpoint(TRUNCATE)"))
+
+
+# ASYNC connections
+async def make_async_engine(database_url: str) -> AsyncEngine:
+    engine = create_async_engine(database_url, connect_args={"check_same_thread": False})
+
+    async with engine.begin() as conn:
+        await conn.run_sync(lambda sync_conn: sync_conn.execute(text("PRAGMA journal_mode=WAL;")))
+
+    return engine
+
+
+async def make_async_session_local(engine: AsyncEngine) -> AsyncSession:
+    return async_sessionmaker(engine, expire_on_commit=False, autoflush=False, autocommit=False)
+
+
+@asynccontextmanager
+async def get_db_async():
+    engine = await make_async_engine(get_settings().ASYNC_DATABASE_PATH)
+    async_session = await make_async_session_local(engine)
+    async with async_session() as session:
+        try: yield session
+        finally: await engine.dispose()

app/shared/db/models.py

@@ -0,0 +1,113 @@
+from sqlalchemy import Column, String, JSON, DateTime, Boolean, Table, ForeignKey
+from sqlalchemy.sql import func
+from sqlalchemy.orm import relationship, declarative_base
+import uuid
+
+
+Base = declarative_base()
+
+
+def generate_uuid():
+    return str(uuid.uuid4())
+
+
+# many to many association tables
+association_table_archive_tags = Table(
+    "mtm_archives_tags",
+    Base.metadata,
+    Column("archive_id", ForeignKey("archives.id")),
+    Column("tag_id", ForeignKey("tags.id")),
+)
+association_table_user_groups = Table(
+    "mtm_users_groups",
+    Base.metadata,
+    Column("user_id", ForeignKey("users.email")),
+    Column("group_id", ForeignKey("groups.id")),
+)
+
+
+# data model tables
+class Archive(Base):
+    __tablename__ = "archives"
+
+    id = Column(String, primary_key=True, index=True)
+    url = Column(String, index=True)
+    result = Column(JSON, default=None)
+    public = Column(Boolean, default=True)  # if public=false, access by group and author
+    deleted = Column(Boolean, default=False)
+    created_at = Column(DateTime(timezone=True), server_default=func.now())
+    updated_at = Column(DateTime(timezone=True), onupdate=func.now())
+    store_until = Column(DateTime(timezone=True), default=None)
+
+    group_id = Column(String, ForeignKey("groups.id"), default=None)
+    author_id = Column(String, ForeignKey("users.email"))
+    sheet_id = Column(String, ForeignKey("sheets.id"), default=None)
+
+    tags = relationship("Tag", back_populates="archives", secondary=association_table_archive_tags)
+    group = relationship("Group", back_populates="archives")
+    author = relationship("User", back_populates="archives")
+    urls = relationship("ArchiveUrl", back_populates="archive")
+    sheet = relationship("Sheet", back_populates="archives")
+
+
+class ArchiveUrl(Base):
+    __tablename__ = "archive_urls"
+
+    url = Column(String, primary_key=True, index=True)
+    archive_id = Column(String, ForeignKey("archives.id"), primary_key=True)
+    key = Column(String, default=None)
+
+    archive = relationship("Archive", back_populates="urls")
+
+
+class Tag(Base):
+    __tablename__ = "tags"
+
+    id = Column(String, primary_key=True, index=True)
+    created_at = Column(DateTime(timezone=True), server_default=func.now())
+
+    archives = relationship("Archive", back_populates="tags", secondary=association_table_archive_tags)
+
+
+class User(Base):
+    __tablename__ = "users"
+
+    email = Column(String, primary_key=True, index=True)
+
+    archives = relationship("Archive", back_populates="author")
+    sheets = relationship("Sheet", back_populates="author")
+    groups = relationship("Group", back_populates="users", secondary=association_table_user_groups)
+
+
+class Group(Base):
+    __tablename__ = "groups"
+
+    id = Column(String, primary_key=True, index=True)
+    description = Column(String, default=None)
+    orchestrator = Column(String, default=None)
+    orchestrator_sheet = Column(String, default=None)
+    permissions = Column(JSON, default={})
+    domains = Column(JSON, default=[])
+
+    archives = relationship("Archive", back_populates="group")
+    sheets = relationship("Sheet", back_populates="group")
+    users = relationship("User", back_populates="groups", secondary=association_table_user_groups)
+
+
+class Sheet(Base):
+    __tablename__ = "sheets"
+
+    id = Column(String, primary_key=True, index=True, doc="Google Sheet ID")
+    name = Column(String, default=None)
+    author_id = Column(String, ForeignKey("users.email"))
+    group_id = Column(String, ForeignKey("groups.id"), doc="Group ID, user must be in a group to create a sheet.")
+    frequency = Column(String, default="daily", doc="Frequency of archiving: hourly, daily, weekly.")
+    # TODO: stats is not needed, is it?
+    stats = Column(JSON, default={}, doc="Sheet statistics like total links, total rows, ...")
+    last_url_archived_at = Column(DateTime(timezone=True), server_default=func.now(), doc="Last time a new link was archived.")
+    created_at = Column(DateTime(timezone=True), server_default=func.now())
+    updated_at = Column(DateTime(timezone=True), onupdate=func.now())
+
+    group = relationship("Group", back_populates="sheets")
+    author = relationship("User", back_populates="sheets")
+    archives = relationship("Archive", back_populates="sheet")

app/shared/db/user_state.py

@@ -0,0 +1,328 @@
+
+from typing import Dict, Set
+import sqlalchemy
+from sqlalchemy.orm import Session
+from sqlalchemy import func
+from datetime import datetime
+
+from app.shared.db import crud, models
+from app.shared.user_groups import GroupInfo, GroupPermissions
+from app.shared.schemas import Usage, UsageResponse
+
+class UserState:
+    """
+    Manage a user's state and permissions
+    """
+
+    def __init__(self, db: Session, email: str):
+        self.db = db
+        self.email = email.lower()
+
+    @property
+    def permissions(self) -> Dict[str, GroupInfo]:
+        """
+        Returns a dict of all group permissions and a special {"all": read/archive_url/archive_sheet} key
+        """
+        if not hasattr(self, '_permissions'):
+            self._permissions = {}
+            self._permissions["all"] = GroupInfo(
+                read=self.read,
+                read_public=self.read_public,
+                archive_url=self.archive_url,
+                archive_sheet=self.archive_sheet,
+                # below are relevant only for /url endpoints
+                max_archive_lifespan_months=self.max_archive_lifespan_months,
+                max_monthly_urls=self.max_monthly_urls,
+                max_monthly_mbs=self.max_monthly_mbs,
+                priority=self.priority
+            )
+            for group in self.user_groups:
+                if not group.permissions: continue
+                self._permissions[group.id] = GroupInfo(**group.permissions, description=group.description)
+        return self._permissions
+
+    @property
+    def user_groups_names(self):
+        if not hasattr(self, '_user_groups_names'):
+            self._user_groups_names = crud.get_user_groups(self.email) + ["default"]
+        return self._user_groups_names
+
+    @property
+    def user_groups(self):
+        if not hasattr(self, '_user_groups'):
+            self._user_groups = self.db.query(models.Group).filter(
+                models.Group.id.in_(self.user_groups_names)
+            ).all()
+        return self._user_groups
+
+    @property
+    def read(self) -> Set[str] | bool:
+        """
+        Read can be a list of group names or True, if all can be read.
+        """
+        if not hasattr(self, '_read'):
+            self._read = set()
+            for group in self.user_groups:
+                if not group.permissions: continue
+                group_read_permissions = group.permissions.get("read", [])
+                if "all" in group_read_permissions:
+                    self._read = True
+                    return self._read
+                else:
+                    self._read.update(group.permissions.get("read", []))
+        return self._read
+
+    @property
+    def read_public(self) -> bool:
+        """
+        Read public permission
+        """
+        if not hasattr(self, '_read_public'):
+            self._read_public = False
+            for group in self.user_groups:
+                if not group.permissions: continue
+                if group.permissions.get("read_public", False):
+                    self._read_public = True
+                    return self._read_public
+        return self._read_public
+
+    @property
+    def archive_url(self) -> bool:
+        """
+        Archive URL permission
+        """
+        if not hasattr(self, '_archive_url'):
+            self._archive_url = False
+            for group in self.user_groups:
+                if not group.permissions: continue
+                if group.permissions.get("archive_url", False):
+                    self._archive_url = True
+                    return self._archive_url
+        return self._archive_url
+
+    @property
+    def archive_sheet(self) -> bool:
+        """
+        Archive sheet permission
+        """
+        if not hasattr(self, '_archive_sheet'):
+            self._archive_sheet = False
+            for group in self.user_groups:
+                if not group.permissions: continue
+                if group.permissions.get("archive_sheet", False):
+                    self._archive_sheet = True
+                    return self._archive_sheet
+        return self._archive_sheet
+
+    @property
+    def sheet_frequency(self):
+        if not hasattr(self, '_sheet_frequency'):
+            self._sheet_frequency = set()
+            for group in self.user_groups:
+                if not group.permissions: continue
+                self._sheet_frequency.update(group.permissions.get("sheet_frequency", None))
+        return self._sheet_frequency
+
+    @property
+    def max_archive_lifespan_months(self) -> int:
+        if not hasattr(self, '_max_archive_lifespan_months'):
+            self._max_archive_lifespan_months = self._helper_for_grouping_max_numerical_permissions("max_archive_lifespan_months")
+        return self._max_archive_lifespan_months
+
+    @property
+    def max_monthly_urls(self) -> int:
+        if not hasattr(self, '_max_monthly_urls'):
+            self._max_monthly_urls = self._helper_for_grouping_max_numerical_permissions("max_monthly_urls")
+        return self._max_monthly_urls
+
+    @property
+    def max_monthly_mbs(self) -> int:
+        if not hasattr(self, '_max_monthly_mbs'):
+            self._max_monthly_mbs = self._helper_for_grouping_max_numerical_permissions("max_monthly_mbs")
+        return self._max_monthly_mbs
+
+    @property
+    def priority(self) -> str:
+        if not hasattr(self, '_priority'):
+            self._priority = "low"
+            for group in self.user_groups:
+                if not group.permissions: continue
+                if group.permissions.get("priority", "low") == "high":
+                    self._priority = "high"
+        return self._priority
+
+    @property
+    def active(self) -> bool:
+        """
+        A user is active if they can read/archive anything
+        """
+        if not hasattr(self, '_active'):
+            self._active = bool(self.read or self.read_public or self.archive_url or self.archive_sheet)
+        return self._active
+
+    def _helper_for_grouping_max_numerical_permissions(self, permission_name: str) -> int:
+        """
+        Iterates one of the numerical permissions where -1 means no restrictions and returns either -1 or the maximum value, defaults according to GroupPermissions
+        """
+        default = GroupPermissions.model_fields[permission_name].default
+        max_value = default
+        for group in self.user_groups:
+            if not group.permissions: continue
+            group_value = group.permissions.get(permission_name, default)
+            if group_value == -1:
+                max_value = -1
+                return max_value
+            max_value = max(max_value, group_value)
+        return max_value
+
+    def in_group(self, group_id: str) -> bool:
+        return group_id in self.user_groups_names
+
+    def usage(self) -> Dict:
+        """
+        returns the monthly quotas for the URLs/MBs and the totals for Sheets
+        """
+        current_month = datetime.now().month
+        current_year = datetime.now().year
+
+        # find and sum all user sheets over this month
+        user_sheets = self.db.query(
+            models.Sheet.group_id,
+            func.count(models.Sheet.id).label('sheet_count')
+        ).filter(models.Sheet.author_id == self.email).group_by(models.Sheet.group_id).all()
+
+        sheets_by_group = {sheet.group_id: sheet.sheet_count for sheet in user_sheets}
+
+        # find and sum all user urls over this month
+        urls_by_group = self.db.query(
+            models.Archive.group_id,
+            func.count(models.Archive.id).label('url_count'),
+            func.coalesce(func.sum(
+                func.coalesce(
+                    func.cast(
+                        func.json_extract(models.Archive.result, '$.metadata.total_bytes'),
+                        sqlalchemy.Integer
+                    ), 0
+                )
+            ), 0).label('total_bytes')
+        ).filter(
+            models.Archive.author_id == self.email,
+            func.extract('month', models.Archive.created_at) == current_month,
+            func.extract('year', models.Archive.created_at) == current_year
+        ).group_by(models.Archive.group_id).all()
+
+        # merge the two queries
+        usage_by_group: Dict[str, Usage] = {
+            (url.group_id or ""):
+            Usage(monthly_urls=url.url_count, monthly_mbs=int(url.total_bytes / 1024 / 1024))
+            for url in urls_by_group
+        }
+        for group_id, sheet_count in sheets_by_group.items():
+            group_id = group_id or ""
+            if group_id in usage_by_group:
+                usage_by_group[group_id].total_sheets = sheet_count
+            else:
+                usage_by_group[group_id] = Usage(total_sheets=sheet_count)
+
+        # calculate totals
+        total_sheets = sum([sheet.sheet_count for sheet in user_sheets])
+        total_bytes = sum([url.total_bytes for url in urls_by_group])
+        total_urls = sum([url.url_count for url in urls_by_group])
+
+        return UsageResponse(
+            monthly_urls=total_urls,
+            monthly_mbs=int(total_bytes / 1024 / 1024),
+            total_sheets=total_sheets,
+            groups=usage_by_group
+        )
+
+    def has_quota_monthly_sheets(self, group_id: str) -> bool:
+        """
+        checks if a user has reached their sheet quota for a given group
+        """
+        if group_id not in self.permissions:
+            return False
+
+        user_sheets = self.db.query(models.Sheet).filter(models.Sheet.author_id == self.email, models.Sheet.group_id == group_id).count()
+
+        sheet_quota = self.permissions[group_id].max_sheets
+        if sheet_quota == -1:
+            return True
+        return user_sheets < sheet_quota
+
+    def has_quota_max_monthly_urls(self, group_id: str) -> bool:
+        """
+        checks if a user has reached their monthly url quota for a group, if global then group should be empty string
+        """
+        quota = 0
+        if not group_id:
+            quota = self.max_monthly_urls
+        else:
+            if group_id not in self.permissions: return False
+            quota = self.permissions[group_id].max_monthly_urls
+        
+        if quota == -1:
+            return True
+
+        current_month = datetime.now().month
+        current_year = datetime.now().year
+        user_urls = self.db.query(models.Archive).filter(
+            models.Archive.author_id == self.email,
+            func.extract('month', models.Archive.created_at) == current_month,
+            func.extract('year', models.Archive.created_at) == current_year
+        ).count()
+
+        return user_urls < quota
+
+    def has_quota_max_monthly_mbs(self, group_id: str) -> bool:
+        """
+        checks if a user has reached their monthly MBs quota for a group, if global then group should be empty string
+        """
+        quota = 0
+        if not group_id:
+            quota = self.max_monthly_mbs
+        else:
+            if group_id not in self.permissions: return False
+            quota = self.permissions[group_id].max_monthly_mbs
+
+        if quota == -1:
+            return True
+        
+        current_month = datetime.now().month
+        current_year = datetime.now().year
+
+        # find and sum all user bytes over this month
+        user_bytes = self.db.query(models.Archive).filter(
+            models.Archive.author_id == self.email,
+            func.extract('month', models.Archive.created_at) == current_month,
+            func.extract('year', models.Archive.created_at) == current_year
+        ).with_entities(func.coalesce(func.sum(
+            func.coalesce(
+                func.cast(
+                    func.json_extract(models.Archive.result, '$.metadata.total_bytes'),
+                    sqlalchemy.Integer
+                ), 0
+            )
+        ), 0).label('total')).scalar()
+
+        # convert bytes to mb
+        user_mbs = int(user_bytes / 1024 / 1024)
+        return user_mbs < quota
+
+    def can_manually_trigger(self, group_id: str) -> bool:
+        """
+        checks if a user is allowed to manually trigger a sheet
+        """
+        if group_id not in self.permissions:
+            return False
+
+        return self.permissions[group_id].manually_trigger_sheet
+
+    def is_sheet_frequency_allowed(self, group_id: str, frequency: str) -> bool:
+        """
+        checks if a user is allowed to create a sheet with this frequency for this group
+        """
+        if group_id not in self.permissions:
+            return False
+
+        return frequency in self.permissions[group_id].sheet_frequency

app/shared/log.py

@@ -0,0 +1,13 @@
+import traceback
+from loguru import logger
+
+
+# logging configurations
+logger.add("logs/api_logs.log", retention="30 days", rotation="3 days")
+logger.add("logs/error_logs.log", retention="30 days", level="ERROR")
+
+    
+def log_error(e: Exception, traceback_str: str = None, extra:str = ""):
+    if not traceback_str: traceback_str = traceback.format_exc()
+    if extra: extra = f"{extra}\n"
+    logger.error(f"{extra}{e.__class__.__name__}: {e}\n{traceback_str}")

app/shared/schemas.py

@@ -0,0 +1,117 @@
+from typing import Annotated
+from annotated_types import Len
+from pydantic import BaseModel
+from datetime import datetime
+
+
+class SubmitSheet(BaseModel):
+    sheet_id: str | None
+    author_id: str | None = None
+    group_id: str = "default"
+    tags: set[str] | None = set()
+    columns: dict | None = {}  # TODO: implement/remove
+
+
+class SubmitManual(BaseModel):  # deprecated
+    result: str  # should be a Metadata.to_json()
+    public: bool = False
+    author_id: str | None = None
+    group_id: str | None = None
+    tags: set[str] | None = set()
+
+# API REQUESTS BELOW
+# TODO: replace existing schemas with these
+
+
+class ArchiveUrl(BaseModel):
+    url: str
+    public: bool = False
+    author_id: str | None
+    group_id: str | None
+    tags: set[str] | None = set()
+
+# API RESPONSES BELOW
+
+
+class ArchiveResult(BaseModel):
+    id: str
+    url: str
+    result: dict
+    created_at: datetime
+    store_until: datetime | None
+
+
+class Task(BaseModel):
+    id: str
+
+
+class TaskResult(Task):
+    status: str
+    result: str
+
+
+class TaskDelete(Task):
+    deleted: bool
+
+
+class ActiveUser(BaseModel):
+    active: bool
+
+
+class SheetAdd(BaseModel):
+    id: str
+    name: str
+    group_id: str
+    frequency: str
+
+
+class SheetResponse(SheetAdd):
+    author_id: str
+    created_at: datetime
+    last_url_archived_at: datetime | None
+
+
+class ArchiveTrigger(BaseModel):
+    author_id: str | None = None
+    url: Annotated[str, Len(min_length=5)]
+    public: bool = False
+    group_id: Annotated[str, Len(min_length=1)] = "default"
+    tags: set[str] | None = None
+
+
+class ArchiveCreate(ArchiveTrigger):
+    id: str | None = None
+    result: dict | None = None
+    sheet_id: str | None = None
+    urls: list | None = None
+    store_until: datetime | None = None
+
+
+class Archive(ArchiveCreate):
+    created_at: datetime
+    updated_at: datetime | None
+    deleted: bool
+
+    model_config = {"from_attributes": True}
+
+
+class Usage(BaseModel):
+    monthly_urls: int = 0
+    monthly_mbs: int = 0
+    total_sheets: int = 0
+
+
+class UsageResponse(Usage):
+    groups: dict[str, Usage]
+
+
+class CelerySheetTask(BaseModel):
+    success: bool
+    sheet_id: str
+    time: datetime
+    stats: dict
+
+
+class SubmitManualArchive(ArchiveTrigger):
+    url: None = None
+    result: str  # should be a Metadata.to_json()

app/shared/settings.py

@@ -0,0 +1,73 @@
+
+from functools import lru_cache
+from fastapi_mail import ConnectionConfig
+from pydantic_settings import BaseSettings
+from pydantic import ConfigDict
+from typing import Annotated, Set
+from annotated_types import Len
+
+
+class Settings(BaseSettings):
+    model_config = ConfigDict(extra='ignore', str_strip_whitespace=True)
+    
+	# general
+    SERVE_LOCAL_ARCHIVE: str = ""
+    USER_GROUPS_FILENAME: str = "user-groups.yaml"
+    SHEET_ORCHESTRATION_YAML : str = "secrets/orchestration-sheet.yaml"
+    
+    # cronjobs
+    #TODO: disable by default?
+    CRON_ARCHIVE_SHEETS: bool = False
+    CRON_DELETE_STALE_SHEETS: bool = True
+    DELETE_STALE_SHEETS_DAYS: int = 14
+    CRON_DELETE_SCHEDULED_ARCHIVES: bool = True
+    DELETE_SCHEDULED_ARCHIVES_NOTIFY_DAYS: int = 14
+
+	# database
+    DATABASE_PATH: str
+    DATABASE_QUERY_LIMIT: int = 100
+    @property
+    def ASYNC_DATABASE_PATH(self) -> str:
+        return self.DATABASE_PATH.replace("sqlite://", "sqlite+aiosqlite://")
+
+    # redis
+    REDIS_PASSWORD: str = ""
+    CELERY_BROKER_URL: str = "redis://localhost:6379"
+    REDIS_EXCEPTIONS_CHANNEL: str = "exceptions-channel"
+    
+	# observability
+    REPEAT_COUNT_METRICS_SECONDS: int = 30
+
+	# security
+    API_BEARER_TOKEN: Annotated[str, Len(min_length=20)]
+    ALLOWED_ORIGINS: Annotated[Set[str], Len(min_length=1)]
+    CHROME_APP_IDS: Annotated[Set[Annotated[str, Len(min_length=10)]], Len(min_length=1)]
+    #TODO: deprecate blocklist?
+    BLOCKED_EMAILS: Annotated[Set[str], Len(min_length=0)] = set()
+
+    # email configuration, if needed
+    MAIL_FROM: str = "noreply@bellingcat.com"
+    MAIL_FROM_NAME: str = "Bellingcat's Auto Archiver"
+    MAIL_USERNAME: str = ""
+    MAIL_PASSWORD: str = ""
+    MAIL_SERVER: str = ""
+    MAIL_PORT: int = 587
+    MAIL_STARTTLS: bool = False
+    MAIL_SSL_TLS: bool = True
+    @property
+    def MAIL_CONFIG(self) -> str:
+        return ConnectionConfig(
+            MAIL_FROM=self.MAIL_FROM,
+            MAIL_FROM_NAME=self.MAIL_FROM_NAME,
+            MAIL_USERNAME=self.MAIL_USERNAME,
+            MAIL_PASSWORD=self.MAIL_PASSWORD,
+            MAIL_SERVER=self.MAIL_SERVER,
+            MAIL_PORT=self.MAIL_PORT,
+            MAIL_STARTTLS=self.MAIL_STARTTLS,
+            MAIL_SSL_TLS=self.MAIL_SSL_TLS,
+        )
+
+
+@lru_cache
+def get_settings():
+    return Settings()

app/shared/task_messaging.py

@@ -0,0 +1,18 @@
+
+from functools import lru_cache
+from celery import Celery
+import redis
+
+from app.shared.settings import get_settings
+
+@lru_cache
+def get_celery(name:str="") -> Celery:
+    return Celery(
+        name,
+        broker_url=get_settings().CELERY_BROKER_URL,
+        result_backend=get_settings().CELERY_BROKER_URL,
+    )
+
+
+def get_redis() -> redis.Redis:
+    return redis.Redis.from_url(get_settings().CELERY_BROKER_URL)

app/shared/user_groups.py

@@ -0,0 +1,140 @@
+import os
+import yaml
+from loguru import logger
+from pydantic import BaseModel, field_validator, Field, model_validator
+from typing import Dict, List, Set
+from typing_extensions import Self
+
+
+class UserGroups:
+    def __init__(self, filename):
+        user_groups = self.read_yaml(filename)
+        self.validate_and_load(user_groups)
+
+    def read_yaml(self, user_groups_filename):
+        # read yaml safely
+        with open(user_groups_filename) as inf:
+            try:
+                return yaml.safe_load(inf)
+            except yaml.YAMLError as e:
+                logger.error(f"could not open user groups filename {user_groups_filename}: {e}")
+                raise e
+
+    def validate_and_load(self, user_groups):
+        try:
+            configs = UserGroupModel(**user_groups)
+            self.users = configs.users
+            self.domains = configs.domains
+            self.groups = configs.groups
+        except Exception as e:
+            logger.error(f"Validation error: {e}")
+            raise e
+
+
+class GroupPermissions(BaseModel):
+    read: Set[str] | bool = Field(default_factory=list)
+    read_public: bool = False
+    archive_url: bool = False
+    archive_sheet: bool = False
+    manually_trigger_sheet: bool = False
+    sheet_frequency: Set[str] = Field(default_factory=list)
+    max_sheets: int = 0
+    max_archive_lifespan_months: int = 12
+    max_monthly_urls: int = 0
+    max_monthly_mbs: int = 0
+    priority: str = "low"
+
+    @field_validator('max_sheets', 'max_archive_lifespan_months', 'max_monthly_urls', 'max_monthly_mbs', mode='before')
+    def validate_max_values(cls, v):
+        if v < -1:
+            raise ValueError("max_* values should be positive integers or -1 (for no limit).")
+        return v
+
+    @field_validator('sheet_frequency', mode='before')
+    def validate_sheet_frequency(cls, v):
+        if not v: return []
+        allowed = ["daily", "hourly"]
+        for k in v:
+            if k not in allowed:
+                raise ValueError(f"Invalid sheet frequency: '{k}', expected one of {allowed}")
+        return v
+
+    @field_validator('priority', mode='before')
+    def validate_priority(cls, v):
+        v = v.lower()
+        if v not in ["low", "high"]:
+            raise ValueError("priority must be either 'low' or 'high'.")
+        return v
+
+
+class GroupModel(BaseModel):
+    description: str
+    orchestrator: str
+    orchestrator_sheet: str
+    permissions: GroupPermissions
+
+    @field_validator('orchestrator', 'orchestrator_sheet', mode='before')
+    def validate_priority(cls, v):
+        if not os.path.exists(v):
+            raise ValueError(f"Orchestrator file not found with this path: {v}")
+        return v
+
+
+class UserGroupModel(BaseModel):
+    users: Dict[str, List[str]] = Field(default_factory=dict)
+    domains: Dict[str, List[str]] = Field(default_factory=dict)
+    groups: Dict[str, GroupModel] = Field(default_factory=dict)
+
+    @field_validator('users', mode='before')
+    @classmethod
+    def validate_emails(cls, v):
+        for email in v.keys():
+            if '@' not in email:
+                raise ValueError(f"Invalid user, it should be an address: {email}")
+            if not v[email]:
+                raise ValueError(f"User {email} has no explicitly listed groups, only include them here if they should be in a group.")
+        # all users belong to the default group
+        return {k.lower().strip(): list(set(["default"] + [g.lower().strip() for g in v])) for k, v in v.items()}
+
+    @field_validator('domains', mode='before')
+    @classmethod
+    def validate_domains(cls, v):
+        for domain, members in v.items():
+            if '.' not in domain:
+                raise ValueError(f"Invalid domain, it should contain a dot: {domain}")
+            if not members:
+                raise ValueError(f"Domain {domain} should have at least one member.")
+        return {k.lower().strip(): list(set([g.lower().strip() for g in v])) for k, v in v.items()}
+
+    @field_validator('groups', mode='before')
+    @classmethod
+    def validate_groups(cls, v):
+        if "default" not in v.keys():
+            raise ValueError("Please include a 'default' group.")
+        if "all" in v.keys():
+            raise ValueError("'all' is a reserved group name.")
+        for group in v.keys():
+            if not group == group.lower():
+                raise ValueError(f"Group names should be lowercase: {group}")
+        return v
+
+    @model_validator(mode='after')
+    def check_groups_consistency(self) -> Self:
+        groups_in_domains = set([g for domain in self.domains for g in self.domains[domain]])
+        groups_in_users = set([g for user in self.users for g in self.users[user]])
+        configured_groups = set(self.groups.keys())
+
+        # groups mentioned in domains and users should be defined, but this is not a ValueError since historical DB data may require it
+        if groups_in_domains - configured_groups:
+            logger.warning(f"These groups are associated to DOMAINS but not defined in the GROUPS section, the domains settings may not work as expected: {groups_in_domains - configured_groups}")
+        if groups_in_users - configured_groups:
+            logger.warning(f"These groups are associated to USERS but not defined in the GROUPS section, the users settings may not work as expected: {groups_in_users - configured_groups}")
+
+        return self
+
+# for the API return values
+
+
+class GroupInfo(GroupPermissions):
+    description: str = ""
+    service_account_emails: list[str] = []

app/shared/utils/misc.py

@@ -0,0 +1,10 @@
+
+def fnv1a_hash_mod(s: str, modulo:int) -> int:
+    # receives a string and returns a number in [0:modulo-1], ensures an even distribution over the modulo range
+    hash = 0x811c9dc5 # FNV offset basis
+    fnv_prime = 0x01000193 # FNV prime
+    for char in s:
+        hash ^= ord(char)
+        hash *= fnv_prime
+        hash &= 0xFFFFFFFF # Keep it 32-bit
+    return (hash if hash < 0x80000000 else hash - 0x100000000) % modulo