major refactor of structure for worker V web: docker/app/secrets/envs/...

app/tests/conftest.py

@@ -0,0 +1,114 @@
+import os
+from fastapi.testclient import TestClient
+import pytest
+from unittest.mock import patch
+from app.shared.config import ALLOW_ANY_EMAIL
+from db.user_state import UserState
+from shared.settings import Settings
+
+
+@pytest.fixture(autouse=True)
+def mock_logger_add():
+    """Fixture to mock loguru.logger.add for all tests."""
+    with patch('loguru.logger.add') as mock_add:
+        yield mock_add  # This makes the mock available to tests
+
+
+@pytest.fixture()
+def get_settings():
+    return Settings(_env_file=".env.test")
+
+
+@pytest.fixture(autouse=True)
+def mock_settings():
+    with patch('shared.settings.Settings', return_value=Settings(_env_file=".env.test")) as mock_settings:
+        yield mock_settings
+
+
+@pytest.fixture()
+def test_db(get_settings: Settings):
+    from db.database import make_engine
+    from db import models
+    from db.crud import get_user_groups
+
+    get_user_groups.cache_clear()
+    make_engine.cache_clear()
+    engine = make_engine(get_settings.DATABASE_PATH)
+
+    fs = get_settings.DATABASE_PATH.replace("sqlite:///", "")
+    if not os.path.exists(fs):
+        open(fs, 'w').close()
+
+    models.Base.metadata.create_all(engine)
+
+    connection = engine.connect()
+    yield connection
+    connection.close()
+
+    models.Base.metadata.drop_all(bind=engine)
+    for suffix in ["", "-wal", "-shm"]:
+        new_fs = fs + suffix
+        if os.path.exists(new_fs):
+            os.remove(new_fs)
+
+
+@pytest.fixture()
+def db_session(test_db):
+    from db.database import make_session_local
+    session_local = make_session_local(test_db)
+    with session_local() as session:
+        yield session
+
+
+@pytest.fixture()
+def app(db_session):
+    from web.main import app_factory
+    from db import crud
+    app = app_factory()
+    crud.upsert_user_groups(db_session)
+    return app
+
+
+@pytest.fixture()
+def client(app):
+    client = TestClient(app)
+    return client
+
+
+@pytest.fixture()
+def app_with_auth(app, db_session):
+    from web.security import get_token_or_user_auth, get_user_auth, get_user_state
+    app.dependency_overrides[get_token_or_user_auth] = lambda: "rick@example.com"
+    app.dependency_overrides[get_user_auth] = lambda: "morty@example.com"
+    app.dependency_overrides[get_user_state] = lambda: UserState(db_session, "MORTY@example.com")
+    return app
+
+
+@pytest.fixture()
+def client_with_auth(app_with_auth):
+    client = TestClient(app_with_auth)
+    return client
+
+
+@pytest.fixture()
+def app_with_token(app):
+    from web.security import token_api_key_auth, get_token_or_user_auth
+    app.dependency_overrides[token_api_key_auth] = lambda: ALLOW_ANY_EMAIL
+    app.dependency_overrides[get_token_or_user_auth] = lambda: ALLOW_ANY_EMAIL
+    return app
+
+
+@pytest.fixture()
+def client_with_token(app_with_token):
+    client = TestClient(app_with_token)
+    return client
+
+
+@pytest.fixture()
+def test_no_auth():
+    # reusable code to ensure a method/endpoint combination is unauthorized
+    def no_auth(http_method, endpoint):
+        response = http_method(endpoint)
+        assert response.status_code == 403
+        assert response.json() == {"detail": "Not authenticated"}
+    return no_auth

app/tests/db/test_crud.py

@@ -0,0 +1,468 @@
+from datetime import datetime
+from unittest.mock import patch
+
+import pytest
+import yaml
+from db import models
+from shared.settings import Settings
+
+authors = ["rick@example.com", "morty@example.com", "jerry@example.com"]
+
+
+@pytest.fixture()
+def test_data(db_session):
+
+    # creates 3 users
+    for email in authors:
+        db_session.add(models.User(email=email))
+    db_session.commit()
+    assert db_session.query(models.User).count() == 3
+
+    # creates 100 archives for 3 users over 2 months with repeating URLs
+    for i in range(100):
+        author = authors[i % 3]
+        archive = models.Archive(
+            id=f"archive-id-456-{i}",
+            url=f"https://example-{i%3}.com",
+            result={},
+            public=author == "jerry@example.com",
+            author_id=author,
+            group_id="spaceship" if author == "morty@example.com" and i % 2 == 0 else None,
+            created_at=datetime(2021, (i % 2) + 1, (i % 25) + 1)
+        )
+        if i % 5 == 0:
+            archive.tags.append(models.Tag(id=f"tag-{i}"))
+        if i % 10 == 0:
+            archive.tags.append(models.Tag(id=f"tag-second-{i}"))
+        if i % 4 == 0:
+            archive.tags.append(models.Tag(id=f"tag-third-{i}"))
+        for j in range(10):
+            archive.urls.append(models.ArchiveUrl(url=f"https://example-{i}.com/{j}", key=f"media_{j}"))
+        db_session.add(archive)
+
+    # creates a sheet for each user
+    for i, email in enumerate(authors):
+        db_session.add(models.Sheet(id=f"sheet-{i}", name=f"sheet-{i}", author_id=email, group_id=None, frequency="daily"))
+        if email == "rick@example.com":
+            db_session.add(models.Sheet(id=f"sheet-{i}-2", name=f"sheet-{i}-2", author_id=email, group_id="spaceship", frequency="hourly"))
+
+    db_session.commit()
+
+    assert db_session.query(models.Archive).count() == 100
+    assert db_session.query(models.Tag).count() == 20 + 10 + 25
+    assert db_session.query(models.ArchiveUrl).count() == 1000
+    assert db_session.query(models.ArchiveUrl).filter(models.ArchiveUrl.archive_id == "archive-id-456-0").count() == 10
+
+    # setup groups
+    assert db_session.query(models.Group).count() == 0
+    from db import crud
+    crud.upsert_user_groups(db_session)
+    assert db_session.query(models.Group).count() == 4
+    assert db_session.query(models.User).count() == 3
+
+
+def test_get_archive(test_data, db_session):
+    from db import crud
+    from app.shared.config import ALLOW_ANY_EMAIL
+
+    print(db_session.query(models.Group).all())
+
+    # each author's archives work
+    assert (a0 := crud.get_archive(db_session, "archive-id-456-0", authors[0])) is not None
+    assert a0.id == "archive-id-456-0"
+    assert a0.url == "https://example-0.com"
+    assert a0.author_id == authors[0]
+    assert a0.public == False
+
+    assert crud.get_archive(db_session, "archive-id-456-1", authors[1]) is not None
+    assert crud.get_archive(db_session, "archive-id-456-2", authors[2]) is not None
+
+    # ALLOW_ANY_EMAIL
+    assert crud.get_archive(db_session, "archive-id-456-0", ALLOW_ANY_EMAIL) is not None
+    assert crud.get_archive(db_session, "archive-id-456-1", ALLOW_ANY_EMAIL) is not None
+
+    # not found
+    assert crud.get_archive(db_session, "archive-missing", authors[0]) is None
+
+    # public
+    assert (a_public := crud.get_archive(db_session, "archive-id-456-2", authors[0])) is not None
+    assert a_public.public == True
+
+    # not public - rick's
+    assert crud.get_archive(db_session, "archive-id-456-0", authors[1]) is None
+
+
+def test_search_archives_by_url(test_data, db_session):
+    from db import crud
+    from app.shared.config import ALLOW_ANY_EMAIL
+
+    # rick's archives are private
+    assert len(crud.search_archives_by_url(db_session, "https://example-0.com", "rick@example.com")) == 34
+    assert len(crud.search_archives_by_url(db_session, "https://example-0.com", ALLOW_ANY_EMAIL)) == 34
+    assert len(crud.search_archives_by_url(db_session, "https://example-0.com", "morty@example.com")) == 0
+
+    # morty's archives are public but half are in spaceship group
+    assert len(crud.search_archives_by_url(db_session, "https://example-1.com", "rick@example.com")) == 16
+
+    # jerry's archives are public
+    assert len(crud.search_archives_by_url(db_session, "https://example-2.com", "jerry@example.com")) == 33
+    assert len(crud.search_archives_by_url(db_session, "https://example-2.com", "rick@example.com")) == 33
+
+    # fuzzy search
+    assert len(crud.search_archives_by_url(db_session, "https://example", ALLOW_ANY_EMAIL)) == 100
+    assert len(crud.search_archives_by_url(db_session, "https://EXAMPLE", ALLOW_ANY_EMAIL)) == 100
+    assert len(crud.search_archives_by_url(db_session, "2.com", ALLOW_ANY_EMAIL)) == 33
+
+    # absolute search
+    assert len(crud.search_archives_by_url(db_session, "example-2.com", ALLOW_ANY_EMAIL, absolute_search=True)) == 0
+    assert len(crud.search_archives_by_url(db_session, "https://example-2.com", ALLOW_ANY_EMAIL, absolute_search=True)) == 33
+
+    # archived_after
+    assert len(crud.search_archives_by_url(db_session, "https://example", ALLOW_ANY_EMAIL, archived_after=datetime(2010, 1, 1))) == 100
+    assert len(crud.search_archives_by_url(db_session, "https://example", ALLOW_ANY_EMAIL, archived_after=datetime(2021, 1, 15))) == 70
+    assert len(crud.search_archives_by_url(db_session, "https://example", ALLOW_ANY_EMAIL, archived_after=datetime(2031, 1, 1))) == 0
+
+    # archived before
+    assert len(crud.search_archives_by_url(db_session, "https://example", ALLOW_ANY_EMAIL, archived_before=datetime(2010, 1, 1))) == 0
+    assert len(crud.search_archives_by_url(db_session, "https://example", ALLOW_ANY_EMAIL, archived_before=datetime(2021, 1, 15))) == 28
+    assert len(crud.search_archives_by_url(db_session, "https://example", ALLOW_ANY_EMAIL, archived_before=datetime(2031, 1, 1))) == 100
+
+    # archived before and after
+    assert len(crud.search_archives_by_url(db_session, "https://example", ALLOW_ANY_EMAIL, archived_after=datetime(2001, 1, 1), archived_before=datetime(2031, 1, 11))) == 100
+    assert len(crud.search_archives_by_url(db_session, "https://example", ALLOW_ANY_EMAIL, archived_after=datetime(2021, 1, 14), archived_before=datetime(2021, 1, 16))) == 2
+
+    # limit
+    assert len(crud.search_archives_by_url(db_session, "https://example", ALLOW_ANY_EMAIL, limit=10)) == 10
+    assert len(crud.search_archives_by_url(db_session, "https://example", ALLOW_ANY_EMAIL, limit=-1)) == 1
+
+    # skip
+    assert len(crud.search_archives_by_url(db_session, "https://example", ALLOW_ANY_EMAIL, skip=10)) == 90
+
+
+def test_search_archives_by_email(test_data, db_session):
+    from app.shared.config import ALLOW_ANY_EMAIL
+    from db import crud
+
+    # lower/upper case
+    assert len(crud.search_archives_by_email(db_session, "rick@example.com")) == 34
+
+    # ALLOW_ANY_EMAIL is not a user
+    assert len(crud.search_archives_by_email(db_session, ALLOW_ANY_EMAIL)) == 0
+
+    # most recent first
+    a1 = crud.search_archives_by_email(db_session, "rick@example.com", limit=1)
+    assert len(a1) == 1
+    assert a1[0].created_at == datetime(2021, 2, 25)
+
+    # earliest is the last
+    a2 = crud.search_archives_by_email(db_session, "rick@example.com", skip=33)
+    assert len(a2) == 1
+    assert a2[0].created_at == datetime(2021, 1, 1)
+
+
+@patch("db.crud.DATABASE_QUERY_LIMIT", new=25)
+def test_max_query_limit(test_data, db_session):
+    from db import crud
+    from app.shared.config import ALLOW_ANY_EMAIL
+
+    assert len(crud.search_archives_by_url(db_session, "https://example", ALLOW_ANY_EMAIL)) == 25
+    assert len(crud.search_archives_by_url(db_session, "https://example", ALLOW_ANY_EMAIL, limit=1000)) == 25
+
+    assert len(crud.search_archives_by_email(db_session, "rick@example.com")) == 25
+    assert len(crud.search_archives_by_email(db_session, "rick@example.com", limit=1000)) == 25
+
+
+def test_create_task(db_session):
+    from db import crud
+    from db import schemas
+
+    task = schemas.ArchiveCreate(
+        id="archive-id-456-101",
+        url="https://example-0.com",
+        result={},
+        public=False,
+        author_id="rick@example.com",
+        group_id="spaceship",
+        tags=[],
+        urls=[]
+    )
+
+    # with tags and urls
+    nt = crud.create_task(db_session, task, [models.Tag(id="tag-101")], [models.ArchiveUrl(url="https://example-0.com/0", key="media_0")])
+
+    assert nt is not None
+    assert nt.id == "archive-id-456-101"
+    assert nt.url == "https://example-0.com"
+    assert nt.author_id == "rick@example.com"
+    assert nt.public == False
+    assert nt.group_id == "spaceship"
+    assert len(nt.tags) == 1
+    assert nt.tags[0].id == "tag-101"
+    assert len(nt.urls) == 1
+    assert nt.urls[0].url == "https://example-0.com/0"
+    assert nt.urls[0].key == "media_0"
+    assert nt.created_at is not None
+
+    # without tags and urls
+    task.id = "archive-id-456-102"
+    nt = crud.create_task(db_session, task, [], [])
+    assert nt is not None
+    assert nt.id == "archive-id-456-102"
+    assert nt.url == "https://example-0.com"
+    assert nt.author_id == "rick@example.com"
+    assert nt.public == False
+    assert nt.group_id == "spaceship"
+    assert len(nt.tags) == 0
+    assert len(nt.urls) == 0
+    assert nt.created_at is not None
+
+
+def test_soft_delete(test_data, db_session):
+    from db import crud
+
+    # none deleted yet
+    assert crud.get_archive(db_session, "archive-id-456-0", "rick@example.com") is not None
+    assert db_session.query(models.Archive).filter(models.Archive.deleted == True).count() == 0
+
+    # delete
+    assert crud.soft_delete_task(db_session, "archive-id-456-0", "rick@example.com") == True
+
+    # ensure soft delete
+    assert db_session.query(models.Archive).filter(models.Archive.deleted == True).count() == 1
+    assert crud.get_archive(db_session, "archive-id-456-0", "rick@example.com") is None
+
+    # already deleted
+    assert crud.soft_delete_task(db_session, "archive-id-456-0", "rick@example.com") == False
+
+
+def test_count_archives(test_data, db_session):
+    from db import crud
+
+    assert crud.count_archives(db_session) == 100
+    db_session.query(models.Archive).filter(models.Archive.id == "archive-id-456-0").delete()
+    db_session.commit()
+    assert crud.count_archives(db_session) == 99
+
+
+def test_count_archive_urls(test_data, db_session):
+    from db import crud
+
+    assert crud.count_archive_urls(db_session) == 1000
+    db_session.query(models.ArchiveUrl).filter(models.ArchiveUrl.url == "https://example-0.com/0").delete()
+    db_session.commit()
+    assert crud.count_archive_urls(db_session) == 999
+
+    db_session.query(models.Archive).filter(models.Archive.id == "archive-id-456-0").delete()
+    db_session.commit()
+    # no Cascade is enabled
+    assert crud.count_archives(db_session) == 99
+    assert crud.count_archive_urls(db_session) == 999
+
+
+def test_count_users(test_data, db_session):
+    from db import crud
+
+    assert crud.count_users(db_session) == 3
+    db_session.query(models.User).filter(models.User.email == "rick@example.com").delete()
+    db_session.commit()
+    assert crud.count_users(db_session) == 2
+
+
+def test_count_by_users_since(test_data, db_session):
+    from db import crud
+
+    # 100y window
+    assert len(cu := crud.count_by_user_since(db_session, 60 * 60 * 24 * 31 * 12 * 100)) == 3
+    assert cu[0].total == 34
+    assert cu[1].total == 33
+    assert cu[2].total == 33
+
+
+def test_create_tag(db_session):
+    from db import crud
+
+    assert db_session.query(models.Tag).count() == 0
+
+    # create first
+    create_tag = crud.create_tag(db_session, "tag-101")
+    assert create_tag is not None
+    assert create_tag.id == "tag-101"
+    assert db_session.query(models.Tag).count() == 1
+    assert db_session.query(models.Tag).filter(models.Tag.id == "tag-101").first() == create_tag
+
+    # same id does not add new db entry
+    existing_tag = crud.create_tag(db_session, "tag-101")
+    assert existing_tag == create_tag
+    assert db_session.query(models.Tag).count() == 1
+
+    # create second
+    second_tag = crud.create_tag(db_session, "tag-102")
+    assert second_tag is not None
+    assert second_tag.id == "tag-102"
+    assert db_session.query(models.Tag).count() == 2
+
+
+def test_is_user_in_group(test_data, db_session):
+    from db import crud
+    from app.shared.config import ALLOW_ANY_EMAIL
+
+    # see user-groups.test.yaml
+    test_pairs = [
+        (ALLOW_ANY_EMAIL, "spaceship", True),
+        (ALLOW_ANY_EMAIL, "non-existant!@#!%!", True),
+
+        ("rick@example.com", "spaceship", True),
+        ("rick@example.com", "SPACESHIP", False),
+        ("rick@example.com", "interdimensional", True),
+        ("rick@example.com", "animated-characters", True),
+        ("rick@example.com", "the-jerrys-club", False),
+
+        ("morty@example.com", "spaceship", True),
+        ("morty@example.com", "interdimensional", False),
+        ("morty@example.com", "the-jerrys-club", False),
+
+        ("jerry@example.com", "spaceship", False),
+        ("jerry@example.com", "interdimensional", False),
+        ("jerry@example.com", "the-jerrys-club", False),  # group not in 'groups'
+
+        ("rick@example.com", "animated-characters", True),
+        ("morty@example.com", "animated-characters", True),
+        ("jerry@example.com", "animated-characters", True),
+        ("anyone@example.com", "animated-characters", True),
+        ("anyone@birdy.com", "animated-characters", True),
+
+        ("summer@herself.com", "animated-characters", False),
+
+        ("rick@example.com", "", False),
+        ("", "spaceship", False),
+        ("bademailexample.com", "spaceship", False),
+    ]
+    for email, group, expected in test_pairs:
+        print(f"{email} in {group} == {expected}")
+        assert crud.is_user_in_group(db_session, email, group) == expected
+
+
+def test_get_group(test_data, db_session):
+    from db import crud
+
+    assert crud.get_group(db_session, "spaceship") is not None
+    assert crud.get_group(db_session, "interdimensional") is not None
+    assert crud.get_group(db_session, "animated-characters") is not None
+    assert crud.get_group(db_session, "non-existent!@#!%!") is None
+
+
+def test_create_or_get_user(test_data, db_session):
+    from db import crud
+
+    assert db_session.query(models.User).count() == 3
+
+    # already exists
+    assert (u1 := crud.create_or_get_user(db_session, "rick@example.com")) is not None
+    assert u1.email == "rick@example.com"
+
+    # new user
+    assert (u2 := crud.create_or_get_user(db_session, "beth@example.com")) is not None
+    assert u2.email == "beth@example.com"
+
+    assert db_session.query(models.User).count() == 4
+
+
+def test_upsert_group(test_data, db_session):
+    from db import crud
+
+    assert db_session.query(models.Group).count() == 4
+
+    repeatable_params = ["desc 1", "orch.yaml", "sheet.yaml", {"read": ["all"]}, ["example.com"]]
+
+    assert (g1 := crud.upsert_group(db_session, "spaceship", *repeatable_params)) is not None
+    assert g1.id == "spaceship"
+    assert g1.description == "desc 1"
+    assert g1.orchestrator == "orch.yaml"
+    assert g1.orchestrator_sheet == "sheet.yaml"
+    assert g1.permissions == {"read": ["all"]}
+    assert g1.domains == ["example.com"]
+    assert len(g1.users) == 2
+    assert [u.email for u in g1.users] == ["rick@example.com", "morty@example.com"]
+
+    assert (g2 := crud.upsert_group(db_session, "interdimensional", *repeatable_params)) is not None
+    assert g2.id == "interdimensional"
+    assert len(g2.users) == 1
+    assert [u.email for u in g2.users] == ["rick@example.com"]
+
+    assert (g3 := crud.upsert_group(db_session, "this-is-a-new-group", *repeatable_params)) is not None
+    assert g3.id == "this-is-a-new-group"
+    assert len(g3.users) == 0
+
+    assert db_session.query(models.Group).count() == 5
+
+
+def test_upsert_user_groups(db_session):
+    from db import crud
+
+    @patch('db.crud.get_settings', new=lambda: bad_setings)
+    def test_missing_yaml(db_session):
+        with pytest.raises(FileNotFoundError):
+            crud.upsert_user_groups(db_session)
+
+    @patch('db.crud.get_settings', new=lambda: bad_setings)
+    def test_broken_yaml(db_session):
+        with pytest.raises(yaml.YAMLError):
+            crud.upsert_user_groups(db_session)
+
+    bad_setings = Settings(_env_file=".env.test")
+
+    bad_setings.USER_GROUPS_FILENAME = "tests/user-groups.test.missing.yaml"
+    test_missing_yaml(db_session)
+
+    bad_setings.USER_GROUPS_FILENAME = "tests/user-groups.test.broken.yaml"
+    test_broken_yaml(db_session)
+
+
+def test_create_sheet(db_session):
+    from db import crud
+
+    assert db_session.query(models.Sheet).count() == 0
+
+    s = crud.create_sheet(db_session, "sheet-id-123", "sheet name", "email@example.com", "group-id", "hourly")
+    assert s is not None
+    assert s.id == "sheet-id-123"
+    assert s.name == "sheet name"
+    assert s.author_id == "email@example.com"
+    assert s.group_id == "group-id"
+    assert s.frequency == "hourly"
+
+    assert db_session.query(models.Sheet).count() == 1
+
+    # duplicate id
+    import sqlalchemy
+    with pytest.raises(sqlalchemy.exc.IntegrityError):
+        crud.create_sheet(db_session, "sheet-id-123", "I thought this was another sheet", "email", "group-id", "hourly")
+
+
+def test_get_user_sheet(test_data, db_session):
+    from db import crud
+
+    assert crud.get_user_sheet(db_session, "", "sheet-0") is None
+    assert crud.get_user_sheet(db_session, "morty@example.com", "sheet-0") is None
+
+    assert crud.get_user_sheet(db_session, "rick@example.com", "sheet-0") is not None
+    assert crud.get_user_sheet(db_session, "rick@example.com", "sheet-0-2") is not None
+    assert crud.get_user_sheet(db_session, "morty@example.com", "sheet-1") is not None
+
+
+def test_get_user_sheets(test_data, db_session):
+    from db import crud
+
+    assert len(crud.get_user_sheets(db_session, "")) == 0
+    rick_sheets = crud.get_user_sheets(db_session, "rick@example.com")
+    assert len(rick_sheets) == 2
+    assert [s.id for s in rick_sheets] == ["sheet-0", "sheet-0-2"]
+    assert len(crud.get_user_sheets(db_session, "morty@example.com")) == 1
+
+def test_delete_sheet(test_data, db_session):
+    from db import crud
+
+    assert crud.delete_sheet(db_session, "sheet-0", "") == False
+    assert crud.delete_sheet(db_session, "sheet-0", "rick@example.com") == True
+    assert crud.delete_sheet(db_session, "sheet-0", "rick@example.com") == False
+

app/tests/db/test_models.py

@@ -0,0 +1,6 @@
+def test_generate_uuid():
+	from db.models import generate_uuid
+
+	assert generate_uuid() != generate_uuid()
+	assert len(generate_uuid()) == 36
+	assert generate_uuid().count("-") == 4

app/tests/endpoints/test_default.py

@@ -0,0 +1,128 @@
+from unittest.mock import AsyncMock, MagicMock, patch
+from fastapi.testclient import TestClient
+import pytest
+from app.shared.config import VERSION
+from tests.db.test_crud import test_data
+
+
+def test_endpoint_home(client_with_auth):
+    r = client_with_auth.get("/")
+    assert r.status_code == 200
+    j = r.json()
+    assert "version" in j and j["version"] == VERSION
+    assert "breakingChanges" in j
+    assert "groups" not in j
+
+
+@patch("endpoints.default.bearer_security", new_callable=AsyncMock)
+@patch("endpoints.default.get_user_auth", new_callable=AsyncMock, return_value="test@example.com")
+@patch("endpoints.default.crud.get_user_groups", return_value=["group1", "group2"])
+def test_endpoint_home_with_groups(m1, m2, m3, client_with_auth):
+    r = client_with_auth.get("/")
+    assert r.status_code == 200
+    j = r.json()
+    assert "version" in j and j["version"] == VERSION
+    assert "breakingChanges" in j
+    assert "groups" in j
+    assert j["groups"] == ["group1", "group2"]
+
+
+@patch("endpoints.default.bearer_security", new_callable=AsyncMock)
+@patch("endpoints.default.get_user_auth", new_callable=AsyncMock, return_value="test@example.com")
+@patch("endpoints.default.crud.get_user_groups", side_effect=Exception('mocked error'))
+def test_endpoint_home_with_groups_exception(m1, m2, m3, client_with_auth):  # mocks call that triggers an internal error
+    r = client_with_auth.get("/")
+    assert r.status_code == 200
+    j = r.json()
+    assert "version" in j and j["version"] == VERSION
+    assert "breakingChanges" in j
+    assert "groups" not in j
+
+
+def test_endpoint_health(client_with_auth):
+    r = client_with_auth.get("/health")
+    assert r.status_code == 200
+    assert r.json() == {"status": "ok"}
+
+
+def test_endpoint_active_no_auth(client, test_no_auth):
+    test_no_auth(client.get, "/user/active")
+
+
+def test_endpoint_active(app):
+    m_user_state = MagicMock()
+
+    from web.security import get_user_state
+    app.dependency_overrides[get_user_state] = lambda: m_user_state
+    
+    # inactive user
+    m_user_state.active = False
+    client = TestClient(app)
+    r = client.get("/user/active")
+    assert r.status_code == 200
+    assert r.json() == {"active": False}
+
+    # active user
+    m_user_state.active = True
+    client = TestClient(app)
+    r = client.get("/user/active")
+    assert r.status_code == 200
+    assert r.json() == {"active": True}
+    
+
+
+def test_no_serve_local_archive_by_default(client_with_auth):
+    r = client_with_auth.get("/app/local_archive_test/temp.txt")
+    assert r.status_code == 404
+
+
+def test_favicon(client_with_auth):
+    r = client_with_auth.get("/favicon.ico")
+    assert r.status_code == 200
+    assert r.headers["content-type"] == "image/vnd.microsoft.icon"
+
+
+def test_endpoint_test_prometheus_no_auth(client, test_no_auth):
+    test_no_auth(client.get, "/metrics")
+
+
+def test_endpoint_test_prometheus_no_user_auth(client_with_auth, test_no_auth):
+    test_no_auth(client_with_auth.get, "/metrics")
+
+
+@pytest.mark.asyncio
+async def test_prometheus_metrics(test_data, client_with_token, get_settings):
+    # before metrics calculation
+    r = client_with_token.get("/metrics")
+    assert r.status_code == 200
+    assert r.headers["content-type"] == "text/plain; version=0.0.4; charset=utf-8"
+    assert "disk_utilization" in r.text
+    assert "database_metrics" in r.text
+    assert "exceptions" in r.text
+    assert "worker_exceptions_total" in r.text
+    assert 'disk_utilization{type="used"}' not in r.text
+
+    # after metrics calculation
+    from web.utils.metrics import measure_regular_metrics
+    await measure_regular_metrics(get_settings.DATABASE_PATH, 60 * 60 * 24 * 31 * 12 * 100)
+    r2 = client_with_token.get("/metrics")
+    assert 'disk_utilization{type="used"}' in r2.text
+    assert 'disk_utilization{type="free"}' in r2.text
+    assert 'disk_utilization{type="database"}' in r2.text
+    assert 'database_metrics{query="count_archives"} 100.0' in r2.text
+    assert 'database_metrics{query="count_archive_urls"} 1000.0' in r2.text
+    assert 'database_metrics{query="count_users"} 3.0' in r2.text
+    assert 'database_metrics_counter_total{query="count_by_user",user="rick@example.com"} 34.0' in r2.text
+    assert 'database_metrics_counter_total{query="count_by_user",user="morty@example.com"} 33.0' in r2.text
+    assert 'database_metrics_counter_total{query="count_by_user",user="jerry@example.com"} 33.0' in r2.text
+
+    # 30s window, should not change the gauges nor the total in the counters
+    from web.utils.metrics import measure_regular_metrics
+    await measure_regular_metrics(get_settings.DATABASE_PATH, 30)
+    r3 = client_with_token.get("/metrics")
+    assert 'database_metrics{query="count_archives"} 100.0' in r3.text
+    assert 'database_metrics{query="count_archive_urls"} 1000.0' in r3.text
+    assert 'database_metrics{query="count_users"} 3.0' in r3.text
+    assert 'database_metrics_counter_total{query="count_by_user",user="rick@example.com"} 34.0' in r3.text
+    assert 'database_metrics_counter_total{query="count_by_user",user="morty@example.com"} 33.0' in r3.text
+    assert 'database_metrics_counter_total{query="count_by_user",user="jerry@example.com"} 33.0' in r3.text

app/tests/endpoints/test_interoperability.py

@@ -0,0 +1,40 @@
+from datetime import datetime
+import json
+from unittest.mock import patch
+
+from app.shared.config import ALLOW_ANY_EMAIL
+from db import crud
+
+
+def test_submit_manual_archive_unauthenticated(client, test_no_auth):
+    test_no_auth(client.post, "/interop/submit-archive")
+
+
+def test_submit_manual_archive_not_user_auth(client_with_auth, test_no_auth):
+    test_no_auth(client_with_auth.post, "/interop/submit-archive")
+
+
+@patch("endpoints.interoperability.get_store_until", return_value=datetime.now())
+def test_submit_manual_archive(m1, client_with_token, db_session):
+    # normal workflow
+    aa_metadata = json.dumps({"status": "test: success", "metadata": {"url": "http://example.com"}, "media": [{"filename": "fn1", "urls": ["http://example.s3.com"]}]})
+    r = client_with_token.post("/interop/submit-archive", json={"result": aa_metadata, "public": True, "author_id": "jerry@gmail.com", "group_id": "spaceship", "tags": ["test"]})
+    assert r.status_code == 201
+    assert "id" in r.json()
+
+    inserted = crud.get_archive(db_session, r.json()["id"], ALLOW_ANY_EMAIL)
+    assert inserted.url == "http://example.com"
+    assert inserted.group_id == "spaceship"
+    assert inserted.author_id == "jerry@gmail.com"
+    assert sorted([t.id for t in inserted.tags]) == sorted(["test", "manual"])
+    assert inserted.public
+    assert type(inserted.result) == dict
+    assert [u.url for u in inserted.urls] == ["http://example.s3.com"]
+    assert type(inserted.store_until) == datetime
+
+
+    # cannot have the same URL twice 
+    aa_metadata = json.dumps({"status": "test: success", "metadata": {"url": "http://example.com"}, "media": [{"filename": "fn1", "urls": ["http://example.com", "http://example.com"]}]})
+    r = client_with_token.post("/interop/submit-archive", json={"result": aa_metadata, "public": False, "author_id": "jerry@gmail.com", "tags": ["test"]})
+    assert r.status_code == 422
+    assert r.json() == {"detail": "Cannot insert into DB due to integrity error, likely duplicate urls."}

app/tests/endpoints/test_sheet.py

@@ -0,0 +1,193 @@
+from datetime import datetime
+import json
+from unittest.mock import MagicMock, patch
+
+from fastapi.testclient import TestClient
+
+from app.shared.schemas import TaskResult
+
+
+def test_endpoints_no_auth(client, test_no_auth):
+    test_no_auth(client.post, "/sheet/create")
+    test_no_auth(client.get, "/sheet/mine")
+    test_no_auth(client.delete, "/sheet/123-sheet-id")
+    test_no_auth(client.post, "/sheet/123-sheet-id/archive")
+
+
+def test_create_sheet_endpoint(app_with_auth, db_session):
+    client_with_auth = TestClient(app_with_auth)
+    good_data = {
+        "id": "123-sheet-id",
+        "name": "Test Sheet",
+        "group_id": "spaceship",
+        "frequency": "daily"
+    }
+
+    # with good data
+    response = client_with_auth.post("/sheet/create", json=good_data)
+    assert response.status_code == 201
+    j = response.json()
+    assert datetime.fromisoformat(j.pop("created_at"))
+    assert datetime.fromisoformat(j.pop("last_url_archived_at"))
+    assert j.pop("author_id") == 'morty@example.com'
+    assert j == good_data
+
+    # already exists
+    response = client_with_auth.post("/sheet/create", json=good_data)
+    assert response.status_code == 400
+    assert response.json() == {"detail": "Sheet with this ID is already being archived."}
+
+    # bad group
+    bad_data = good_data.copy()
+    bad_data["group_id"] = "not a group"
+    response = client_with_auth.post("/sheet/create", json=bad_data)
+    assert response.status_code == 403
+    assert response.json() == {"detail": "User does not have access to this group."}
+
+    # switch to jerry who's got less quota/permissions
+    from web.security import get_user_state
+    from db.user_state import UserState
+    app_with_auth.dependency_overrides[get_user_state] = lambda: UserState(db_session, "jerry@example.com")
+    client_jerry = TestClient(app_with_auth)
+
+    # frequency not allowed
+    jerry_data = good_data.copy()
+    jerry_data["group_id"] = "animated-characters"
+    jerry_data["frequency"] = "hourly"
+    jerry_data["id"] = "jerry-sheet-id"
+    response = client_jerry.post("/sheet/create", json=jerry_data)
+    assert response.status_code == 422
+    assert response.json() == {"detail": "Invalid frequency selected for this group."}
+
+    jerry_data["frequency"] = "daily"
+    # success for the first sheet, bad quota on second
+    response = client_jerry.post("/sheet/create", json=jerry_data)
+    assert response.status_code == 201
+
+    response = client_jerry.post("/sheet/create", json=jerry_data)
+    assert response.status_code == 429
+    assert response.json() == {"detail": "User has reached their sheet quota for this group."}
+
+
+def test_get_user_sheets_endpoint(client_with_auth, db_session):
+    # no data
+    response = client_with_auth.get("/sheet/mine")
+    assert response.status_code == 200
+    assert response.json() == []
+
+    # with data
+    from db import models
+    db_session.add(
+        models.Sheet(id="123", name="Test Sheet 1", author_id="morty@example.com", group_id="spaceship", frequency="hourly")
+    )
+    db_session.commit()
+    db_session.add_all([
+        models.Sheet(id="456", name="Test Sheet 2", author_id="morty@example.com", group_id="interdimensional", frequency="daily"),
+        models.Sheet(id="789", name="Test Sheet 3", author_id="rick@example.com", group_id="interdimensional", frequency="hourly"),
+    ])
+    db_session.commit()
+
+    response = client_with_auth.get("/sheet/mine")
+    assert response.status_code == 200
+    r = response.json()
+    assert isinstance(r, list)
+    assert len(r) == 2
+    assert datetime.fromisoformat(r[0].pop("created_at"))
+    assert datetime.fromisoformat(r[0].pop("last_url_archived_at"))
+    assert datetime.fromisoformat(r[1].pop("created_at"))
+    assert datetime.fromisoformat(r[1].pop("last_url_archived_at"))
+    assert r[0] == {
+        'id': '123',
+        'author_id': 'morty@example.com',
+        'frequency': 'hourly',
+        'group_id': 'spaceship',
+        'name': 'Test Sheet 1',
+    }
+    assert r[1] == {
+        'id': '456',
+        'author_id': 'morty@example.com',
+        'frequency': 'daily',
+        'group_id': 'interdimensional',
+        'name': 'Test Sheet 2',
+    }
+
+
+def test_delete_sheet_endpoint(client_with_auth, db_session):
+    # missing sheet
+    response = client_with_auth.delete("/sheet/123-sheet-id")
+    assert response.status_code == 200
+    assert response.json() == {
+        "id": "123-sheet-id",
+        "deleted": False
+    }
+
+    # add sheets for deletion
+    from db import models
+    db_session.add_all([
+        models.Sheet(id="123-sheet-id", name="Test Sheet 1", author_id="morty@example.com", group_id="interdimensional", frequency="daily"),
+        models.Sheet(id="456-sheet-id", name="Test Sheet 2", author_id="rick@example.com", group_id="spaceship", frequency="hourly"),
+    ])
+    db_session.commit()
+
+    # morty can delete his
+    response = client_with_auth.delete("/sheet/123-sheet-id")
+    assert response.status_code == 200
+    assert response.json() == {"id": "123-sheet-id", "deleted": True}
+    # but only once
+    response = client_with_auth.delete("/sheet/123-sheet-id")
+    assert response.status_code == 200
+    assert response.json() == {"id": "123-sheet-id", "deleted": False}
+    # and not rick's
+    response = client_with_auth.delete("/sheet/456-sheet-id")
+    assert response.status_code == 200
+    assert response.json() == {"id": "456-sheet-id", "deleted": False}
+
+
+class TestArchiveUserSheetEndpoint:
+    @patch("endpoints.sheet.celery", return_value=MagicMock())
+    def test_normal_flow(self, m_celery, client_with_auth, db_session):
+        from db import models
+        db_session.add(models.Sheet(id="123-sheet-id", name="Test Sheet 1", author_id="morty@example.com", group_id="spaceship", frequency="hourly"))
+        db_session.commit()
+
+        m_signature = MagicMock()
+        m_signature.delay.return_value = TaskResult(id="123-taskid", status="PENDING", result="")
+        m_celery.signature.return_value = m_signature
+
+        r = client_with_auth.post("/sheet/123-sheet-id/archive")
+        assert r.status_code == 201
+        assert r.json() == {"id": "123-taskid"}
+        m_celery.signature.assert_called_once()
+        m_signature.delay.assert_called_once()
+
+    def test_token_auth(self, client_with_token, test_no_auth):
+        test_no_auth(client_with_token.post, "/sheet/123-sheet-id/archive")
+
+    def test_missing_data(self, client_with_auth):
+        r = client_with_auth.post("/sheet/123-sheet-id/archive")
+        assert r.status_code == 403
+        assert r.json() == {"detail": "No access to this sheet."}
+
+    def test_no_access(self, client_with_auth, db_session):
+        from db import models
+        db_session.add(models.Sheet(id="123-sheet-id", name="Test Sheet 1", author_id="rick@example.com", group_id="spaceship", frequency="hourly"))
+        db_session.commit()
+        r = client_with_auth.post("/sheet/123-sheet-id/archive")
+        assert r.status_code == 403
+        assert r.json() == {"detail": "No access to this sheet."}
+
+    def test_user_not_in_group(self, client_with_auth, db_session):
+        from db import models
+        db_session.add(models.Sheet(id="123-sheet-id", name="Test Sheet 1", author_id="morty@example.com", group_id="interdimensional", frequency="hourly"))
+        db_session.commit()
+        r = client_with_auth.post("/sheet/123-sheet-id/archive")
+        assert r.status_code == 403
+        assert r.json() == {"detail": "User does not have access to this group."}
+
+    def test_user_cannot_manually_trigger(self, client_with_auth, db_session):
+        from db import models
+        db_session.add(models.Sheet(id="123-sheet-id", name="Test Sheet 1", author_id="morty@example.com", group_id="default", frequency="hourly"))
+        db_session.commit()
+        r = client_with_auth.post("/sheet/123-sheet-id/archive")
+        assert r.status_code == 429
+        assert r.json() == {"detail": "User cannot manually trigger sheet archiving in this group."}

app/tests/endpoints/test_task.py

@@ -0,0 +1,51 @@
+from unittest.mock import patch
+
+
+def test_endpoint_task_status_no_auth(client, test_no_auth):
+    test_no_auth(client.get, "/task/test-task-id")
+
+
+@patch("endpoints.task.AsyncResult")
+def test_get_status_success(mock_async_result, client_with_auth):
+    mock_async_result.return_value.status = "SUCCESS"
+    mock_async_result.return_value.result = {"data": "some result"}
+
+    response = client_with_auth.get("/task/test-task-id")
+
+    assert response.status_code == 200
+    assert response.json() == {
+        "id": "test-task-id",
+        "status": "SUCCESS",
+        "result": {"data": "some result"}
+    }
+
+
+@patch("endpoints.task.AsyncResult")
+def test_get_status_failure(mock_async_result, client_with_auth):
+
+    mock_async_result.return_value.status = "FAILURE"
+    mock_async_result.return_value.result = Exception("Some error")
+
+    response = client_with_auth.get("/task/test-task-id")
+
+    assert response.status_code == 200
+    assert response.json() == {
+        "id": "test-task-id",
+        "status": "FAILURE",
+        "result": {"error": "Some error"}
+    }
+
+
+@patch("endpoints.task.AsyncResult")
+def test_get_status_pending(mock_async_result, client_with_auth):
+    mock_async_result.return_value.status = "PENDING"
+    mock_async_result.return_value.result = None
+
+    response = client_with_auth.get("/task/test-task-id")
+
+    assert response.status_code == 200
+    assert response.json() == {
+        "id": "test-task-id",
+        "status": "PENDING",
+        "result": None
+    }

app/tests/endpoints/test_url.py

@@ -0,0 +1,192 @@
+import json
+from unittest.mock import MagicMock, patch
+
+from app.shared.schemas import ArchiveCreate, TaskResult
+
+
+def test_archive_url_unauthenticated(client, test_no_auth):
+    test_no_auth(client.post, "/url/archive")
+
+
+@patch("endpoints.url.UserState")
+@patch("endpoints.url.celery", return_value=MagicMock())
+def test_archive_url(m_celery, m2, client_with_auth):
+    m_signature = MagicMock()
+    m_signature.delay.return_value = TaskResult(id="123-456-789", status="PENDING", result="")
+    m_celery.signature.return_value = m_signature
+
+    m_user_state = MagicMock()
+    m2.return_value = m_user_state
+
+    # url is too short
+    response = client_with_auth.post("/url/archive", json={"url": "bad"})
+    assert response.status_code == 422
+    assert response.json()["detail"][0]["msg"] == 'String should have at least 5 characters'
+    m_celery.signature.assert_not_called()
+
+    # url is invalid
+    response = client_with_auth.post("/url/archive", json={"url": "example.com"})
+    assert response.status_code == 400
+    assert response.json()["detail"] == "Invalid URL received."
+
+    # valid request
+    m_user_state.has_quota_max_monthly_urls.return_value = True
+    m_user_state.has_quota_max_monthly_mbs.return_value = True
+    response = client_with_auth.post("/url/archive", json={"url": "https://example.com"})
+    assert response.status_code == 201
+    assert response.json() == {'id': '123-456-789'}
+    m_celery.signature.assert_called_once()
+    m_signature.delay.assert_called_once()
+    called_val = m_celery.signature.call_args
+    assert called_val[0][0] == "create_archive_task"
+    assert json.loads(called_val[1]['args'][0]) ==  {"id": None, "url": "https://example.com", "result": None, "public": False, "author_id": "rick@example.com", "group_id": "default", "tags": None, "sheet_id": None, "store_until": None, "urls": None}
+    m_user_state.has_quota_max_monthly_urls.assert_called_once()
+    m_user_state.has_quota_max_monthly_mbs.assert_called_once()
+    m_user_state.in_group.assert_called_once_with("default")
+
+    # user is not in group
+    m_user_state.in_group.return_value = False
+    response = client_with_auth.post("/url/archive", json={"url": "https://example.com", "group_id": "new-group"})
+    assert response.status_code == 403
+    assert response.json()["detail"] == "User does not have access to this group."
+    m_user_state.in_group.assert_called_with("new-group")
+
+    # user is in group
+    m_user_state.in_group.return_value = True
+    response = client_with_auth.post("/url/archive", json={"url": "https://example.com", "group_id": "spaceship"})
+    assert response.status_code == 201
+    assert response.json() == {'id': '123-456-789'}
+    assert m_celery.signature.call_count == 2
+    assert m_signature.delay.call_count == 2
+    called_val = m_celery.signature.call_args
+    assert json.loads(called_val[1]['args'][0])["group_id"] == "spaceship"
+    m_user_state.in_group.assert_called_with("spaceship")
+
+    # user is over monthly URL quota
+    m_user_state.has_quota_max_monthly_urls.return_value = False
+    m_user_state.has_quota_max_monthly_mbs.return_value = True
+    response = client_with_auth.post("/url/archive", json={"url": "https://example.com", "group_id": "spaceship"})
+    assert response.status_code == 429
+    assert response.json()["detail"] == "User has reached their monthly URL quota."
+    m_user_state.has_quota_max_monthly_urls.assert_called_with("spaceship")
+
+    # user is over monthly MB quota
+    m_user_state.has_quota_max_monthly_urls.return_value = True
+    m_user_state.has_quota_max_monthly_mbs.return_value = False
+    response = client_with_auth.post("/url/archive", json={"url": "https://example.com", "group_id": "spacesuit"})
+    assert response.status_code == 429
+    assert response.json()["detail"] == "User has reached their monthly MB quota."
+    m_user_state.has_quota_max_monthly_mbs.assert_called_with("spacesuit")
+    assert m_celery.signature.call_count == 2
+    assert m_signature.delay.call_count == 2
+
+
+@patch("endpoints.url.UserState")
+def test_archive_url_quotas(m1, client_with_auth):
+    m_user_state = MagicMock()
+    m1.return_value = m_user_state
+
+    # misses on monthly URLs quota
+    m_user_state.has_quota_max_monthly_urls.return_value = False
+    response = client_with_auth.post("/url/archive", json={"url": "https://example.com"})
+    assert response.status_code == 429
+    assert response.json()["detail"] == "User has reached their monthly URL quota."
+    m_user_state.has_quota_max_monthly_urls.assert_called_once()
+
+    # misses on monthly MBs quota
+    m_user_state.has_quota_max_monthly_urls.return_value = True
+    m_user_state.has_quota_max_monthly_mbs.return_value = False
+    response = client_with_auth.post("/url/archive", json={"url": "https://example.com"})
+    assert response.status_code == 429
+    assert response.json()["detail"] == "User has reached their monthly MB quota."
+    m_user_state.has_quota_max_monthly_mbs.assert_called_once()
+
+
+@patch("endpoints.url.celery", return_value=MagicMock())
+def test_archive_url_with_api_token(m_celery, client_with_token):
+    m_signature = MagicMock()
+    m_signature.delay.return_value = TaskResult(id="123-456-789", status="PENDING", result="")
+    m_celery.signature.return_value = m_signature
+    response = client_with_token.post("/url/archive", json={"url": "https://example.com"})
+    assert response.status_code == 201
+    assert response.json() == {'id': '123-456-789'}
+    m_celery.signature.assert_called_once()
+    m_signature.delay.assert_called_once()
+    called_val = m_celery.signature.call_args
+    assert called_val[0][0] == "create_archive_task"
+
+
+def test_search_by_url_unauthenticated(client, test_no_auth):
+    test_no_auth(client.get, "/url/search")
+
+
+def test_search_by_url(client_with_auth, client_with_token, db_session):
+    # tests the search endpoint, including through some db data for the endpoint params
+    response = client_with_auth.get("/url/search")
+    assert response.status_code == 422
+    assert response.json()["detail"][0]["msg"] == "Field required"
+
+    response = client_with_auth.get("/url/search?url=https://example.com")
+    assert response.status_code == 200
+    assert response.json() == []
+
+    from db import crud, schemas
+    for i in range(11):
+        crud.create_task(db_session, ArchiveCreate(id=f"url-456-{i}", url="https://example.com" if i < 10 else "https://something-else.com", result={}, public=True, author_id="rick@example.com"), [], [])
+        # NB: this insertion is too fast for the ordering to be correct as they are within the same second
+
+    response = client_with_auth.get("/url/search?url=https://example.com")
+    assert response.status_code == 200
+    assert len(j := response.json()) == 10
+    assert "url-456-0" in [i["id"] for i in j]
+    assert "url-456-9" in [i["id"] for i in j]
+    assert "url-456-10" not in [i["id"] for i in j]
+    assert j[0].keys() == schemas.ArchiveResult.model_fields.keys()
+
+    response = client_with_auth.get("/url/search?url=https://example.com&limit=5")
+    assert response.status_code == 200
+    assert len(response.json()) == 5
+
+    response = client_with_auth.get("/url/search?url=https://example.com&skip=5&limit=2")
+    assert response.status_code == 200
+    assert len(response.json()) == 2
+
+    response = client_with_auth.get("/url/search?url=https://example.com&archived_before=2010-01-01")
+    assert response.status_code == 200
+    assert len(response.json()) == 0
+
+    response = client_with_auth.get("/url/search?url=https://example.com&archived_after=2010-01-01")
+    assert response.status_code == 200
+    assert len(response.json()) == 10
+
+    # API token will also work
+    response = client_with_token.get("/url/search?url=https://example.com&archived_after=2010-01-01")
+    assert response.status_code == 200
+    assert len(response.json()) == 10
+
+
+@patch("endpoints.url.UserState")
+def test_search_no_read_access(mock_user_state, client_with_auth):
+    mock_user_state.return_value.read = False
+    mock_user_state.return_value.read_public = False
+
+    response = client_with_auth.get("/url/search?url=https://example.com")
+    assert response.status_code == 403
+    assert response.json() == {"detail": "User does not have read access."}
+
+
+def test_delete_task_unauthenticated(client, test_no_auth):
+    test_no_auth(client.delete, "/url/123-456-789")
+
+
+def test_delete_task(client_with_auth, db_session):
+    response = client_with_auth.delete("/url/delete-123-456-789")
+    assert response.status_code == 200
+    assert response.json() == {"id": "delete-123-456-789", "deleted": False}
+
+    from db import crud
+    crud.create_task(db_session, ArchiveCreate(id="delete-123-456-789", url="https://example.com", result={}, public=True, author_id="morty@example.com"), [], [])
+
+    response = client_with_auth.delete("/url/delete-123-456-789")
+    assert response.status_code == 200
+    assert response.json() == {"id": "delete-123-456-789", "deleted": True}

app/tests/orchestration.test.yaml

@@ -0,0 +1,24 @@
+steps:
+  feeder: cli_feeder
+  archivers: # order matters
+    - youtubedl_archiver
+  enrichers:
+    - hash_enricher
+
+  formatter: html_formatter # defaults to mute_formatter
+  storages:
+    - local_storage
+  databases:
+    - console_db
+
+configurations:
+  cli_feeder:
+    urls: 
+      - "url1"
+  hash_enricher:
+    algorithm: "SHA-256"
+  local_storage:
+    save_to: "./local_archive"
+    save_absolute: true
+    filename_generator: static
+    path_generator: flat

app/tests/user-groups.test.broken.yaml

@@ -0,0 +1,6 @@
+broken: True
+This is just an invalid yaml for testing
+
+still broken: True
+  - one
+  - two

app/tests/user-groups.test.yaml

@@ -0,0 +1,87 @@
+# NOTE: all emails should be lower-cased
+users:
+  rick@example.com:
+    - spaceship
+    - interdimensional
+  morty@example.com:
+    - spaceship
+  jerry@example.com:
+    - the-jerrys-club
+  # summer@herself.com:
+  # badyemail.com:
+
+domains:
+  example.com:
+    - animated-characters
+  birdy.com:
+    - animated-characters
+    - this-does-not-exist
+
+
+orchestrators:
+  spaceship: tests/orchestration.test.yaml
+  interdimensional: tests/orchestration.test.yaml
+  default: tests/orchestration.test.yaml
+
+default_orchestrator: tests/orchestration.test.yaml
+
+groups:
+  spaceship:
+    description: "The spaceship crew"
+    orchestrator: tests/orchestration.test.yaml
+    orchestrator_sheet: tests/orchestration.test.yaml
+    permissions:
+      read: ["all"]
+      archive_url: true
+      archive_sheet: true
+      manually_trigger_sheet: true
+      sheet_frequency: ["hourly", "daily"]
+      max_sheets: -1
+      max_archive_lifespan_months: -1
+      max_monthly_urls: -1
+      max_monthly_mbs: -1
+      priority: "high"
+  interdimensional:
+    description: "Interdimensional travelers"
+    orchestrator: tests/orchestration.test.yaml
+    orchestrator_sheet: tests/orchestration.test.yaml
+    permissions:
+      read: ["interdimensional", "animated-characters"]
+      archive_url: true
+      archive_sheet: true
+      manually_trigger_sheet: true
+      sheet_frequency: ["hourly", "daily"]
+      max_sheets: 5
+      max_archive_lifespan_months: 12
+      max_monthly_urls: 1000
+      max_monthly_mbs: 1000
+      priority: "high"
+  animated-characters:
+    description: "Animated characters"
+    orchestrator: tests/orchestration.test.yaml
+    orchestrator_sheet: tests/orchestration.test.yaml
+    permissions:
+      read: ["animated-characters"]
+      archive_url: true
+      archive_sheet: true
+      sheet_frequency: ["daily"]
+      max_sheets: 1
+      max_archive_lifespan_months: 12
+      max_monthly_urls: 2
+      max_monthly_mbs: 10
+      priority: "low"
+  default:
+    description: "Public access"
+    orchestrator: tests/orchestration.test.yaml
+    orchestrator_sheet: tests/orchestration.test.yaml
+    permissions:
+      # read: []
+      archive_url: true
+      # manually_trigger_sheet: false
+      # archive_sheet: false
+      # sheet_frequency: []
+      # max_sheets: 0
+      # max_archive_lifespan_months: 12
+      max_monthly_urls: 1
+      # max_monthly_mbs: 50
+      priority: "low"

app/tests/web/test_main.py

@@ -0,0 +1,49 @@
+import os
+from unittest.mock import patch
+from fastapi.testclient import TestClient
+
+import shutil
+
+import pytest
+
+def test_lifespan(app):
+    with TestClient(app) as client:
+        r = client.get("/health")
+        assert r.status_code == 200
+        assert r.json() == {"status": "ok"}
+
+def test_alembic(db_session):
+    import alembic.config
+    alembic.config.main(argv=['--raiseerr', 'upgrade', 'head'])
+    alembic.config.main(argv=['--raiseerr', 'downgrade', 'base'])
+
+@patch("endpoints.default.crud.soft_delete_task", side_effect=Exception('mocked error'))
+def test_logging_middleware(m1, client_with_auth):
+    from web.utils.metrics import EXCEPTION_COUNTER
+    assert len(EXCEPTION_COUNTER.collect()[0].samples) == 0
+    with pytest.raises(Exception, match="mocked error"):
+        client_with_auth.delete("/url/123")
+    # creates one empty and one from above
+    assert len(EXCEPTION_COUNTER.collect()[0].samples) == 2
+    
+
+def test_serve_local_archive_logic(get_settings):
+    # create a test file first
+    os.makedirs("local_archive_test", exist_ok=True)
+    with open("local_archive_test/temp.txt", "w") as f:
+        f.write("test")
+
+    try:
+        # modify the settings
+        get_settings.SERVE_LOCAL_ARCHIVE = "/app/local_archive_test"
+        from web.main import app_factory
+        app = app_factory(get_settings)
+        
+        # test
+        client = TestClient(app)
+        r = client.get("/app/local_archive_test/temp.txt")
+        assert r.status_code == 200
+        assert r.text == "test"
+    finally:
+        # cleanup
+        shutil.rmtree("local_archive_test")

app/tests/web/test_security.py

@@ -0,0 +1,108 @@
+from unittest.mock import patch
+
+from fastapi import HTTPException
+from fastapi.security import HTTPAuthorizationCredentials
+import pytest
+
+from app.shared.config import ALLOW_ANY_EMAIL
+
+
+def test_secure_compare():
+    from web.security import secure_compare
+
+    assert secure_compare("test", "test")
+    assert not secure_compare("test", "test2")
+
+
+@pytest.mark.asyncio
+async def test_get_token_or_user_auth_with_api():
+    from web.security import get_token_or_user_auth
+    mock_api = HTTPAuthorizationCredentials(scheme="lorem", credentials="this_is_the_test_api_token")
+    assert await get_token_or_user_auth(mock_api) == ALLOW_ANY_EMAIL
+
+
+@pytest.mark.asyncio
+async def test_get_token_or_user_auth_with_user():
+    from web.security import get_token_or_user_auth
+    bad_user = HTTPAuthorizationCredentials(scheme="ipsum", credentials="invalid")
+    e: pytest.ExceptionInfo = None
+    with pytest.raises(HTTPException) as e:
+        await get_token_or_user_auth(bad_user)
+    assert e.value.status_code == 401
+    assert e.value.detail == "invalid access_token"
+
+
+@patch("web.security.authenticate_user", return_value=(True, "summer@example.com"))
+@pytest.mark.asyncio
+async def test_get_user_auth(m1):
+    from web.security import get_user_auth
+    good_user = HTTPAuthorizationCredentials(scheme="ipsum", credentials="valid-and-good")
+    assert await get_user_auth(good_user) == "summer@example.com"
+
+
+@patch("web.security.secure_compare", return_value=False)
+@pytest.mark.asyncio
+async def test_token_api_key_auth_exception(m1):
+    from web.security import token_api_key_auth
+
+    e: pytest.ExceptionInfo = None
+    with pytest.raises(HTTPException) as e:
+        await token_api_key_auth(HTTPAuthorizationCredentials(scheme="ipsum", credentials="does-not-matter"), auto_error=True)
+    assert e.value.status_code == 401
+    assert e.value.detail == "Wrong auth credentials"
+
+
+@pytest.mark.asyncio
+async def test_authenticate_user():
+    from web.security import authenticate_user
+
+    assert authenticate_user("test") == (False, "invalid access_token")
+    assert authenticate_user(123) == (False, "invalid access_token")
+
+    with patch("web.security.requests.get") as mock_get:
+        # bad response from oauth2
+        mock_get.return_value.status_code = 403
+        assert authenticate_user("this-will-call-requests") == (False, "invalid token")
+        assert mock_get.call_count == 1
+
+        # 200 but invalid json
+        mock_get.return_value.status_code = 200
+        assert authenticate_user("this-will-call-requests") == (False, "token does not belong to valid APP_ID")
+        assert mock_get.call_count == 2
+
+        # 200 but invalid azp and aud
+        mock_get.return_value.json.return_value = {"email": "summer@example.com", "azp": "not_an_app"}
+        assert authenticate_user("this-will-call-requests") == (False, "token does not belong to valid APP_ID")
+
+        mock_get.return_value.json.return_value = {"email": "summer@example.com", "aud": "not_an_app"}
+        assert authenticate_user("this-will-call-requests") == (False, "token does not belong to valid APP_ID")
+
+        mock_get.return_value.json.return_value = {"email": "summer@example.com", "azp": "not_an_app", "aud": "not_an_app"}
+        assert authenticate_user("this-will-call-requests") == (False, "token does not belong to valid APP_ID")
+
+        # blocked email
+        mock_get.return_value.json.return_value = {"email": "blocked@example.com", "azp": "test_app_id_1", "aud": "not_an_app"}
+        assert authenticate_user("this-will-call-requests") == (False, "email 'blocked@example.com' not allowed")
+
+        # not verified
+        mock_get.return_value.json.return_value = {"email": "summer@example.com", "azp": "not_an_app", "aud": "test_app_id_1"}
+        assert authenticate_user("this-will-call-requests") == (False, "email 'summer@example.com' not verified")
+
+        # token expired
+        mock_get.return_value.json.return_value = {"email": "summer@example.com", "azp": "test_app_id_2", "email_verified": "true"}
+        assert authenticate_user("this-will-call-requests") == (False, "Token expired")
+
+        # 200 and valid azp and aup and verified
+        mock_get.return_value.json.return_value = {"email": "summer@example.com", "azp": "test_app_id_2", "email_verified": "true", "expires_in": 100}
+        assert authenticate_user("this-will-call-requests") == (True, "summer@example.com")
+        assert mock_get.call_count == 9
+
+
+@pytest.mark.asyncio
+async def test_authenticate_user_exception():
+    from web.security import authenticate_user
+
+    with patch("web.security.requests.get") as mock_get:
+        mock_get.return_value.status_code = 200
+        mock_get.return_value.json.side_effect = Exception("mocked error")
+        assert authenticate_user("this-will-call-requests") == (False, "exception occurred")

app/tests/worker/test_worker_main.py

@@ -0,0 +1,138 @@
+from datetime import datetime
+from unittest import mock
+
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from db import models, schemas
+from auto_archiver import Metadata
+from auto_archiver.core import Media
+
+
+
+class Test_create_archive_task():
+    URL = "https://example-live.com"
+    archive = schemas.ArchiveCreate(url=URL, tags=["tag-celery"], public=True, author_id="rick@example.com", group_id="interstellar")
+
+    @patch("worker.main.insert_result_into_db")
+    @patch("worker.main.get_store_until", return_value=datetime.now())
+    @patch("worker.main.load_orchestrator")
+    @patch("celery.app.task.Task.request")
+    def test_success(self, m_req, m_load, m_store, m_insert, db_session):
+        from worker.main import create_archive_task
+
+        m_req.id = "this-just-in"
+        mock_orchestrator = self.mock_orchestrator_choice(m_load)
+
+        task = create_archive_task(self.archive.model_dump_json())
+
+        m_load.assert_called_once_with("interstellar")
+        m_store.assert_called_once_with("interstellar")
+        m_insert.assert_called_once()
+        mock_orchestrator.feed_item.assert_called_once()
+
+        assert task["status"] == "success"
+        assert task["metadata"]["url"] == self.URL
+        assert len(task["media"]) == 0
+
+    def test_raise_invalid(self):
+        from worker.main import create_archive_task
+        with pytest.raises(Exception):
+            create_archive_task(self.archive.model_dump_json())
+
+    @patch("worker.main.insert_result_into_db", side_effect=Exception)
+    @patch("worker.main.load_orchestrator")
+    def test_raise_db_error(self, m_load, m_insert):
+        from worker.main import create_archive_task
+        mock_orchestrator = self.mock_orchestrator_choice(m_load)
+
+        with pytest.raises(Exception):
+            create_archive_task(self.archive.model_dump_json())
+        mock_orchestrator.feed_item.assert_called_once()
+
+
+    @patch("worker.main.insert_result_into_db", return_value=None)
+    @patch("worker.main.load_orchestrator")
+    def test_raise_empty_result(self, m_load, m_insert):
+        from worker.main import create_archive_task
+        mock_orchestrator = self.mock_orchestrator_choice(m_load)
+
+        with pytest.raises(Exception) as e:
+            create_archive_task(self.archive.model_dump_json())
+            assert "UNABLE TO archive" in str(e)
+        mock_orchestrator.feed_item.assert_called_once()
+
+    def mock_orchestrator_choice(self, m_load):
+        mock_orchestrator = mock.MagicMock()
+        mock_orchestrator.configure_mock(feed_item=mock.MagicMock(return_value=Metadata().set_url(self.URL).success()))
+        m_load.return_value = mock_orchestrator
+        return mock_orchestrator
+
+
+class Test_create_sheet_task():
+    URL = "https://example-live.com"
+    sheet = schemas.SubmitSheet(sheet_id="123", author_id="rick@example.com", group_id="interstellar", tags=["spaceship"])
+
+    @patch("worker.main.models.generate_uuid", return_value="constant-uuid")
+    @patch("worker.main.get_store_until", return_value=datetime.now())
+    @patch("worker.main.load_orchestrator")
+    def test_success(self, m_load, m_store, m_uuid, db_session):
+        from worker.main import create_sheet_task
+
+        assert db_session.query(models.Archive).filter(models.Archive.url == self.URL).count() == 0
+
+        mock_metadata = Metadata().set_url(self.URL).success()
+        mock_metadata.add_media(Media("fn1.txt", urls=["outcome1.com"]))
+        m_orch = MagicMock()
+        m_orch.feed.return_value = iter([False, mock_metadata, mock_metadata])
+        m_load.return_value = m_orch
+
+        res = create_sheet_task(self.sheet.model_dump_json())
+
+        m_load.assert_called_once_with("interstellar", True, {'configurations': {'gsheet_feeder': {'sheet_id': '123'}}})
+        m_orch.feed.assert_called_once()
+        m_store.assert_called_with("interstellar")
+        m_store.call_count == 2
+        m_uuid.call_count == 2
+        assert type(res) == dict
+        assert res["stats"]["archived"] == 1
+        assert res["stats"]["failed"] == 1
+        assert len(res["stats"]["errors"]) == 1
+        assert res["sheet_id"] == "123"
+        assert res["success"]
+        assert type(res["time"]) == datetime
+
+        # query created archive entry
+        inserted = db_session.query(models.Archive).filter(models.Archive.url == self.URL).one()
+        assert inserted is not None
+        assert inserted.url == self.URL
+        assert len(inserted.tags) == 1
+        assert inserted.tags[0].id == "spaceship"
+        assert inserted.group_id == "interstellar"
+        assert inserted.author_id == "rick@example.com"
+        assert inserted.public == False
+
+
+def test_get_all_urls(db_session):
+    from worker.main import get_all_urls
+    from auto_archiver import Metadata
+
+    meta = Metadata().set_url("https://example.com")
+    m1 = meta.add_media(Media("fn1.txt", urls=["outcome1.com"]))
+    m2 = meta.add_media(Media("fn2.txt", urls=["outcome2.com"]))
+    m3 = meta.add_media(Media("fn3.txt", urls=["outcome3.com"]))
+    m1.set("screenshot", Media("screenshot.png", urls=["screenshot.com"]))
+    m2.set("thumbnails", [Media("thumb1.png", urls=["thumb1.com"]), Media("thumb2.png", urls=["thumb2.com"])])
+    m3.set("ssl_data", Media("ssl_data.txt", urls=["ssl_data.com"]).to_dict())
+    m3.set("bad_data", {"bad": "dict is ignored"})
+
+    urls = [u.url for u in get_all_urls(meta)]
+    assert len(urls) == 7
+    assert "outcome1.com" in urls
+    assert "outcome2.com" in urls
+    assert "outcome3.com" in urls
+    assert "screenshot.com" in urls
+    assert "thumb1.com" in urls
+    assert "thumb2.com" in urls
+    assert "ssl_data.com" in urls