further worker/web separation and tests fixed

app/tests/conftest.py

@@ -3,8 +3,8 @@ from fastapi.testclient import TestClient
 import pytest
 from unittest.mock import patch
 from app.shared.config import ALLOW_ANY_EMAIL
-from app.web.db.user_state import UserState
 from app.shared.settings import Settings
+from app.web.db.user_state import UserState
 
 
 @pytest.fixture(autouse=True)
@@ -21,7 +21,7 @@ def get_settings():
 
 @pytest.fixture(autouse=True)
 def mock_settings():
-    with patch('shared.settings.Settings', return_value=Settings(_env_file=".env.test")) as mock_settings:
+    with patch('app.shared.settings.Settings', return_value=Settings(_env_file=".env.test")) as mock_settings:
         yield mock_settings
 
 
@@ -29,7 +29,7 @@ def mock_settings():
 def test_db(get_settings: Settings):
     from app.shared.db import models
     from app.shared.db.database import make_engine
-    from app.shared.db.crud import get_user_groups
+    from app.web.db.crud import get_user_groups
 
     get_user_groups.cache_clear()
     make_engine.cache_clear()
@@ -62,8 +62,8 @@ def db_session(test_db):
 
 @pytest.fixture()
 def app(db_session):
-    from web.main import app_factory
-    from app.shared.db import crud
+    from app.web.main import app_factory
+    from app.web.db import crud
     app = app_factory()
     crud.upsert_user_groups(db_session)
     return app
@@ -77,7 +77,7 @@ def client(app):
 
 @pytest.fixture()
 def app_with_auth(app, db_session):
-    from web.security import get_token_or_user_auth, get_user_auth, get_user_state
+    from app.web.security import get_token_or_user_auth, get_user_auth, get_user_state
     app.dependency_overrides[get_token_or_user_auth] = lambda: "rick@example.com"
     app.dependency_overrides[get_user_auth] = lambda: "morty@example.com"
     app.dependency_overrides[get_user_state] = lambda: UserState(db_session, "MORTY@example.com")
@@ -92,7 +92,7 @@ def client_with_auth(app_with_auth):
 
 @pytest.fixture()
 def app_with_token(app):
-    from web.security import token_api_key_auth, get_token_or_user_auth
+    from app.web.security import token_api_key_auth, get_token_or_user_auth
     app.dependency_overrides[token_api_key_auth] = lambda: ALLOW_ANY_EMAIL
     app.dependency_overrides[get_token_or_user_auth] = lambda: ALLOW_ANY_EMAIL
     return app

app/tests/shared/db/test_worker_crud.py

@@ -0,0 +1,98 @@
+from app.shared.db import models
+
+
+from app.tests.web.db.test_crud import test_data
+
+
+def test_get_group(test_data, db_session):
+    from app.shared.db import worker_crud
+
+    assert worker_crud.get_group(db_session, "spaceship") is not None
+    assert worker_crud.get_group(db_session, "interdimensional") is not None
+    assert worker_crud.get_group(db_session, "animated-characters") is not None
+    assert worker_crud.get_group(db_session, "non-existent!@#!%!") is None
+
+
+def test_create_or_get_user(test_data, db_session):
+    from app.shared.db import worker_crud
+
+    assert db_session.query(models.User).count() == 3
+
+    # already exists
+    assert (u1 := worker_crud.create_or_get_user(db_session, "rick@example.com")) is not None
+    assert u1.email == "rick@example.com"
+
+    # new user
+    assert (u2 := worker_crud.create_or_get_user(db_session, "beth@example.com")) is not None
+    assert u2.email == "beth@example.com"
+
+    assert db_session.query(models.User).count() == 4
+
+
+def test_create_tag(db_session):
+    from app.shared.db import worker_crud
+
+    assert db_session.query(models.Tag).count() == 0
+
+    # create first
+    create_tag = worker_crud.create_tag(db_session, "tag-101")
+    assert create_tag is not None
+    assert create_tag.id == "tag-101"
+    assert db_session.query(models.Tag).count() == 1
+    assert db_session.query(models.Tag).filter(models.Tag.id == "tag-101").first() == create_tag
+
+    # same id does not add new db entry
+    existing_tag = worker_crud.create_tag(db_session, "tag-101")
+    assert existing_tag == create_tag
+    assert db_session.query(models.Tag).count() == 1
+
+    # create second
+    second_tag = worker_crud.create_tag(db_session, "tag-102")
+    assert second_tag is not None
+    assert second_tag.id == "tag-102"
+    assert db_session.query(models.Tag).count() == 2
+
+
+def test_create_task(db_session):
+    from app.shared.db import worker_crud
+    from app.shared import schemas
+
+    task = schemas.ArchiveCreate(
+        id="archive-id-456-101",
+        url="https://example-0.com",
+        result={},
+        public=False,
+        author_id="rick@example.com",
+        group_id="spaceship",
+        tags=[],
+        urls=[]
+    )
+
+    # with tags and urls
+    nt = worker_crud.create_task(db_session, task, [models.Tag(id="tag-101")], [models.ArchiveUrl(url="https://example-0.com/0", key="media_0")])
+
+    assert nt is not None
+    assert nt.id == "archive-id-456-101"
+    assert nt.url == "https://example-0.com"
+    assert nt.author_id == "rick@example.com"
+    assert nt.public == False
+    assert nt.group_id == "spaceship"
+    assert len(nt.tags) == 1
+    assert nt.tags[0].id == "tag-101"
+    assert len(nt.urls) == 1
+    assert nt.urls[0].url == "https://example-0.com/0"
+    assert nt.urls[0].key == "media_0"
+    assert nt.created_at is not None
+
+    # without tags and urls
+    task.id = "archive-id-456-102"
+    nt = worker_crud.create_task(db_session, task, [], [])
+    assert nt is not None
+    assert nt.id == "archive-id-456-102"
+    assert nt.url == "https://example-0.com"
+    assert nt.author_id == "rick@example.com"
+    assert nt.public == False
+    assert nt.group_id == "spaceship"
+    assert len(nt.tags) == 0
+    assert len(nt.urls) == 0
+    assert nt.created_at is not None

app/tests/user-groups.test.yaml

@@ -19,17 +19,17 @@ domains:
 
 
 orchestrators:
-  spaceship: tests/orchestration.test.yaml
-  interdimensional: tests/orchestration.test.yaml
-  default: tests/orchestration.test.yaml
+  spaceship: app/tests/orchestration.test.yaml
+  interdimensional: app/tests/orchestration.test.yaml
+  default: app/tests/orchestration.test.yaml
 
-default_orchestrator: tests/orchestration.test.yaml
+default_orchestrator: app/tests/orchestration.test.yaml
 
 groups:
   spaceship:
     description: "The spaceship crew"
-    orchestrator: tests/orchestration.test.yaml
-    orchestrator_sheet: tests/orchestration.test.yaml
+    orchestrator: app/tests/orchestration.test.yaml
+    orchestrator_sheet: app/tests/orchestration.test.yaml
     permissions:
       read: ["all"]
       archive_url: true
@@ -43,8 +43,8 @@ groups:
       priority: "high"
   interdimensional:
     description: "Interdimensional travelers"
-    orchestrator: tests/orchestration.test.yaml
-    orchestrator_sheet: tests/orchestration.test.yaml
+    orchestrator: app/tests/orchestration.test.yaml
+    orchestrator_sheet: app/tests/orchestration.test.yaml
     permissions:
       read: ["interdimensional", "animated-characters"]
       archive_url: true
@@ -58,8 +58,8 @@ groups:
       priority: "high"
   animated-characters:
     description: "Animated characters"
-    orchestrator: tests/orchestration.test.yaml
-    orchestrator_sheet: tests/orchestration.test.yaml
+    orchestrator: app/tests/orchestration.test.yaml
+    orchestrator_sheet: app/tests/orchestration.test.yaml
     permissions:
       read: ["animated-characters"]
       archive_url: true
@@ -72,8 +72,8 @@ groups:
       priority: "low"
   default:
     description: "Public access"
-    orchestrator: tests/orchestration.test.yaml
-    orchestrator_sheet: tests/orchestration.test.yaml
+    orchestrator: app/tests/orchestration.test.yaml
+    orchestrator_sheet: app/tests/orchestration.test.yaml
     permissions:
       # read: []
       archive_url: true

app/tests/web/db/test_crud.py

@@ -4,7 +4,7 @@ from unittest.mock import patch
 import pytest
 import yaml
 from app.shared.db import models
-from shared.settings import Settings
+from app.shared.settings import Settings
 
 authors = ["rick@example.com", "morty@example.com", "jerry@example.com"]
 
@@ -55,18 +55,16 @@ def test_data(db_session):
 
     # setup groups
     assert db_session.query(models.Group).count() == 0
-    from app.shared.db import crud
+    from app.web.db import crud
     crud.upsert_user_groups(db_session)
     assert db_session.query(models.Group).count() == 4
     assert db_session.query(models.User).count() == 3
 
 
 def test_get_archive(test_data, db_session):
-    from app.shared.db import crud
+    from app.web.db import crud
     from app.shared.config import ALLOW_ANY_EMAIL
 
-    print(db_session.query(models.Group).all())
-
     # each author's archives work
     assert (a0 := crud.get_archive(db_session, "archive-id-456-0", authors[0])) is not None
     assert a0.id == "archive-id-456-0"
@@ -93,7 +91,7 @@ def test_get_archive(test_data, db_session):
 
 
 def test_search_archives_by_url(test_data, db_session):
-    from app.shared.db import crud
+    from app.web.db import crud
     from app.shared.config import ALLOW_ANY_EMAIL
 
     # rick's archives are private
@@ -141,7 +139,7 @@ def test_search_archives_by_url(test_data, db_session):
 
 def test_search_archives_by_email(test_data, db_session):
     from app.shared.config import ALLOW_ANY_EMAIL
-    from app.shared.db import crud
+    from app.web.db import crud
 
     # lower/upper case
     assert len(crud.search_archives_by_email(db_session, "rick@example.com")) == 34
@@ -160,9 +158,9 @@ def test_search_archives_by_email(test_data, db_session):
     assert a2[0].created_at == datetime(2021, 1, 1)
 
 
-@patch("db.crud.DATABASE_QUERY_LIMIT", new=25)
+@patch("app.web.db.crud.DATABASE_QUERY_LIMIT", new=25)
 def test_max_query_limit(test_data, db_session):
-    from app.shared.db import crud
+    from app.web.db import crud
     from app.shared.config import ALLOW_ANY_EMAIL
 
     assert len(crud.search_archives_by_url(db_session, "https://example", ALLOW_ANY_EMAIL)) == 25
@@ -172,53 +170,8 @@ def test_max_query_limit(test_data, db_session):
     assert len(crud.search_archives_by_email(db_session, "rick@example.com", limit=1000)) == 25
 
 
-def test_create_task(db_session):
-    from app.shared.db import crud
-    from app.shared import schemas
-
-    task = schemas.ArchiveCreate(
-        id="archive-id-456-101",
-        url="https://example-0.com",
-        result={},
-        public=False,
-        author_id="rick@example.com",
-        group_id="spaceship",
-        tags=[],
-        urls=[]
-    )
-
-    # with tags and urls
-    nt = crud.create_task(db_session, task, [models.Tag(id="tag-101")], [models.ArchiveUrl(url="https://example-0.com/0", key="media_0")])
-
-    assert nt is not None
-    assert nt.id == "archive-id-456-101"
-    assert nt.url == "https://example-0.com"
-    assert nt.author_id == "rick@example.com"
-    assert nt.public == False
-    assert nt.group_id == "spaceship"
-    assert len(nt.tags) == 1
-    assert nt.tags[0].id == "tag-101"
-    assert len(nt.urls) == 1
-    assert nt.urls[0].url == "https://example-0.com/0"
-    assert nt.urls[0].key == "media_0"
-    assert nt.created_at is not None
-
-    # without tags and urls
-    task.id = "archive-id-456-102"
-    nt = crud.create_task(db_session, task, [], [])
-    assert nt is not None
-    assert nt.id == "archive-id-456-102"
-    assert nt.url == "https://example-0.com"
-    assert nt.author_id == "rick@example.com"
-    assert nt.public == False
-    assert nt.group_id == "spaceship"
-    assert len(nt.tags) == 0
-    assert len(nt.urls) == 0
-    assert nt.created_at is not None
-
-
 def test_soft_delete(test_data, db_session):
-    from app.shared.db import crud
+    from app.web.db import crud
 
     # none deleted yet
     assert crud.get_archive(db_session, "archive-id-456-0", "rick@example.com") is not None
@@ -236,7 +189,7 @@ def test_soft_delete(test_data, db_session):
 
 
 def test_count_archives(test_data, db_session):
-    from app.shared.db import crud
+    from app.web.db import crud
 
     assert crud.count_archives(db_session) == 100
     db_session.query(models.Archive).filter(models.Archive.id == "archive-id-456-0").delete()
@@ -245,7 +198,7 @@ def test_count_archives(test_data, db_session):
 
 
 def test_count_archive_urls(test_data, db_session):
-    from app.shared.db import crud
+    from app.web.db import crud
 
     assert crud.count_archive_urls(db_session) == 1000
     db_session.query(models.ArchiveUrl).filter(models.ArchiveUrl.url == "https://example-0.com/0").delete()
@@ -260,7 +213,7 @@ def test_count_archive_urls(test_data, db_session):
 
 
 def test_count_users(test_data, db_session):
-    from app.shared.db import crud
+    from app.web.db import crud
 
     assert crud.count_users(db_session) == 3
     db_session.query(models.User).filter(models.User.email == "rick@example.com").delete()
@@ -269,7 +222,7 @@ def test_count_users(test_data, db_session):
 
 
 def test_count_by_users_since(test_data, db_session):
-    from app.shared.db import crud
+    from app.web.db import crud
 
     # 100y window
     assert len(cu := crud.count_by_user_since(db_session, 60 * 60 * 24 * 31 * 12 * 100)) == 3
@@ -278,32 +231,8 @@ def test_count_by_users_since(test_data, db_session):
     assert cu[2].total == 33
 
 
-def test_create_tag(db_session):
-    from app.shared.db import crud
-
-    assert db_session.query(models.Tag).count() == 0
-
-    # create first
-    create_tag = crud.create_tag(db_session, "tag-101")
-    assert create_tag is not None
-    assert create_tag.id == "tag-101"
-    assert db_session.query(models.Tag).count() == 1
-    assert db_session.query(models.Tag).filter(models.Tag.id == "tag-101").first() == create_tag
-
-    # same id does not add new db entry
-    existing_tag = crud.create_tag(db_session, "tag-101")
-    assert existing_tag == create_tag
-    assert db_session.query(models.Tag).count() == 1
-
-    # create second
-    second_tag = crud.create_tag(db_session, "tag-102")
-    assert second_tag is not None
-    assert second_tag.id == "tag-102"
-    assert db_session.query(models.Tag).count() == 2
-
-
 def test_is_user_in_group(test_data, db_session):
-    from app.shared.db import crud
+    from app.web.db import crud
     from app.shared.config import ALLOW_ANY_EMAIL
 
     # see user-groups.test.yaml
@@ -339,36 +268,12 @@ def test_is_user_in_group(test_data, db_session):
     ]
     for email, group, expected in test_pairs:
         print(f"{email} in {group} == {expected}")
-        assert crud.is_user_in_group(db_session, email, group) == expected
+        assert crud.is_user_in_group(email, group) == expected
 
 
-def test_get_group(test_data, db_session):
-    from app.shared.db import crud
-
-    assert crud.get_group(db_session, "spaceship") is not None
-    assert crud.get_group(db_session, "interdimensional") is not None
-    assert crud.get_group(db_session, "animated-characters") is not None
-    assert crud.get_group(db_session, "non-existent!@#!%!") is None
-
-
-def test_create_or_get_user(test_data, db_session):
-    from app.shared.db import crud
-
-    assert db_session.query(models.User).count() == 3
-
-    # already exists
-    assert (u1 := crud.create_or_get_user(db_session, "rick@example.com")) is not None
-    assert u1.email == "rick@example.com"
-
-    # new user
-    assert (u2 := crud.create_or_get_user(db_session, "beth@example.com")) is not None
-    assert u2.email == "beth@example.com"
-
-    assert db_session.query(models.User).count() == 4
-
 
 def test_upsert_group(test_data, db_session):
-    from app.shared.db import crud
+    from app.web.db import crud
 
     assert db_session.query(models.Group).count() == 4
 
@@ -397,29 +302,29 @@ def test_upsert_group(test_data, db_session):
 
 
 def test_upsert_user_groups(db_session):
-    from app.shared.db import crud
+    from app.web.db import crud
 
-    @patch('db.crud.get_settings', new=lambda: bad_setings)
+    @patch('app.web.db.crud.get_settings', new=lambda: bad_setings)
     def test_missing_yaml(db_session):
         with pytest.raises(FileNotFoundError):
             crud.upsert_user_groups(db_session)
 
-    @patch('db.crud.get_settings', new=lambda: bad_setings)
+    @patch('app.web.db.crud.get_settings', new=lambda: bad_setings)
     def test_broken_yaml(db_session):
         with pytest.raises(yaml.YAMLError):
             crud.upsert_user_groups(db_session)
 
     bad_setings = Settings(_env_file=".env.test")
 
-    bad_setings.USER_GROUPS_FILENAME = "tests/user-groups.test.missing.yaml"
+    bad_setings.USER_GROUPS_FILENAME = "app/tests/user-groups.test.missing.yaml"
     test_missing_yaml(db_session)
 
-    bad_setings.USER_GROUPS_FILENAME = "tests/user-groups.test.broken.yaml"
+    bad_setings.USER_GROUPS_FILENAME = "app/tests/user-groups.test.broken.yaml"
     test_broken_yaml(db_session)
 
 
 def test_create_sheet(db_session):
-    from app.shared.db import crud
+    from app.web.db import crud
 
     assert db_session.query(models.Sheet).count() == 0
 
@@ -440,7 +345,7 @@ def test_create_sheet(db_session):
 
 
 def test_get_user_sheet(test_data, db_session):
-    from app.shared.db import crud
+    from app.web.db import crud
 
     assert crud.get_user_sheet(db_session, "", "sheet-0") is None
     assert crud.get_user_sheet(db_session, "morty@example.com", "sheet-0") is None
@@ -451,7 +356,7 @@ def test_get_user_sheet(test_data, db_session):
 
 
 def test_get_user_sheets(test_data, db_session):
-    from app.shared.db import crud
+    from app.web.db import crud
 
     assert len(crud.get_user_sheets(db_session, "")) == 0
     rick_sheets = crud.get_user_sheets(db_session, "rick@example.com")
@@ -459,10 +364,10 @@ def test_get_user_sheets(test_data, db_session):
     assert [s.id for s in rick_sheets] == ["sheet-0", "sheet-0-2"]
     assert len(crud.get_user_sheets(db_session, "morty@example.com")) == 1
 
+
 def test_delete_sheet(test_data, db_session):
-    from app.shared.db import crud
+    from app.web.db import crud
 
     assert crud.delete_sheet(db_session, "sheet-0", "") == False
     assert crud.delete_sheet(db_session, "sheet-0", "rick@example.com") == True
     assert crud.delete_sheet(db_session, "sheet-0", "rick@example.com") == False
-

app/tests/web/endpoints/test_default.py

@@ -2,7 +2,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
 from fastapi.testclient import TestClient
 import pytest
 from app.shared.config import VERSION
-from tests.db.test_crud import test_data
+from app.tests.web.db.test_crud import test_data
 
 
 def test_endpoint_home(client_with_auth):
@@ -14,9 +14,9 @@ def test_endpoint_home(client_with_auth):
     assert "groups" not in j
 
 
-@patch("endpoints.default.bearer_security", new_callable=AsyncMock)
-@patch("endpoints.default.get_user_auth", new_callable=AsyncMock, return_value="test@example.com")
-@patch("endpoints.default.crud.get_user_groups", return_value=["group1", "group2"])
+@patch("app.web.endpoints.default.bearer_security", new_callable=AsyncMock)
+@patch("app.web.endpoints.default.get_user_auth", new_callable=AsyncMock, return_value="test@example.com")
+@patch("app.web.endpoints.default.crud.get_user_groups", return_value=["group1", "group2"])
 def test_endpoint_home_with_groups(m1, m2, m3, client_with_auth):
     r = client_with_auth.get("/")
     assert r.status_code == 200
@@ -27,9 +27,9 @@ def test_endpoint_home_with_groups(m1, m2, m3, client_with_auth):
     assert j["groups"] == ["group1", "group2"]
 
 
-@patch("endpoints.default.bearer_security", new_callable=AsyncMock)
-@patch("endpoints.default.get_user_auth", new_callable=AsyncMock, return_value="test@example.com")
-@patch("endpoints.default.crud.get_user_groups", side_effect=Exception('mocked error'))
+@patch("app.web.endpoints.default.bearer_security", new_callable=AsyncMock)
+@patch("app.web.endpoints.default.get_user_auth", new_callable=AsyncMock, return_value="test@example.com")
+@patch("app.web.endpoints.default.crud.get_user_groups", side_effect=Exception('mocked error'))
 def test_endpoint_home_with_groups_exception(m1, m2, m3, client_with_auth):  # mocks call that triggers an internal error
     r = client_with_auth.get("/")
     assert r.status_code == 200
@@ -52,7 +52,7 @@ def test_endpoint_active_no_auth(client, test_no_auth):
 def test_endpoint_active(app):
     m_user_state = MagicMock()
 
-    from web.security import get_user_state
+    from app.web.security import get_user_state
     app.dependency_overrides[get_user_state] = lambda: m_user_state
     
     # inactive user
@@ -103,7 +103,7 @@ async def test_prometheus_metrics(test_data, client_with_token, get_settings):
     assert 'disk_utilization{type="used"}' not in r.text
 
     # after metrics calculation
-    from web.utils.metrics import measure_regular_metrics
+    from app.web.utils.metrics import measure_regular_metrics
     await measure_regular_metrics(get_settings.DATABASE_PATH, 60 * 60 * 24 * 31 * 12 * 100)
     r2 = client_with_token.get("/metrics")
     assert 'disk_utilization{type="used"}' in r2.text
@@ -117,7 +117,7 @@ async def test_prometheus_metrics(test_data, client_with_token, get_settings):
     assert 'database_metrics_counter_total{query="count_by_user",user="jerry@example.com"} 33.0' in r2.text
 
     # 30s window, should not change the gauges nor the total in the counters
-    from web.utils.metrics import measure_regular_metrics
+    from app.web.utils.metrics import measure_regular_metrics
     await measure_regular_metrics(get_settings.DATABASE_PATH, 30)
     r3 = client_with_token.get("/metrics")
     assert 'database_metrics{query="count_archives"} 100.0' in r3.text

app/tests/web/endpoints/test_interoperability.py

@@ -1,9 +1,9 @@
 from datetime import datetime
 import json
-from unittest.mock import patch
+from unittest.mock import MagicMock, patch
 
 from app.shared.config import ALLOW_ANY_EMAIL
-from app.shared.db import crud
+from app.web.db import crud
 
 
 def test_submit_manual_archive_unauthenticated(client, test_no_auth):
@@ -14,11 +14,11 @@ def test_submit_manual_archive_not_user_auth(client_with_auth, test_no_auth):
     test_no_auth(client_with_auth.post, "/interop/submit-archive")
 
 
-@patch("endpoints.interoperability.get_store_until", return_value=datetime.now())
+@patch("app.web.endpoints.interoperability.business_logic", return_value=MagicMock(get_store_archive_until=MagicMock(return_value=datetime)))
 def test_submit_manual_archive(m1, client_with_token, db_session):
     # normal workflow
     aa_metadata = json.dumps({"status": "test: success", "metadata": {"url": "http://example.com"}, "media": [{"filename": "fn1", "urls": ["http://example.s3.com"]}]})
-    r = client_with_token.post("/interop/submit-archive", json={"result": aa_metadata, "public": True, "author_id": "jerry@gmail.com", "group_id": "spaceship", "tags": ["test"]})
+    r = client_with_token.post("/interop/submit-archive", json={"result": aa_metadata, "public": True, "author_id": "jerry@gmail.com", "group_id": "spaceship", "tags": ["test"], "url": "http://example.com"})
     assert r.status_code == 201
     assert "id" in r.json()
 
@@ -35,6 +35,6 @@ def test_submit_manual_archive(m1, client_with_token, db_session):
 
     # cannot have the same URL twice 
     aa_metadata = json.dumps({"status": "test: success", "metadata": {"url": "http://example.com"}, "media": [{"filename": "fn1", "urls": ["http://example.com", "http://example.com"]}]})
-    r = client_with_token.post("/interop/submit-archive", json={"result": aa_metadata, "public": False, "author_id": "jerry@gmail.com", "tags": ["test"]})
+    r = client_with_token.post("/interop/submit-archive", json={"result": aa_metadata, "public": False, "author_id": "jerry@gmail.com", "tags": ["test"], "url": "http://example.com"})
     assert r.status_code == 422
     assert r.json() == {"detail": "Cannot insert into DB due to integrity error, likely duplicate urls."}

app/tests/web/endpoints/test_sheet.py

@@ -45,7 +45,7 @@ def test_create_sheet_endpoint(app_with_auth, db_session):
     assert response.json() == {"detail": "User does not have access to this group."}
 
     # switch to jerry who's got less quota/permissions
-    from web.security import get_user_state
+    from app.web.security import get_user_state
     from app.web.db.user_state import UserState
     app_with_auth.dependency_overrides[get_user_state] = lambda: UserState(db_session, "jerry@example.com")
     client_jerry = TestClient(app_with_auth)
@@ -144,7 +144,7 @@ def test_delete_sheet_endpoint(client_with_auth, db_session):
 
 
 class TestArchiveUserSheetEndpoint:
-    @patch("endpoints.sheet.celery", return_value=MagicMock())
+    @patch("app.web.endpoints.sheet.celery", return_value=MagicMock())
     def test_normal_flow(self, m_celery, client_with_auth, db_session):
         from app.shared.db import models
         db_session.add(models.Sheet(id="123-sheet-id", name="Test Sheet 1", author_id="morty@example.com", group_id="spaceship", frequency="hourly"))

app/tests/web/endpoints/test_task.py

@@ -5,7 +5,7 @@ def test_endpoint_task_status_no_auth(client, test_no_auth):
     test_no_auth(client.get, "/task/test-task-id")
 
 
-@patch("endpoints.task.AsyncResult")
+@patch("app.web.endpoints.task.AsyncResult")
 def test_get_status_success(mock_async_result, client_with_auth):
     mock_async_result.return_value.status = "SUCCESS"
     mock_async_result.return_value.result = {"data": "some result"}
@@ -20,7 +20,7 @@ def test_get_status_success(mock_async_result, client_with_auth):
     }
 
 
-@patch("endpoints.task.AsyncResult")
+@patch("app.web.endpoints.task.AsyncResult")
 def test_get_status_failure(mock_async_result, client_with_auth):
 
     mock_async_result.return_value.status = "FAILURE"
@@ -36,7 +36,7 @@ def test_get_status_failure(mock_async_result, client_with_auth):
     }
 
 
-@patch("endpoints.task.AsyncResult")
+@patch("app.web.endpoints.task.AsyncResult")
 def test_get_status_pending(mock_async_result, client_with_auth):
     mock_async_result.return_value.status = "PENDING"
     mock_async_result.return_value.result = None

app/tests/web/endpoints/test_url.py

@@ -8,8 +8,8 @@ def test_archive_url_unauthenticated(client, test_no_auth):
     test_no_auth(client.post, "/url/archive")
 
 
-@patch("endpoints.url.UserState")
-@patch("endpoints.url.celery", return_value=MagicMock())
+@patch("app.web.endpoints.url.UserState")
+@patch("app.web.endpoints.url.celery", return_value=MagicMock())
 def test_archive_url(m_celery, m2, client_with_auth):
     m_signature = MagicMock()
     m_signature.delay.return_value = TaskResult(id="123-456-789", status="PENDING", result="")
@@ -81,7 +81,7 @@ def test_archive_url(m_celery, m2, client_with_auth):
     assert m_signature.delay.call_count == 2
 
 
-@patch("endpoints.url.UserState")
+@patch("app.web.endpoints.url.UserState")
 def test_archive_url_quotas(m1, client_with_auth):
     m_user_state = MagicMock()
     m1.return_value = m_user_state
@@ -102,7 +102,7 @@ def test_archive_url_quotas(m1, client_with_auth):
     m_user_state.has_quota_max_monthly_mbs.assert_called_once()
 
 
-@patch("endpoints.url.celery", return_value=MagicMock())
+@patch("app.web.endpoints.url.celery", return_value=MagicMock())
 def test_archive_url_with_api_token(m_celery, client_with_token):
     m_signature = MagicMock()
     m_signature.delay.return_value = TaskResult(id="123-456-789", status="PENDING", result="")
@@ -130,9 +130,11 @@ def test_search_by_url(client_with_auth, client_with_token, db_session):
     assert response.status_code == 200
     assert response.json() == []
 
-    from app.shared.db import crud, schemas
+    from app.shared import schemas
+    from app.shared.db import worker_crud
     for i in range(11):
-        crud.create_task(db_session, ArchiveCreate(id=f"url-456-{i}", url="https://example.com" if i < 10 else "https://something-else.com", result={}, public=True, author_id="rick@example.com"), [], [])
+        #TODO: fix as this method is gone to shared.db
+        worker_crud.create_task(db_session, ArchiveCreate(id=f"url-456-{i}", url="https://example.com" if i < 10 else "https://something-else.com", result={}, public=True, author_id="rick@example.com"), [], [])
         # NB: this insertion is too fast for the ordering to be correct as they are within the same second
 
     response = client_with_auth.get("/url/search?url=https://example.com")
@@ -165,7 +167,7 @@ def test_search_by_url(client_with_auth, client_with_token, db_session):
     assert len(response.json()) == 10
 
 
-@patch("endpoints.url.UserState")
+@patch("app.web.endpoints.url.UserState")
 def test_search_no_read_access(mock_user_state, client_with_auth):
     mock_user_state.return_value.read = False
     mock_user_state.return_value.read_public = False
@@ -184,8 +186,8 @@ def test_delete_task(client_with_auth, db_session):
     assert response.status_code == 200
     assert response.json() == {"id": "delete-123-456-789", "deleted": False}
 
-    from app.shared.db import crud
-    crud.create_task(db_session, ArchiveCreate(id="delete-123-456-789", url="https://example.com", result={}, public=True, author_id="morty@example.com"), [], [])
+    from app.shared.db import worker_crud
+    worker_crud.create_task(db_session, ArchiveCreate(id="delete-123-456-789", url="https://example.com", result={}, public=True, author_id="morty@example.com"), [], [])
 
     response = client_with_auth.delete("/url/delete-123-456-789")
     assert response.status_code == 200

app/tests/web/test_main.py

@@ -17,9 +17,9 @@ def test_alembic(db_session):
     alembic.config.main(argv=['--raiseerr', 'upgrade', 'head'])
     alembic.config.main(argv=['--raiseerr', 'downgrade', 'base'])
 
-@patch("endpoints.default.crud.soft_delete_task", side_effect=Exception('mocked error'))
+@patch("app.web.endpoints.default.crud.soft_delete_task", side_effect=Exception('mocked error'))
 def test_logging_middleware(m1, client_with_auth):
-    from web.utils.metrics import EXCEPTION_COUNTER
+    from app.web.utils.metrics import EXCEPTION_COUNTER
     assert len(EXCEPTION_COUNTER.collect()[0].samples) == 0
     with pytest.raises(Exception, match="mocked error"):
         client_with_auth.delete("/url/123")
@@ -36,7 +36,7 @@ def test_serve_local_archive_logic(get_settings):
     try:
         # modify the settings
         get_settings.SERVE_LOCAL_ARCHIVE = "/app/local_archive_test"
-        from web.main import app_factory
+        from app.web.main import app_factory
         app = app_factory(get_settings)
         
         # test

app/tests/web/test_security.py

@@ -8,7 +8,7 @@ from app.shared.config import ALLOW_ANY_EMAIL
 
 
 def test_secure_compare():
-    from web.security import secure_compare
+    from app.web.security import secure_compare
 
     assert secure_compare("test", "test")
     assert not secure_compare("test", "test2")
@@ -16,14 +16,14 @@ def test_secure_compare():
 
 @pytest.mark.asyncio
 async def test_get_token_or_user_auth_with_api():
-    from web.security import get_token_or_user_auth
+    from app.web.security import get_token_or_user_auth
     mock_api = HTTPAuthorizationCredentials(scheme="lorem", credentials="this_is_the_test_api_token")
     assert await get_token_or_user_auth(mock_api) == ALLOW_ANY_EMAIL
 
 
 @pytest.mark.asyncio
 async def test_get_token_or_user_auth_with_user():
-    from web.security import get_token_or_user_auth
+    from app.web.security import get_token_or_user_auth
     bad_user = HTTPAuthorizationCredentials(scheme="ipsum", credentials="invalid")
     e: pytest.ExceptionInfo = None
     with pytest.raises(HTTPException) as e:
@@ -32,18 +32,18 @@ async def test_get_token_or_user_auth_with_user():
     assert e.value.detail == "invalid access_token"
 
 
-@patch("web.security.authenticate_user", return_value=(True, "summer@example.com"))
+@patch("app.web.security.authenticate_user", return_value=(True, "summer@example.com"))
 @pytest.mark.asyncio
 async def test_get_user_auth(m1):
-    from web.security import get_user_auth
+    from app.web.security import get_user_auth
     good_user = HTTPAuthorizationCredentials(scheme="ipsum", credentials="valid-and-good")
     assert await get_user_auth(good_user) == "summer@example.com"
 
 
-@patch("web.security.secure_compare", return_value=False)
+@patch("app.web.security.secure_compare", return_value=False)
 @pytest.mark.asyncio
 async def test_token_api_key_auth_exception(m1):
-    from web.security import token_api_key_auth
+    from app.web.security import token_api_key_auth
 
     e: pytest.ExceptionInfo = None
     with pytest.raises(HTTPException) as e:
@@ -54,12 +54,12 @@ async def test_token_api_key_auth_exception(m1):
 
 @pytest.mark.asyncio
 async def test_authenticate_user():
-    from web.security import authenticate_user
+    from app.web.security import authenticate_user
 
     assert authenticate_user("test") == (False, "invalid access_token")
     assert authenticate_user(123) == (False, "invalid access_token")
 
-    with patch("web.security.requests.get") as mock_get:
+    with patch("app.web.security.requests.get") as mock_get:
         # bad response from oauth2
         mock_get.return_value.status_code = 403
         assert authenticate_user("this-will-call-requests") == (False, "invalid token")
@@ -100,9 +100,9 @@ async def test_authenticate_user():
 
 @pytest.mark.asyncio
 async def test_authenticate_user_exception():
-    from web.security import authenticate_user
+    from app.web.security import authenticate_user
 
-    with patch("web.security.requests.get") as mock_get:
+    with patch("app.web.security.requests.get") as mock_get:
         mock_get.return_value.status_code = 200
         mock_get.return_value.json.side_effect = Exception("mocked error")
         assert authenticate_user("this-will-call-requests") == (False, "exception occurred")

app/tests/worker/test_worker_main.py

@@ -16,12 +16,12 @@ class Test_create_archive_task():
     URL = "https://example-live.com"
     archive = schemas.ArchiveCreate(url=URL, tags=["tag-celery"], public=True, author_id="rick@example.com", group_id="interstellar")
 
-    @patch("worker.main.insert_result_into_db")
-    @patch("worker.main.get_store_until", return_value=datetime.now())
-    @patch("worker.main.load_orchestrator")
+    @patch("app.worker.main.insert_result_into_db")
+    @patch("app.worker.main.get_store_until", return_value=datetime.now())
+    @patch("app.worker.main.load_orchestrator")
     @patch("celery.app.task.Task.request")
     def test_success(self, m_req, m_load, m_store, m_insert, db_session):
-        from worker.main import create_archive_task
+        from app.worker.main import create_archive_task
 
         m_req.id = "this-just-in"
         mock_orchestrator = self.mock_orchestrator_choice(m_load)
@@ -38,14 +38,14 @@ class Test_create_archive_task():
         assert len(task["media"]) == 0
 
     def test_raise_invalid(self):
-        from worker.main import create_archive_task
+        from app.worker.main import create_archive_task
         with pytest.raises(Exception):
             create_archive_task(self.archive.model_dump_json())
 
-    @patch("worker.main.insert_result_into_db", side_effect=Exception)
-    @patch("worker.main.load_orchestrator")
+    @patch("app.worker.main.insert_result_into_db", side_effect=Exception)
+    @patch("app.worker.main.load_orchestrator")
     def test_raise_db_error(self, m_load, m_insert):
-        from worker.main import create_archive_task
+        from app.worker.main import create_archive_task
         mock_orchestrator = self.mock_orchestrator_choice(m_load)
 
         with pytest.raises(Exception):
@@ -53,10 +53,10 @@ class Test_create_archive_task():
         mock_orchestrator.feed_item.assert_called_once()
 
 
-    @patch("worker.main.insert_result_into_db", return_value=None)
-    @patch("worker.main.load_orchestrator")
+    @patch("app.worker.main.insert_result_into_db", return_value=None)
+    @patch("app.worker.main.load_orchestrator")
     def test_raise_empty_result(self, m_load, m_insert):
-        from worker.main import create_archive_task
+        from app.worker.main import create_archive_task
         mock_orchestrator = self.mock_orchestrator_choice(m_load)
 
         with pytest.raises(Exception) as e:
@@ -75,11 +75,11 @@ class Test_create_sheet_task():
     URL = "https://example-live.com"
     sheet = schemas.SubmitSheet(sheet_id="123", author_id="rick@example.com", group_id="interstellar", tags=["spaceship"])
 
-    @patch("worker.main.models.generate_uuid", return_value="constant-uuid")
-    @patch("worker.main.get_store_until", return_value=datetime.now())
-    @patch("worker.main.load_orchestrator")
+    @patch("app.worker.main.models.generate_uuid", return_value="constant-uuid")
+    @patch("app.worker.main.get_store_until", return_value=datetime.now())
+    @patch("app.worker.main.load_orchestrator")
     def test_success(self, m_load, m_store, m_uuid, db_session):
-        from worker.main import create_sheet_task
+        from app.worker.main import create_sheet_task
 
         assert db_session.query(models.Archive).filter(models.Archive.url == self.URL).count() == 0
 
@@ -116,7 +116,7 @@ class Test_create_sheet_task():
 
 
 def test_get_all_urls(db_session):
-    from worker.main import get_all_urls
+    from app.worker.main import get_all_urls
     from auto_archiver import Metadata
 
     meta = Metadata().set_url("https://example.com")