bellingcat/auto-archiver-api
1
0
Fork 0
mirror of https://github.com/bellingcat/auto-archiver-api.git synced 2026-06-12 05:28:34 +03:00
Code Issues Packages Projects Releases Wiki Activity

WIP decoupling worker/web, cleaning worker code

Browse Source
This commit is contained in:
msramalho
2025-02-08 00:40:35 +00:00
parent 46a5c1a260
commit 9a62f3ff59
19 changed files with 194 additions and 247 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
src/tests/endpoints/test_sheet.py
View File

@@ -1,6 +1,6 @@
from datetime import datetime
import json
from unittest.mock import patch
from unittest.mock import MagicMock, patch
from fastapi.testclient import TestClient
@@ -145,15 +145,21 @@ def test_delete_sheet_endpoint(client_with_auth, db_session):
class TestArchiveUserSheetEndpoint:
@patch("worker.main.create_sheet_task.delay", return_value=TaskResult(id="123-taskid", status="PENDING", result=""))
def test_normal_flow(self, m1, client_with_auth, db_session):
@patch("endpoints.sheet.celery", return_value=MagicMock())
def test_normal_flow(self, m_celery, client_with_auth, db_session):
from db import models
db_session.add(models.Sheet(id="123-sheet-id", name="Test Sheet 1", author_id="morty@example.com", group_id="spaceship", frequency="hourly"))
db_session.commit()
m_signature = MagicMock()
m_signature.delay.return_value = TaskResult(id="123-taskid", status="PENDING", result="")
m_celery.signature.return_value = m_signature
r = client_with_auth.post("/sheet/123-sheet-id/archive")
assert r.status_code == 201
assert r.json() == {"id": "123-taskid"}
m1.assert_called_once()
m_celery.signature.assert_called_once()
m_signature.delay.assert_called_once()
def test_token_auth(self, client_with_token, test_no_auth):
test_no_auth(client_with_token.post, "/sheet/123-sheet-id/archive")
@@ -198,23 +204,30 @@ class TestTokenArchiveEndpoint:
assert r.status_code == 422
assert r.json() == {"detail": "sheet id is required"}
@patch("worker.main.create_sheet_task.delay", return_value=TaskResult(id="123-456-789", status="PENDING", result=""))
def test_normal_flow(self, m1, client_with_token):
@patch("endpoints.sheet.celery", return_value=MagicMock())
def test_normal_flow(self, m_celery, client_with_token):
m_signature = MagicMock()
m_signature.delay.return_value = TaskResult(id="123-456-789", status="PENDING", result="")
m_celery.signature.return_value = m_signature
# minimum data
response = client_with_token.post("/sheet/archive", json={"sheet_id": "123-sheet-id"})
assert response.status_code == 201
assert response.json() == {'id': '123-456-789'}
m1.assert_called_once()
called_val = m1.call_args.args[0]
assert json.loads(called_val) == {"sheet_id": "123-sheet-id", "sheet_name": None, "public": False, "author_id": "api-endpoint", "group_id": None, "tags": [], "columns": {}, "header": 1}
m_celery.signature.assert_called_once()
m_signature.delay.assert_called_once()
called_val = m_celery.signature.call_args
assert called_val[0][0] == "create_sheet_task"
assert json.loads(called_val[1]['args'][0]) == {"sheet_id": "123-sheet-id", "sheet_name": None, "public": False, "author_id": "api-endpoint", "group_id": None, "tags": [], "columns": {}, "header": 1}
# maximum data
response = client_with_token.post("/sheet/archive", json={"sheet_id": "123-sheet-id", "sheet_name": "768-sheet-name", "author_id": "birdman@example.com", "header": 2, "public": True, "group_id": "456-group-id", "tags": ["tag1"], "columns": {"col1": "type1"}})
assert response.status_code == 201
assert response.json() == {'id': '123-456-789'}
m1.call_count == 2
called_val = m1.call_args.args[0]
assert json.loads(called_val) == {"sheet_id": "123-sheet-id", "sheet_name": "768-sheet-name", "public": True, "author_id": "birdman@example.com", "group_id": "456-group-id", "tags": ["tag1"], "columns": {"col1": "type1"}, "header": 2}
m_celery.signature.call_count == 2
m_signature.delay.call_count == 2
called_val = m_celery.signature.call_args
assert called_val[0][0] == "create_sheet_task"
assert json.loads(called_val[1]['args'][0]) == {"sheet_id": "123-sheet-id", "sheet_name": "768-sheet-name", "public": True, "author_id": "birdman@example.com", "group_id": "456-group-id", "tags": ["tag1"], "columns": {"col1": "type1"}, "header": 2}

46
src/tests/endpoints/test_url.py
View File

@@ -3,13 +3,18 @@ from unittest.mock import MagicMock, patch
from db.schemas import ArchiveCreate, TaskResult
def test_archive_url_unauthenticated(client, test_no_auth):
test_no_auth(client.post, "/url/archive")
@patch("endpoints.url.UserState")
@patch("worker.main.create_archive_task.delay", return_value=TaskResult(id="123-456-789", status="PENDING", result=""))
def test_archive_url(m1, m2, client_with_auth):
@patch("endpoints.url.celery", return_value=MagicMock())
def test_archive_url(m_celery, m2, client_with_auth):
m_signature = MagicMock()
m_signature.delay.return_value = TaskResult(id="123-456-789", status="PENDING", result="")
m_celery.signature.return_value = m_signature
m_user_state = MagicMock()
m2.return_value = m_user_state
@@ -17,7 +22,7 @@ def test_archive_url(m1, m2, client_with_auth):
response = client_with_auth.post("/url/archive", json={"url": "bad"})
assert response.status_code == 422
assert response.json()["detail"][0]["msg"] == 'String should have at least 5 characters'
m1.assert_not_called()
m_celery.signature.assert_not_called()
# url is invalid
response = client_with_auth.post("/url/archive", json={"url": "example.com"})
@@ -30,9 +35,11 @@ def test_archive_url(m1, m2, client_with_auth):
response = client_with_auth.post("/url/archive", json={"url": "https://example.com"})
assert response.status_code == 201
assert response.json() == {'id': '123-456-789'}
m1.assert_called_once()
called_val = m1.call_args.args[0]
assert json.loads(called_val) == {"id": None, "url": "https://example.com", "result": None, "public": True, "author_id": "rick@example.com", "group_id": None, "tags": [], "rearchive": True, "sheet_id":None}
m_celery.signature.assert_called_once()
m_signature.delay.assert_called_once()
called_val = m_celery.signature.call_args
assert called_val[0][0] == "create_archive_task"
assert json.loads(called_val[1]['args'][0]) == {"id": None, "url": "https://example.com", "result": None, "public": True, "author_id": "rick@example.com", "group_id": None, "tags": [], "sheet_id": None}
m_user_state.has_quota_max_monthly_urls.assert_called_once()
m_user_state.has_quota_max_monthly_mbs.assert_called_once()
@@ -48,9 +55,10 @@ def test_archive_url(m1, m2, client_with_auth):
response = client_with_auth.post("/url/archive", json={"url": "https://example.com", "group_id": "spaceship"})
assert response.status_code == 201
assert response.json() == {'id': '123-456-789'}
assert m1.call_count == 2
called_val = m1.call_args.args[0]
assert json.loads(called_val)["group_id"] == "spaceship"
assert m_celery.signature.call_count == 2
assert m_signature.delay.call_count == 2
called_val = m_celery.signature.call_args
assert json.loads(called_val[1]['args'][0])["group_id"] == "spaceship"
m_user_state.in_group.assert_called_with("spaceship")
# user is over monthly URL quota
@@ -68,6 +76,9 @@ def test_archive_url(m1, m2, client_with_auth):
assert response.status_code == 429
assert response.json()["detail"] == "User has reached their monthly MB quota."
m_user_state.has_quota_max_monthly_mbs.assert_called_with("spacesuit")
assert m_celery.signature.call_count == 2
assert m_signature.delay.call_count == 2
@patch("endpoints.url.UserState")
def test_archive_url_quotas(m1, client_with_auth):
@@ -89,15 +100,25 @@ def test_archive_url_quotas(m1, client_with_auth):
assert response.json()["detail"] == "User has reached their monthly MB quota."
m_user_state.has_quota_max_monthly_mbs.assert_called_once()
@patch("worker.main.create_archive_task.delay", return_value=TaskResult(id="123-456-789", status="PENDING", result=""))
def test_archive_url_with_api_token(m1, client_with_token):
@patch("endpoints.url.celery", return_value=MagicMock())
def test_archive_url_with_api_token(m_celery, client_with_token):
m_signature = MagicMock()
m_signature.delay.return_value = TaskResult(id="123-456-789", status="PENDING", result="")
m_celery.signature.return_value = m_signature
response = client_with_token.post("/url/archive", json={"url": "https://example.com"})
assert response.status_code == 201
assert response.json() == {'id': '123-456-789'}
m_celery.signature.assert_called_once()
m_signature.delay.assert_called_once()
called_val = m_celery.signature.call_args
assert called_val[0][0] == "create_archive_task"
def test_search_by_url_unauthenticated(client, test_no_auth):
test_no_auth(client.get, "/url/search")
def test_search_by_url(client_with_auth, client_with_token, db_session):
# tests the search endpoint, including through some db data for the endpoint params
response = client_with_auth.get("/url/search")
@@ -111,7 +132,7 @@ def test_search_by_url(client_with_auth, client_with_token, db_session):
from db import crud, schemas
for i in range(11):
crud.create_task(db_session, ArchiveCreate(id=f"url-456-{i}", url="https://example.com" if i < 10 else "https://something-else.com", result={}, public=True, author_id="rick@example.com", group_id=None), [], [])
#NB: this insertion is too fast for the ordering to be correct as they are within the same second
# NB: this insertion is too fast for the ordering to be correct as they are within the same second
response = client_with_auth.get("/url/search?url=https://example.com")
assert response.status_code == 200
@@ -142,6 +163,7 @@ def test_search_by_url(client_with_auth, client_with_token, db_session):
assert response.status_code == 200
assert len(response.json()) == 10
@patch("endpoints.url.UserState")
def test_search_no_read_access(mock_user_state, client_with_auth):
mock_user_state.return_value.read = False