bellingcat/auto-archiver
1
0
Fork 0
mirror of https://github.com/bellingcat/auto-archiver.git synced 2026-06-12 05:08:28 +03:00
Code Issues Packages Projects Releases Wiki Activity

Ruff format with defaults.

Browse Source
This commit is contained in:
erinhmclark
2025-03-10 18:44:54 +00:00
parent cbb0414e5f
commit 85abe1837a
155 changed files with 2539 additions and 1908 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
tests/databases/test_api_db.py
View File

@@ -41,9 +41,16 @@ def test_fetch(api_db, metadata, mocker):
mock_datetime = mocker.patch("auto_archiver.core.metadata.datetime.datetime")
mock_datetime.now.return_value = "2021-01-01T00:00:00"
mock_get.return_value.status_code = 200
mock_get.return_value.json.return_value = [{"result": {}}, {"result":
{'media': [], 'metadata': {'_processed_at': '2021-01-01T00:00:00', 'url': 'https://example.com'},
'status': 'no archiver'}}]
mock_get.return_value.json.return_value = [
{"result": {}},
{
"result": {
"media": [],
"metadata": {"_processed_at": "2021-01-01T00:00:00", "url": "https://example.com"},
"status": "no archiver",
}
},
]
assert api_db.fetch(metadata) == metadata
@@ -52,8 +59,15 @@ def test_done_success(api_db, metadata, mocker):
mock_post.return_value.status_code = 201
api_db.done(metadata)
mock_post.assert_called_once()
mock_post.assert_called_once_with("https://api.example.com/interop/submit-archive",
json={'author_id': 'Someone', 'url': 'https://example.com',
'public': False, 'group_id': '123', 'tags': ['[', ']'], 'result': '{"status": "no archiver", "metadata": {"_processed_at": "2021-01-01T00:00:00", "url": "https://example.com"}, "media": []}'},
headers={'Authorization': 'Bearer test-token'})
mock_post.assert_called_once_with(
"https://api.example.com/interop/submit-archive",
json={
"author_id": "Someone",
"url": "https://example.com",
"public": False,
"group_id": "123",
"tags": ["[", "]"],
"result": '{"status": "no archiver", "metadata": {"_processed_at": "2021-01-01T00:00:00", "url": "https://example.com"}, "media": []}',
},
headers={"Authorization": "Bearer test-token"},
)

4
tests/databases/test_atlos_db.py
View File

@@ -50,9 +50,7 @@ def test_failed_with_atlos_id(atlos_db, metadata, mocker):
post_mock = mocker.patch.object(atlos_db, "_post", return_value=fake_resp)
atlos_db.failed(metadata, "failure reason")
expected_endpoint = f"/api/v2/source_material/metadata/42/auto_archiver"
expected_json = {
"metadata": {"processed": True, "status": "error", "error": "failure reason"}
}
expected_json = {"metadata": {"processed": True, "status": "error", "error": "failure reason"}}
post_mock.assert_called_once_with(expected_endpoint, json=expected_json)

16
tests/databases/test_csv_db.py
View File

@@ -1,4 +1,3 @@
from auto_archiver.modules.csv_db import CSVDb
from auto_archiver.core import Metadata
@@ -9,12 +8,21 @@ def test_store_item(tmp_path, setup_module):
temp_db = tmp_path / "temp_db.csv"
db = setup_module(CSVDb, {"csv_file": temp_db.as_posix()})
item = Metadata().set_url("http://example.com").set_title("Example").set_content("Example content").success("my-archiver")
item = (
Metadata()
.set_url("http://example.com")
.set_title("Example")
.set_content("Example content")
.success("my-archiver")
)
db.done(item)
with open(temp_db, "r", encoding="utf-8") as f:
assert f.read().strip() == f"status,metadata,media\nmy-archiver: success,\"{{'_processed_at': {repr(item.get('_processed_at'))}, 'url': 'http://example.com', 'title': 'Example', 'content': 'Example content'}}\",[]"
assert (
f.read().strip()
== f"status,metadata,media\nmy-archiver: success,\"{{'_processed_at': {repr(item.get('_processed_at'))}, 'url': 'http://example.com', 'title': 'Example', 'content': 'Example content'}}\",[]"
)
# TODO: csv db doesn't have a fetch method - need to add it (?)
# assert db.fetch(item) == item
# assert db.fetch(item) == item

65
tests/databases/test_gsheet_db.py
View File

@@ -28,6 +28,7 @@ def mock_metadata(mocker):
metadata.get_first_image.return_value = None
return metadata
@pytest.fixture
def metadata():
metadata = Metadata()
@@ -51,6 +52,7 @@ def mock_media(mocker):
mock_media.get.return_value = "not-calculated"
return mock_media
@pytest.fixture
def gsheets_db(mock_gworksheet, setup_module, mocker):
mocker.patch("gspread.service_account")
@@ -59,7 +61,22 @@ def gsheets_db(mock_gworksheet, setup_module, mocker):
"sheet_id": None,
"header": 1,
"service_account": "test/service_account.json",
"columns": {'url': 'link', 'status': 'archive status', 'folder': 'destination folder', 'archive': 'archive location', 'date': 'archive date', 'thumbnail': 'thumbnail', 'timestamp': 'upload timestamp', 'title': 'upload title', 'text': 'text content', 'screenshot': 'screenshot', 'hash': 'hash', 'pdq_hash': 'perceptual hashes', 'wacz': 'wacz', 'replaywebpage': 'replaywebpage'},
"columns": {
"url": "link",
"status": "archive status",
"folder": "destination folder",
"archive": "archive location",
"date": "archive date",
"thumbnail": "thumbnail",
"timestamp": "upload timestamp",
"title": "upload title",
"text": "text content",
"screenshot": "screenshot",
"hash": "hash",
"pdq_hash": "perceptual hashes",
"wacz": "wacz",
"replaywebpage": "replaywebpage",
},
"allow_worksheets": set(),
"block_worksheets": set(),
"use_sheet_names_in_stored_paths": True,
@@ -78,20 +95,21 @@ def fixed_timestamp():
@pytest.fixture
def expected_calls(mock_media, fixed_timestamp):
"""Fixture for the expected cell updates."""
return [
(1, 'status', 'my-archiver: success'),
(1, 'archive', 'http://example.com/screenshot.png'),
(1, 'date', '2025-02-01T00:00:00+00:00'),
(1, 'title', 'Example Title'),
(1, 'text', 'Example Content'),
(1, 'timestamp', '2025-01-01T00:00:00+00:00'),
(1, 'hash', 'not-calculated'),
return [
(1, "status", "my-archiver: success"),
(1, "archive", "http://example.com/screenshot.png"),
(1, "date", "2025-02-01T00:00:00+00:00"),
(1, "title", "Example Title"),
(1, "text", "Example Content"),
(1, "timestamp", "2025-01-01T00:00:00+00:00"),
(1, "hash", "not-calculated"),
# (1, 'screenshot', 'http://example.com/screenshot.png'),
# (1, 'thumbnail', '=IMAGE("http://example.com/thumbnail.png")'),
# (1, 'wacz', 'http://example.com/browsertrix.wacz'),
# (1, 'replaywebpage', 'https://replayweb.page/?source=http%3A%2F%2Fexample.com%2Fbrowsertrix.wacz#view=pages&url=')
]
def test_retrieve_gsheet(gsheets_db, metadata, mock_gworksheet):
gw, row = gsheets_db._retrieve_gsheet(metadata)
assert gw == mock_gworksheet
@@ -100,27 +118,34 @@ def test_retrieve_gsheet(gsheets_db, metadata, mock_gworksheet):
def test_started(gsheets_db, mock_metadata, mock_gworksheet):
gsheets_db.started(mock_metadata)
mock_gworksheet.set_cell.assert_called_once_with(1, 'status', 'Archive in progress')
mock_gworksheet.set_cell.assert_called_once_with(1, "status", "Archive in progress")
def test_failed(gsheets_db, mock_metadata, mock_gworksheet):
reason = "Test failure"
gsheets_db.failed(mock_metadata, reason)
mock_gworksheet.set_cell.assert_called_once_with(1, 'status', f'Archive failed {reason}')
mock_gworksheet.set_cell.assert_called_once_with(1, "status", f"Archive failed {reason}")
def test_aborted(gsheets_db, mock_metadata, mock_gworksheet):
gsheets_db.aborted(mock_metadata)
mock_gworksheet.set_cell.assert_called_once_with(1, 'status', '')
mock_gworksheet.set_cell.assert_called_once_with(1, "status", "")
def test_done(gsheets_db, metadata, mock_gworksheet, expected_calls, mocker):
mocker.patch("auto_archiver.modules.gsheet_feeder_db.gsheet_feeder_db.get_current_timestamp", return_value='2025-02-01T00:00:00+00:00')
mocker.patch(
"auto_archiver.modules.gsheet_feeder_db.gsheet_feeder_db.get_current_timestamp",
return_value="2025-02-01T00:00:00+00:00",
)
gsheets_db.done(metadata)
mock_gworksheet.batch_set_cell.assert_called_once_with(expected_calls)
def test_done_cached(gsheets_db, metadata, mock_gworksheet, mocker):
mocker.patch("auto_archiver.modules.gsheet_feeder_db.gsheet_feeder_db.get_current_timestamp", return_value='2025-02-01T00:00:00+00:00')
mocker.patch(
"auto_archiver.modules.gsheet_feeder_db.gsheet_feeder_db.get_current_timestamp",
return_value="2025-02-01T00:00:00+00:00",
)
gsheets_db.done(metadata, cached=True)
# Verify the status message includes "[cached]"
@@ -131,15 +156,17 @@ def test_done_cached(gsheets_db, metadata, mock_gworksheet, mocker):
def test_done_missing_media(gsheets_db, metadata, mock_gworksheet, mocker):
# clear media from metadata
metadata.media = []
mocker.patch("auto_archiver.modules.gsheet_feeder_db.gsheet_feeder_db.get_current_timestamp", return_value='2025-02-01T00:00:00+00:00')
mocker.patch(
"auto_archiver.modules.gsheet_feeder_db.gsheet_feeder_db.get_current_timestamp",
return_value="2025-02-01T00:00:00+00:00",
)
gsheets_db.done(metadata)
# Verify nothing media-related gets updated
call_args = mock_gworksheet.batch_set_cell.call_args[0][0]
media_fields = {'archive', 'screenshot', 'thumbnail', 'wacz', 'replaywebpage'}
media_fields = {"archive", "screenshot", "thumbnail", "wacz", "replaywebpage"}
assert all(call[1] not in media_fields for call in call_args)
def test_safe_status_update(gsheets_db, metadata, mock_gworksheet):
gsheets_db._safe_status_update(metadata, "Test status")
mock_gworksheet.set_cell.assert_called_once_with(1, 'status', 'Test status')
mock_gworksheet.set_cell.assert_called_once_with(1, "status", "Test status")