bellingcat/auto-archiver
1
0
Fork 0
mirror of https://github.com/bellingcat/auto-archiver.git synced 2026-06-08 03:18:28 +03:00
Code Issues Packages Projects Releases Wiki Activity

implementing default metadata omission/user metadata selection

Browse Source
This commit is contained in:
mgaughan
2025-07-23 14:22:11 -04:00
parent 1256fde159
commit 94e0803fb3
3 changed files with 90 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
src/auto_archiver/core/orchestrator.py
View File

@@ -274,6 +274,14 @@ Here's how that would look: \n\nsteps:\n extractors:\n - [your_extractor_name_
default=False,
)
parser.add_argument(
"--metadata",
dest="requested_metadata",
help="An array of specific metadata fields to select from the collected content.",
default=[],
nargs="?",
)
def add_individual_module_args(
self, modules: list[LazyBaseModule] = None, parser: argparse.ArgumentParser = None
) -> None:

38
src/auto_archiver/modules/metadata_enricher/metadata_enricher.py
View File

@@ -11,19 +11,21 @@ class MetadataEnricher(Enricher):
Extracts metadata information from files using exiftool.
"""
def enrich(self, to_enrich: Metadata) -> None:
def enrich(self, to_enrich: Metadata, md_grocery_list=["author", "datetimes", "location"]) -> None:
logger.debug("Extracting EXIF metadata")
for i, m in enumerate(to_enrich.media):
if len(md := self.get_metadata(m.filename)):
to_enrich.media[i].set("metadata", md)
# feature flag has this currently turned on
specified_md = self.select_metadata(md, md_grocery_list)
to_enrich.media[i].set("metadata", specified_md)
# to_enrich.media[i].set("metadata", md)
def get_metadata(self, filename: str) -> dict:
try:
# Run ExifTool command to extract metadata from the file
cmd = ["exiftool", filename]
result = subprocess.run(cmd, capture_output=True, text=True)
# Process the output to extract individual metadata fields
metadata = {}
for line in result.stdout.splitlines():
@@ -35,3 +37,33 @@ class MetadataEnricher(Enricher):
except Exception as e:
logger.error(f"Error occurred: {e}: {traceback.format_exc()}")
return {}
def select_metadata(self, all_md, md_grocery_list):
"""
coordinates the selection of metadata from the general exiftool output to the user-specified grocery list
"""
# defining the batches of metadata that get pulled for special terms
author_key_terms = ["author", "producer", "creator"]
datetime_key_terms = ["date", "time"]
location_key_terms = ["gps", "latitude", "longitude"]
specified_md = {}
for md_key in all_md.keys():
md_key_lower = md_key.lower()
# checking for special baskets within the grocery list of requested metadata
if ("author" in md_grocery_list) and any(
term in md_key_lower and len(all_md[md_key]) for term in author_key_terms
):
specified_md[md_key] = all_md[md_key]
if ("datetime" in md_grocery_list) and any(
term in md_key_lower and len(all_md[md_key]) for term in datetime_key_terms
):
specified_md[md_key] = all_md[md_key]
if ("location" in md_grocery_list) and any(
term in md_key_lower and len(all_md[md_key]) for term in location_key_terms
):
specified_md[md_key] = all_md[md_key]
# if the metadata value is requested directly
if md_key_lower in md_grocery_list or md_key in md_grocery_list and len(all_md[md_key]):
specified_md[md_key] = all_md[md_key]
return specified_md

49
tests/enrichers/test_metadata_enricher.py
View File

@@ -49,13 +49,25 @@ def test_enrich_sets_metadata(enricher, mocker):
metadata.media = [media1, media2]
enricher.get_metadata = lambda f: {"key": "value"} if f == "img1.jpg" else {}
enricher.enrich(metadata)
enricher.enrich(metadata, ["key"])
media1.set.assert_called_once_with("metadata", {"key": "value"})
media2.set.assert_not_called()
assert metadata.media == [media1, media2]
def test_enrich_no_metadata_selection(enricher, mocker):
media1 = mocker.Mock(filename="img1.jpg")
media2 = mocker.Mock(filename="img2.jpg")
metadata = mocker.Mock()
metadata.media = [media1, media2]
enricher.get_metadata = lambda f: {"key": "value"} if f == "img1.jpg" else {}
enricher.enrich(metadata)
media1.set.assert_called_once_with("metadata", {})
media2.set.assert_not_called()
assert metadata.media == [media1, media2]
def test_enrich_empty_media(enricher, mocker):
metadata = mocker.Mock()
metadata.media = []
@@ -71,7 +83,9 @@ def test_get_metadata_error_handling(enricher, mocker):
assert "Error occurred: " in mock_log.call_args[0][0]
def test_metadata_pickle(enricher, unpickle, mocker):
# TODO depends on the expected functionality
"""
def test_default_metadata_pickle(enricher, unpickle, mocker):
mock_run = mocker.patch("subprocess.run")
# Uses pickled values
mock_run.return_value = unpickle("metadata_enricher_exif.pickle")
@@ -79,6 +93,37 @@ def test_metadata_pickle(enricher, unpickle, mocker):
expected = unpickle("metadata_enricher_ytshort_expected.pickle")
enricher.enrich(metadata)
expected_media = expected.media
print(expected_media)
actual_media = metadata.media
assert len(expected_media) == len(actual_media)
assert actual_media[0].properties.get("metadata") == expected_media[0].properties.get("metadata")
"""
def test_metadata_pickle_megapixel(enricher, unpickle, mocker):
mock_run = mocker.patch("subprocess.run")
mock_run.return_value = unpickle("metadata_enricher_exif.pickle")
metadata = unpickle("metadata_enricher_ytshort_input.pickle")
enricher.enrich(metadata, ["megapixels"])
actual_media = metadata.media
assert actual_media[0].properties.get("metadata") == {"Megapixels": "0.922"}
def test_metadata_specify_datetime_and_metapixels(enricher, unpickle, mocker):
mock_run = mocker.patch("subprocess.run")
mock_run.return_value = unpickle("metadata_enricher_exif.pickle")
metadata = unpickle("metadata_enricher_ytshort_input.pickle")
# expected_md = {"Metapixels":"0.922", "File Inode Change Date/Time":"2025:02:18 19:42:50+00:00"}
enricher.enrich(metadata, ["datetime", "megapixels"])
actual_media = metadata.media
assert actual_media[0].properties.get("metadata") == {
"File Modification Date/Time": "2025:02:18 19:42:50+00:00",
"File Access Date/Time": "2025:02:18 19:42:50+00:00",
"File Inode Change Date/Time": "2025:02:18 19:42:50+00:00",
"Megapixels": "0.922",
}