diff --git a/src/auto_archiver/core/module.py b/src/auto_archiver/core/module.py index 0888378..cb380cf 100644 --- a/src/auto_archiver/core/module.py +++ b/src/auto_archiver/core/module.py @@ -143,6 +143,7 @@ def available_modules(with_manifest: bool=False, limit_to_modules: List[str]= [] if _LAZY_LOADED_MODULES.get(possible_module): continue lazy_module = LazyBaseModule(possible_module, possible_module_path) + _LAZY_LOADED_MODULES[possible_module] = lazy_module all_modules.append(lazy_module) @@ -229,6 +230,9 @@ class LazyBaseModule: # check external dependencies are installed def check_deps(deps, check): for dep in deps: + if not len(dep): + # clear out any empty strings that a user may have erroneously added + continue if not check(dep): logger.error(f"Module '{self.name}' requires external dependency '{dep}' which is not available/setup. Have you installed the required dependencies for the '{self.name}' module? See the README for more information.") exit(1) diff --git a/src/auto_archiver/modules/api_db/__manifest__.py b/src/auto_archiver/modules/api_db/__manifest__.py index c89165f..d22fa59 100644 --- a/src/auto_archiver/modules/api_db/__manifest__.py +++ b/src/auto_archiver/modules/api_db/__manifest__.py @@ -3,7 +3,7 @@ "type": ["database"], "entry_point": "api_db:AAApiDb", "requires_setup": True, - "external_dependencies": { + "dependencies": { "python": ["requests", "loguru"], }, diff --git a/src/auto_archiver/modules/atlos/__manifest__.py b/src/auto_archiver/modules/atlos/__manifest__.py index 459fefe..7ba2f72 100644 --- a/src/auto_archiver/modules/atlos/__manifest__.py +++ b/src/auto_archiver/modules/atlos/__manifest__.py @@ -2,7 +2,7 @@ "name": "atlos_storage", "type": ["storage"], "requires_setup": True, - "external_dependencies": {"python": ["loguru", "requests"], "bin": [""]}, + "dependencies": {"python": ["loguru", "requests"], "bin": [""]}, "configs": { "path_generator": { "default": "url", diff --git a/src/auto_archiver/modules/atlos_db/__manifest__.py b/src/auto_archiver/modules/atlos_db/__manifest__.py index 42ce560..8f9473f 100644 --- a/src/auto_archiver/modules/atlos_db/__manifest__.py +++ b/src/auto_archiver/modules/atlos_db/__manifest__.py @@ -3,7 +3,7 @@ "type": ["database"], "entry_point": "atlos_db:AtlosDb", "requires_setup": True, - "external_dependencies": + "dependencies": {"python": ["loguru", ""], "bin": [""]}, diff --git a/src/auto_archiver/modules/atlos_feeder/__manifest__.py b/src/auto_archiver/modules/atlos_feeder/__manifest__.py index 0d90c8b..f2772f2 100644 --- a/src/auto_archiver/modules/atlos_feeder/__manifest__.py +++ b/src/auto_archiver/modules/atlos_feeder/__manifest__.py @@ -2,7 +2,7 @@ "name": "Atlos Feeder", "type": ["feeder"], "requires_setup": True, - "external_dependencies": { + "dependencies": { "python": ["loguru", "requests"], }, "configs": { diff --git a/src/auto_archiver/modules/cli_feeder/__manifest__.py b/src/auto_archiver/modules/cli_feeder/__manifest__.py index fe784c3..cf5c1b7 100644 --- a/src/auto_archiver/modules/cli_feeder/__manifest__.py +++ b/src/auto_archiver/modules/cli_feeder/__manifest__.py @@ -2,7 +2,7 @@ "name": "CLI Feeder", "type": ["feeder"], "requires_setup": False, - "external_dependencies": { + "dependencies": { "python": ["loguru"], }, 'entry_point': 'cli_feeder::CLIFeeder', diff --git a/src/auto_archiver/modules/console_db/__manifest__.py b/src/auto_archiver/modules/console_db/__manifest__.py index cd40496..a1d0d48 100644 --- a/src/auto_archiver/modules/console_db/__manifest__.py +++ b/src/auto_archiver/modules/console_db/__manifest__.py @@ -2,7 +2,7 @@ "name": "Console Database", "type": ["database"], "requires_setup": False, - "external_dependencies": { + "dependencies": { "python": ["loguru"], }, "description": """ diff --git a/src/auto_archiver/modules/csv_db/__manifest__.py b/src/auto_archiver/modules/csv_db/__manifest__.py index 3131188..507ce14 100644 --- a/src/auto_archiver/modules/csv_db/__manifest__.py +++ b/src/auto_archiver/modules/csv_db/__manifest__.py @@ -2,7 +2,7 @@ "name": "CSV Database", "type": ["database"], "requires_setup": False, - "external_dependencies": {"python": ["loguru"] + "dependencies": {"python": ["loguru"] }, 'entry_point': 'csv_db::CSVDb', "configs": { diff --git a/src/auto_archiver/modules/csv_feeder/__manifest__.py b/src/auto_archiver/modules/csv_feeder/__manifest__.py index 81c4dcd..b062ee6 100644 --- a/src/auto_archiver/modules/csv_feeder/__manifest__.py +++ b/src/auto_archiver/modules/csv_feeder/__manifest__.py @@ -2,7 +2,7 @@ "name": "CSV Feeder", "type": ["feeder"], "requires_setup": False, - "external_dependencies": { + "dependencies": { "python": ["loguru"], "bin": [""] }, diff --git a/src/auto_archiver/modules/gdrive_storage/__manifest__.py b/src/auto_archiver/modules/gdrive_storage/__manifest__.py index b81b717..e24f21b 100644 --- a/src/auto_archiver/modules/gdrive_storage/__manifest__.py +++ b/src/auto_archiver/modules/gdrive_storage/__manifest__.py @@ -2,7 +2,7 @@ "name": "Google Drive Storage", "type": ["storage"], "requires_setup": True, - "external_dependencies": { + "dependencies": { "python": [ "loguru", "google-api-python-client", diff --git a/src/auto_archiver/modules/gsheet_db/__manifest__.py b/src/auto_archiver/modules/gsheet_db/__manifest__.py index f2f1c35..f926adc 100644 --- a/src/auto_archiver/modules/gsheet_db/__manifest__.py +++ b/src/auto_archiver/modules/gsheet_db/__manifest__.py @@ -3,7 +3,7 @@ "type": ["database"], "entry_point": "gsheet_db::GsheetsDb", "requires_setup": True, - "external_dependencies": { + "dependencies": { "python": ["loguru", "gspread", "python-slugify"], }, "configs": { diff --git a/src/auto_archiver/modules/gsheet_feeder/__manifest__.py b/src/auto_archiver/modules/gsheet_feeder/__manifest__.py index 3d9cb08..1c9acab 100644 --- a/src/auto_archiver/modules/gsheet_feeder/__manifest__.py +++ b/src/auto_archiver/modules/gsheet_feeder/__manifest__.py @@ -3,7 +3,7 @@ "type": ["feeder"], "entry_point": "gsheet_feeder::GsheetsFeeder", "requires_setup": True, - "external_dependencies": { + "dependencies": { "python": ["loguru", "gspread", "python-slugify"], }, "configs": { diff --git a/src/auto_archiver/modules/hash_enricher/__manifest__.py b/src/auto_archiver/modules/hash_enricher/__manifest__.py index f306808..c7a023e 100644 --- a/src/auto_archiver/modules/hash_enricher/__manifest__.py +++ b/src/auto_archiver/modules/hash_enricher/__manifest__.py @@ -2,7 +2,7 @@ "name": "Hash Enricher", "type": ["enricher"], "requires_setup": False, - "external_dependencies": { + "dependencies": { "python": ["loguru"], }, "configs": { diff --git a/src/auto_archiver/modules/html_formatter/__manifest__.py b/src/auto_archiver/modules/html_formatter/__manifest__.py index 259a3d1..ec19cf8 100644 --- a/src/auto_archiver/modules/html_formatter/__manifest__.py +++ b/src/auto_archiver/modules/html_formatter/__manifest__.py @@ -2,8 +2,8 @@ "name": "HTML Formatter", "type": ["formatter"], "requires_setup": False, - "external_dependencies": { - "python": ["loguru", "jinja2"], + "dependencies": { + "python": ["hash_enricher", "loguru", "jinja2"], "bin": [""] }, "configs": { diff --git a/src/auto_archiver/modules/html_formatter/html_formatter.py b/src/auto_archiver/modules/html_formatter/html_formatter.py index 570fc6f..8f006e0 100644 --- a/src/auto_archiver/modules/html_formatter/html_formatter.py +++ b/src/auto_archiver/modules/html_formatter/html_formatter.py @@ -53,6 +53,7 @@ class HtmlFormatter(Formatter): outf.write(content) final_media = Media(filename=html_path, _mimetype="text/html") + # get the already instantiated hash_enricher module he = get_module('hash_enricher', self.config) if len(hd := he.calculate_hash(final_media.filename)): final_media.set("hash", f"{he.algorithm}:{hd}") diff --git a/src/auto_archiver/modules/instagram_api_extractor/__manifest__.py b/src/auto_archiver/modules/instagram_api_extractor/__manifest__.py index cdaf635..57f378e 100644 --- a/src/auto_archiver/modules/instagram_api_extractor/__manifest__.py +++ b/src/auto_archiver/modules/instagram_api_extractor/__manifest__.py @@ -1,7 +1,7 @@ { "name": "Instagram API Extractor", "type": ["extractor"], - "external_dependencies": + "dependencies": {"python": ["requests", "loguru", "retrying", diff --git a/src/auto_archiver/modules/instagram_extractor/__manifest__.py b/src/auto_archiver/modules/instagram_extractor/__manifest__.py index f1857c2..6e7518e 100644 --- a/src/auto_archiver/modules/instagram_extractor/__manifest__.py +++ b/src/auto_archiver/modules/instagram_extractor/__manifest__.py @@ -1,7 +1,7 @@ { "name": "Instagram Extractor", "type": ["extractor"], - "external_dependencies": { + "dependencies": { "python": [ "instaloader", "loguru", diff --git a/src/auto_archiver/modules/instagram_tbot_extractor/__manifest__.py b/src/auto_archiver/modules/instagram_tbot_extractor/__manifest__.py index 95d6808..8a1f74f 100644 --- a/src/auto_archiver/modules/instagram_tbot_extractor/__manifest__.py +++ b/src/auto_archiver/modules/instagram_tbot_extractor/__manifest__.py @@ -1,7 +1,7 @@ { "name": "Instagram Telegram Bot Extractor", "type": ["extractor"], - "external_dependencies": {"python": ["loguru", + "dependencies": {"python": ["loguru", "telethon",], }, "requires_setup": True, diff --git a/src/auto_archiver/modules/local_storage/__manifest__.py b/src/auto_archiver/modules/local_storage/__manifest__.py index ce00953..6d9cf53 100644 --- a/src/auto_archiver/modules/local_storage/__manifest__.py +++ b/src/auto_archiver/modules/local_storage/__manifest__.py @@ -2,7 +2,7 @@ "name": "Local Storage", "type": ["storage"], "requires_setup": False, - "external_dependencies": { + "dependencies": { "python": ["loguru"], }, "configs": { diff --git a/src/auto_archiver/modules/meta_enricher/__manifest__.py b/src/auto_archiver/modules/meta_enricher/__manifest__.py index 10acf71..37c9201 100644 --- a/src/auto_archiver/modules/meta_enricher/__manifest__.py +++ b/src/auto_archiver/modules/meta_enricher/__manifest__.py @@ -2,7 +2,7 @@ "name": "Archive Metadata Enricher", "type": ["enricher"], "requires_setup": False, - "external_dependencies": { + "dependencies": { "python": ["loguru"], }, "description": """ diff --git a/src/auto_archiver/modules/metadata_enricher/__manifest__.py b/src/auto_archiver/modules/metadata_enricher/__manifest__.py index 50064e9..f8ccdc6 100644 --- a/src/auto_archiver/modules/metadata_enricher/__manifest__.py +++ b/src/auto_archiver/modules/metadata_enricher/__manifest__.py @@ -2,7 +2,7 @@ "name": "Media Metadata Enricher", "type": ["enricher"], "requires_setup": True, - "external_dependencies": { + "dependencies": { "python": ["loguru"], "bin": ["exiftool"] }, diff --git a/src/auto_archiver/modules/mute_formatter/__manifest__.py b/src/auto_archiver/modules/mute_formatter/__manifest__.py index 77f2784..e81dc4c 100644 --- a/src/auto_archiver/modules/mute_formatter/__manifest__.py +++ b/src/auto_archiver/modules/mute_formatter/__manifest__.py @@ -2,7 +2,7 @@ "name": "Mute Formatter", "type": ["formatter"], "requires_setup": True, - "external_dependencies": { + "dependencies": { }, "description": """ Default formatter. """, diff --git a/src/auto_archiver/modules/pdq_hash_enricher/__manifest__.py b/src/auto_archiver/modules/pdq_hash_enricher/__manifest__.py index 7b418b1..6353d12 100644 --- a/src/auto_archiver/modules/pdq_hash_enricher/__manifest__.py +++ b/src/auto_archiver/modules/pdq_hash_enricher/__manifest__.py @@ -2,7 +2,7 @@ "name": "PDQ Hash Enricher", "type": ["enricher"], "requires_setup": False, - "external_dependencies": { + "dependencies": { "python": ["loguru", "pdqhash", "numpy", "Pillow"], }, "description": """ diff --git a/src/auto_archiver/modules/s3_storage/__manifest__.py b/src/auto_archiver/modules/s3_storage/__manifest__.py index 811c703..16ac7bd 100644 --- a/src/auto_archiver/modules/s3_storage/__manifest__.py +++ b/src/auto_archiver/modules/s3_storage/__manifest__.py @@ -2,7 +2,7 @@ "name": "S3 Storage", "type": ["storage"], "requires_setup": True, - "external_dependencies": { + "dependencies": { "python": ["boto3", "loguru"], }, "configs": { diff --git a/src/auto_archiver/modules/screenshot_enricher/__manifest__.py b/src/auto_archiver/modules/screenshot_enricher/__manifest__.py index c1a30e7..52842c9 100644 --- a/src/auto_archiver/modules/screenshot_enricher/__manifest__.py +++ b/src/auto_archiver/modules/screenshot_enricher/__manifest__.py @@ -2,7 +2,7 @@ "name": "Screenshot Enricher", "type": ["enricher"], "requires_setup": True, - "external_dependencies": { + "dependencies": { "python": ["loguru", "selenium"], "bin": ["chromedriver"] }, diff --git a/src/auto_archiver/modules/ssl_enricher/__manifest__.py b/src/auto_archiver/modules/ssl_enricher/__manifest__.py index ccde957..0fb7cd9 100644 --- a/src/auto_archiver/modules/ssl_enricher/__manifest__.py +++ b/src/auto_archiver/modules/ssl_enricher/__manifest__.py @@ -2,7 +2,7 @@ "name": "SSL Certificate Enricher", "type": ["enricher"], "requires_setup": False, - "external_dependencies": { + "dependencies": { "python": ["loguru", "python-slugify"], }, 'entry_point': 'ssl_enricher::SSLEnricher', diff --git a/src/auto_archiver/modules/telegram_extractor/__manifest__.py b/src/auto_archiver/modules/telegram_extractor/__manifest__.py index 86b5e0f..e1c49c2 100644 --- a/src/auto_archiver/modules/telegram_extractor/__manifest__.py +++ b/src/auto_archiver/modules/telegram_extractor/__manifest__.py @@ -2,7 +2,7 @@ "name": "Telegram Extractor", "type": ["extractor"], "requires_setup": False, - "external_dependencies": { + "dependencies": { "python": [ "requests", "bs4", diff --git a/src/auto_archiver/modules/telethon_extractor/__manifest__.py b/src/auto_archiver/modules/telethon_extractor/__manifest__.py index 2cf1e42..6b37654 100644 --- a/src/auto_archiver/modules/telethon_extractor/__manifest__.py +++ b/src/auto_archiver/modules/telethon_extractor/__manifest__.py @@ -2,7 +2,7 @@ "name": "telethon_extractor", "type": ["extractor"], "requires_setup": True, - "external_dependencies": { + "dependencies": { "python": ["telethon", "loguru", "tqdm", diff --git a/src/auto_archiver/modules/thumbnail_enricher/__manifest__.py b/src/auto_archiver/modules/thumbnail_enricher/__manifest__.py index 2b0f167..bd7836d 100644 --- a/src/auto_archiver/modules/thumbnail_enricher/__manifest__.py +++ b/src/auto_archiver/modules/thumbnail_enricher/__manifest__.py @@ -2,7 +2,7 @@ "name": "Thumbnail Enricher", "type": ["enricher"], "requires_setup": False, - "external_dependencies": { + "dependencies": { "python": ["loguru", "ffmpeg-python"], "bin": ["ffmpeg"] }, diff --git a/src/auto_archiver/modules/timestamping_enricher/__manifest__.py b/src/auto_archiver/modules/timestamping_enricher/__manifest__.py index 496d211..6ad9c57 100644 --- a/src/auto_archiver/modules/timestamping_enricher/__manifest__.py +++ b/src/auto_archiver/modules/timestamping_enricher/__manifest__.py @@ -2,7 +2,7 @@ "name": "Timestamping Enricher", "type": ["enricher"], "requires_setup": True, - "external_dependencies": { + "dependencies": { "python": [ "loguru", "slugify", diff --git a/src/auto_archiver/modules/twitter_api_extractor/__manifest__.py b/src/auto_archiver/modules/twitter_api_extractor/__manifest__.py index 02d0d6c..05d1ac0 100644 --- a/src/auto_archiver/modules/twitter_api_extractor/__manifest__.py +++ b/src/auto_archiver/modules/twitter_api_extractor/__manifest__.py @@ -2,7 +2,7 @@ "name": "Twitter API Extractor", "type": ["extractor"], "requires_setup": True, - "external_dependencies": { + "dependencies": { "python": ["requests", "loguru", "pytwitter", diff --git a/src/auto_archiver/modules/vk_extractor/__manifest__.py b/src/auto_archiver/modules/vk_extractor/__manifest__.py index bdcaf99..116b430 100644 --- a/src/auto_archiver/modules/vk_extractor/__manifest__.py +++ b/src/auto_archiver/modules/vk_extractor/__manifest__.py @@ -3,7 +3,7 @@ "type": ["extractor"], "requires_setup": True, "depends": ["core", "utils"], - "external_dependencies": { + "dependencies": { "python": ["loguru", "vk_url_scraper"], }, diff --git a/src/auto_archiver/modules/wacz_enricher/__manifest__.py b/src/auto_archiver/modules/wacz_enricher/__manifest__.py index 07983d9..bb9d290 100644 --- a/src/auto_archiver/modules/wacz_enricher/__manifest__.py +++ b/src/auto_archiver/modules/wacz_enricher/__manifest__.py @@ -2,7 +2,7 @@ "name": "WACZ Enricher", "type": ["enricher", "archiver"], "requires_setup": True, - "external_dependencies": { + "dependencies": { "python": [ "loguru", "jsonlines", diff --git a/src/auto_archiver/modules/wayback_enricher/__manifest__.py b/src/auto_archiver/modules/wayback_enricher/__manifest__.py index bff10af..5d1fe25 100644 --- a/src/auto_archiver/modules/wayback_enricher/__manifest__.py +++ b/src/auto_archiver/modules/wayback_enricher/__manifest__.py @@ -2,7 +2,7 @@ "name": "Wayback Machine Enricher", "type": ["enricher", "archiver"], "requires_setup": True, - "external_dependencies": { + "dependencies": { "python": ["loguru", "requests"], }, "entry_point": "wayback_enricher::WaybackExtractorEnricher", diff --git a/src/auto_archiver/modules/whisper_enricher/__manifest__.py b/src/auto_archiver/modules/whisper_enricher/__manifest__.py index 25eae25..0adf9ff 100644 --- a/src/auto_archiver/modules/whisper_enricher/__manifest__.py +++ b/src/auto_archiver/modules/whisper_enricher/__manifest__.py @@ -2,7 +2,7 @@ "name": "Whisper Enricher", "type": ["enricher"], "requires_setup": True, - "external_dependencies": { + "dependencies": { "python": ["loguru", "requests"], }, "configs": { diff --git a/tests/data/example_module/__manifest__.py b/tests/data/example_module/__manifest__.py index ca3a678..19a85f9 100644 --- a/tests/data/example_module/__manifest__.py +++ b/tests/data/example_module/__manifest__.py @@ -2,7 +2,7 @@ "name": "Example Module", "type": ["extractor"], "requires_setup": False, - "external_dependencies": {"python": ["loguru"] + "dependencies": {"python": ["loguru"] }, "configs": { "csv_file": {"default": "db.csv", "help": "CSV file name"} diff --git a/tests/data/example_module/example_module.py b/tests/data/example_module/example_module.py index b752743..bce8ba4 100644 --- a/tests/data/example_module/example_module.py +++ b/tests/data/example_module/example_module.py @@ -1,4 +1,4 @@ from auto_archiver.core.extractor import Extractor - class ExampleModule(Extractor): - pass \ No newline at end of file + def download(self, item): + print("do something") \ No newline at end of file diff --git a/tests/test_modules.py b/tests/test_modules.py index 109bc52..decc616 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -1,13 +1,24 @@ import sys import pytest from auto_archiver.core.module import get_module_lazy, BaseModule, LazyBaseModule, _LAZY_LOADED_MODULES -from auto_archiver.core.extractor import Extractor @pytest.fixture def example_module(): - yield get_module_lazy("example_module", ["tests/data/"]) + import auto_archiver + + previous_path = auto_archiver.modules.__path__ + auto_archiver.modules.__path__.append("tests/data/") + + module = get_module_lazy("example_module") + yield module # cleanup - _LAZY_LOADED_MODULES.pop("example_module") + try: + del module._manifest + except AttributeError: + pass + del _LAZY_LOADED_MODULES["example_module"] + sys.modules.pop("auto_archiver.modules.example_module.example_module", None) + auto_archiver.modules.__path__ = previous_path def test_get_module_lazy(example_module): assert example_module.name == "example_module" @@ -15,18 +26,34 @@ def test_get_module_lazy(example_module): assert example_module.manifest is not None +def test_python_dependency_check(example_module): + # example_module requires loguru, which is not installed + # monkey patch the manifest to include a nonexistnet dependency + example_module.manifest["dependencies"]["python"] = ["does_not_exist"] -def test_load_module_abc_check(example_module): - - # example_module is an extractor but doesn't have the 'download' method, should raise an ABC error - with pytest.raises(TypeError) as load_error: + with pytest.raises(SystemExit) as load_error: example_module.load({}) - assert "Can't instantiate abstract class ExampleModule with abstract method download" in str(load_error.value) - -def test_load_module(example_module, monkeypatch): - # hack - remove the 'download' method from the required methods of Extractor - monkeypatch.setattr(Extractor, "__abstractmethods__", set()) + assert load_error.value.code == 1 + +def test_binary_dependency_check(example_module): + # example_module requires ffmpeg, which is not installed + # monkey patch the manifest to include a nonexistnet dependency + example_module.manifest["dependencies"]["binary"] = ["does_not_exist"] + +def test_module_dependency_check_loads_module(example_module): + # example_module requires cli_feeder, which is not installed + # monkey patch the manifest to include a nonexistnet dependency + example_module.manifest["dependencies"]["python"] = ["hash_enricher"] + + loaded_module = example_module.load({}) + assert loaded_module is not None + + # check the dependency is loaded + assert _LAZY_LOADED_MODULES["hash_enricher"] is not None + assert _LAZY_LOADED_MODULES["hash_enricher"]._instance is not None + +def test_load_module(example_module): # setup the module, and check that config is set to the default values loaded_module = example_module.load({})