Fix running 'cleanup' method on extractors that fail to start

tests/data/test_modules/example_extractor/example_extractor.py

@@ -1,6 +1,11 @@
 from auto_archiver.core import Extractor
 
+from loguru import logger
+
 
 class ExampleExtractor(Extractor):
     def download(self, item):
-        print("download")
+        logger.info("download")
+
+    def cleanup(self):
+        logger.info("cleanup")

tests/data/test_modules/example_module/example_module.py

@@ -1,27 +1,29 @@
 from auto_archiver.core import Extractor, Enricher, Feeder, Database, Storage, Formatter, Metadata
 
+from loguru import logger
+
 
 class ExampleModule(Extractor, Enricher, Feeder, Database, Storage, Formatter):
     def download(self, item):
-        print("download")
+        logger.info("download")
 
     def __iter__(self):
         yield Metadata().set_url("https://example.com")
 
     def done(self, result):
-        print("done")
+        logger.info("done")
 
     def enrich(self, to_enrich):
-        print("enrich")
+        logger.info("enrich")
 
     def get_cdn_url(self, media):
         return "nice_url"
 
     def save(self, item):
-        print("save")
+        logger.info("save")
 
     def uploadf(self, file, key, **kwargs):
-        print("uploadf")
+        logger.info("uploadf")
 
     def format(self, item):
-        print("format")
+        logger.info("format")

tests/test_orchestrator.py

@@ -237,3 +237,23 @@ def test_wrong_step_type(test_args, caplog):
     with pytest.raises(SetupError) as err:
         orchestrator.setup(args)
         assert "Module 'example_extractor' is not a feeder" in str(err.value)
+
+
+def test_load_failed_extractor_cleanup(test_args, mocker, caplog):
+    orchestrator = ArchivingOrchestrator()
+
+    # hack to set up the paths so we can patch properly
+    orchestrator.module_factory.setup_paths([TEST_MODULES])
+
+    # patch example_module.setup to throw an exception
+    mocker.patch(
+        "auto_archiver.modules.example_extractor.example_extractor.ExampleExtractor.setup",
+        side_effect=Exception("Test exception"),
+    )
+
+    with pytest.raises(Exception):
+        orchestrator.setup(test_args + ["--extractors", "example_extractor"])
+
+    assert "Error during setup of modules: Test exception" in caplog.text
+    # make sure the 'cleanup' is called
+    assert "cleanup" in caplog.text