Merge branch 'load_modules' into timestamping_rewrite

tests/data/csv_no_headers.csv

@@ -0,0 +1,2 @@
+https://example.com/1/,data 1
+https://example.com/2/,data 2

tests/data/csv_with_headers.csv

@@ -0,0 +1,3 @@
+webpages,other data
+https://example.com/1/,data 1
+https://example.com/2/,data 2

tests/data/test_modules/example_module/__init__.py

@@ -0,0 +1 @@
+from .example_module import ExampleModule

tests/data/test_modules/example_module/__manifest__.py

@@ -0,0 +1,11 @@
+{
+    "name": "Example Module",
+    "type": ["extractor", "feeder", "formatter", "storage", "enricher", "database"],
+    "requires_setup": False,
+    "dependencies": {"python": ["loguru"]
+                              },
+    "configs": {
+            "csv_file": {"default": "db.csv", "help": "CSV file name"},
+            "required_field": {"required": True, "help": "required field in the CSV file"},
+        },
+}

tests/data/test_modules/example_module/example_module.py

@@ -0,0 +1,28 @@
+from auto_archiver.core import Extractor, Enricher, Feeder, Database, Storage, Formatter, Metadata
+
+class ExampleModule(Extractor, Enricher, Feeder, Database, Storage, Formatter):
+    def download(self, item):
+        print("download")
+
+    def __iter__(self):
+        yield Metadata().set_url("https://example.com")
+
+    
+    def done(self, result):
+        print("done")
+
+    def enrich(self, to_enrich):
+        print("enrich")
+
+    def get_cdn_url(self, media):
+        return "nice_url"
+    
+    def save(self, item):
+        print("save")
+    
+    def uploadf(self, file, key, **kwargs):
+        print("uploadf")
+
+    
+    def format(self, item):
+        print("format")

tests/data/test_orchestration.yaml

@@ -0,0 +1,16 @@
+steps:
+  feeders:
+  - example_module
+  extractors:
+  - example_module
+  formatters:
+  - example_module
+  storages:
+  - example_module
+  databases:
+  - example_module
+  enrichers:
+  - example_module
+
+
+# Global configuration