mirror of
https://github.com/bellingcat/auto-archiver.git
synced 2026-06-11 04:38:29 +03:00
Further tidyups + refactoring for new structure
* Add implementation tests for orchestrator + logging tests * Standardise method/class vars for extractors to see if they are suitable * Fix bugs with removing default loguru logger (allows further customisation) * Fix bug loading required fields from file *
This commit is contained in:
1
tests/data/test_modules/example_module/__init__.py
Normal file
1
tests/data/test_modules/example_module/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
from .example_module import ExampleModule
|
||||
11
tests/data/test_modules/example_module/__manifest__.py
Normal file
11
tests/data/test_modules/example_module/__manifest__.py
Normal file
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"name": "Example Module",
|
||||
"type": ["extractor", "feeder", "formatter", "storage", "enricher", "database"],
|
||||
"requires_setup": False,
|
||||
"dependencies": {"python": ["loguru"]
|
||||
},
|
||||
"configs": {
|
||||
"csv_file": {"default": "db.csv", "help": "CSV file name"},
|
||||
"required_field": {"required": True, "help": "required field in the CSV file"},
|
||||
},
|
||||
}
|
||||
28
tests/data/test_modules/example_module/example_module.py
Normal file
28
tests/data/test_modules/example_module/example_module.py
Normal file
@@ -0,0 +1,28 @@
|
||||
from auto_archiver.core import Extractor, Enricher, Feeder, Database, Storage, Formatter, Metadata
|
||||
|
||||
class ExampleModule(Extractor, Enricher, Feeder, Database, Storage, Formatter):
|
||||
def download(self, item):
|
||||
print("download")
|
||||
|
||||
def __iter__(self):
|
||||
yield Metadata().set_url("https://example.com")
|
||||
|
||||
|
||||
def done(self, result):
|
||||
print("done")
|
||||
|
||||
def enrich(self, to_enrich):
|
||||
print("enrich")
|
||||
|
||||
def get_cdn_url(self, media):
|
||||
return "nice_url"
|
||||
|
||||
def save(self, item):
|
||||
print("save")
|
||||
|
||||
def uploadf(self, file, key, **kwargs):
|
||||
print("uploadf")
|
||||
|
||||
|
||||
def format(self, item):
|
||||
print("format")
|
||||
Reference in New Issue
Block a user