Refactor loader + step into module, use LazyBaseModule and BaseModule

This commit is contained in:
Patrick Robertson
2025-01-27 14:01:36 +01:00
parent 7fd95866a1
commit f68e2726f2
16 changed files with 232 additions and 231 deletions

View File

@@ -8,9 +8,9 @@
'entry_point': 'cli_feeder::CLIFeeder',
"configs": {
"urls": {
"default": None,
"help": "URL(s) to archive, either a single URL or a list of urls, should not come from config.yaml",
"nargs": "+",
"required": True,
},
},
"description": """

View File

@@ -5,11 +5,10 @@ from auto_archiver.core import Metadata, ArchivingContext
class CLIFeeder(Feeder):
name = "cli_feeder"
def __iter__(self) -> Metadata:
for url in self.urls:
logger.debug(f"Processing {url}")
logger.debug(f"Processing URL: '{url}'")
yield Metadata().set_url(url)
ArchivingContext.set("folder", "cli")