mirror of
https://github.com/bellingcat/auto-archiver.git
synced 2026-06-12 05:08:28 +03:00
Ruff format with defaults.
This commit is contained in:
@@ -1,16 +1,16 @@
|
||||
{
|
||||
'name': 'Command Line Feeder',
|
||||
'type': ['feeder'],
|
||||
'entry_point': 'cli_feeder::CLIFeeder',
|
||||
'requires_setup': False,
|
||||
'description': 'Feeds URLs to orchestrator from the command line',
|
||||
'configs': {
|
||||
'urls': {
|
||||
'default': None,
|
||||
'help': 'URL(s) to archive, either a single URL or a list of urls, should not come from config.yaml',
|
||||
"name": "Command Line Feeder",
|
||||
"type": ["feeder"],
|
||||
"entry_point": "cli_feeder::CLIFeeder",
|
||||
"requires_setup": False,
|
||||
"description": "Feeds URLs to orchestrator from the command line",
|
||||
"configs": {
|
||||
"urls": {
|
||||
"default": None,
|
||||
"help": "URL(s) to archive, either a single URL or a list of urls, should not come from config.yaml",
|
||||
},
|
||||
},
|
||||
'description': """
|
||||
"description": """
|
||||
The Command Line Feeder is the default enabled feeder for the Auto Archiver. It allows you to pass URLs directly to the orchestrator from the command line
|
||||
without the need to specify any additional configuration or command line arguments:
|
||||
|
||||
@@ -20,4 +20,4 @@ You can pass multiple URLs by separating them with a space. The URLs will be pro
|
||||
|
||||
`auto-archiver --feeder cli_feeder -- https://example.com/1/ https://example.com/2/`
|
||||
""",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,19 +3,21 @@ from loguru import logger
|
||||
from auto_archiver.core.feeder import Feeder
|
||||
from auto_archiver.core.metadata import Metadata
|
||||
|
||||
class CLIFeeder(Feeder):
|
||||
|
||||
class CLIFeeder(Feeder):
|
||||
def setup(self) -> None:
|
||||
self.urls = self.config['urls']
|
||||
self.urls = self.config["urls"]
|
||||
if not self.urls:
|
||||
raise ValueError("No URLs provided. Please provide at least one URL via the command line, or set up an alternative feeder. Use --help for more information.")
|
||||
raise ValueError(
|
||||
"No URLs provided. Please provide at least one URL via the command line, or set up an alternative feeder. Use --help for more information."
|
||||
)
|
||||
|
||||
def __iter__(self) -> Metadata:
|
||||
urls = self.config['urls']
|
||||
urls = self.config["urls"]
|
||||
for url in urls:
|
||||
logger.debug(f"Processing {url}")
|
||||
m = Metadata().set_url(url)
|
||||
m.set_context("folder", "cli")
|
||||
yield m
|
||||
|
||||
logger.success(f"Processed {len(urls)} URL(s)")
|
||||
logger.success(f"Processed {len(urls)} URL(s)")
|
||||
|
||||
Reference in New Issue
Block a user