mirror of
https://github.com/bellingcat/auto-archiver.git
synced 2026-06-12 05:08:28 +03:00
770f4c8a3d
1. Fix some bugs in local_storage 2. Refactor unit tests to not set Media.key explicitly (unless it's well-known beforehand, which it isn't) 3. Limit length of URL for 'url' type path_generator 4. Throw an error if 'save_to' of local storage is too long 5. A few other tidyups
20 lines
672 B
Python
20 lines
672 B
Python
from loguru import logger
|
|
|
|
from auto_archiver.core.feeder import Feeder
|
|
from auto_archiver.core.metadata import Metadata
|
|
|
|
class CLIFeeder(Feeder):
|
|
|
|
def setup(self) -> None:
|
|
self.urls = self.config['urls']
|
|
if not self.urls:
|
|
raise ValueError("No URLs provided. Please provide at least one URL via the command line, or set up an alternative feeder. Use --help for more information.")
|
|
|
|
def __iter__(self) -> Metadata:
|
|
urls = self.config['urls']
|
|
for url in urls:
|
|
logger.debug(f"Processing {url}")
|
|
m = Metadata().set_url(url)
|
|
yield m
|
|
|
|
logger.success(f"Processed {len(urls)} URL(s)") |