mirror of
https://github.com/bellingcat/auto-archiver.git
synced 2026-06-12 05:08:28 +03:00
Allow framework for config settings that should not be stored in config (e.g. cli_feeder.urls
Use 'do_not_store': True in the config settings to apply this. Also: fix up generic archiver dropins loading + local_storage defaults (same as what's in example orchestration)
This commit is contained in:
@@ -11,6 +11,7 @@
|
||||
"help": "URL(s) to archive, either a single URL or a list of urls, should not come from config.yaml",
|
||||
"nargs": "+",
|
||||
"required": True,
|
||||
"do_not_store": True,
|
||||
},
|
||||
},
|
||||
"description": """
|
||||
|
||||
@@ -121,6 +121,7 @@ class GenericExtractor(Extractor):
|
||||
|
||||
ie_instance = info_extractor(downloader=ydl)
|
||||
dropin = self.dropin_for_name(info_extractor.ie_key())
|
||||
|
||||
if not dropin:
|
||||
# TODO: add a proper link to 'how to create your own dropin'
|
||||
logger.debug(f"""Could not find valid dropin for {info_extractor.IE_NAME}.
|
||||
@@ -172,6 +173,8 @@ class GenericExtractor(Extractor):
|
||||
|
||||
def dropin_for_name(self, dropin_name: str, additional_paths = [], package=__package__) -> Type[InfoExtractor]:
|
||||
|
||||
dropin_name = dropin_name.lower()
|
||||
|
||||
if dropin_name == "generic":
|
||||
# no need for a dropin for the generic extractor (?)
|
||||
return None
|
||||
|
||||
@@ -7,16 +7,16 @@
|
||||
},
|
||||
"configs": {
|
||||
"path_generator": {
|
||||
"default": "url",
|
||||
"default": "flat",
|
||||
"help": "how to store the file in terms of directory structure: 'flat' sets to root; 'url' creates a directory based on the provided URL; 'random' creates a random directory.",
|
||||
"choices": ["flat", "url", "random"],
|
||||
},
|
||||
"filename_generator": {
|
||||
"default": "random",
|
||||
"default": "static",
|
||||
"help": "how to name stored files: 'random' creates a random string; 'static' uses a replicable strategy such as a hash.",
|
||||
"choices": ["random", "static"],
|
||||
},
|
||||
"save_to": {"default": "./archived", "help": "folder where to save archived content"},
|
||||
"save_to": {"default": "./local_archive", "help": "folder where to save archived content"},
|
||||
"save_absolute": {"default": False, "help": "whether the path to the stored file is absolute or relative in the output result inc. formatters (WARN: leaks the file structure)"},
|
||||
},
|
||||
"description": """
|
||||
|
||||
Reference in New Issue
Block a user