Further fixes/changes to loading 'types' for config + manifest edits

This commit is contained in:
Patrick Robertson
2025-01-27 11:48:04 +01:00
parent 14e2479599
commit 7fd95866a1
29 changed files with 39 additions and 143 deletions

View File

@@ -6,16 +6,12 @@ flexible setup in various environments.
"""
import argparse
from ruamel.yaml import YAML, CommentedMap
from ruamel.yaml.comments import CommentedMap
from ruamel.yaml import YAML, CommentedMap, add_representer
from dataclasses import dataclass, field
from collections import OrderedDict
from collections.abc import Iterable
from copy import deepcopy
from .loader import MODULE_TYPES
from typing import Any, List
from typing import Any, List, Type
# configurable_parents = [
# Feeder,

View File

@@ -152,12 +152,12 @@ class ArchivingOrchestrator:
if not modules:
modules = available_modules(with_manifest=True)
module: Module
for module in modules:
if not module.configs:
# this module has no configs, don't show anything in the help
# (TODO: do we want to show something about this module though, like a description?)
continue
group = parser.add_argument_group(module.display_name or module.name, f"{module.description[:100]}...")
for name, kwargs in module.configs.items():
# TODO: go through all the manifests and make sure we're not breaking anything with removing cli_set
@@ -165,8 +165,8 @@ class ArchivingOrchestrator:
kwargs.pop('cli_set', None)
kwargs['dest'] = f"{module.name}.{kwargs.pop('dest', name)}"
try:
kwargs['type'] = getattr(__builtins__, kwargs.get('type', 'str'))
except AttributeError:
kwargs['type'] = __builtins__.get(kwargs.get('type'), str)
except KeyError:
kwargs['type'] = getattr(validators, kwargs['type'])
group.add_argument(f"--{module.name}.{name}", **kwargs)
@@ -207,7 +207,7 @@ class ArchivingOrchestrator:
exit()
if (module_type == 'feeder' or module_type == 'formatter') and len(step_items) > 1:
logger.error(f"Only one feeder is allowed, found {len(step_items)} {module_type}s. Please remove one of the following from your configuration file: {modules_to_load}")
logger.error(f"Only one {module_type} is allowed, found {len(step_items)} {module_type}s. Please remove one of the following from your configuration file: {modules_to_load}")
exit()
for i, module in enumerate(modules_to_load):