metadata.json hardcode in storage. add new metadata_json_enricher. log level change in orchestrator

This commit is contained in:
Dave Mateer
2025-06-17 09:51:19 +01:00
parent ba3f1a52e8
commit b3adc5603a
5 changed files with 66 additions and 2 deletions

View File

@@ -541,7 +541,7 @@ Here's how that would look: \n\nsteps:\n extractors:\n - [your_extractor_name_
yield self.feed_item(item)
url_count += 1
logger.success(f"Processed {url_count} URL(s)")
logger.info(f"Processed {url_count} URL(s)")
self.cleanup()
def feed_item(self, item: Metadata) -> Metadata:

View File

@@ -100,7 +100,12 @@ class Storage(BaseModule):
# Handle filename_generator logic
filename_generator = self.filename_generator
if filename_generator == "random":
# DM 9th Jun 25 - special case for metadata.json file in metadata_json_enricher
# where we want the filename to remain metadata.json
# TODO - should this be a config option to keep the original filename? Is it useful anywhere else?
if filename.endswith('metadata'):
filename = 'metadata'
elif filename_generator == "random":
filename = random_str(24)
elif filename_generator == "static":
# load the hash_enricher module