Set up feeder manifests (not merged by source yet)

This commit is contained in:
erinhmclark
2025-01-23 09:16:42 +00:00
parent c517d35bdf
commit 79684f8348
82 changed files with 721 additions and 730 deletions

View File

@@ -0,0 +1,22 @@
{
"name": "csv_db",
"type": ["database"],
"requires_setup": False,
"external_dependencies": {"python": ["loguru"]
},
"configs": {
"csv_file": {"default": "db.csv", "help": "CSV file name"}
},
"description": """
Handles exporting archival results to a CSV file.
### Features
- Saves archival metadata as rows in a CSV file.
- Automatically creates the CSV file with a header if it does not exist.
- Appends new metadata entries to the existing file.
### Setup
Required config:
- csv_file: Path to the CSV file where results will be stored (default: "db.csv").
""",
}

View File

@@ -0,0 +1,29 @@
import os
from loguru import logger
from csv import DictWriter
from dataclasses import asdict
from auto_archiver.databases import Database
from auto_archiver.core import Metadata
class CSVDb(Database):
"""
Outputs results to a CSV file
"""
name = "csv_db"
def __init__(self, config: dict) -> None:
# without this STEP.__init__ is not called
super().__init__(config)
self.assert_valid_string("csv_file")
def done(self, item: Metadata, cached: bool=False) -> None:
"""archival result ready - should be saved to DB"""
logger.success(f"DONE {item}")
is_empty = not os.path.isfile(self.csv_file) or os.path.getsize(self.csv_file) == 0
with open(self.csv_file, "a", encoding="utf-8") as outf:
writer = DictWriter(outf, fieldnames=asdict(Metadata()))
if is_empty: writer.writeheader()
writer.writerow(asdict(item))