final naming cleanup + new feeders/dbs

This commit is contained in:
msramalho
2023-01-21 19:44:12 +00:00
parent 753039240f
commit b763fc4188
34 changed files with 322 additions and 927 deletions

View File

@@ -0,0 +1,33 @@
import os
from loguru import logger
from . import Database
from ..core import Metadata
from csv import DictWriter
from dataclasses import asdict
class CSVDb(Database):
"""
Outputs results to a CSV file
"""
name = "csv_db"
def __init__(self, config: dict) -> None:
# without this STEP.__init__ is not called
super().__init__(config)
self.assert_valid_string("csv_file")
@staticmethod
def configs() -> dict:
return {
"csv_file": {"default": "db.csv", "help": "CSV file name"}
}
def done(self, item: Metadata) -> None:
"""archival result ready - should be saved to DB"""
logger.success(f"DONE {item}")
is_empty = not os.path.isfile(self.csv_file) or os.path.getsize(self.csv_file) == 0
with open(self.csv_file, "a", encoding="utf-8") as outf:
writer = DictWriter(outf, fieldnames=asdict(Metadata()))
if is_empty: writer.writeheader()
writer.writerow(asdict(item))