mirror of
https://github.com/bellingcat/auto-archiver.git
synced 2026-06-12 21:28:29 +03:00
Set up feeder manifests (not merged by source yet)
This commit is contained in:
0
src/auto_archiver/modules/csv_db/__init__.py
Normal file
0
src/auto_archiver/modules/csv_db/__init__.py
Normal file
22
src/auto_archiver/modules/csv_db/__manifest__.py
Normal file
22
src/auto_archiver/modules/csv_db/__manifest__.py
Normal file
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"name": "csv_db",
|
||||
"type": ["database"],
|
||||
"requires_setup": False,
|
||||
"external_dependencies": {"python": ["loguru"]
|
||||
},
|
||||
"configs": {
|
||||
"csv_file": {"default": "db.csv", "help": "CSV file name"}
|
||||
},
|
||||
"description": """
|
||||
Handles exporting archival results to a CSV file.
|
||||
|
||||
### Features
|
||||
- Saves archival metadata as rows in a CSV file.
|
||||
- Automatically creates the CSV file with a header if it does not exist.
|
||||
- Appends new metadata entries to the existing file.
|
||||
|
||||
### Setup
|
||||
Required config:
|
||||
- csv_file: Path to the CSV file where results will be stored (default: "db.csv").
|
||||
""",
|
||||
}
|
||||
29
src/auto_archiver/modules/csv_db/csv_db.py
Normal file
29
src/auto_archiver/modules/csv_db/csv_db.py
Normal file
@@ -0,0 +1,29 @@
|
||||
import os
|
||||
from loguru import logger
|
||||
from csv import DictWriter
|
||||
from dataclasses import asdict
|
||||
|
||||
from auto_archiver.databases import Database
|
||||
from auto_archiver.core import Metadata
|
||||
|
||||
|
||||
class CSVDb(Database):
|
||||
"""
|
||||
Outputs results to a CSV file
|
||||
"""
|
||||
name = "csv_db"
|
||||
|
||||
def __init__(self, config: dict) -> None:
|
||||
# without this STEP.__init__ is not called
|
||||
super().__init__(config)
|
||||
self.assert_valid_string("csv_file")
|
||||
|
||||
|
||||
def done(self, item: Metadata, cached: bool=False) -> None:
|
||||
"""archival result ready - should be saved to DB"""
|
||||
logger.success(f"DONE {item}")
|
||||
is_empty = not os.path.isfile(self.csv_file) or os.path.getsize(self.csv_file) == 0
|
||||
with open(self.csv_file, "a", encoding="utf-8") as outf:
|
||||
writer = DictWriter(outf, fieldnames=asdict(Metadata()))
|
||||
if is_empty: writer.writeheader()
|
||||
writer.writerow(asdict(item))
|
||||
Reference in New Issue
Block a user