bellingcat/auto-archiver
1
0
Fork 0
mirror of https://github.com/bellingcat/auto-archiver.git synced 2026-06-13 05:38:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

Google sheets feeder and database implemented.

Browse Source
This commit is contained in:
erinhmclark
2025-01-27 20:13:12 +00:00
parent 6c67effd8c
commit 57b3bec935
3 changed files with 39 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
src/auto_archiver/modules/gsheet_feeder/gsheet_feeder.py
View File

@@ -21,41 +21,13 @@ from . import GWorksheet
class GsheetsFeeder(Feeder):
def __init__(self) -> None:
"""
Initializes the GsheetsFeeder with preloaded configurations.
"""
super().__init__()
# Initialize the gspread client with the provided service account file
# self.gsheets_client = gspread.service_account(filename=self.config["service_account"])
#
# # Set up feeder-specific configurations from the config
# self.sheet_name = config.get("sheet")
# self.sheet_id = config.get("sheet_id")
# self.header = config.get("header", 1)
# self.columns = config.get("columns", {})
# assert self.sheet_name or self.sheet_id, (
# "You need to define either a 'sheet' name or a 'sheet_id' in your manifest."
# )
# # Configuration attributes
# self.sheet = config.get("sheet")
# self.sheet_id = config.get("sheet_id")
# self.header = config.get("header", 1)
# self.columns = config.get("columns", {})
# self.allow_worksheets = config.get("allow_worksheets", set())
# self.block_worksheets = config.get("block_worksheets", set())
# self.use_sheet_names_in_stored_paths = config.get("use_sheet_names_in_stored_paths", True)
# Ensure the header is an integer
# try:
# self.header = int(self.header)
# except ValueError:
# pass
# assert isinstance(self.header, int), f"Header must be an integer, got {type(self.header)}"
# assert self.sheet or self.sheet_id, "Either 'sheet' or 'sheet_id' must be defined."
#
def setup(self, config: dict):
super().setup(config)
self.gsheets_client = gspread.service_account(filename=self.service_account)
# TODO mv to validators
assert self.sheet or self.sheet_id, (
"You need to define either a 'sheet' name or a 'sheet_id' in your manifest."
)
def open_sheet(self):
if self.sheet:
@@ -63,7 +35,6 @@ class GsheetsFeeder(Feeder):
else: # self.sheet_id
return self.gsheets_client.open_by_key(self.sheet_id)
def __iter__(self) -> Metadata:
sh = self.open_sheet()
for ii, wks in enumerate(sh.worksheets()):