bellingcat/auto-archiver
1
0
Fork 0
mirror of https://github.com/bellingcat/auto-archiver.git synced 2026-06-12 05:08:28 +03:00
Code Issues Packages Projects Releases Wiki Activity

WIP refactor logging

Browse Source
This commit is contained in:
msramalho
2025-06-21 15:54:51 +01:00
parent ade7feb5a0
commit ce4d7ac649
54 changed files with 298 additions and 207 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
src/auto_archiver/modules/csv_feeder/csv_feeder.py
View File

@@ -1,4 +1,4 @@
from loguru import logger
from auto_archiver.utils.custom_logger import logger
import csv
from auto_archiver.core import Feeder
@@ -20,20 +20,19 @@ class CSVFeeder(Feeder):
url_column = first_row.index(url_column)
except ValueError:
logger.error(
f"Column {url_column} not found in header row: {first_row}. Did you set the 'column' config correctly?"
f"column {url_column} not found in header row: {first_row}. Did you set the 'column' config correctly?"
)
return
elif not (url_or_none(first_row[url_column])):
# it's a header row, but we've been given a column number already
logger.debug(f"Skipping header row: {first_row}")
logger.debug(f"skipping header row: {first_row}")
else:
# first row isn't a header row, rewind the file
f.seek(0)
for row in reader:
if not url_or_none(row[url_column]):
logger.warning(f"Not a valid URL in row: {row}, skipping")
logger.warning(f"not a valid URL in row: {row}, skipping")
continue
url = row[url_column]
logger.debug(f"Processing {url}")
yield Metadata().set_url(url)