bellingcat/auto-archiver-api
1
0
Fork 0
mirror of https://github.com/bellingcat/auto-archiver-api.git synced 2026-06-08 03:28:35 +03:00
Code Issues Packages Projects Releases Wiki Activity

bumps auto-archiver to 0.13.4

Browse Source
This commit is contained in:
msramalho
2025-02-19 18:24:46 +00:00
parent 7eb6f652be
commit d9f40c8e08
4 changed files with 21 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
app/worker/main.py
View File

@@ -33,10 +33,10 @@ def create_archive_task(self, archive_json: str):
# call auto-archiver
args = get_orchestrator_args(archive.group_id, False, [archive.url])
# args = get_orchestrator_args(archive.group_id, False, [archive.url, "--extractors", "generic_extractor"])
logger.debug(args)
try:
result = next(ArchivingOrchestrator().run(args), None)
orchestrator = ArchivingOrchestrator()
orchestrator.setup(args)
result = next(orchestrator.feed())
except SystemExit as e:
log_error(e, f"create_archive_task: SystemExit from AA")
except Exception as e:
@@ -61,11 +61,12 @@ def create_sheet_task(self, sheet_json: str):
logger.info(f"[queue={queue_name}] SHEET START {sheet=}")
args = get_orchestrator_args(sheet.group_id, True, ["--gsheet_feeder.sheet_id", sheet.sheet_id])
logger.info(f"[queue={queue_name}] {args=}")
orchestrator = ArchivingOrchestrator()
orchestrator.setup(args)
stats = {"archived": 0, "failed": 0, "errors": []}
try:
for result in ArchivingOrchestrator().run(args):
for result in orchestrator.feed():
try:
assert result, f"ERROR archiving URL for sheet {sheet.sheet_id}"
archive = schemas.ArchiveCreate(