WIP: code working, dependency conflict

docker-compose.yml

@@ -33,12 +33,13 @@ services:
 
   redis:
     image: redis:6-alpine
+    # command: redis-server /conf/redis.conf # local dev
     command: redis-server /conf/redis.conf --requirepass ${REDIS_PASSWORD}
     volumes:
       - "./redis/data:/data"
       - "./redis/config:/conf"
     restart: always
-    # DEV ONLY
+    # TODO: DEV ONLY
     # ports:
     #   - 6379:6379
 

src/Pipfile

@@ -8,7 +8,7 @@ aiofiles = "==0.6.0"
 celery = "==4.4.7"
 fastapi = "*"
 flower = "==0.9.7"
-jinja2 = ">=3.0.3"
+jinja2 = "*"
 pytest = "==6.2.4"
 redis = "==3.5.3"
 requests = ">=2.25.1"
@@ -17,7 +17,7 @@ aiosqlite = "*"
 python-dotenv = "*"
 loguru = "*"
 sqlalchemy = "*"
-auto-archiver = "*"
+auto-archiver = ">=0.5.12"
 alembic = "*"
 fastapi-utils = "*"
 

src/db/schemas.py

@@ -28,4 +28,9 @@ class Archive(ArchiveCreate):
 # class Tag(TagCreate):
 #     created_at: datetime
 #     # class Config:
-#     #     orm_mode = True
+#     #     orm_mode = True
+
+class SubmitSheet(BaseModel):
+    sheet_name: str
+    header: int = 1
+    

src/main.py

@@ -10,7 +10,7 @@ from dotenv import load_dotenv
 import traceback, os, logging
 from loguru import logger
 
-from worker import create_archive_task, celery
+from worker import create_archive_task, create_sheet_task, celery
 
 from db import crud, models, schemas
 from db.database import engine, SessionLocal
@@ -138,6 +138,12 @@ def get_status(task_id, db: Session = Depends(get_db), email = Depends(get_beare
         "deleted": crud.soft_delete_task(db, task_id, email)
     })
 
+@app.post("/sheet", status_code=201)
+def run_task(sheet:schemas.SubmitSheet, basic_auth = Depends(get_basic_auth)):
+    logger.info("LAUNCHING SHEET TASK")
+    task = create_sheet_task.delay(sheet.json())
+    return JSONResponse({"id": task.id})
+
 # Basic protected logic to allow access to 1 static file
 SF = os.environ.get("STATIC_FILE", "")
 if len(SF) > 1 and os.path.isfile(SF):

src/worker.py

@@ -6,6 +6,7 @@ from celery.exceptions import Ignore
 from celery.signals import task_failure
 from auto_archiver import Config, ArchivingOrchestrator, Metadata
 # from auto_archiver.enrichers import ScreenshotEnricher
+from auto_archiver.feeders import GsheetsFeeder
 from loguru import logger
 
 from db import crud, schemas, models
@@ -57,6 +58,24 @@ def create_archive_task(self, archive_json: str):
         logger.debug(f"Added {db_task.id=} to database on {db_task.created_at}")
     return result_json
 
+
+@celery.task(name="create_sheet_task", bind=True, autoretry_for=(Exception,), retry_backoff=True, retry_kwargs={'max_retries': 0}) 
+def create_sheet_task(self, sheet_json: str):
+    logger.info(f"STARTING {sheet_json}")
+    sheet = schemas.SubmitSheet.parse_raw(sheet_json)
+    orchestrator = choose_orchestrator("bellingcat", "")
+    #TODO: modify archiver to accept sheetId too
+    config = Config()
+    config.parse(use_cli=False, yaml_config_filename="secrets/orchestration-sheet.yaml")
+    config.feeder.sheet = sheet.sheet_name
+    config.feeder.header = sheet.header
+    orchestrator = ArchivingOrchestrator(config)
+    #TODO: make auto-archiver config.parse receive an overriding dict
+    result = orchestrator.feed()
+    # TODO: save into local DB
+
+    return result.to_json()
+
 @task_failure.connect(sender=create_archive_task)
 def task_failure_notifier(sender=None, **kwargs):
     logger.warning("😅 From task_failure_notifier ==> Task failed successfully! ")