adds health check and new env logic

.gitignore

@@ -7,11 +7,14 @@ secrets
 __pycache
 .pytest_cach
 .env
+.env.dev
+.env.prod
 *.db
 redis/data/*
 .ipynb_checkpoints*
 #temp
 tests
 src/user-groups.yaml
+src/user-groups.dev.yaml
 wit*
 src/crawls

README.md

@@ -6,6 +6,8 @@ An api that uses celery workers to process URL archive requests via [bellingcat/
 ## Development
 http://localhost:8004
 
+TODO: update .env file instructions, should use .env.prod and .env.dev and only use .env for always overwriting dev/prod settings.
+
 requires `src/.env`
 
 cd /src
@@ -37,7 +39,7 @@ Auto-archiver orchestrator files configurations. For each archiving task an orch
 orchestrators:
   group1: secrets/orchestration-group1.yaml
   group2: secrets/orchestration-group2.yaml
-  default: secrets/orchestration-default:.yaml
+  default: secrets/orchestration-default:orchestration.yaml
 ```
 
 ## Database migrations

docker-compose.dev.yml

@@ -1,15 +1,19 @@
 services:
   web:
     restart: "no"
+    env_file: src/.env.dev
     environment:
       - SERVE_LOCAL_ARCHIVE=/app/local_archive # See orchestration.yaml local_storage.save_to
       - ALLOWED_ORIGINS=http://localhost:8004,chrome-extension://ojcimmjndnlmmlgnjaeojoebaceokpdp
       - API_BEARER_TOKEN=dev-api-bearer-token
+      - USER_GROUPS_FILENAME=user-groups.dev.yaml
 
   worker:
     restart: "no"
+    env_file: src/.env.dev
 
   redis:
     restart: "no"
+    env_file: src/.env.dev
     ports:
       - 6379:6379

docker-compose.yml

@@ -4,7 +4,7 @@ x-broker-url: &broker-url "redis://:${REDIS_PASSWORD}@redis:6379/0"
 x-base-setup: &base-setup
   build: ./src
   restart: always
-  env_file: src/.env
+  env_file: src/.env.prod
   environment:
     CELERY_BROKER_URL: *broker-url
     CELERY_RESULT_BACKEND: *broker-url
@@ -23,10 +23,15 @@ services:
       - ./src:/app
     depends_on:
       - redis
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
 
   worker:
     <<: *base-setup
-    command: celery worker --app=worker.celery --loglevel=info --logfile=logs/celery.log
+    command: celery --app=worker.celery worker --loglevel=info --logfile=logs/celery.log
     volumes:
       - ./src:/app
       - /var/run/docker.sock:/var/run/docker.sock
@@ -41,6 +46,11 @@ services:
     depends_on:
       - web
       - redis
+    healthcheck:
+      test: ["CMD", "pipenv", "run", "celery", "-A", "worker.celery", "status"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
 
   redis:
     image: redis:6-alpine
@@ -49,6 +59,11 @@ services:
     volumes:
       - "./redis/data:/data"
       - "./redis/config:/conf"
+    healthcheck:
+      test: ["CMD", "redis-cli", "-a", "${REDIS_PASSWORD}", "ping"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
 
   # dashboard service will only launch the dashboard if "--profile flower" is passed to docker compose; or if explicitly called "docker compose up dashboard"
   dashboard:

src/endpoints/default.py

@@ -26,6 +26,10 @@ async def home(request: Request):
     return JSONResponse(status)
 
 
+@default_router.get("/health")
+async def health(request: Request):
+    return JSONResponse({"status": "ok"})
+
 @default_router.get("/groups", response_model=list[str])
 def get_user_groups(db: Session = Depends(get_db), email=Depends(get_user_auth)):
     return crud.get_user_groups(db, email)