Merge pull request #11 from bellingcat/wacz-dood

Makefile

@@ -0,0 +1,17 @@
+clean-dev:
+	@echo -n "Are you sure? [yes/N] (this will delete volumes) " && read ans && [ $${ans:-N} = yes ]
+	docker compose -f docker-compose.yml -f docker-compose.dev.yml down --volumes --remove-orphans
+
+dev:
+	docker compose -f docker-compose.yml -f docker-compose.dev.yml build
+	docker compose -f docker-compose.yml -f docker-compose.dev.yml up --remove-orphans
+
+stop-dev:
+	docker compose -f docker-compose.yml -f docker-compose.dev.yml down --volumes
+
+prod:
+	docker compose build
+	docker compose up -d --remove-orphans
+
+stop-prod:
+	docker compose down

docker-compose.dev.yml

@@ -2,10 +2,15 @@ version: '3.8'
 
 services:
   web:
+    restart: "no"
     environment:
       - SERVE_LOCAL_ARCHIVE=/app/local_archive # See orchestration.yaml local_storage.save_to
       - ALLOWED_ORIGINS=http://localhost:8004
+
+  worker:
+    restart: "no"
+
   redis:
-    command: redis-server /conf/redis.conf
+    restart: "no"
     ports:
       - 6379:6379

docker-compose.yml

@@ -1,54 +1,64 @@
+# reusable YAML variables
+x-broker-url: &broker-url "redis://:${REDIS_PASSWORD}@redis:6379/0"
+
+x-base-setup: &base-setup
+  build: ./src
+  restart: always
+  env_file: src/.env
+  environment:
+    CELERY_BROKER_URL: *broker-url
+    CELERY_RESULT_BACKEND: *broker-url
+
 version: '3.8'
 
-services:
+volumes:
+  crawls:
 
+name: "auto-archiver-api"
+services:
   web:
-    build: ./src
-    restart: always
+    <<: *base-setup
     ports:
       - 8004:8000
     command: uvicorn main:app --host 0.0.0.0 --reload
     volumes:
       - ./src:/app
-    env_file: src/.env
-    environment:
-      - CELERY_BROKER_URL=redis://:${REDIS_PASSWORD}@redis:6379/0
-      - CELERY_RESULT_BACKEND=redis://:${REDIS_PASSWORD}@redis:6379/0
     depends_on:
       - redis
 
   worker:
-    build: ./src
-    restart: always
+    <<: *base-setup
     command: celery worker --app=worker.celery --loglevel=info --logfile=logs/celery.log
     volumes:
       - ./src:/app
-    env_file: src/.env
+      - /var/run/docker.sock:/var/run/docker.sock
+      - crawls:/crawls # BROWSERTRIX_HOME_HOST:BROWSERTRIX_HOME_CONTAINER
     environment:
-      - CELERY_BROKER_URL=redis://:${REDIS_PASSWORD}@redis:6379/0
-      - CELERY_RESULT_BACKEND=redis://:${REDIS_PASSWORD}@redis:6379/0
+      CELERY_BROKER_URL: *broker-url
+      CELERY_RESULT_BACKEND: *broker-url
+      WACZ_ENABLE_DOCKER: 1 # Enable calling docker from this container
+      BROWSERTRIX_HOME_HOST: auto-archiver-api_crawls
+      BROWSERTRIX_HOME_CONTAINER: /crawls
     depends_on:
       - web
       - redis
 
   redis:
     image: redis:6-alpine
+    restart: always
     command: redis-server /conf/redis.conf --requirepass ${REDIS_PASSWORD}
     volumes:
       - "./redis/data:/data"
       - "./redis/config:/conf"
-    restart: always
 
+  # dashboard service will only launch the dashboard if "--profile flower" is passed to docker compose; or if explicitly called "docker compose up flower"
   dashboard:
-    build: ./src
-    restart: always
-    command:  flower --app=worker.celery --port=5555 --broker=redis://:${REDIS_PASSWORD}@redis:6379/0 --basic_auth=${FLOWER_USERNAME}:${FLOWER_PASSWORD}
-    env_file: src/.env
+    <<: *base-setup
+    profiles:
+      - flower
+    command: ["flower", "--app=worker.celery", "--port=5555", "--broker", *broker-url, "--basic_auth=${FLOWER_USERNAME}:${FLOWER_PASSWORD}"]
     ports:
       - 5556:5555
-    environment:
-      - CELERY_BROKER_URL=redis://:${REDIS_PASSWORD}@redis:6379/0
-      - CELERY_RESULT_BACKEND=redis://:${REDIS_PASSWORD}@redis:6379/0
     depends_on:
       - web
       - redis

src/Dockerfile

@@ -4,6 +4,8 @@ FROM bellingcat/auto-archiver
 # set work directory
 WORKDIR /app
 
+RUN curl -fsSL https://get.docker.com -o get-docker.sh && \
+    sh get-docker.sh
 # set environment variables
 ENV PYTHONUNBUFFERED 1
 ENV PYTHONDONTWRITEBYTECODE 1

src/Pipfile.lock

@@ -1926,7 +1926,7 @@
                 "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36",
                 "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"
             ],
-            "markers": "python_version >= '3.7'",
+            "markers": "python_version < '3.11'",
             "version": "==4.7.1"
         },
         "typing-inspect": {