diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..5290c69 --- /dev/null +++ b/Makefile @@ -0,0 +1,17 @@ +clean-dev: + @echo -n "Are you sure? [yes/N] (this will delete volumes) " && read ans && [ $${ans:-N} = yes ] + docker compose -f docker-compose.yml -f docker-compose.dev.yml down --volumes --remove-orphans + +dev: + docker compose -f docker-compose.yml -f docker-compose.dev.yml build + docker compose -f docker-compose.yml -f docker-compose.dev.yml up --remove-orphans + +stop-dev: + docker compose -f docker-compose.yml -f docker-compose.dev.yml down --volumes + +prod: + docker compose build + docker compose up -d --remove-orphans + +stop-prod: + docker compose down \ No newline at end of file diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 3dfa717..cb51d8d 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -2,10 +2,15 @@ version: '3.8' services: web: + restart: "no" environment: - SERVE_LOCAL_ARCHIVE=/app/local_archive # See orchestration.yaml local_storage.save_to - ALLOWED_ORIGINS=http://localhost:8004 + + worker: + restart: "no" + redis: - command: redis-server /conf/redis.conf + restart: "no" ports: - 6379:6379 diff --git a/docker-compose.yml b/docker-compose.yml index fa99160..32cd109 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,54 +1,64 @@ +# reusable YAML variables +x-broker-url: &broker-url "redis://:${REDIS_PASSWORD}@redis:6379/0" + +x-base-setup: &base-setup + build: ./src + restart: always + env_file: src/.env + environment: + CELERY_BROKER_URL: *broker-url + CELERY_RESULT_BACKEND: *broker-url + version: '3.8' -services: +volumes: + crawls: +name: "auto-archiver-api" +services: web: - build: ./src - restart: always + <<: *base-setup ports: - 8004:8000 command: uvicorn main:app --host 0.0.0.0 --reload volumes: - ./src:/app - env_file: src/.env - environment: - - CELERY_BROKER_URL=redis://:${REDIS_PASSWORD}@redis:6379/0 - - CELERY_RESULT_BACKEND=redis://:${REDIS_PASSWORD}@redis:6379/0 depends_on: - redis worker: - build: ./src - restart: always + <<: *base-setup command: celery worker --app=worker.celery --loglevel=info --logfile=logs/celery.log volumes: - ./src:/app - env_file: src/.env + - /var/run/docker.sock:/var/run/docker.sock + - crawls:/crawls # BROWSERTRIX_HOME_HOST:BROWSERTRIX_HOME_CONTAINER environment: - - CELERY_BROKER_URL=redis://:${REDIS_PASSWORD}@redis:6379/0 - - CELERY_RESULT_BACKEND=redis://:${REDIS_PASSWORD}@redis:6379/0 + CELERY_BROKER_URL: *broker-url + CELERY_RESULT_BACKEND: *broker-url + WACZ_ENABLE_DOCKER: 1 # Enable calling docker from this container + BROWSERTRIX_HOME_HOST: auto-archiver-api_crawls + BROWSERTRIX_HOME_CONTAINER: /crawls depends_on: - web - redis redis: image: redis:6-alpine + restart: always command: redis-server /conf/redis.conf --requirepass ${REDIS_PASSWORD} volumes: - "./redis/data:/data" - "./redis/config:/conf" - restart: always + # dashboard service will only launch the dashboard if "--profile flower" is passed to docker compose; or if explicitly called "docker compose up flower" dashboard: - build: ./src - restart: always - command: flower --app=worker.celery --port=5555 --broker=redis://:${REDIS_PASSWORD}@redis:6379/0 --basic_auth=${FLOWER_USERNAME}:${FLOWER_PASSWORD} - env_file: src/.env + <<: *base-setup + profiles: + - flower + command: ["flower", "--app=worker.celery", "--port=5555", "--broker", *broker-url, "--basic_auth=${FLOWER_USERNAME}:${FLOWER_PASSWORD}"] ports: - 5556:5555 - environment: - - CELERY_BROKER_URL=redis://:${REDIS_PASSWORD}@redis:6379/0 - - CELERY_RESULT_BACKEND=redis://:${REDIS_PASSWORD}@redis:6379/0 depends_on: - web - redis diff --git a/src/Dockerfile b/src/Dockerfile index 7e01b7e..bd44d70 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -4,6 +4,8 @@ FROM bellingcat/auto-archiver # set work directory WORKDIR /app +RUN curl -fsSL https://get.docker.com -o get-docker.sh && \ + sh get-docker.sh # set environment variables ENV PYTHONUNBUFFERED 1 ENV PYTHONDONTWRITEBYTECODE 1 diff --git a/src/Pipfile.lock b/src/Pipfile.lock index 6770c6d..8edb4d0 100644 --- a/src/Pipfile.lock +++ b/src/Pipfile.lock @@ -1926,7 +1926,7 @@ "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36", "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2" ], - "markers": "python_version >= '3.7'", + "markers": "python_version < '3.11'", "version": "==4.7.1" }, "typing-inspect": {