docker updates

README.md

@@ -1,6 +1,8 @@
 ## development
 http://localhost:8004
 
+requires `src/.env`
+
 cd /src
 * `pipenv install --editable ../../auto-archiver`
 * console 1 - `docker compose up  web redis dashboard `

docker-compose.yml

@@ -4,11 +4,12 @@ services:
 
   web:
     build: ./src
+    restart: always
     ports:
       - 8004:8000
     command: uvicorn main:app --host 0.0.0.0 --reload
     volumes:
-      - ./project:/usr/src/app
+      - ./src:/usr/src/app
     environment:
       - CELERY_BROKER_URL=redis://redis:6379/0
       - CELERY_RESULT_BACKEND=redis://redis:6379/0
@@ -17,6 +18,7 @@ services:
 
   worker:
     build: ./src
+    restart: always
     command: celery worker --app=worker.celery --loglevel=info --logfile=logs/celery.log
     volumes:
       - ./src:/usr/src/app
@@ -29,12 +31,14 @@ services:
 
   redis:
     image: redis:6-alpine
+    restart: always
     # DEV ONLY
     ports:
       - 6379:6379
 
   dashboard:
     build: ./src
+    restart: always
     command:  flower --app=worker.celery --port=5555 --broker=redis://redis:6379/0
     ports:
       - 5556:5555

src/Dockerfile

@@ -1,5 +1,6 @@
 # pull official base image 3.9
-FROM python:3.9.5-slim-buster
+# FROM python:3.9.5-slim-buster
+From python:3.10
 
 # set work directory
 WORKDIR /usr/src/app
@@ -9,7 +10,13 @@ ENV PYTHONUNBUFFERED 1
 ENV PYTHONDONTWRITEBYTECODE 1
 
 # install dependencies
-RUN pip install --upgrade pip
+RUN pip install --upgrade pip && \
+	apt-get update && \
+	apt-get install -y gcc ffmpeg fonts-noto firefox-esr && \
+	wget https://github.com/mozilla/geckodriver/releases/download/v0.32.0/geckodriver-v0.32.0-linux64.tar.gz && \
+	tar -xvzf geckodriver* -C /usr/local/bin && \
+	chmod +x /usr/local/bin/geckodriver && \
+	rm geckodriver-v* 
 COPY ./requirements.txt .
 RUN pip install -r requirements.txt
 

src/Pipfile

@@ -8,16 +8,16 @@ aiofiles = "==0.6.0"
 celery = "==4.4.7"
 fastapi = "*"
 flower = "==0.9.7"
-jinja2 = "==3.0.3"
+jinja2 = ">=3.0.3"
 pytest = "==6.2.4"
 redis = "==3.5.3"
-requests = "==2.25.1"
-uvicorn = "==0.13.4"
+requests = ">=2.25.1"
+uvicorn = ">=0.13.4"
 aiosqlite = "*"
 python-dotenv = "*"
-auto-archiver = {editable = true, path = "./../../auto-archiver"}
 loguru = "*"
 sqlalchemy = "*"
+auto-archiver = "*"
 
 [dev-packages]
 

src/main.py

@@ -80,7 +80,7 @@ def home():
 @app.on_event("startup")
 async def on_startup():
 #     # Not needed if you setup a migration system like Alembic
-#     await create_db_and_tables()
+#     await create_db_and_tables()https://github.com/bellingcat/auto-archiver/tree/dockerize
     models.Base.metadata.create_all(bind=engine)
 
 #### helper methods

src/requirements.txt

@@ -1,9 +1,130 @@
+#
+# These requirements were autogenerated by pipenv
+# To regenerate from the project's Pipfile, run:
+#
+#    pipenv lock --requirements
+#
+
+# -i https://pypi.org/simple
 aiofiles==0.6.0
+aiosqlite==0.18.0
+amqp==2.6.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+anyio==3.6.2; python_full_version >= '3.6.2'
+argparse==1.4.0
+async-generator==1.10; python_version >= '3.5'
+attrs==22.2.0; python_version >= '3.6'
+authlib==0.15.6
+auto-archiver==0.4.2
+beautifulsoup4==4.11.2; python_version >= '3.6'
+billiard==3.6.4.0
+boto3==1.26.78; python_version >= '3.7'
+botocore==1.29.78; python_version >= '3.7'
+brotli==1.0.9; platform_python_implementation == 'CPython'
+bs4==0.0.1
+cachetools==5.3.0; python_version ~= '3.7'
 celery==4.4.7
-fastapi
+certifi==2022.12.7; python_version >= '3.6'
+cffi==1.15.1
+charset-normalizer==3.0.1
+click==8.1.3; python_version >= '3.7'
+cloudscraper==1.2.68
+cryptography==38.0.4; python_version >= '3.6'
+dataclasses-json==0.5.7; python_version >= '3.6'
+dateparser==1.1.7; python_version >= '3.7'
+exceptiongroup==1.1.0; python_version < '3.11'
+fastapi==0.92.0
+ffmpeg-python==0.2.0
+filelock==3.9.0; python_version >= '3.7'
+flask==2.2.3; python_version >= '3.7'
 flower==0.9.7
-Jinja2==3.0.3
+future==0.18.3; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'
+google-api-core==2.11.0; python_version >= '3.7'
+google-api-python-client==2.79.0; python_version >= '3.7'
+google-auth-httplib2==0.1.0
+google-auth-oauthlib==1.0.0; python_version >= '3.6'
+google-auth==2.16.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
+googleapis-common-protos==1.58.0; python_version >= '3.7'
+greenlet==2.0.2; platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32')))))
+gspread==5.7.2; python_version not in '3.0, 3.1, 3.2, 3.3' and python_version >= '3.6'
+h11==0.14.0; python_version >= '3.7'
+httplib2==0.21.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+humanize==4.6.0; python_version >= '3.7'
+idna==3.4; python_version >= '3.5'
+iniconfig==2.0.0; python_version >= '3.7'
+instaloader==4.9.6; python_version >= '3.8'
+itsdangerous==2.1.2; python_version >= '3.7'
+jinja2==3.1.2
+jmespath==1.0.1; python_version >= '3.7'
+kombu==4.6.11; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+loguru==0.6.0
+lxml==4.9.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+markdown-it-py==2.2.0; python_version >= '3.7'
+markupsafe==2.1.2; python_version >= '3.7'
+marshmallow-enum==1.5.1
+marshmallow==3.19.0; python_version >= '3.7'
+mdurl==0.1.2; python_version >= '3.7'
+mutagen==1.46.0; python_version >= '3.7'
+mypy-extensions==1.0.0; python_version >= '3.5'
+oauth2client==4.1.3
+oauthlib==3.2.2; python_version >= '3.6'
+outcome==1.2.0; python_version >= '3.7'
+packaging==23.0; python_version >= '3.7'
+pluggy==0.13.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+prometheus-client==0.8.0
+protobuf==4.22.0; python_version >= '3.7'
+py==1.11.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+pyaes==1.6.1
+pyasn1-modules==0.2.8
+pyasn1==0.4.8
+pycparser==2.21
+pycryptodomex==3.17; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+pydantic==1.10.5; python_version >= '3.7'
+pygments==2.14.0; python_version >= '3.6'
+pyparsing==3.0.9; python_version > '3.0'
+pysocks==1.7.1
 pytest==6.2.4
+python-dateutil==2.8.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'
+python-dotenv==1.0.0
+python-slugify==8.0.0; python_version >= '3.7'
+python-twitter-v2==0.8.1; python_version >= '3.6' and python_version < '4.0'
+pytz-deprecation-shim==0.1.0.post0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
+pytz==2022.7.1
+pyyaml==6.0; python_version >= '3.6'
 redis==3.5.3
-requests==2.25.1
-uvicorn==0.13.4
+regex==2022.10.31; python_version >= '3.6'
+requests-oauthlib==1.3.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+requests-toolbelt==0.10.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+requests==2.28.2
+rich==13.3.1; python_version >= '3.7'
+rsa==4.9; python_version >= '3.6' and python_version < '4.0'
+s3transfer==0.6.0; python_version >= '3.7'
+selenium==4.8.2; python_version >= '3.7'
+six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'
+sniffio==1.3.0; python_version >= '3.7'
+snscrape==0.5.0.20230113; python_version ~= '3.8'
+sortedcontainers==2.4.0
+soupsieve==2.4; python_version >= '3.7'
+sqlalchemy==2.0.4
+starlette==0.25.0; python_version >= '3.7'
+telethon==1.27.0; python_version >= '3.5'
+text-unidecode==1.3
+tiktok-downloader==0.3.4
+toml==0.10.2; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'
+tornado==6.2; python_version >= '3.5.2'
+tqdm==4.64.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+trio-websocket==0.9.2; python_version >= '3.5'
+trio==0.22.0; python_version >= '3.7'
+typing-extensions==4.5.0; python_version >= '3.7'
+typing-inspect==0.8.0
+tzdata==2022.7; python_version >= '3.6'
+tzlocal==4.2; python_version >= '3.6'
+uritemplate==4.1.1; python_version >= '3.6'
+urllib3==1.26.14; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
+uvicorn==0.20.0
+vine==1.3.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+vk-api==11.9.9
+vk-url-scraper==0.3.15; python_version >= '3.7'
+websockets==10.4; python_version >= '3.7'
+werkzeug==2.2.3; python_version >= '3.7'
+wsproto==1.2.0; python_version >= '3.7'
+yt-dlp==2023.2.17; python_version >= '3.7'

src/worker.py

@@ -32,7 +32,6 @@ def create_archive_task(self, url: str , user_email:str=""):
     global orchestrator
     if not orchestrator: orchestrator = ArchivingOrchestrator(config)
     result = orchestrator.feed_item(Metadata().set_url(url)).to_json()
-    # result = orchestrator.feed_item(Metadata().set_url(url))
     with get_db() as session:
         db_task = crud.create_task(session, task=schemas.TaskCreate(id=self.request.id, url=url, author=user_email, result=json.loads(result)))
         logger.debug(f"Added {db_task.id=} to database on {db_task.created_at}")