Move alembic.ini to migrations directory (#66)

app/migrations/alembic.ini

@@ -2,7 +2,7 @@
 
 [alembic]
 # path to migration scripts
-script_location = app/migrations
+script_location = ./app/migrations
 
 # template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
 # Uncomment the line below if you want the files to be prepended with date and time
@@ -10,10 +10,6 @@ script_location = app/migrations
 # for all available tokens
 # file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
 
-# sys.path path, will be prepended to sys.path if present.
-# defaults to the current working directory.
-prepend_sys_path = .
-
 # timezone to use when rendering the date within the migration file
 # as well as the filename.
 # If specified, requires the python-dateutil library that can be

app/tests/web/test_main.py

@@ -18,8 +18,24 @@ def test_lifespan(app):
 
 
 def test_alembic(db_session):
-    alembic.config.main(argv=["--raiseerr", "upgrade", "head"])
-    alembic.config.main(argv=["--raiseerr", "downgrade", "base"])
+    alembic.config.main(
+        argv=[
+            "-c",
+            "./app/migrations/alembic.ini",
+            "--raiseerr",
+            "upgrade",
+            "head",
+        ]
+    )
+    alembic.config.main(
+        argv=[
+            "-c",
+            "./app/migrations/alembic.ini",
+            "--raiseerr",
+            "downgrade",
+            "base",
+        ]
+    )
 
 
 @patch(

app/web/events.py

@@ -38,9 +38,22 @@ async def lifespan(app: FastAPI):
     # STARTUP
     engine = make_engine(get_settings().DATABASE_PATH)
     models.Base.metadata.create_all(bind=engine)
-    alembic.config.main(prog="alembic", argv=['--raiseerr', 'upgrade', 'head'])
+    alembic.config.main(
+        prog="alembic",
+        argv=[
+            "-c",
+            "./app/migrations/alembic.ini",
+            "--raiseerr",
+            "upgrade",
+            "head",
+        ],
+    )
     logging.getLogger("uvicorn.access").disabled = True  # loguru
-    asyncio.create_task(redis_subscribe_worker_exceptions(get_settings().REDIS_EXCEPTIONS_CHANNEL))
+    asyncio.create_task(
+        redis_subscribe_worker_exceptions(
+            get_settings().REDIS_EXCEPTIONS_CHANNEL
+        )
+    )
     asyncio.create_task(repeat_measure_regular_metrics())
     with get_db() as db:
         crud.upsert_user_groups(db)
@@ -71,41 +84,74 @@ async def lifespan(app: FastAPI):
 
 
 # CRON JOBS
-@repeat_every(seconds=get_settings().REPEAT_COUNT_METRICS_SECONDS, on_exception=increase_exceptions_counter)
+@repeat_every(
+    seconds=get_settings().REPEAT_COUNT_METRICS_SECONDS,
+    on_exception=increase_exceptions_counter,
+)
 async def repeat_measure_regular_metrics():
-    await measure_regular_metrics(get_settings().DATABASE_PATH, get_settings().REPEAT_COUNT_METRICS_SECONDS)
+    await measure_regular_metrics(
+        get_settings().DATABASE_PATH,
+        get_settings().REPEAT_COUNT_METRICS_SECONDS,
+    )
 
 
-@repeat_every(seconds=60, wait_first=120, on_exception=increase_exceptions_counter)
+@repeat_every(
+    seconds=60, wait_first=120, on_exception=increase_exceptions_counter
+)
 async def archive_hourly_sheets_cronjob():
     await archive_sheets_cronjob("hourly", 60, datetime.datetime.now().minute)
 
 
-@repeat_every(seconds=3600, wait_first=120, on_exception=increase_exceptions_counter)
+@repeat_every(
+    seconds=3600, wait_first=120, on_exception=increase_exceptions_counter
+)
 async def archive_daily_sheets_cronjob():
     await archive_sheets_cronjob("daily", 24, datetime.datetime.now().hour)
 
 
-async def archive_sheets_cronjob(frequency: str, interval: int, current_time_unit: int):
+async def archive_sheets_cronjob(
+    frequency: str, interval: int, current_time_unit: int
+):
     triggered_jobs = []
 
     async with get_db_async() as db:
-        sheets = await crud.get_sheets_by_id_hash(db, frequency, interval, current_time_unit)
+        sheets = await crud.get_sheets_by_id_hash(
+            db, frequency, str(interval), current_time_unit
+        )
         for s in sheets:
             group_queue = await crud.get_group_priority_async(db, s.group_id)
-            task = celery.signature("create_sheet_task", args=[schemas.SubmitSheet(sheet_id=s.id, author_id=s.author_id, group_id=s.group_id).model_dump_json()]).apply_async(**group_queue)
+            task = celery.signature(
+                "create_sheet_task",
+                args=[
+                    schemas.SubmitSheet(
+                        sheet_id=s.id,
+                        author_id=s.author_id,
+                        group_id=s.group_id,
+                    ).model_dump_json()
+                ],
+            ).apply_async(**group_queue)
 
             triggered_jobs.append({"sheet_id": s.id, "task_id": task.id})
-    logger.debug(f"[CRON {frequency.upper()}:{current_time_unit}] Triggered {len(triggered_jobs)} sheet tasks: {triggered_jobs}")
+    logger.debug(
+        f"[CRON {frequency.upper()}:{current_time_unit}] Triggered {len(triggered_jobs)} sheet tasks: {triggered_jobs}"
+    )
 
 
 # TODO: on exception should logerror but also prometheus counter
-DELETE_WINDOW = get_settings().DELETE_SCHEDULED_ARCHIVES_CHECK_EVERY_N_DAYS * 24 * 60 * 60
+DELETE_WINDOW = (
+    get_settings().DELETE_SCHEDULED_ARCHIVES_CHECK_EVERY_N_DAYS * 24 * 60 * 60
+)
 
 
-@repeat_every(seconds=DELETE_WINDOW, wait_first=180, on_exception=increase_exceptions_counter)
+@repeat_every(
+    seconds=DELETE_WINDOW,
+    wait_first=180,
+    on_exception=increase_exceptions_counter,
+)
 async def notify_about_expired_archives():
-    notify_from = datetime.datetime.now() + datetime.timedelta(days=get_settings().DELETE_SCHEDULED_ARCHIVES_CHECK_EVERY_N_DAYS)
+    notify_from = datetime.datetime.now() + datetime.timedelta(
+        days=get_settings().DELETE_SCHEDULED_ARCHIVES_CHECK_EVERY_N_DAYS
+    )
     async with get_db_async() as db:
         scheduled_deletions = await crud.find_by_store_until(db, notify_from)
 
@@ -117,7 +163,12 @@ async def notify_about_expired_archives():
         fastmail = FastMail(get_settings().MAIL_CONFIG)
         # notify users
         for email in user_archives:
-            list_of_archives = "\n".join([f'{a.url}, {a.id}, {a.store_until.isoformat()}<br/>' for a in user_archives[email]])
+            list_of_archives = "\n".join(
+                [
+                    f"{a.url}, {a.id}, {a.store_until.isoformat()}<br/>"
+                    for a in user_archives[email]
+                ]
+            )
             # TODO: how can users download them in bulk?
             message = MessageSchema(
                 subject="Auto Archiver: Archives Scheduled for Deletion",
@@ -137,16 +188,23 @@ async def notify_about_expired_archives():
                 </body>
                 </html>
                 """,
-                subtype=MessageType.html
+                subtype=MessageType.html,
             )
             await fastmail.send_message(message)
-            logger.debug(f"[CRON] Email sent to {email} about {len(user_archives[email])} scheduled archives deletion.")
+            logger.debug(
+                f"[CRON] Email sent to {email} about {len(user_archives[email])} scheduled archives deletion."
+            )
 
     # now schedule the deletion event
     asyncio.create_task(delete_expired_archives())
 
 
-@repeat_every(max_repetitions=1, wait_first=10, seconds=0, on_exception=increase_exceptions_counter)
+@repeat_every(
+    max_repetitions=1,
+    wait_first=10,
+    seconds=0,
+    on_exception=increase_exceptions_counter,
+)
 async def delete_expired_archives():
     async with get_db_async() as db:
         count_deleted = await crud.soft_delete_expired_archives(db)
@@ -154,19 +212,27 @@ async def delete_expired_archives():
             logger.debug(f"[CRON] Deleted {count_deleted} archives.")
 
 
-@repeat_every(seconds=86400, wait_first=150, on_exception=increase_exceptions_counter)
+@repeat_every(
+    seconds=86400, wait_first=150, on_exception=increase_exceptions_counter
+)
 async def delete_stale_sheets():
     STALE_DAYS = get_settings().DELETE_STALE_SHEETS_DAYS
     logger.debug(f"[CRON] Deleting stale sheets older than {STALE_DAYS} days.")
     async with get_db_async() as db:
         user_sheets = await crud.delete_stale_sheets(db, STALE_DAYS)
 
-    if not user_sheets: return
+    if not user_sheets:
+        return
 
     fastmail = FastMail(get_settings().MAIL_CONFIG)
     # notify users
     for email in user_sheets:
-        list_of_sheets = "\n".join([f'<li><a href="https://docs.google.com/spreadsheets/d/{s.id}">{s.name}</a></li>' for s in user_sheets[email]])
+        list_of_sheets = "\n".join(
+            [
+                f'<li><a href="https://docs.google.com/spreadsheets/d/{s.id}">{s.name}</a></li>'
+                for s in user_sheets[email]
+            ]
+        )
         message = MessageSchema(
             subject="Auto Archiver: Stale Sheets Removed",
             recipients=[email],
@@ -183,14 +249,16 @@ async def delete_stale_sheets():
             </body>
             </html>
             """,
-            subtype=MessageType.html
+            subtype=MessageType.html,
         )
         await fastmail.send_message(message)
-        logger.debug(f"[CRON] Email sent to {email} about stale sheets deletion.")
+        logger.debug(
+            f"[CRON] Email sent to {email} about stale sheets deletion."
+        )
 
 
 # @repeat_at
 async def generate_users_export_csv():
-    #TODO: implement a cronjob that regularly requested user data to a CSV file
+    # TODO: implement a cronjob that regularly requested user data to a CSV file
     # see https://colab.research.google.com/drive/1QDbo3QXHPBdiTuANlA1AWVvN-rqxuCPa?authuser=0#scrollTo=4nPXeSdK8RBT
     pass

docker/web/Dockerfile

@@ -14,7 +14,6 @@ COPY ../../pyproject.toml ../../poetry.lock ../../README.md ./
 RUN poetry install --with web --no-interaction --no-ansi --no-cache
 
 # Copy the application code and configurations
-COPY ../../alembic.ini ./
 COPY ../../app ./app/
 COPY ../../user-groups.* ./app/
 

docker/worker/Dockerfile

@@ -26,7 +26,6 @@ RUN ./poetry-venv/bin/poetry install --without dev --no-root --no-cache
 # install dependencies
 
 # copy source code and .env files over
-COPY ../../alembic.ini ./
 COPY ../../app ./app/
 COPY ../../user-groups.* ./app/