Move alembic.ini to migrations directory (#66)

app/migrations/alembic.ini

@@ -0,0 +1,102 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = ./app/migrations
+
+# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
+# Uncomment the line below if you want the files to be prepended with date and time
+# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file
+# for all available tokens
+# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
+
+# timezone to use when rendering the date within the migration file
+# as well as the filename.
+# If specified, requires the python-dateutil library that can be
+# installed by adding `alembic[tz]` to the pip requirements
+# string value is passed to dateutil.tz.gettz()
+# leave blank for localtime
+# timezone =
+
+# max length of characters to apply to the
+# "slug" field
+# truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; This defaults
+# to migrations/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path.
+# The path separator used here should be the separator specified by "version_path_separator" below.
+# version_locations = %(here)s/bar:%(here)s/bat:migrations/versions
+
+# version path separator; As mentioned above, this is the character used to split
+# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep.
+# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas.
+# Valid values for version_path_separator are:
+#
+# version_path_separator = :
+# version_path_separator = ;
+# version_path_separator = space
+version_path_separator = os  # Use os.pathsep. Default configuration used for new projects.
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+# passed from .env
+# sqlalchemy.url = driver://user:pass@localhost/dbname
+
+
+[post_write_hooks]
+# post_write_hooks defines scripts or Python functions that are run
+# on newly generated revision scripts.  See the documentation for further
+# detail and examples
+
+# format using "black" - use the console_scripts runner, against the "black" entrypoint
+# hooks = black
+# black.type = console_scripts
+# black.entrypoint = black
+# black.options = -l 79 REVISION_SCRIPT_FILENAME
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

app/tests/web/test_main.py

@@ -18,8 +18,24 @@ def test_lifespan(app):
 
 
 def test_alembic(db_session):
-    alembic.config.main(argv=["--raiseerr", "upgrade", "head"])
-    alembic.config.main(argv=["--raiseerr", "downgrade", "base"])
+    alembic.config.main(
+        argv=[
+            "-c",
+            "./app/migrations/alembic.ini",
+            "--raiseerr",
+            "upgrade",
+            "head",
+        ]
+    )
+    alembic.config.main(
+        argv=[
+            "-c",
+            "./app/migrations/alembic.ini",
+            "--raiseerr",
+            "downgrade",
+            "base",
+        ]
+    )
 
 
 @patch(

app/web/events.py

@@ -38,9 +38,22 @@ async def lifespan(app: FastAPI):
     # STARTUP
     engine = make_engine(get_settings().DATABASE_PATH)
     models.Base.metadata.create_all(bind=engine)
-    alembic.config.main(prog="alembic", argv=['--raiseerr', 'upgrade', 'head'])
+    alembic.config.main(
+        prog="alembic",
+        argv=[
+            "-c",
+            "./app/migrations/alembic.ini",
+            "--raiseerr",
+            "upgrade",
+            "head",
+        ],
+    )
     logging.getLogger("uvicorn.access").disabled = True  # loguru
-    asyncio.create_task(redis_subscribe_worker_exceptions(get_settings().REDIS_EXCEPTIONS_CHANNEL))
+    asyncio.create_task(
+        redis_subscribe_worker_exceptions(
+            get_settings().REDIS_EXCEPTIONS_CHANNEL
+        )
+    )
     asyncio.create_task(repeat_measure_regular_metrics())
     with get_db() as db:
         crud.upsert_user_groups(db)
@@ -71,41 +84,74 @@ async def lifespan(app: FastAPI):
 
 
 # CRON JOBS
-@repeat_every(seconds=get_settings().REPEAT_COUNT_METRICS_SECONDS, on_exception=increase_exceptions_counter)
+@repeat_every(
+    seconds=get_settings().REPEAT_COUNT_METRICS_SECONDS,
+    on_exception=increase_exceptions_counter,
+)
 async def repeat_measure_regular_metrics():
-    await measure_regular_metrics(get_settings().DATABASE_PATH, get_settings().REPEAT_COUNT_METRICS_SECONDS)
+    await measure_regular_metrics(
+        get_settings().DATABASE_PATH,
+        get_settings().REPEAT_COUNT_METRICS_SECONDS,
+    )
 
 
-@repeat_every(seconds=60, wait_first=120, on_exception=increase_exceptions_counter)
+@repeat_every(
+    seconds=60, wait_first=120, on_exception=increase_exceptions_counter
+)
 async def archive_hourly_sheets_cronjob():
     await archive_sheets_cronjob("hourly", 60, datetime.datetime.now().minute)
 
 
-@repeat_every(seconds=3600, wait_first=120, on_exception=increase_exceptions_counter)
+@repeat_every(
+    seconds=3600, wait_first=120, on_exception=increase_exceptions_counter
+)
 async def archive_daily_sheets_cronjob():
     await archive_sheets_cronjob("daily", 24, datetime.datetime.now().hour)
 
 
-async def archive_sheets_cronjob(frequency: str, interval: int, current_time_unit: int):
+async def archive_sheets_cronjob(
+    frequency: str, interval: int, current_time_unit: int
+):
     triggered_jobs = []
 
     async with get_db_async() as db:
-        sheets = await crud.get_sheets_by_id_hash(db, frequency, interval, current_time_unit)
+        sheets = await crud.get_sheets_by_id_hash(
+            db, frequency, str(interval), current_time_unit
+        )
         for s in sheets:
             group_queue = await crud.get_group_priority_async(db, s.group_id)
-            task = celery.signature("create_sheet_task", args=[schemas.SubmitSheet(sheet_id=s.id, author_id=s.author_id, group_id=s.group_id).model_dump_json()]).apply_async(**group_queue)
+            task = celery.signature(
+                "create_sheet_task",
+                args=[
+                    schemas.SubmitSheet(
+                        sheet_id=s.id,
+                        author_id=s.author_id,
+                        group_id=s.group_id,
+                    ).model_dump_json()
+                ],
+            ).apply_async(**group_queue)
 
             triggered_jobs.append({"sheet_id": s.id, "task_id": task.id})
-    logger.debug(f"[CRON {frequency.upper()}:{current_time_unit}] Triggered {len(triggered_jobs)} sheet tasks: {triggered_jobs}")
+    logger.debug(
+        f"[CRON {frequency.upper()}:{current_time_unit}] Triggered {len(triggered_jobs)} sheet tasks: {triggered_jobs}"
+    )
 
 
 # TODO: on exception should logerror but also prometheus counter
-DELETE_WINDOW = get_settings().DELETE_SCHEDULED_ARCHIVES_CHECK_EVERY_N_DAYS * 24 * 60 * 60
+DELETE_WINDOW = (
+    get_settings().DELETE_SCHEDULED_ARCHIVES_CHECK_EVERY_N_DAYS * 24 * 60 * 60
+)
 
 
-@repeat_every(seconds=DELETE_WINDOW, wait_first=180, on_exception=increase_exceptions_counter)
+@repeat_every(
+    seconds=DELETE_WINDOW,
+    wait_first=180,
+    on_exception=increase_exceptions_counter,
+)
 async def notify_about_expired_archives():
-    notify_from = datetime.datetime.now() + datetime.timedelta(days=get_settings().DELETE_SCHEDULED_ARCHIVES_CHECK_EVERY_N_DAYS)
+    notify_from = datetime.datetime.now() + datetime.timedelta(
+        days=get_settings().DELETE_SCHEDULED_ARCHIVES_CHECK_EVERY_N_DAYS
+    )
     async with get_db_async() as db:
         scheduled_deletions = await crud.find_by_store_until(db, notify_from)
 
@@ -117,7 +163,12 @@ async def notify_about_expired_archives():
         fastmail = FastMail(get_settings().MAIL_CONFIG)
         # notify users
         for email in user_archives:
-            list_of_archives = "\n".join([f'{a.url}, {a.id}, {a.store_until.isoformat()}<br/>' for a in user_archives[email]])
+            list_of_archives = "\n".join(
+                [
+                    f"{a.url}, {a.id}, {a.store_until.isoformat()}<br/>"
+                    for a in user_archives[email]
+                ]
+            )
             # TODO: how can users download them in bulk?
             message = MessageSchema(
                 subject="Auto Archiver: Archives Scheduled for Deletion",
@@ -137,16 +188,23 @@ async def notify_about_expired_archives():
                 </body>
                 </html>
                 """,
-                subtype=MessageType.html
+                subtype=MessageType.html,
             )
             await fastmail.send_message(message)
-            logger.debug(f"[CRON] Email sent to {email} about {len(user_archives[email])} scheduled archives deletion.")
+            logger.debug(
+                f"[CRON] Email sent to {email} about {len(user_archives[email])} scheduled archives deletion."
+            )
 
     # now schedule the deletion event
     asyncio.create_task(delete_expired_archives())
 
 
-@repeat_every(max_repetitions=1, wait_first=10, seconds=0, on_exception=increase_exceptions_counter)
+@repeat_every(
+    max_repetitions=1,
+    wait_first=10,
+    seconds=0,
+    on_exception=increase_exceptions_counter,
+)
 async def delete_expired_archives():
     async with get_db_async() as db:
         count_deleted = await crud.soft_delete_expired_archives(db)
@@ -154,19 +212,27 @@ async def delete_expired_archives():
             logger.debug(f"[CRON] Deleted {count_deleted} archives.")
 
 
-@repeat_every(seconds=86400, wait_first=150, on_exception=increase_exceptions_counter)
+@repeat_every(
+    seconds=86400, wait_first=150, on_exception=increase_exceptions_counter
+)
 async def delete_stale_sheets():
     STALE_DAYS = get_settings().DELETE_STALE_SHEETS_DAYS
     logger.debug(f"[CRON] Deleting stale sheets older than {STALE_DAYS} days.")
     async with get_db_async() as db:
         user_sheets = await crud.delete_stale_sheets(db, STALE_DAYS)
 
-    if not user_sheets: return
+    if not user_sheets:
+        return
 
     fastmail = FastMail(get_settings().MAIL_CONFIG)
     # notify users
     for email in user_sheets:
-        list_of_sheets = "\n".join([f'<li><a href="https://docs.google.com/spreadsheets/d/{s.id}">{s.name}</a></li>' for s in user_sheets[email]])
+        list_of_sheets = "\n".join(
+            [
+                f'<li><a href="https://docs.google.com/spreadsheets/d/{s.id}">{s.name}</a></li>'
+                for s in user_sheets[email]
+            ]
+        )
         message = MessageSchema(
             subject="Auto Archiver: Stale Sheets Removed",
             recipients=[email],
@@ -183,14 +249,16 @@ async def delete_stale_sheets():
             </body>
             </html>
             """,
-            subtype=MessageType.html
+            subtype=MessageType.html,
         )
         await fastmail.send_message(message)
-        logger.debug(f"[CRON] Email sent to {email} about stale sheets deletion.")
+        logger.debug(
+            f"[CRON] Email sent to {email} about stale sheets deletion."
+        )
 
 
 # @repeat_at
 async def generate_users_export_csv():
-    #TODO: implement a cronjob that regularly requested user data to a CSV file
+    # TODO: implement a cronjob that regularly requested user data to a CSV file
     # see https://colab.research.google.com/drive/1QDbo3QXHPBdiTuANlA1AWVvN-rqxuCPa?authuser=0#scrollTo=4nPXeSdK8RBT
     pass