major refactor of structure for worker V web: docker/app/secrets/envs/...

app/migrations/env.py

@@ -0,0 +1,79 @@
+from logging.config import fileConfig
+from sqlalchemy import engine_from_config
+from sqlalchemy import pool
+
+from alembic import context
+
+from app.shared.settings import get_settings
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+config.set_main_option('sqlalchemy.url', get_settings().DATABASE_PATH)
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+if config.config_file_name is not None:
+    fileConfig(config.config_file_name, disable_existing_loggers=False) # disable_existing_loggers prevents loguru disabling
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+target_metadata = None
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def run_migrations_offline() -> None:
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(
+        url=url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+    )
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online() -> None:
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    connectable = engine_from_config(
+        config.get_section(config.config_ini_section, {}),
+        prefix="sqlalchemy.",
+        poolclass=pool.NullPool,
+    )
+
+    with connectable.connect() as connection:
+        context.configure(
+            connection=connection, target_metadata=target_metadata
+        )
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

app/migrations/script.py.mako

@@ -0,0 +1,24 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+branch_labels = ${repr(branch_labels)}
+depends_on = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    ${downgrades if downgrades else "pass"}

app/migrations/versions/02b2f6d17ed0_create_archives_store_until_column.py

@@ -0,0 +1,34 @@
+"""create archives.store_until column
+
+Revision ID: 02b2f6d17ed0
+Revises: 1636724ec4b1
+Create Date: 2025-02-08 15:22:20.392522
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '02b2f6d17ed0'
+down_revision = '1636724ec4b1'
+branch_labels = None
+depends_on = None
+STORE_UNTIL_COL = "store_until"
+
+
+def upgrade() -> None:
+    conn = op.get_bind()
+    inspector = sa.inspect(conn)
+    columns = [col['name'] for col in inspector.get_columns('archives')]
+
+    if STORE_UNTIL_COL not in columns:
+        op.add_column('archives', sa.Column(STORE_UNTIL_COL, sa.DateTime(), nullable=True, default=None))
+
+
+def downgrade() -> None:
+    conn = op.get_bind()
+    inspector = sa.inspect(conn)
+    columns = [col['name'] for col in inspector.get_columns('archives')]
+    if STORE_UNTIL_COL in columns:
+        op.drop_column('archives', STORE_UNTIL_COL)

app/migrations/versions/1636724ec4b1_rename_sheets_last_archived_col.py

@@ -0,0 +1,32 @@
+"""rename sheets last_archived col
+
+Revision ID: 1636724ec4b1
+Revises: a23aaf3ae930
+Create Date: 2025-02-05 19:19:01.984396
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '1636724ec4b1'
+down_revision = 'a23aaf3ae930'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    conn = op.get_bind()
+    inspector = sa.inspect(conn)
+    columns = [col['name'] for col in inspector.get_columns('sheets')]
+    if 'last_archived_at' in columns:
+        op.alter_column('sheets', 'last_archived_at', new_column_name='last_url_archived_at')
+
+
+def downgrade() -> None:
+    conn = op.get_bind()
+    inspector = sa.inspect(conn)
+    columns = [col['name'] for col in inspector.get_columns('sheets')]
+    if 'last_url_archived_at' in columns:
+        op.alter_column('sheets', 'last_url_archived_at', new_column_name='last_archived_at')

app/migrations/versions/89121d2c96d8_add_sheet_id_to_archive_table.py

@@ -0,0 +1,42 @@
+"""add sheet_id to archive table
+
+Revision ID: 89121d2c96d8
+Revises: fa012ec405b8
+Create Date: 2024-11-04 11:12:30.237299
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.engine.reflection import Inspector
+
+
+# revision identifiers, used by Alembic.
+revision = '89121d2c96d8'
+down_revision = 'fa012ec405b8'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    conn = op.get_bind()
+    inspector = sa.inspect(conn)
+    columns = [col['name'] for col in inspector.get_columns('archives')]
+
+    if 'sheet_id' not in columns:
+        with op.batch_alter_table('archives') as batch_op:
+            batch_op.add_column(sa.Column('sheet_id', sa.String(), nullable=True, default=None))
+            batch_op.create_foreign_key('fk_sheet_id', 'sheets', ['sheet_id'], ['id'])
+
+
+def downgrade() -> None:
+    conn = op.get_bind()
+    inspector = sa.inspect(conn)
+    foreign_keys = [fk['name'] for fk in inspector.get_foreign_keys('archives')]
+    columns = [col['name'] for col in inspector.get_columns('archives')]
+
+    with op.batch_alter_table('archives') as batch_op:
+        if 'fk_sheet_id' in foreign_keys:
+            batch_op.drop_constraint('fk_sheet_id', type_='foreignkey')
+
+        if 'sheet_id' in columns:
+            batch_op.drop_column('sheet_id')

app/migrations/versions/9369a264945b_modify_archive_url_to_have_uuid_id_.py

@@ -0,0 +1,27 @@
+"""modify archive url to have uuid id instead of url unique constraint
+
+Revision ID: 9369a264945b
+Revises: 
+Create Date: 2023-12-20 17:24:59.320691
+
+"""
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = '9369a264945b'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # since the primary key constraint is not named, we have to recreate it first
+    with op.batch_alter_table("archive_urls") as batch_op:
+        batch_op.create_primary_key("pk_url", ["url"])
+        batch_op.drop_constraint("pk_url", type_='primary')
+        batch_op.create_primary_key("pk_url_archive_id", ["url", "archive_id"])
+
+def downgrade() -> None:
+    with op.batch_alter_table("archive_urls") as batch_op:
+        batch_op.drop_constraint("pk_url_archive_id", type_='primary')
+        batch_op.create_primary_key("url", ["url"])

app/migrations/versions/93a611e4c066_vacuum_database_if_there_s_enough_space.py

@@ -0,0 +1,28 @@
+"""vacuum database (if there's enough space)
+
+Revision ID: 93a611e4c066
+Revises: 9369a264945b
+Create Date: 2023-12-20 18:33:27.132566
+
+"""
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = '93a611e4c066'
+down_revision = '9369a264945b'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    try:
+        with op.get_context().autocommit_block():
+            op.execute("VACUUM")
+    except Exception as e:
+        print("Unable to run vacuum, maybe there's not enough disk space. it should be 2x the size of the database")
+        print(e)
+
+
+def downgrade() -> None:
+    pass

app/migrations/versions/a23aaf3ae930_drop_active_column.py

@@ -0,0 +1,34 @@
+"""drop active column
+
+Revision ID: a23aaf3ae930
+Revises: 89121d2c96d8
+Create Date: 2025-02-04 12:19:20.753570
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'a23aaf3ae930'
+down_revision = '89121d2c96d8'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    conn = op.get_bind()
+    inspector = sa.inspect(conn)
+    columns = [col['name'] for col in inspector.get_columns('users')]
+
+    if 'is_active' in columns:
+        op.drop_column('users', 'is_active')
+
+
+def downgrade() -> None:
+    conn = op.get_bind()
+    inspector = sa.inspect(conn)
+    columns = [col['name'] for col in inspector.get_columns('users')]
+
+    if 'is_active' not in columns:
+        op.add_column('users', sa.Column('is_active', sa.Boolean(), nullable=False, server_default=sa.false()))

app/migrations/versions/fa012ec405b8_add_columns_to_groups_table.py

@@ -0,0 +1,45 @@
+"""add columns to groups table
+
+Revision ID: fa012ec405b8
+Revises: 93a611e4c066
+Create Date: 2024-10-31 09:36:50.360710
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.engine.reflection import Inspector
+
+
+# revision identifiers, used by Alembic.
+revision = 'fa012ec405b8'
+down_revision = '93a611e4c066'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    conn = op.get_bind()
+    inspector = sa.inspect(conn)
+    columns = [col['name'] for col in inspector.get_columns('groups')]
+
+    if 'description' not in columns:
+        op.add_column('groups', sa.Column('description', sa.String(), nullable=True))
+    if 'orchestrator' not in columns:
+        op.add_column('groups', sa.Column('orchestrator', sa.String(), nullable=True))
+    if 'orchestrator_sheet' not in columns:
+        op.add_column('groups', sa.Column('orchestrator_sheet', sa.String(), nullable=True))
+    if 'permissions' not in columns:
+        op.add_column('groups', sa.Column('permissions', sa.JSON(), nullable=True))
+    if 'domains' not in columns:
+        op.add_column('groups', sa.Column('domains', sa.JSON(), nullable=True))
+
+
+def downgrade() -> None:
+    conn = op.get_bind()
+    inspector = sa.inspect(conn)
+    columns = [col['name'] for col in inspector.get_columns('groups')]
+
+    column_names = ['description', 'orchestrator', 'orchestrator_sheet', 'permissions', 'domains']
+    for column_name in column_names:
+        if column_name in columns:
+            op.drop_column('groups', column_name)