major refactor of structure for worker V web: docker/app/secrets/envs/...

app/migrations/versions/02b2f6d17ed0_create_archives_store_until_column.py

@@ -0,0 +1,34 @@
+"""create archives.store_until column
+
+Revision ID: 02b2f6d17ed0
+Revises: 1636724ec4b1
+Create Date: 2025-02-08 15:22:20.392522
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '02b2f6d17ed0'
+down_revision = '1636724ec4b1'
+branch_labels = None
+depends_on = None
+STORE_UNTIL_COL = "store_until"
+
+
+def upgrade() -> None:
+    conn = op.get_bind()
+    inspector = sa.inspect(conn)
+    columns = [col['name'] for col in inspector.get_columns('archives')]
+
+    if STORE_UNTIL_COL not in columns:
+        op.add_column('archives', sa.Column(STORE_UNTIL_COL, sa.DateTime(), nullable=True, default=None))
+
+
+def downgrade() -> None:
+    conn = op.get_bind()
+    inspector = sa.inspect(conn)
+    columns = [col['name'] for col in inspector.get_columns('archives')]
+    if STORE_UNTIL_COL in columns:
+        op.drop_column('archives', STORE_UNTIL_COL)

app/migrations/versions/1636724ec4b1_rename_sheets_last_archived_col.py

@@ -0,0 +1,32 @@
+"""rename sheets last_archived col
+
+Revision ID: 1636724ec4b1
+Revises: a23aaf3ae930
+Create Date: 2025-02-05 19:19:01.984396
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '1636724ec4b1'
+down_revision = 'a23aaf3ae930'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    conn = op.get_bind()
+    inspector = sa.inspect(conn)
+    columns = [col['name'] for col in inspector.get_columns('sheets')]
+    if 'last_archived_at' in columns:
+        op.alter_column('sheets', 'last_archived_at', new_column_name='last_url_archived_at')
+
+
+def downgrade() -> None:
+    conn = op.get_bind()
+    inspector = sa.inspect(conn)
+    columns = [col['name'] for col in inspector.get_columns('sheets')]
+    if 'last_url_archived_at' in columns:
+        op.alter_column('sheets', 'last_url_archived_at', new_column_name='last_archived_at')

app/migrations/versions/89121d2c96d8_add_sheet_id_to_archive_table.py

@@ -0,0 +1,42 @@
+"""add sheet_id to archive table
+
+Revision ID: 89121d2c96d8
+Revises: fa012ec405b8
+Create Date: 2024-11-04 11:12:30.237299
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.engine.reflection import Inspector
+
+
+# revision identifiers, used by Alembic.
+revision = '89121d2c96d8'
+down_revision = 'fa012ec405b8'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    conn = op.get_bind()
+    inspector = sa.inspect(conn)
+    columns = [col['name'] for col in inspector.get_columns('archives')]
+
+    if 'sheet_id' not in columns:
+        with op.batch_alter_table('archives') as batch_op:
+            batch_op.add_column(sa.Column('sheet_id', sa.String(), nullable=True, default=None))
+            batch_op.create_foreign_key('fk_sheet_id', 'sheets', ['sheet_id'], ['id'])
+
+
+def downgrade() -> None:
+    conn = op.get_bind()
+    inspector = sa.inspect(conn)
+    foreign_keys = [fk['name'] for fk in inspector.get_foreign_keys('archives')]
+    columns = [col['name'] for col in inspector.get_columns('archives')]
+
+    with op.batch_alter_table('archives') as batch_op:
+        if 'fk_sheet_id' in foreign_keys:
+            batch_op.drop_constraint('fk_sheet_id', type_='foreignkey')
+
+        if 'sheet_id' in columns:
+            batch_op.drop_column('sheet_id')

app/migrations/versions/9369a264945b_modify_archive_url_to_have_uuid_id_.py

@@ -0,0 +1,27 @@
+"""modify archive url to have uuid id instead of url unique constraint
+
+Revision ID: 9369a264945b
+Revises: 
+Create Date: 2023-12-20 17:24:59.320691
+
+"""
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = '9369a264945b'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # since the primary key constraint is not named, we have to recreate it first
+    with op.batch_alter_table("archive_urls") as batch_op:
+        batch_op.create_primary_key("pk_url", ["url"])
+        batch_op.drop_constraint("pk_url", type_='primary')
+        batch_op.create_primary_key("pk_url_archive_id", ["url", "archive_id"])
+
+def downgrade() -> None:
+    with op.batch_alter_table("archive_urls") as batch_op:
+        batch_op.drop_constraint("pk_url_archive_id", type_='primary')
+        batch_op.create_primary_key("url", ["url"])

app/migrations/versions/93a611e4c066_vacuum_database_if_there_s_enough_space.py

@@ -0,0 +1,28 @@
+"""vacuum database (if there's enough space)
+
+Revision ID: 93a611e4c066
+Revises: 9369a264945b
+Create Date: 2023-12-20 18:33:27.132566
+
+"""
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision = '93a611e4c066'
+down_revision = '9369a264945b'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    try:
+        with op.get_context().autocommit_block():
+            op.execute("VACUUM")
+    except Exception as e:
+        print("Unable to run vacuum, maybe there's not enough disk space. it should be 2x the size of the database")
+        print(e)
+
+
+def downgrade() -> None:
+    pass

app/migrations/versions/a23aaf3ae930_drop_active_column.py

@@ -0,0 +1,34 @@
+"""drop active column
+
+Revision ID: a23aaf3ae930
+Revises: 89121d2c96d8
+Create Date: 2025-02-04 12:19:20.753570
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'a23aaf3ae930'
+down_revision = '89121d2c96d8'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    conn = op.get_bind()
+    inspector = sa.inspect(conn)
+    columns = [col['name'] for col in inspector.get_columns('users')]
+
+    if 'is_active' in columns:
+        op.drop_column('users', 'is_active')
+
+
+def downgrade() -> None:
+    conn = op.get_bind()
+    inspector = sa.inspect(conn)
+    columns = [col['name'] for col in inspector.get_columns('users')]
+
+    if 'is_active' not in columns:
+        op.add_column('users', sa.Column('is_active', sa.Boolean(), nullable=False, server_default=sa.false()))

app/migrations/versions/fa012ec405b8_add_columns_to_groups_table.py

@@ -0,0 +1,45 @@
+"""add columns to groups table
+
+Revision ID: fa012ec405b8
+Revises: 93a611e4c066
+Create Date: 2024-10-31 09:36:50.360710
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.engine.reflection import Inspector
+
+
+# revision identifiers, used by Alembic.
+revision = 'fa012ec405b8'
+down_revision = '93a611e4c066'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    conn = op.get_bind()
+    inspector = sa.inspect(conn)
+    columns = [col['name'] for col in inspector.get_columns('groups')]
+
+    if 'description' not in columns:
+        op.add_column('groups', sa.Column('description', sa.String(), nullable=True))
+    if 'orchestrator' not in columns:
+        op.add_column('groups', sa.Column('orchestrator', sa.String(), nullable=True))
+    if 'orchestrator_sheet' not in columns:
+        op.add_column('groups', sa.Column('orchestrator_sheet', sa.String(), nullable=True))
+    if 'permissions' not in columns:
+        op.add_column('groups', sa.Column('permissions', sa.JSON(), nullable=True))
+    if 'domains' not in columns:
+        op.add_column('groups', sa.Column('domains', sa.JSON(), nullable=True))
+
+
+def downgrade() -> None:
+    conn = op.get_bind()
+    inspector = sa.inspect(conn)
+    columns = [col['name'] for col in inspector.get_columns('groups')]
+
+    column_names = ['description', 'orchestrator', 'orchestrator_sheet', 'permissions', 'domains']
+    for column_name in column_names:
+        if column_name in columns:
+            op.drop_column('groups', column_name)