From 01af0ead1d1508c17a26f8e20dca83fcbe5f57d2 Mon Sep 17 00:00:00 2001
From: michplunkett <5885605+michplunkett@users.noreply.github.com>
Date: Fri, 21 Feb 2025 14:29:02 -0600
Subject: [PATCH 01/40] Update .gitignore
---
.gitignore | 148 +++++++++++++++++++++++++++++++++++++++++++++++++----
1 file changed, 137 insertions(+), 11 deletions(-)
diff --git a/.gitignore b/.gitignore
index 562885d..15bb465 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,16 +1,10 @@
+# Misc.
user-groups.dev.yaml
user-groups.yaml
orchestration.yaml
my-archives
*.pyc
-.DS_Store
secrets/*
-*.log
-__pycache__
-.pytest_cache
-.env
-.env.dev
-.env.prod
*.db
redis/data/*
.ipynb_checkpoints*
@@ -18,8 +12,6 @@ app/user-groups.yaml
app/user-groups.dev.yaml
wit*
app/crawls
-.coverage
-.pytest_cache/
htmlcov
local_archive
local_archive_test
@@ -27,6 +19,140 @@ local_archive_test
*db-shm
copy-files.sh
temp/
-.python-version
orchestration2.yaml
-database
\ No newline at end of file
+database
+
+# IDE files
+.idea
+.vscode
+**/.DS_Store
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+# Usually these files are written by a python script from a template
+# before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+# However, in case of collaboration, if having platform-specific dependencies or dependencies
+# having no cross-platform support, pipenv may install dependencies that don't work, or not
+# install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env*
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
From 7d04b69488ef8136e0de838e7acf8762599f80e8 Mon Sep 17 00:00:00 2001
From: michplunkett <5885605+michplunkett@users.noreply.github.com>
Date: Fri, 21 Feb 2025 14:29:57 -0600
Subject: [PATCH 02/40] Create pull_request_template.md
---
.github/pull_request_template.md | 23 +++++++++++++++++++++++
1 file changed, 23 insertions(+)
create mode 100644 .github/pull_request_template.md
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
new file mode 100644
index 0000000..3ec80e9
--- /dev/null
+++ b/.github/pull_request_template.md
@@ -0,0 +1,23 @@
+
+## Describe your changes
+
+
+## Non-obvious technical information
+
+
+## Checklist before requesting a review
+
+- [ ] The code runs successfully.
+
+```commandline
+HERE IS SOME COMMAND LINE OUTPUT
+```
From 793e8db808eaba1e52b9ab12184bfeed544a5be4 Mon Sep 17 00:00:00 2001
From: michplunkett <5885605+michplunkett@users.noreply.github.com>
Date: Fri, 21 Feb 2025 15:00:00 -0600
Subject: [PATCH 03/40] Update poetry.lock
---
poetry.lock | 269 ++++++++++++++++++++++++++--------------------------
1 file changed, 135 insertions(+), 134 deletions(-)
diff --git a/poetry.lock b/poetry.lock
index 79f7907..ceb824a 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 2.0.1 and should not be changed by hand.
+# This file is automatically @generated by Poetry 2.1.1 and should not be changed by hand.
[[package]]
name = "aiosmtplib"
@@ -53,7 +53,7 @@ SQLAlchemy = ">=1.3.0"
typing-extensions = ">=4"
[package.extras]
-tz = ["backports.zoneinfo", "tzdata"]
+tz = ["backports.zoneinfo ; python_version < \"3.9\"", "tzdata"]
[[package]]
name = "amqp"
@@ -102,7 +102,7 @@ typing_extensions = {version = ">=4.5", markers = "python_version < \"3.13\""}
[package.extras]
doc = ["Sphinx (>=7.4,<8.0)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx_rtd_theme"]
-test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "trustme", "truststore (>=0.9.1)", "uvloop (>=0.21)"]
+test = ["anyio[trio]", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "trustme", "truststore (>=0.9.1) ; python_version >= \"3.10\"", "uvloop (>=0.21) ; platform_python_implementation == \"CPython\" and platform_system != \"Windows\" and python_version < \"3.14\""]
trio = ["trio (>=0.26.1)"]
[[package]]
@@ -130,12 +130,12 @@ files = [
]
[package.extras]
-benchmark = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins", "pytest-xdist[psutil]"]
-cov = ["cloudpickle", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"]
-dev = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pre-commit-uv", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"]
+benchmark = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"]
+cov = ["cloudpickle ; platform_python_implementation == \"CPython\"", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"]
+dev = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pre-commit-uv", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"]
docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier (<24.7)"]
-tests = ["cloudpickle", "hypothesis", "mypy (>=1.11.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"]
-tests-mypy = ["mypy (>=1.11.1)", "pytest-mypy-plugins"]
+tests = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-xdist[psutil]"]
+tests-mypy = ["mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\"", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.10\""]
[[package]]
name = "authlib"
@@ -250,18 +250,18 @@ files = [
[[package]]
name = "boto3"
-version = "1.36.21"
+version = "1.36.26"
description = "The AWS SDK for Python"
optional = false
python-versions = ">=3.8"
groups = ["main"]
files = [
- {file = "boto3-1.36.21-py3-none-any.whl", hash = "sha256:f94faa7cf932d781f474d87f8b4c14a033af95ac1460136b40d75e7a30086ef0"},
- {file = "boto3-1.36.21.tar.gz", hash = "sha256:41eb2b73eb612d300e629e3328b83f1ffea0fc6633e75c241a72a76746c1db26"},
+ {file = "boto3-1.36.26-py3-none-any.whl", hash = "sha256:f67d014a7c5a3cd540606d64d7cb9eec3600cf42acab1ac0518df9751ae115e2"},
+ {file = "boto3-1.36.26.tar.gz", hash = "sha256:523b69457eee55ac15aa707c0e768b2a45ca1521f95b2442931090633ec72458"},
]
[package.dependencies]
-botocore = ">=1.36.21,<1.37.0"
+botocore = ">=1.36.26,<1.37.0"
jmespath = ">=0.7.1,<2.0.0"
s3transfer = ">=0.11.0,<0.12.0"
@@ -270,14 +270,14 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"]
[[package]]
name = "botocore"
-version = "1.36.21"
+version = "1.36.26"
description = "Low-level, data-driven core of boto 3."
optional = false
python-versions = ">=3.8"
groups = ["main"]
files = [
- {file = "botocore-1.36.21-py3-none-any.whl", hash = "sha256:24a7052e792639dc2726001bd474cd0aaa959c1e18ddd92c17f3adc6efa1b132"},
- {file = "botocore-1.36.21.tar.gz", hash = "sha256:da746240e2ad64fd4997f7f3664a0a8e303d18075fc1d473727cb6375080ea16"},
+ {file = "botocore-1.36.26-py3-none-any.whl", hash = "sha256:4e3f19913887a58502e71ef8d696fe7eaa54de7813ff73390cd5883f837dfa6e"},
+ {file = "botocore-1.36.26.tar.gz", hash = "sha256:4a63bcef7ecf6146fd3a61dc4f9b33b7473b49bdaf1770e9aaca6eee0c9eab62"},
]
[package.dependencies]
@@ -441,14 +441,14 @@ beautifulsoup4 = "*"
[[package]]
name = "cachetools"
-version = "5.5.1"
+version = "5.5.2"
description = "Extensible memoizing collections and decorators"
optional = false
python-versions = ">=3.7"
groups = ["main"]
files = [
- {file = "cachetools-5.5.1-py3-none-any.whl", hash = "sha256:b76651fdc3b24ead3c648bbdeeb940c1b04d365b38b4af66788f9ec4a81d42bb"},
- {file = "cachetools-5.5.1.tar.gz", hash = "sha256:70f238fbba50383ef62e55c6aff6d9673175fe59f7c6782c7a0b9e38f4a9df95"},
+ {file = "cachetools-5.5.2-py3-none-any.whl", hash = "sha256:d26a22bcc62eb95c3beabd9f1ee5e820d3d2704fe2967cbe350e20c8ffcd3f0a"},
+ {file = "cachetools-5.5.2.tar.gz", hash = "sha256:1a661caa9175d26759571b2e19580f9d6393969e5dfca11fdb1f947a23e640d4"},
]
[[package]]
@@ -478,32 +478,32 @@ vine = ">=5.1.0,<6.0"
arangodb = ["pyArango (>=2.0.2)"]
auth = ["cryptography (==42.0.5)"]
azureblockblob = ["azure-storage-blob (>=12.15.0)"]
-brotli = ["brotli (>=1.0.0)", "brotlipy (>=0.7.0)"]
+brotli = ["brotli (>=1.0.0) ; platform_python_implementation == \"CPython\"", "brotlipy (>=0.7.0) ; platform_python_implementation == \"PyPy\""]
cassandra = ["cassandra-driver (>=3.25.0,<4)"]
consul = ["python-consul2 (==0.1.5)"]
cosmosdbsql = ["pydocumentdb (==2.3.5)"]
-couchbase = ["couchbase (>=3.0.0)"]
+couchbase = ["couchbase (>=3.0.0) ; platform_python_implementation != \"PyPy\" and (platform_system != \"Windows\" or python_version < \"3.10\")"]
couchdb = ["pycouchdb (==1.14.2)"]
django = ["Django (>=2.2.28)"]
dynamodb = ["boto3 (>=1.26.143)"]
elasticsearch = ["elastic-transport (<=8.13.0)", "elasticsearch (<=8.13.0)"]
-eventlet = ["eventlet (>=0.32.0)"]
+eventlet = ["eventlet (>=0.32.0) ; python_version < \"3.10\""]
gcs = ["google-cloud-storage (>=2.10.0)"]
gevent = ["gevent (>=1.5.0)"]
-librabbitmq = ["librabbitmq (>=2.0.0)"]
-memcache = ["pylibmc (==1.6.3)"]
+librabbitmq = ["librabbitmq (>=2.0.0) ; python_version < \"3.11\""]
+memcache = ["pylibmc (==1.6.3) ; platform_system != \"Windows\""]
mongodb = ["pymongo[srv] (>=4.0.2)"]
msgpack = ["msgpack (==1.0.8)"]
pymemcache = ["python-memcached (>=1.61)"]
-pyro = ["pyro4 (==4.82)"]
+pyro = ["pyro4 (==4.82) ; python_version < \"3.11\""]
pytest = ["pytest-celery[all] (>=1.0.0)"]
redis = ["redis (>=4.5.2,!=4.5.5,<6.0.0)"]
s3 = ["boto3 (>=1.26.143)"]
slmq = ["softlayer-messaging (>=1.0.3)"]
-solar = ["ephem (==4.1.5)"]
+solar = ["ephem (==4.1.5) ; platform_python_implementation != \"PyPy\""]
sqlalchemy = ["sqlalchemy (>=1.4.48,<2.1)"]
-sqs = ["boto3 (>=1.26.143)", "kombu[sqs] (>=5.3.4)", "pycurl (>=7.43.0.5)", "urllib3 (>=1.26.16)"]
-tblib = ["tblib (>=1.3.0)", "tblib (>=1.5.0)"]
+sqs = ["boto3 (>=1.26.143)", "kombu[sqs] (>=5.3.4)", "pycurl (>=7.43.0.5) ; sys_platform != \"win32\" and platform_python_implementation == \"CPython\"", "urllib3 (>=1.26.16)"]
+tblib = ["tblib (>=1.3.0) ; python_version < \"3.8.0\"", "tblib (>=1.5.0) ; python_version >= \"3.8.0\""]
yaml = ["PyYAML (>=3.10)"]
zookeeper = ["kazoo (>=1.3.1)"]
zstd = ["zstandard (==0.22.0)"]
@@ -872,7 +872,7 @@ files = [
]
[package.extras]
-toml = ["tomli"]
+toml = ["tomli ; python_full_version <= \"3.11.0a6\""]
[[package]]
name = "cryptography"
@@ -1130,7 +1130,7 @@ requests = ">=2.18.0,<3.0.0.dev0"
[package.extras]
async-rest = ["google-auth[aiohttp] (>=2.35.0,<3.0.dev0)"]
-grpc = ["grpcio (>=1.33.2,<2.0dev)", "grpcio (>=1.49.1,<2.0dev)", "grpcio-status (>=1.33.2,<2.0.dev0)", "grpcio-status (>=1.49.1,<2.0.dev0)"]
+grpc = ["grpcio (>=1.33.2,<2.0dev)", "grpcio (>=1.49.1,<2.0dev) ; python_version >= \"3.11\"", "grpcio-status (>=1.33.2,<2.0.dev0)", "grpcio-status (>=1.49.1,<2.0.dev0) ; python_version >= \"3.11\""]
grpcgcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"]
grpcio-gcp = ["grpcio-gcp (>=0.2.2,<1.0.dev0)"]
@@ -1215,14 +1215,14 @@ tool = ["click (>=6.0.0)"]
[[package]]
name = "googleapis-common-protos"
-version = "1.67.0"
+version = "1.68.0"
description = "Common protobufs used in Google APIs"
optional = false
python-versions = ">=3.7"
groups = ["main"]
files = [
- {file = "googleapis_common_protos-1.67.0-py2.py3-none-any.whl", hash = "sha256:579de760800d13616f51cf8be00c876f00a9f146d3e6510e19d1f4111758b741"},
- {file = "googleapis_common_protos-1.67.0.tar.gz", hash = "sha256:21398025365f138be356d5923e9168737d94d46a72aefee4a6110a1f23463c86"},
+ {file = "googleapis_common_protos-1.68.0-py2.py3-none-any.whl", hash = "sha256:aaf179b2f81df26dfadac95def3b16a95064c76a5f45f07e4c68a21bb371c4ac"},
+ {file = "googleapis_common_protos-1.68.0.tar.gz", hash = "sha256:95d38161f4f9af0d9423eed8fb7b64ffd2568c3464eb542ff02c5bfa1953ab3c"},
]
[package.dependencies]
@@ -1403,7 +1403,7 @@ httpcore = "==1.*"
idna = "*"
[package.extras]
-brotli = ["brotli", "brotlicffi"]
+brotli = ["brotli ; platform_python_implementation == \"CPython\"", "brotlicffi ; platform_python_implementation != \"CPython\""]
cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"]
http2 = ["h2 (>=3,<5)"]
socks = ["socksio (==1.*)"]
@@ -1521,7 +1521,7 @@ azureservicebus = ["azure-servicebus (>=7.10.0)"]
azurestoragequeues = ["azure-identity (>=1.12.0)", "azure-storage-queue (>=12.6.0)"]
confluentkafka = ["confluent-kafka (>=2.2.0)"]
consul = ["python-consul2 (==0.1.5)"]
-librabbitmq = ["librabbitmq (>=2.0.0)"]
+librabbitmq = ["librabbitmq (>=2.0.0) ; python_version < \"3.11\""]
mongodb = ["pymongo (>=4.1.1)"]
msgpack = ["msgpack (==1.1.0)"]
pyro = ["pyro4 (==4.82)"]
@@ -1529,7 +1529,7 @@ qpid = ["qpid-python (>=0.26)", "qpid-tools (>=0.26)"]
redis = ["redis (>=4.5.2,!=4.5.5,!=5.0.2)"]
slmq = ["softlayer-messaging (>=1.0.3)"]
sqlalchemy = ["sqlalchemy (>=1.4.48,<2.1)"]
-sqs = ["boto3 (>=1.26.143)", "pycurl (>=7.43.0.5)", "urllib3 (>=1.26.16)"]
+sqs = ["boto3 (>=1.26.143)", "pycurl (>=7.43.0.5) ; sys_platform != \"win32\" and platform_python_implementation == \"CPython\"", "urllib3 (>=1.26.16)"]
yaml = ["PyYAML (>=3.10)"]
zookeeper = ["kazoo (>=2.8.0)"]
@@ -1550,7 +1550,7 @@ colorama = {version = ">=0.3.4", markers = "sys_platform == \"win32\""}
win32-setctime = {version = ">=1.0.0", markers = "sys_platform == \"win32\""}
[package.extras]
-dev = ["Sphinx (==8.1.3)", "build (==1.2.2)", "colorama (==0.4.5)", "colorama (==0.4.6)", "exceptiongroup (==1.1.3)", "freezegun (==1.1.0)", "freezegun (==1.5.0)", "mypy (==v0.910)", "mypy (==v0.971)", "mypy (==v1.13.0)", "mypy (==v1.4.1)", "myst-parser (==4.0.0)", "pre-commit (==4.0.1)", "pytest (==6.1.2)", "pytest (==8.3.2)", "pytest-cov (==2.12.1)", "pytest-cov (==5.0.0)", "pytest-cov (==6.0.0)", "pytest-mypy-plugins (==1.9.3)", "pytest-mypy-plugins (==3.1.0)", "sphinx-rtd-theme (==3.0.2)", "tox (==3.27.1)", "tox (==4.23.2)", "twine (==6.0.1)"]
+dev = ["Sphinx (==8.1.3) ; python_version >= \"3.11\"", "build (==1.2.2) ; python_version >= \"3.11\"", "colorama (==0.4.5) ; python_version < \"3.8\"", "colorama (==0.4.6) ; python_version >= \"3.8\"", "exceptiongroup (==1.1.3) ; python_version >= \"3.7\" and python_version < \"3.11\"", "freezegun (==1.1.0) ; python_version < \"3.8\"", "freezegun (==1.5.0) ; python_version >= \"3.8\"", "mypy (==v0.910) ; python_version < \"3.6\"", "mypy (==v0.971) ; python_version == \"3.6\"", "mypy (==v1.13.0) ; python_version >= \"3.8\"", "mypy (==v1.4.1) ; python_version == \"3.7\"", "myst-parser (==4.0.0) ; python_version >= \"3.11\"", "pre-commit (==4.0.1) ; python_version >= \"3.9\"", "pytest (==6.1.2) ; python_version < \"3.8\"", "pytest (==8.3.2) ; python_version >= \"3.8\"", "pytest-cov (==2.12.1) ; python_version < \"3.8\"", "pytest-cov (==5.0.0) ; python_version == \"3.8\"", "pytest-cov (==6.0.0) ; python_version >= \"3.9\"", "pytest-mypy-plugins (==1.9.3) ; python_version >= \"3.6\" and python_version < \"3.8\"", "pytest-mypy-plugins (==3.1.0) ; python_version >= \"3.8\"", "sphinx-rtd-theme (==3.0.2) ; python_version >= \"3.11\"", "tox (==3.27.1) ; python_version < \"3.8\"", "tox (==4.23.2) ; python_version >= \"3.8\"", "twine (==6.0.1) ; python_version >= \"3.11\""]
[[package]]
name = "mako"
@@ -1978,7 +1978,7 @@ docs = ["furo", "olefile", "sphinx (>=8.1)", "sphinx-copybutton", "sphinx-inline
fpx = ["olefile"]
mic = ["olefile"]
tests = ["check-manifest", "coverage (>=7.4.2)", "defusedxml", "markdown2", "olefile", "packaging", "pyroma", "pytest", "pytest-cov", "pytest-timeout", "trove-classifiers (>=2024.10.12)"]
-typing = ["typing-extensions"]
+typing = ["typing-extensions ; python_version < \"3.10\""]
xmp = ["defusedxml"]
[[package]]
@@ -2109,7 +2109,7 @@ files = [
]
[package.extras]
-test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"]
+test = ["enum34 ; python_version <= \"3.4\"", "ipaddress ; python_version < \"3.0\"", "mock ; python_version < \"3.0\"", "pywin32 ; sys_platform == \"win32\"", "wmi ; sys_platform == \"win32\""]
[[package]]
name = "pyaes"
@@ -2222,7 +2222,7 @@ typing-extensions = ">=4.12.2"
[package.extras]
email = ["email-validator (>=2.0.0)"]
-timezone = ["tzdata"]
+timezone = ["tzdata ; python_version >= \"3.9\" and platform_system == \"Windows\""]
[[package]]
name = "pydantic-core"
@@ -2339,14 +2339,14 @@ typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0"
[[package]]
name = "pydantic-settings"
-version = "2.7.1"
+version = "2.8.0"
description = "Settings management using Pydantic"
optional = false
python-versions = ">=3.8"
groups = ["main", "web"]
files = [
- {file = "pydantic_settings-2.7.1-py3-none-any.whl", hash = "sha256:590be9e6e24d06db33a4262829edef682500ef008565a969c73d39d5f8bfb3fd"},
- {file = "pydantic_settings-2.7.1.tar.gz", hash = "sha256:10c9caad35e64bfb3c2fbf70a078c0e25cc92499782e5200747f942a065dec93"},
+ {file = "pydantic_settings-2.8.0-py3-none-any.whl", hash = "sha256:c782c7dc3fb40e97b238e713c25d26f64314aece2e91abcff592fcac15f71820"},
+ {file = "pydantic_settings-2.8.0.tar.gz", hash = "sha256:88e2ca28f6e68ea102c99c3c401d6c9078e68a5df600e97b43891c34e089500a"},
]
[package.dependencies]
@@ -2936,14 +2936,14 @@ crt = ["botocore[crt] (>=1.36.0,<2.0a.0)"]
[[package]]
name = "selenium"
-version = "4.28.1"
+version = "4.29.0"
description = "Official Python bindings for Selenium WebDriver"
optional = false
python-versions = ">=3.9"
groups = ["main"]
files = [
- {file = "selenium-4.28.1-py3-none-any.whl", hash = "sha256:4238847e45e24e4472cfcf3554427512c7aab9443396435b1623ef406fff1cc1"},
- {file = "selenium-4.28.1.tar.gz", hash = "sha256:0072d08670d7ec32db901bd0107695a330cecac9f196e3afb3fa8163026e022a"},
+ {file = "selenium-4.29.0-py3-none-any.whl", hash = "sha256:ce5d26f1ddc1111641113653af33694c13947dd36c2df09cdd33f554351d372e"},
+ {file = "selenium-4.29.0.tar.gz", hash = "sha256:3a62f7ec33e669364a6c0562a701deb69745b569c50d55f1a912bf8eb33358ba"},
]
[package.dependencies]
@@ -2967,13 +2967,13 @@ files = [
]
[package.extras]
-check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)", "ruff (>=0.8.0)"]
-core = ["importlib_metadata (>=6)", "jaraco.collections", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging", "packaging (>=24.2)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"]
+check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\"", "ruff (>=0.8.0) ; sys_platform != \"cygwin\""]
+core = ["importlib_metadata (>=6) ; python_version < \"3.10\"", "jaraco.collections", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging", "packaging (>=24.2)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1) ; python_version < \"3.11\"", "wheel (>=0.43.0)"]
cover = ["pytest-cov"]
doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"]
enabler = ["pytest-enabler (>=2.2)"]
-test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.7.2)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"]
-type = ["importlib_metadata (>=7.0.2)", "jaraco.develop (>=7.21)", "mypy (==1.14.*)", "pytest-mypy"]
+test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21) ; python_version >= \"3.9\" and sys_platform != \"cygwin\"", "jaraco.envs (>=2.2)", "jaraco.path (>=3.7.2)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf ; sys_platform != \"cygwin\"", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"]
+type = ["importlib_metadata (>=7.0.2) ; python_version < \"3.10\"", "jaraco.develop (>=7.21) ; sys_platform != \"cygwin\"", "mypy (==1.14.*)", "pytest-mypy"]
[[package]]
name = "six"
@@ -3139,14 +3139,14 @@ full = ["httpx (>=0.27.0,<0.29.0)", "itsdangerous", "jinja2", "python-multipart
[[package]]
name = "telethon"
-version = "1.38.1"
+version = "1.39.0"
description = "Full-featured Telegram client library for Python 3"
optional = false
python-versions = ">=3.5"
groups = ["main"]
files = [
- {file = "Telethon-1.38.1-py3-none-any.whl", hash = "sha256:30c187017501bfb982b8af5659f864dda4108f77ea49cfce61e8f6fdb8a18d6e"},
- {file = "Telethon-1.38.1.tar.gz", hash = "sha256:f9866c1e37197a0894e0c02aa56a6359bffb14a585e88e18e3e819df4fda399a"},
+ {file = "Telethon-1.39.0-py3-none-any.whl", hash = "sha256:aa9f394b94be144799a6f6a93ab463867bc7c63503ede9631751940a98f6c703"},
+ {file = "telethon-1.39.0.tar.gz", hash = "sha256:35d4795d8c91deac515fb0bcb3723866b924de1c724e1d5c230460e96f284a63"},
]
[package.dependencies]
@@ -3256,18 +3256,19 @@ sortedcontainers = "*"
[[package]]
name = "trio-websocket"
-version = "0.11.1"
+version = "0.12.1"
description = "WebSocket library for Trio"
optional = false
-python-versions = ">=3.7"
+python-versions = ">=3.8"
groups = ["main"]
files = [
- {file = "trio-websocket-0.11.1.tar.gz", hash = "sha256:18c11793647703c158b1f6e62de638acada927344d534e3c7628eedcb746839f"},
- {file = "trio_websocket-0.11.1-py3-none-any.whl", hash = "sha256:520d046b0d030cf970b8b2b2e00c4c2245b3807853ecd44214acd33d74581638"},
+ {file = "trio_websocket-0.12.1-py3-none-any.whl", hash = "sha256:608ec746bb287e5d5a66baf483e41194193c5cf05ffaad6240e7d1fcd80d1e6f"},
+ {file = "trio_websocket-0.12.1.tar.gz", hash = "sha256:d55ccd4d3eae27c494f3fdae14823317839bdcb8214d1173eacc4d42c69fc91b"},
]
[package.dependencies]
exceptiongroup = {version = "*", markers = "python_version < \"3.11\""}
+outcome = ">=1.2.0"
trio = ">=0.11"
wsproto = ">=0.14"
@@ -3377,7 +3378,7 @@ files = [
pysocks = {version = ">=1.5.6,<1.5.7 || >1.5.7,<2.0", optional = true, markers = "extra == \"socks\""}
[package.extras]
-brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"]
+brotli = ["brotli (>=1.0.9) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\""]
h2 = ["h2 (>=4,<5)"]
socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"]
zstd = ["zstandard (>=0.18.0)"]
@@ -3400,7 +3401,7 @@ h11 = ">=0.8"
typing-extensions = {version = ">=4.0", markers = "python_version < \"3.11\""}
[package.extras]
-standard = ["colorama (>=0.4)", "httptools (>=0.6.3)", "python-dotenv (>=0.13)", "pyyaml (>=5.1)", "uvloop (>=0.14.0,!=0.15.0,!=0.15.1)", "watchfiles (>=0.13)", "websockets (>=10.4)"]
+standard = ["colorama (>=0.4) ; sys_platform == \"win32\"", "httptools (>=0.6.3)", "python-dotenv (>=0.13)", "pyyaml (>=5.1)", "uvloop (>=0.14.0,!=0.15.0,!=0.15.1) ; sys_platform != \"win32\" and sys_platform != \"cygwin\" and platform_python_implementation != \"PyPy\"", "watchfiles (>=0.13)", "websockets (>=10.4)"]
[[package]]
name = "vine"
@@ -3554,81 +3555,81 @@ test = ["websockets"]
[[package]]
name = "websockets"
-version = "14.2"
+version = "15.0"
description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)"
optional = false
python-versions = ">=3.9"
groups = ["main"]
files = [
- {file = "websockets-14.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e8179f95323b9ab1c11723e5d91a89403903f7b001828161b480a7810b334885"},
- {file = "websockets-14.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0d8c3e2cdb38f31d8bd7d9d28908005f6fa9def3324edb9bf336d7e4266fd397"},
- {file = "websockets-14.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:714a9b682deb4339d39ffa674f7b674230227d981a37d5d174a4a83e3978a610"},
- {file = "websockets-14.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2e53c72052f2596fb792a7acd9704cbc549bf70fcde8a99e899311455974ca3"},
- {file = "websockets-14.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e3fbd68850c837e57373d95c8fe352203a512b6e49eaae4c2f4088ef8cf21980"},
- {file = "websockets-14.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b27ece32f63150c268593d5fdb82819584831a83a3f5809b7521df0685cd5d8"},
- {file = "websockets-14.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4daa0faea5424d8713142b33825fff03c736f781690d90652d2c8b053345b0e7"},
- {file = "websockets-14.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:bc63cee8596a6ec84d9753fd0fcfa0452ee12f317afe4beae6b157f0070c6c7f"},
- {file = "websockets-14.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:7a570862c325af2111343cc9b0257b7119b904823c675b22d4ac547163088d0d"},
- {file = "websockets-14.2-cp310-cp310-win32.whl", hash = "sha256:75862126b3d2d505e895893e3deac0a9339ce750bd27b4ba515f008b5acf832d"},
- {file = "websockets-14.2-cp310-cp310-win_amd64.whl", hash = "sha256:cc45afb9c9b2dc0852d5c8b5321759cf825f82a31bfaf506b65bf4668c96f8b2"},
- {file = "websockets-14.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3bdc8c692c866ce5fefcaf07d2b55c91d6922ac397e031ef9b774e5b9ea42166"},
- {file = "websockets-14.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c93215fac5dadc63e51bcc6dceca72e72267c11def401d6668622b47675b097f"},
- {file = "websockets-14.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1c9b6535c0e2cf8a6bf938064fb754aaceb1e6a4a51a80d884cd5db569886910"},
- {file = "websockets-14.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a52a6d7cf6938e04e9dceb949d35fbdf58ac14deea26e685ab6368e73744e4c"},
- {file = "websockets-14.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9f05702e93203a6ff5226e21d9b40c037761b2cfb637187c9802c10f58e40473"},
- {file = "websockets-14.2-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:22441c81a6748a53bfcb98951d58d1af0661ab47a536af08920d129b4d1c3473"},
- {file = "websockets-14.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:efd9b868d78b194790e6236d9cbc46d68aba4b75b22497eb4ab64fa640c3af56"},
- {file = "websockets-14.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1a5a20d5843886d34ff8c57424cc65a1deda4375729cbca4cb6b3353f3ce4142"},
- {file = "websockets-14.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:34277a29f5303d54ec6468fb525d99c99938607bc96b8d72d675dee2b9f5bf1d"},
- {file = "websockets-14.2-cp311-cp311-win32.whl", hash = "sha256:02687db35dbc7d25fd541a602b5f8e451a238ffa033030b172ff86a93cb5dc2a"},
- {file = "websockets-14.2-cp311-cp311-win_amd64.whl", hash = "sha256:862e9967b46c07d4dcd2532e9e8e3c2825e004ffbf91a5ef9dde519ee2effb0b"},
- {file = "websockets-14.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:1f20522e624d7ffbdbe259c6b6a65d73c895045f76a93719aa10cd93b3de100c"},
- {file = "websockets-14.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:647b573f7d3ada919fd60e64d533409a79dcf1ea21daeb4542d1d996519ca967"},
- {file = "websockets-14.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6af99a38e49f66be5a64b1e890208ad026cda49355661549c507152113049990"},
- {file = "websockets-14.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:091ab63dfc8cea748cc22c1db2814eadb77ccbf82829bac6b2fbe3401d548eda"},
- {file = "websockets-14.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b374e8953ad477d17e4851cdc66d83fdc2db88d9e73abf755c94510ebddceb95"},
- {file = "websockets-14.2-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a39d7eceeea35db85b85e1169011bb4321c32e673920ae9c1b6e0978590012a3"},
- {file = "websockets-14.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0a6f3efd47ffd0d12080594f434faf1cd2549b31e54870b8470b28cc1d3817d9"},
- {file = "websockets-14.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:065ce275e7c4ffb42cb738dd6b20726ac26ac9ad0a2a48e33ca632351a737267"},
- {file = "websockets-14.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e9d0e53530ba7b8b5e389c02282f9d2aa47581514bd6049d3a7cffe1385cf5fe"},
- {file = "websockets-14.2-cp312-cp312-win32.whl", hash = "sha256:20e6dd0984d7ca3037afcb4494e48c74ffb51e8013cac71cf607fffe11df7205"},
- {file = "websockets-14.2-cp312-cp312-win_amd64.whl", hash = "sha256:44bba1a956c2c9d268bdcdf234d5e5ff4c9b6dc3e300545cbe99af59dda9dcce"},
- {file = "websockets-14.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:6f1372e511c7409a542291bce92d6c83320e02c9cf392223272287ce55bc224e"},
- {file = "websockets-14.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:4da98b72009836179bb596a92297b1a61bb5a830c0e483a7d0766d45070a08ad"},
- {file = "websockets-14.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f8a86a269759026d2bde227652b87be79f8a734e582debf64c9d302faa1e9f03"},
- {file = "websockets-14.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:86cf1aaeca909bf6815ea714d5c5736c8d6dd3a13770e885aafe062ecbd04f1f"},
- {file = "websockets-14.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9b0f6c3ba3b1240f602ebb3971d45b02cc12bd1845466dd783496b3b05783a5"},
- {file = "websockets-14.2-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:669c3e101c246aa85bc8534e495952e2ca208bd87994650b90a23d745902db9a"},
- {file = "websockets-14.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:eabdb28b972f3729348e632ab08f2a7b616c7e53d5414c12108c29972e655b20"},
- {file = "websockets-14.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:2066dc4cbcc19f32c12a5a0e8cc1b7ac734e5b64ac0a325ff8353451c4b15ef2"},
- {file = "websockets-14.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ab95d357cd471df61873dadf66dd05dd4709cae001dd6342edafc8dc6382f307"},
- {file = "websockets-14.2-cp313-cp313-win32.whl", hash = "sha256:a9e72fb63e5f3feacdcf5b4ff53199ec8c18d66e325c34ee4c551ca748623bbc"},
- {file = "websockets-14.2-cp313-cp313-win_amd64.whl", hash = "sha256:b439ea828c4ba99bb3176dc8d9b933392a2413c0f6b149fdcba48393f573377f"},
- {file = "websockets-14.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7cd5706caec1686c5d233bc76243ff64b1c0dc445339bd538f30547e787c11fe"},
- {file = "websockets-14.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ec607328ce95a2f12b595f7ae4c5d71bf502212bddcea528290b35c286932b12"},
- {file = "websockets-14.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:da85651270c6bfb630136423037dd4975199e5d4114cae6d3066641adcc9d1c7"},
- {file = "websockets-14.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c3ecadc7ce90accf39903815697917643f5b7cfb73c96702318a096c00aa71f5"},
- {file = "websockets-14.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1979bee04af6a78608024bad6dfcc0cc930ce819f9e10342a29a05b5320355d0"},
- {file = "websockets-14.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2dddacad58e2614a24938a50b85969d56f88e620e3f897b7d80ac0d8a5800258"},
- {file = "websockets-14.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:89a71173caaf75fa71a09a5f614f450ba3ec84ad9fca47cb2422a860676716f0"},
- {file = "websockets-14.2-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:6af6a4b26eea4fc06c6818a6b962a952441e0e39548b44773502761ded8cc1d4"},
- {file = "websockets-14.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:80c8efa38957f20bba0117b48737993643204645e9ec45512579132508477cfc"},
- {file = "websockets-14.2-cp39-cp39-win32.whl", hash = "sha256:2e20c5f517e2163d76e2729104abc42639c41cf91f7b1839295be43302713661"},
- {file = "websockets-14.2-cp39-cp39-win_amd64.whl", hash = "sha256:b4c8cef610e8d7c70dea92e62b6814a8cd24fbd01d7103cc89308d2bfe1659ef"},
- {file = "websockets-14.2-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:d7d9cafbccba46e768be8a8ad4635fa3eae1ffac4c6e7cb4eb276ba41297ed29"},
- {file = "websockets-14.2-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:c76193c1c044bd1e9b3316dcc34b174bbf9664598791e6fb606d8d29000e070c"},
- {file = "websockets-14.2-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fd475a974d5352390baf865309fe37dec6831aafc3014ffac1eea99e84e83fc2"},
- {file = "websockets-14.2-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2c6c0097a41968b2e2b54ed3424739aab0b762ca92af2379f152c1aef0187e1c"},
- {file = "websockets-14.2-pp310-pypy310_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d7ff794c8b36bc402f2e07c0b2ceb4a2424147ed4785ff03e2a7af03711d60a"},
- {file = "websockets-14.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:dec254fcabc7bd488dab64846f588fc5b6fe0d78f641180030f8ea27b76d72c3"},
- {file = "websockets-14.2-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:bbe03eb853e17fd5b15448328b4ec7fb2407d45fb0245036d06a3af251f8e48f"},
- {file = "websockets-14.2-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:a3c4aa3428b904d5404a0ed85f3644d37e2cb25996b7f096d77caeb0e96a3b42"},
- {file = "websockets-14.2-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:577a4cebf1ceaf0b65ffc42c54856214165fb8ceeba3935852fc33f6b0c55e7f"},
- {file = "websockets-14.2-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ad1c1d02357b7665e700eca43a31d52814ad9ad9b89b58118bdabc365454b574"},
- {file = "websockets-14.2-pp39-pypy39_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f390024a47d904613577df83ba700bd189eedc09c57af0a904e5c39624621270"},
- {file = "websockets-14.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:3c1426c021c38cf92b453cdf371228d3430acd775edee6bac5a4d577efc72365"},
- {file = "websockets-14.2-py3-none-any.whl", hash = "sha256:7a6ceec4ea84469f15cf15807a747e9efe57e369c384fa86e022b3bea679b79b"},
- {file = "websockets-14.2.tar.gz", hash = "sha256:5059ed9c54945efb321f097084b4c7e52c246f2c869815876a69d1efc4ad6eb5"},
+ {file = "websockets-15.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5e6ee18a53dd5743e6155b8ff7e8e477c25b29b440f87f65be8165275c87fef0"},
+ {file = "websockets-15.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ee06405ea2e67366a661ed313e14cf2a86e84142a3462852eb96348f7219cee3"},
+ {file = "websockets-15.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8711682a629bbcaf492f5e0af72d378e976ea1d127a2d47584fa1c2c080b436b"},
+ {file = "websockets-15.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:94c4a9b01eede952442c088d415861b0cf2053cbd696b863f6d5022d4e4e2453"},
+ {file = "websockets-15.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:45535fead66e873f411c1d3cf0d3e175e66f4dd83c4f59d707d5b3e4c56541c4"},
+ {file = "websockets-15.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e389efe46ccb25a1f93d08c7a74e8123a2517f7b7458f043bd7529d1a63ffeb"},
+ {file = "websockets-15.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:67a04754d121ea5ca39ddedc3f77071651fb5b0bc6b973c71c515415b44ed9c5"},
+ {file = "websockets-15.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:bd66b4865c8b853b8cca7379afb692fc7f52cf898786537dfb5e5e2d64f0a47f"},
+ {file = "websockets-15.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:a4cc73a6ae0a6751b76e69cece9d0311f054da9b22df6a12f2c53111735657c8"},
+ {file = "websockets-15.0-cp310-cp310-win32.whl", hash = "sha256:89da58e4005e153b03fe8b8794330e3f6a9774ee9e1c3bd5bc52eb098c3b0c4f"},
+ {file = "websockets-15.0-cp310-cp310-win_amd64.whl", hash = "sha256:4ff380aabd7a74a42a760ee76c68826a8f417ceb6ea415bd574a035a111fd133"},
+ {file = "websockets-15.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:dd24c4d256558429aeeb8d6c24ebad4e982ac52c50bc3670ae8646c181263965"},
+ {file = "websockets-15.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f83eca8cbfd168e424dfa3b3b5c955d6c281e8fc09feb9d870886ff8d03683c7"},
+ {file = "websockets-15.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4095a1f2093002c2208becf6f9a178b336b7572512ee0a1179731acb7788e8ad"},
+ {file = "websockets-15.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb915101dfbf318486364ce85662bb7b020840f68138014972c08331458d41f3"},
+ {file = "websockets-15.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:45d464622314973d78f364689d5dbb9144e559f93dca11b11af3f2480b5034e1"},
+ {file = "websockets-15.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ace960769d60037ca9625b4c578a6f28a14301bd2a1ff13bb00e824ac9f73e55"},
+ {file = "websockets-15.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c7cd4b1015d2f60dfe539ee6c95bc968d5d5fad92ab01bb5501a77393da4f596"},
+ {file = "websockets-15.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:4f7290295794b5dec470867c7baa4a14182b9732603fd0caf2a5bf1dc3ccabf3"},
+ {file = "websockets-15.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3abd670ca7ce230d5a624fd3d55e055215d8d9b723adee0a348352f5d8d12ff4"},
+ {file = "websockets-15.0-cp311-cp311-win32.whl", hash = "sha256:110a847085246ab8d4d119632145224d6b49e406c64f1bbeed45c6f05097b680"},
+ {file = "websockets-15.0-cp311-cp311-win_amd64.whl", hash = "sha256:8d7bbbe2cd6ed80aceef2a14e9f1c1b61683194c216472ed5ff33b700e784e37"},
+ {file = "websockets-15.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:cccc18077acd34c8072578394ec79563664b1c205f7a86a62e94fafc7b59001f"},
+ {file = "websockets-15.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d4c22992e24f12de340ca5f824121a5b3e1a37ad4360b4e1aaf15e9d1c42582d"},
+ {file = "websockets-15.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1206432cc6c644f6fc03374b264c5ff805d980311563202ed7fef91a38906276"},
+ {file = "websockets-15.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d3cc75ef3e17490042c47e0523aee1bcc4eacd2482796107fd59dd1100a44bc"},
+ {file = "websockets-15.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b89504227a5311610e4be16071465885a0a3d6b0e82e305ef46d9b064ce5fb72"},
+ {file = "websockets-15.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:56e3efe356416bc67a8e093607315951d76910f03d2b3ad49c4ade9207bf710d"},
+ {file = "websockets-15.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0f2205cdb444a42a7919690238fb5979a05439b9dbb73dd47c863d39640d85ab"},
+ {file = "websockets-15.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:aea01f40995fa0945c020228ab919b8dfc93fc8a9f2d3d705ab5b793f32d9e99"},
+ {file = "websockets-15.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a9f8e33747b1332db11cf7fcf4a9512bef9748cb5eb4d3f7fbc8c30d75dc6ffc"},
+ {file = "websockets-15.0-cp312-cp312-win32.whl", hash = "sha256:32e02a2d83f4954aa8c17e03fe8ec6962432c39aca4be7e8ee346b05a3476904"},
+ {file = "websockets-15.0-cp312-cp312-win_amd64.whl", hash = "sha256:ffc02b159b65c05f2ed9ec176b715b66918a674bd4daed48a9a7a590dd4be1aa"},
+ {file = "websockets-15.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:d2244d8ab24374bed366f9ff206e2619345f9cd7fe79aad5225f53faac28b6b1"},
+ {file = "websockets-15.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:3a302241fbe825a3e4fe07666a2ab513edfdc6d43ce24b79691b45115273b5e7"},
+ {file = "websockets-15.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:10552fed076757a70ba2c18edcbc601c7637b30cdfe8c24b65171e824c7d6081"},
+ {file = "websockets-15.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c53f97032b87a406044a1c33d1e9290cc38b117a8062e8a8b285175d7e2f99c9"},
+ {file = "websockets-15.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1caf951110ca757b8ad9c4974f5cac7b8413004d2f29707e4d03a65d54cedf2b"},
+ {file = "websockets-15.0-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8bf1ab71f9f23b0a1d52ec1682a3907e0c208c12fef9c3e99d2b80166b17905f"},
+ {file = "websockets-15.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:bfcd3acc1a81f106abac6afd42327d2cf1e77ec905ae11dc1d9142a006a496b6"},
+ {file = "websockets-15.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:c8c5c8e1bac05ef3c23722e591ef4f688f528235e2480f157a9cfe0a19081375"},
+ {file = "websockets-15.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:86bfb52a9cfbcc09aba2b71388b0a20ea5c52b6517c0b2e316222435a8cdab72"},
+ {file = "websockets-15.0-cp313-cp313-win32.whl", hash = "sha256:26ba70fed190708551c19a360f9d7eca8e8c0f615d19a574292b7229e0ae324c"},
+ {file = "websockets-15.0-cp313-cp313-win_amd64.whl", hash = "sha256:ae721bcc8e69846af00b7a77a220614d9b2ec57d25017a6bbde3a99473e41ce8"},
+ {file = "websockets-15.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c348abc5924caa02a62896300e32ea80a81521f91d6db2e853e6b1994017c9f6"},
+ {file = "websockets-15.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5294fcb410ed0a45d5d1cdedc4e51a60aab5b2b3193999028ea94afc2f554b05"},
+ {file = "websockets-15.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c24ba103ecf45861e2e1f933d40b2d93f5d52d8228870c3e7bf1299cd1cb8ff1"},
+ {file = "websockets-15.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cc8821a03bcfb36e4e4705316f6b66af28450357af8a575dc8f4b09bf02a3dee"},
+ {file = "websockets-15.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffc5ae23ada6515f31604f700009e2df90b091b67d463a8401c1d8a37f76c1d7"},
+ {file = "websockets-15.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ac67b542505186b3bbdaffbc303292e1ee9c8729e5d5df243c1f20f4bb9057e"},
+ {file = "websockets-15.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:c86dc2068f1c5ca2065aca34f257bbf4f78caf566eb230f692ad347da191f0a1"},
+ {file = "websockets-15.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:30cff3ef329682b6182c01c568f551481774c476722020b8f7d0daacbed07a17"},
+ {file = "websockets-15.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:98dcf978d4c6048965d1762abd534c9d53bae981a035bfe486690ba11f49bbbb"},
+ {file = "websockets-15.0-cp39-cp39-win32.whl", hash = "sha256:37d66646f929ae7c22c79bc73ec4074d6db45e6384500ee3e0d476daf55482a9"},
+ {file = "websockets-15.0-cp39-cp39-win_amd64.whl", hash = "sha256:24d5333a9b2343330f0f4eb88546e2c32a7f5c280f8dd7d3cc079beb0901781b"},
+ {file = "websockets-15.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:b499caef4bca9cbd0bd23cd3386f5113ee7378094a3cb613a2fa543260fe9506"},
+ {file = "websockets-15.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:17f2854c6bd9ee008c4b270f7010fe2da6c16eac5724a175e75010aacd905b31"},
+ {file = "websockets-15.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:89f72524033abbfde880ad338fd3c2c16e31ae232323ebdfbc745cbb1b3dcc03"},
+ {file = "websockets-15.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1657a9eecb29d7838e3b415458cc494e6d1b194f7ac73a34aa55c6fb6c72d1f3"},
+ {file = "websockets-15.0-pp310-pypy310_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e413352a921f5ad5d66f9e2869b977e88d5103fc528b6deb8423028a2befd842"},
+ {file = "websockets-15.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:8561c48b0090993e3b2a54db480cab1d23eb2c5735067213bb90f402806339f5"},
+ {file = "websockets-15.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:190bc6ef8690cd88232a038d1b15714c258f79653abad62f7048249b09438af3"},
+ {file = "websockets-15.0-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:327adab7671f3726b0ba69be9e865bba23b37a605b585e65895c428f6e47e766"},
+ {file = "websockets-15.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2bd8ef197c87afe0a9009f7a28b5dc613bfc585d329f80b7af404e766aa9e8c7"},
+ {file = "websockets-15.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:789c43bf4a10cd067c24c321238e800b8b2716c863ddb2294d2fed886fa5a689"},
+ {file = "websockets-15.0-pp39-pypy39_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7394c0b7d460569c9285fa089a429f58465db930012566c03046f9e3ab0ed181"},
+ {file = "websockets-15.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:2ea4f210422b912ebe58ef0ad33088bc8e5c5ff9655a8822500690abc3b1232d"},
+ {file = "websockets-15.0-py3-none-any.whl", hash = "sha256:51ffd53c53c4442415b613497a34ba0aa7b99ac07f1e4a62db5dcd640ae6c3c3"},
+ {file = "websockets-15.0.tar.gz", hash = "sha256:ca36151289a15b39d8d683fd8b7abbe26fc50be311066c5f8dcf3cb8cee107ab"},
]
[[package]]
@@ -3645,7 +3646,7 @@ files = [
]
[package.extras]
-dev = ["black (>=19.3b0)", "pytest (>=4.6.2)"]
+dev = ["black (>=19.3b0) ; python_version >= \"3.6\"", "pytest (>=4.6.2)"]
[[package]]
name = "wsproto"
@@ -3664,20 +3665,20 @@ h11 = ">=0.9.0,<1"
[[package]]
name = "yt-dlp"
-version = "2025.1.26"
+version = "2025.2.19"
description = "A feature-rich command-line audio/video downloader"
optional = false
python-versions = ">=3.9"
groups = ["main"]
files = [
- {file = "yt_dlp-2025.1.26-py3-none-any.whl", hash = "sha256:3e76bd896b9f96601021ca192ca0fbdd195e3c3dcc28302a3a34c9bc4979da7b"},
- {file = "yt_dlp-2025.1.26.tar.gz", hash = "sha256:1c9738266921ad43c568ad01ac3362fb7c7af549276fbec92bd72f140da16240"},
+ {file = "yt_dlp-2025.2.19-py3-none-any.whl", hash = "sha256:3ed218eaeece55e9d715afd41abc450dc406ee63bf79355169dfde312d38fdb8"},
+ {file = "yt_dlp-2025.2.19.tar.gz", hash = "sha256:f33ca76df2e4db31880f2fe408d44f5058d9f135015b13e50610dfbe78245bea"},
]
[package.extras]
build = ["build", "hatchling", "pip", "setuptools (>=71.0.2)", "wheel"]
-curl-cffi = ["curl-cffi (==0.5.10)", "curl-cffi (>=0.5.10,!=0.6.*,<0.7.2)"]
-default = ["brotli", "brotlicffi", "certifi", "mutagen", "pycryptodomex", "requests (>=2.32.2,<3)", "urllib3 (>=1.26.17,<3)", "websockets (>=13.0)"]
+curl-cffi = ["curl-cffi (==0.5.10) ; os_name == \"nt\" and implementation_name == \"cpython\"", "curl-cffi (>=0.5.10,!=0.6.*,<0.7.2) ; os_name != \"nt\" and implementation_name == \"cpython\""]
+default = ["brotli ; implementation_name == \"cpython\"", "brotlicffi ; implementation_name != \"cpython\"", "certifi", "mutagen", "pycryptodomex", "requests (>=2.32.2,<3)", "urllib3 (>=1.26.17,<3)", "websockets (>=13.0)"]
dev = ["autopep8 (>=2.0,<3.0)", "pre-commit", "pytest (>=8.1,<9.0)", "pytest-rerunfailures (>=14.0,<15.0)", "ruff (>=0.9.0,<0.10.0)"]
pyinstaller = ["pyinstaller (>=6.11.1)"]
secretstorage = ["cffi", "secretstorage"]
From 5c30f263940961495bb1f64c05a348ee161fa3ab Mon Sep 17 00:00:00 2001
From: michplunkett <5885605+michplunkett@users.noreply.github.com>
Date: Fri, 21 Feb 2025 15:10:06 -0600
Subject: [PATCH 04/40] Create CODEOWNERS
---
CODEOWNERS | 1 +
1 file changed, 1 insertion(+)
create mode 100644 CODEOWNERS
diff --git a/CODEOWNERS b/CODEOWNERS
new file mode 100644
index 0000000..b5ffe06
--- /dev/null
+++ b/CODEOWNERS
@@ -0,0 +1 @@
+* @msramalho
From 9cf449a68f74728fa52563035095fd797cbc0e24 Mon Sep 17 00:00:00 2001
From: michplunkett <5885605+michplunkett@users.noreply.github.com>
Date: Fri, 21 Feb 2025 15:14:00 -0600
Subject: [PATCH 05/40] Update .gitignore
---
.gitignore | 7 -------
1 file changed, 7 deletions(-)
diff --git a/.gitignore b/.gitignore
index 15bb465..ebe19c0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -113,13 +113,6 @@ ipython_config.py
# pyenv
.python-version
-# pipenv
-# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
-# However, in case of collaboration, if having platform-specific dependencies or dependencies
-# having no cross-platform support, pipenv may install dependencies that don't work, or not
-# install all needed dependencies.
-#Pipfile.lock
-
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
__pypackages__/
From d9f36957cd52c604ba0b7b3692e8720a53ef1b94 Mon Sep 17 00:00:00 2001
From: Michael Plunkett <5885605+michplunkett@users.noreply.github.com>
Date: Wed, 26 Feb 2025 10:40:04 -0600
Subject: [PATCH 06/40] Update `Makefile` (#55)
* Update Makefile
* Update pyproject.toml
* Update Makefile
* Update pyproject.toml
---
Makefile | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/Makefile b/Makefile
index 2fd462c..bfcb3d3 100644
--- a/Makefile
+++ b/Makefile
@@ -1,19 +1,23 @@
+.PHONY: clean-dev
clean-dev:
@echo -n "Are you sure? [yes/N] (this will delete volumes) " && read ans && [ $${ans:-N} = yes ]
docker compose -f docker-compose.yml -f docker-compose.dev.yml down --volumes --remove-orphans
+.PHONY: dev
dev:
docker compose --env-file .env.dev -f docker-compose.yml -f docker-compose.dev.yml build
docker compose --env-file .env.dev -f docker-compose.yml -f docker-compose.dev.yml up --remove-orphans
-
+.PHONY: dev-redis-only
dev-redis-only:
docker compose --env-file .env.dev -f docker-compose.yml -f docker-compose.dev.yml build redis
docker compose --env-file .env.dev -f docker-compose.yml -f docker-compose.dev.yml up --remove-orphans redis
+.PHONY: stop-dev
stop-dev:
docker compose -f docker-compose.yml -f docker-compose.dev.yml down --volumes
+.PHONY: prod
prod:
docker compose --env-file .env.prod build
docker compose --env-file .env.prod up -d --remove-orphans
@@ -21,5 +25,6 @@ prod:
docker image prune -f
docker system df
+.PHONY: stop-prod
stop-prod:
- docker compose down
\ No newline at end of file
+ docker compose down
From 7e48f706df3902818b6ed5b4334d1f3b908d89f6 Mon Sep 17 00:00:00 2001
From: Michael Plunkett <5885605+michplunkett@users.noreply.github.com>
Date: Wed, 26 Feb 2025 10:42:28 -0600
Subject: [PATCH 07/40] Add `pre-commit` with GiHub Action (#56)
* Update pyproject.toml
* add pre-commit
* Create .pre-commit-config.yaml
* Comment out ruff
* Update .pre-commit-config.yaml
* General formatting
* Create format-and-fail.yml
* Update ci.yml
* Add pre-commit to dev dependencies
* Update pyproject.toml
---
.env.alembic | 2 +-
.env.example | 2 +-
.env.test | 2 +-
.github/workflows/ci.yml | 10 +-
.github/workflows/format-and-fail.yml | 16 +++
.pre-commit-config.yaml | 79 +++++++++++
LICENSE | 2 +-
Makefile | 4 +
README.md | 2 +-
app/migrations/env.py | 4 +-
...7ed0_create_archives_store_until_column.py | 2 +-
...24ec4b1_rename_sheets_last_archived_col.py | 2 +-
...dd_new_service_account_email_column_to_.py | 2 +-
...21d2c96d8_add_sheet_id_to_archive_table.py | 2 +-
...45b_modify_archive_url_to_have_uuid_id_.py | 3 +-
.../a23aaf3ae930_drop_active_column.py | 2 +-
...a012ec405b8_add_columns_to_groups_table.py | 2 +-
app/shared/aa_utils.py | 5 +-
app/shared/business_logic.py | 1 +
app/shared/db/database.py | 14 +-
app/shared/db/models.py | 15 +-
app/shared/db/worker_crud.py | 6 +-
app/shared/log.py | 3 +-
app/shared/schemas.py | 3 +-
app/shared/settings.py | 15 +-
app/shared/task_messaging.py | 5 +-
app/shared/user_groups.py | 11 +-
app/shared/utils/misc.py | 2 +-
app/tests/conftest.py | 23 ++--
app/tests/fake_service_account.json | 4 +-
app/tests/orchestration.test.yaml | 2 +-
app/tests/shared/db/test_models.py | 2 +-
app/tests/shared/db/test_worker_crud.py | 11 +-
app/tests/shared/test_business_logic.py | 7 +-
app/tests/shared/utils/test_misc.py | 4 +-
app/tests/user-groups.test.broken.yaml | 2 +-
app/tests/user-groups.test.yaml | 2 +-
app/tests/web/db/test_crud.py | 7 +-
app/tests/web/db/test_user_state.py | 1 +
app/tests/web/endpoints/test_default.py | 6 +-
.../web/endpoints/test_interoperability.py | 2 +-
app/tests/web/endpoints/test_sheet.py | 4 +-
app/tests/web/test_main.py | 10 +-
app/tests/web/test_security.py | 4 +-
app/tests/worker/test_worker_main.py | 7 +-
app/web/__init__.py | 3 +-
app/web/config.py | 2 +-
app/web/db/crud.py | 18 +--
app/web/db/user_state.py | 11 +-
app/web/endpoints/default.py | 8 +-
app/web/endpoints/interoperability.py | 14 +-
app/web/endpoints/sheet.py | 12 +-
app/web/endpoints/task.py | 4 +-
app/web/endpoints/url.py | 20 +--
app/web/events.py | 24 +++-
app/web/main.py | 27 ++--
app/web/middleware.py | 8 +-
app/web/security.py | 15 +-
app/web/utils/metrics.py | 3 +-
app/web/utils/misc.py | 1 +
app/worker/main.py | 19 +--
app/worker/worker_log.py | 8 +-
docker-compose.dev.yml | 2 +-
docker-compose.yml | 6 +-
poetry.lock | 129 +++++++++++++++++-
pyproject.toml | 2 +-
user-groups.example.yaml | 1 -
worker.Dockerfile | 2 +-
68 files changed, 473 insertions(+), 182 deletions(-)
create mode 100644 .github/workflows/format-and-fail.yml
create mode 100644 .pre-commit-config.yaml
diff --git a/.env.alembic b/.env.alembic
index 8691557..11bf2aa 100644
--- a/.env.alembic
+++ b/.env.alembic
@@ -2,4 +2,4 @@ CHROME_APP_IDS='["1234567890"]'
ALLOWED_ORIGINS='["allowed"]'
BLOCKED_EMAILS='[]'
DATABASE_PATH="sqlite:///./database/auto-archiver.db"
-API_BEARER_TOKEN=THIS_API_TOKEN_SHOULD_NEVER_BE_USED
\ No newline at end of file
+API_BEARER_TOKEN=THIS_API_TOKEN_SHOULD_NEVER_BE_USED
diff --git a/.env.example b/.env.example
index ef3935a..ea544ef 100644
--- a/.env.example
+++ b/.env.example
@@ -35,4 +35,4 @@ MAIL_SSL_TLS=True
# celery workers config
-CONCURRENCY=2
\ No newline at end of file
+CONCURRENCY=2
diff --git a/.env.test b/.env.test
index 32318f0..360f40e 100644
--- a/.env.test
+++ b/.env.test
@@ -5,4 +5,4 @@ BLOCKED_EMAILS='["blocked@example.com"]'
DATABASE_PATH="sqlite:///auto-archiver.test.db"
API_BEARER_TOKEN=this_is_the_test_api_token
-USER_GROUPS_FILENAME=app/tests/user-groups.test.yaml
\ No newline at end of file
+USER_GROUPS_FILENAME=app/tests/user-groups.test.yaml
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 9b63544..4135ac2 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -1,13 +1,9 @@
name: CI
on:
push:
- branches:
- - main
- - dev
+ branches: [ main, dev ]
pull_request:
- branches:
- - main
- - dev
+ branches: [ main, dev ]
jobs:
test:
@@ -41,4 +37,4 @@ jobs:
run: poetry run coverage run -m pytest -v -ra --color=yes app/tests/
- name: Report coverage
- run: poetry run coverage report
\ No newline at end of file
+ run: poetry run coverage report
diff --git a/.github/workflows/format-and-fail.yml b/.github/workflows/format-and-fail.yml
new file mode 100644
index 0000000..f1c01f8
--- /dev/null
+++ b/.github/workflows/format-and-fail.yml
@@ -0,0 +1,16 @@
+name: Format and Fail
+on:
+ push:
+ branches: [ main, dev ]
+ pull_request:
+ branches: [ main, dev ]
+
+jobs:
+ pre-commit:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-python@v4
+ with:
+ python-version: "3.11"
+ - uses: pre-commit/action@v3.0.0
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
new file mode 100644
index 0000000..73765d5
--- /dev/null
+++ b/.pre-commit-config.yaml
@@ -0,0 +1,79 @@
+repos:
+ - repo: https://github.com/nbQA-dev/nbQA
+ rev: 1.8.5
+ hooks:
+ - id: nbqa-ruff
+ args:
+ - --fix
+ - --target-version=py311
+ - --ignore=E721,E722
+ - --line-length=80
+ - id: nbqa-black
+ args:
+ - --line-length=80
+ - id: nbqa-isort
+ args:
+ - --float-to-top
+ - --profile=black
+
+ - repo: https://github.com/pre-commit/pre-commit-hooks
+ rev: v4.4.0
+ hooks:
+ - id: trailing-whitespace
+ - id: check-docstring-first
+ - id: check-executables-have-shebangs
+ - id: check-json
+ - id: check-case-conflict
+ - id: check-toml
+ - id: check-merge-conflict
+ - id: check-xml
+ - id: check-yaml
+ exclude: app/tests/user-groups.test.broken.yaml
+ - id: end-of-file-fixer
+ - id: check-symlinks
+ - id: mixed-line-ending
+ - id: sort-simple-yaml
+ - id: fix-encoding-pragma
+ args:
+ - --remove
+ - id: pretty-format-json
+ args:
+ - --autofix
+
+ - repo: https://github.com/pre-commit/pygrep-hooks
+ rev: v1.10.0
+ hooks:
+ - id: python-check-blanket-noqa
+ - id: python-check-mock-methods
+ - id: python-no-eval
+ - id: python-no-log-warn
+
+ - repo: https://github.com/PyCQA/isort
+ rev: 5.12.0
+ hooks:
+ - id: isort
+ name: Run isort to sort imports
+ files: \.py$
+ # To keep consistent with the global isort skip config defined in setup.cfg
+ exclude: ^build/.*$|^.tox/.*$|^venv/.*$
+ args:
+ - --lines-after-imports=2
+ - --profile=black
+ - --line-length=80
+
+# - repo: https://github.com/astral-sh/ruff-pre-commit
+# rev: v0.4.10
+# hooks:
+# - id: ruff
+# types_or: [python,pyi]
+# args:
+# - --fix
+# - --target-version=py311
+# - --select=B,C,E,F,W,B9
+# - --line-length=80
+# - --ignore=E203,E402,E501,E261
+# - id: ruff-format
+# types_or: [ python,pyi]
+# args:
+# - --target-version=py311
+# - --line-length=80
diff --git a/LICENSE b/LICENSE
index e10dcd9..c5bae4c 100644
--- a/LICENSE
+++ b/LICENSE
@@ -18,4 +18,4 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
\ No newline at end of file
+SOFTWARE.
diff --git a/Makefile b/Makefile
index bfcb3d3..ddda086 100644
--- a/Makefile
+++ b/Makefile
@@ -1,3 +1,7 @@
+.PHONY: lint
+lint:
+ poetry run pre-commit run --all-files
+
.PHONY: clean-dev
clean-dev:
@echo -n "Are you sure? [yes/N] (this will delete volumes) " && read ans && [ $${ans:-N} = yes ]
diff --git a/README.md b/README.md
index 2c87342..da402e3 100644
--- a/README.md
+++ b/README.md
@@ -14,7 +14,7 @@ To properly set up the API you need to install `docker` and to have these files,
2. go through the example file and configure it according to your needs.
3. you will need to create and reference at least one `secrets/orchestration.yaml` file, you can do so by following the instructions in the [auto-archiver](https://github.com/bellingcat/auto-archiver#installation) that automatically generates one for you. If you use the archive sheets feature you will need to create a `orchestrationsheets-sheets.yaml` file as well that should have the `gsheet_feeder` and `gsheet_db` enabled and configured, the auto-archiver has [extensive documentation](https://auto-archiver.readthedocs.io/en/latest/) on how to set this up.
-Do not commit those files, they are .gitignored by default.
+Do not commit those files, they are .gitignored by default.
We also advise you to keep any sensitive files in the `secrets/` folder which is pinned and gitignored.
We have examples for both of those files (`.env.example` and `user-groups.example.yaml`), and here's how to set them up whether you're in development or production:
diff --git a/app/migrations/env.py b/app/migrations/env.py
index 870ef18..1f579ab 100644
--- a/app/migrations/env.py
+++ b/app/migrations/env.py
@@ -1,11 +1,11 @@
from logging.config import fileConfig
-from sqlalchemy import engine_from_config
-from sqlalchemy import pool
from alembic import context
+from sqlalchemy import engine_from_config, pool
from app.shared.settings import get_settings
+
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
config = context.config
diff --git a/app/migrations/versions/02b2f6d17ed0_create_archives_store_until_column.py b/app/migrations/versions/02b2f6d17ed0_create_archives_store_until_column.py
index d00fa2c..8642f2b 100644
--- a/app/migrations/versions/02b2f6d17ed0_create_archives_store_until_column.py
+++ b/app/migrations/versions/02b2f6d17ed0_create_archives_store_until_column.py
@@ -5,8 +5,8 @@ Revises: 1636724ec4b1
Create Date: 2025-02-08 15:22:20.392522
"""
-from alembic import op
import sqlalchemy as sa
+from alembic import op
# revision identifiers, used by Alembic.
diff --git a/app/migrations/versions/1636724ec4b1_rename_sheets_last_archived_col.py b/app/migrations/versions/1636724ec4b1_rename_sheets_last_archived_col.py
index 6c109f3..324f75a 100644
--- a/app/migrations/versions/1636724ec4b1_rename_sheets_last_archived_col.py
+++ b/app/migrations/versions/1636724ec4b1_rename_sheets_last_archived_col.py
@@ -5,8 +5,8 @@ Revises: a23aaf3ae930
Create Date: 2025-02-05 19:19:01.984396
"""
-from alembic import op
import sqlalchemy as sa
+from alembic import op
# revision identifiers, used by Alembic.
diff --git a/app/migrations/versions/63ac79df4ad0_add_new_service_account_email_column_to_.py b/app/migrations/versions/63ac79df4ad0_add_new_service_account_email_column_to_.py
index 7067746..572905d 100644
--- a/app/migrations/versions/63ac79df4ad0_add_new_service_account_email_column_to_.py
+++ b/app/migrations/versions/63ac79df4ad0_add_new_service_account_email_column_to_.py
@@ -5,8 +5,8 @@ Revises: 02b2f6d17ed0
Create Date: 2025-02-11 21:53:23.293274
"""
-from alembic import op
import sqlalchemy as sa
+from alembic import op
# revision identifiers, used by Alembic.
diff --git a/app/migrations/versions/89121d2c96d8_add_sheet_id_to_archive_table.py b/app/migrations/versions/89121d2c96d8_add_sheet_id_to_archive_table.py
index 3011cf6..892e853 100644
--- a/app/migrations/versions/89121d2c96d8_add_sheet_id_to_archive_table.py
+++ b/app/migrations/versions/89121d2c96d8_add_sheet_id_to_archive_table.py
@@ -5,8 +5,8 @@ Revises: fa012ec405b8
Create Date: 2024-11-04 11:12:30.237299
"""
-from alembic import op
import sqlalchemy as sa
+from alembic import op
from sqlalchemy.engine.reflection import Inspector
diff --git a/app/migrations/versions/9369a264945b_modify_archive_url_to_have_uuid_id_.py b/app/migrations/versions/9369a264945b_modify_archive_url_to_have_uuid_id_.py
index a2b708a..bdfe474 100644
--- a/app/migrations/versions/9369a264945b_modify_archive_url_to_have_uuid_id_.py
+++ b/app/migrations/versions/9369a264945b_modify_archive_url_to_have_uuid_id_.py
@@ -1,12 +1,13 @@
"""modify archive url to have uuid id instead of url unique constraint
Revision ID: 9369a264945b
-Revises:
+Revises:
Create Date: 2023-12-20 17:24:59.320691
"""
from alembic import op
+
# revision identifiers, used by Alembic.
revision = '9369a264945b'
down_revision = None
diff --git a/app/migrations/versions/a23aaf3ae930_drop_active_column.py b/app/migrations/versions/a23aaf3ae930_drop_active_column.py
index 912f408..ebc85e7 100644
--- a/app/migrations/versions/a23aaf3ae930_drop_active_column.py
+++ b/app/migrations/versions/a23aaf3ae930_drop_active_column.py
@@ -5,8 +5,8 @@ Revises: 89121d2c96d8
Create Date: 2025-02-04 12:19:20.753570
"""
-from alembic import op
import sqlalchemy as sa
+from alembic import op
# revision identifiers, used by Alembic.
diff --git a/app/migrations/versions/fa012ec405b8_add_columns_to_groups_table.py b/app/migrations/versions/fa012ec405b8_add_columns_to_groups_table.py
index f0577ea..c3169c3 100644
--- a/app/migrations/versions/fa012ec405b8_add_columns_to_groups_table.py
+++ b/app/migrations/versions/fa012ec405b8_add_columns_to_groups_table.py
@@ -5,8 +5,8 @@ Revises: 93a611e4c066
Create Date: 2024-10-31 09:36:50.360710
"""
-from alembic import op
import sqlalchemy as sa
+from alembic import op
from sqlalchemy.engine.reflection import Inspector
diff --git a/app/shared/aa_utils.py b/app/shared/aa_utils.py
index 393a975..466d003 100644
--- a/app/shared/aa_utils.py
+++ b/app/shared/aa_utils.py
@@ -1,11 +1,13 @@
# TODO: code in this file should eventually be moved to the auto-archiver code base
from typing import List
-from loguru import logger
+
from auto_archiver.core import Media, Metadata
+from loguru import logger
from app.shared.db import models
+
def get_all_urls(result: Metadata) -> List[models.ArchiveUrl]:
db_urls = []
for m in result.media:
@@ -29,4 +31,3 @@ def convert_if_media(media):
except Exception as e:
logger.debug(f"error parsing {media} : {e}")
return False
-
diff --git a/app/shared/business_logic.py b/app/shared/business_logic.py
index d179fda..b4291b6 100644
--- a/app/shared/business_logic.py
+++ b/app/shared/business_logic.py
@@ -2,6 +2,7 @@
import datetime
+
from sqlalchemy.orm import Session
from app.shared.db import worker_crud
diff --git a/app/shared/db/database.py b/app/shared/db/database.py
index 171b97b..51c235a 100644
--- a/app/shared/db/database.py
+++ b/app/shared/db/database.py
@@ -1,8 +1,14 @@
-from functools import lru_cache
-from sqlalchemy import Engine, create_engine, event, text
-from sqlalchemy.orm import sessionmaker
from contextlib import asynccontextmanager, contextmanager
-from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession, AsyncEngine, async_sessionmaker
+from functools import lru_cache
+
+from sqlalchemy import Engine, create_engine, event, text
+from sqlalchemy.ext.asyncio import (
+ AsyncEngine,
+ AsyncSession,
+ async_sessionmaker,
+ create_async_engine,
+)
+from sqlalchemy.orm import sessionmaker
from app.shared.settings import get_settings
diff --git a/app/shared/db/models.py b/app/shared/db/models.py
index 1736224..aa93034 100644
--- a/app/shared/db/models.py
+++ b/app/shared/db/models.py
@@ -1,8 +1,17 @@
-from sqlalchemy import Column, String, JSON, DateTime, Boolean, Table, ForeignKey
-from sqlalchemy.sql import func
-from sqlalchemy.orm import relationship, declarative_base
import uuid
+from sqlalchemy import (
+ JSON,
+ Boolean,
+ Column,
+ DateTime,
+ ForeignKey,
+ String,
+ Table,
+)
+from sqlalchemy.orm import declarative_base, relationship
+from sqlalchemy.sql import func
+
Base = declarative_base()
diff --git a/app/shared/db/worker_crud.py b/app/shared/db/worker_crud.py
index 814689a..82f3380 100644
--- a/app/shared/db/worker_crud.py
+++ b/app/shared/db/worker_crud.py
@@ -1,8 +1,10 @@
-from sqlalchemy.orm import Session
from datetime import datetime
-from app.shared.db import models
+from sqlalchemy.orm import Session
+
from app.shared import schemas
+from app.shared.db import models
+
# TODO: isolate database operations away from worker and into WEB
# ONLY WORKER
diff --git a/app/shared/log.py b/app/shared/log.py
index 68587e2..734c368 100644
--- a/app/shared/log.py
+++ b/app/shared/log.py
@@ -1,4 +1,5 @@
import traceback
+
from loguru import logger
@@ -6,7 +7,7 @@ from loguru import logger
logger.add("logs/api_logs.log", retention="30 days")
logger.add("logs/error_logs.log", retention="30 days", level="ERROR")
-
+
def log_error(e: Exception, traceback_str: str = None, extra:str = ""):
if not traceback_str: traceback_str = traceback.format_exc()
if extra: extra = f"{extra}\n"
diff --git a/app/shared/schemas.py b/app/shared/schemas.py
index 66119f7..e8479f5 100644
--- a/app/shared/schemas.py
+++ b/app/shared/schemas.py
@@ -1,7 +1,8 @@
+from datetime import datetime
from typing import Annotated
+
from annotated_types import Len
from pydantic import BaseModel
-from datetime import datetime
class SubmitSheet(BaseModel):
diff --git a/app/shared/settings.py b/app/shared/settings.py
index d884f80..2f68ae7 100644
--- a/app/shared/settings.py
+++ b/app/shared/settings.py
@@ -1,14 +1,15 @@
-from functools import lru_cache
import os
+from functools import lru_cache
+from typing import Annotated, Set
+
+from annotated_types import Len
from fastapi_mail import ConnectionConfig
from pydantic_settings import BaseSettings, SettingsConfigDict
-from typing import Annotated, Set
-from annotated_types import Len
class Settings(BaseSettings):
-
+
model_config = SettingsConfigDict(env_file=os.environ.get("ENVIRONMENT_FILE") , env_file_encoding='utf-8', extra='ignore', str_strip_whitespace=True)
# general
@@ -37,14 +38,14 @@ class Settings(BaseSettings):
if self.REDIS_PASSWORD:
return f"redis://:{self.REDIS_PASSWORD}@{self.REDIS_HOSTNAME}:6379"
return f"redis://{self.REDIS_HOSTNAME}:6379"
-
+
# cronjobs
CRON_ARCHIVE_SHEETS: bool = False
CRON_DELETE_STALE_SHEETS: bool = False
DELETE_STALE_SHEETS_DAYS: int = 14
CRON_DELETE_SCHEDULED_ARCHIVES: bool = False
DELETE_SCHEDULED_ARCHIVES_CHECK_EVERY_N_DAYS: int = 7
-
+
# observability
REPEAT_COUNT_METRICS_SECONDS: int = 30
@@ -73,4 +74,4 @@ class Settings(BaseSettings):
@lru_cache
def get_settings():
- return Settings()
\ No newline at end of file
+ return Settings()
diff --git a/app/shared/task_messaging.py b/app/shared/task_messaging.py
index 21fb3d1..88d0057 100644
--- a/app/shared/task_messaging.py
+++ b/app/shared/task_messaging.py
@@ -1,8 +1,9 @@
from functools import lru_cache
-from celery import Celery
-import redis
+from celery import Celery
+
+import redis
from app.shared.settings import get_settings
diff --git a/app/shared/user_groups.py b/app/shared/user_groups.py
index 592e012..480b84e 100644
--- a/app/shared/user_groups.py
+++ b/app/shared/user_groups.py
@@ -1,9 +1,16 @@
import json
import os
+from typing import Dict, List, Set
+
import yaml
from loguru import logger
-from pydantic import BaseModel, computed_field, field_validator, Field, model_validator
-from typing import Dict, List, Set
+from pydantic import (
+ BaseModel,
+ Field,
+ computed_field,
+ field_validator,
+ model_validator,
+)
from typing_extensions import Self
diff --git a/app/shared/utils/misc.py b/app/shared/utils/misc.py
index 562b2c3..6c5940d 100644
--- a/app/shared/utils/misc.py
+++ b/app/shared/utils/misc.py
@@ -7,4 +7,4 @@ def fnv1a_hash_mod(s: str, modulo:int) -> int:
hash ^= ord(char)
hash *= fnv_prime
hash &= 0xFFFFFFFF # Keep it 32-bit
- return (hash if hash < 0x80000000 else hash - 0x100000000) % modulo
\ No newline at end of file
+ return (hash if hash < 0x80000000 else hash - 0x100000000) % modulo
diff --git a/app/tests/conftest.py b/app/tests/conftest.py
index afa76f9..f7da39e 100644
--- a/app/tests/conftest.py
+++ b/app/tests/conftest.py
@@ -1,12 +1,14 @@
import os
from typing import AsyncGenerator
-from fastapi.testclient import TestClient
-import pytest
from unittest.mock import patch
+
+import pytest
import pytest_asyncio
-from sqlalchemy.ext.asyncio import AsyncSession, AsyncEngine
-from app.web.config import ALLOW_ANY_EMAIL
+from fastapi.testclient import TestClient
+from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession
+
from app.shared.settings import Settings
+from app.web.config import ALLOW_ANY_EMAIL
from app.web.db.user_state import UserState
@@ -65,10 +67,11 @@ def db_session(test_db):
@pytest_asyncio.fixture()
async def async_test_db(get_settings: Settings):
+ import asyncio
+
from app.shared.db import models
from app.shared.db.database import make_async_engine
from app.web.db.crud import get_user_group_names
- import asyncio
get_user_group_names.cache_clear()
engine = await make_async_engine(get_settings.ASYNC_DATABASE_PATH)
@@ -108,8 +111,8 @@ async def async_db_session(async_test_db: AsyncEngine) -> AsyncGenerator[AsyncSe
@pytest.fixture()
def app(db_session):
- from app.web.main import app_factory
from app.web.db import crud
+ from app.web.main import app_factory
app = app_factory()
crud.upsert_user_groups(db_session)
return app
@@ -123,7 +126,11 @@ def client(app):
@pytest.fixture()
def app_with_auth(app, db_session):
- from app.web.security import get_token_or_user_auth, get_user_auth, get_user_state
+ from app.web.security import (
+ get_token_or_user_auth,
+ get_user_auth,
+ get_user_state,
+ )
app.dependency_overrides[get_token_or_user_auth] = lambda: "rick@example.com"
app.dependency_overrides[get_user_auth] = lambda: "morty@example.com"
app.dependency_overrides[get_user_state] = lambda: UserState(db_session, "MORTY@example.com")
@@ -138,7 +145,7 @@ def client_with_auth(app_with_auth):
@pytest.fixture()
def app_with_token(app):
- from app.web.security import token_api_key_auth, get_token_or_user_auth
+ from app.web.security import get_token_or_user_auth, token_api_key_auth
app.dependency_overrides[token_api_key_auth] = lambda: ALLOW_ANY_EMAIL
app.dependency_overrides[get_token_or_user_auth] = lambda: ALLOW_ANY_EMAIL
return app
diff --git a/app/tests/fake_service_account.json b/app/tests/fake_service_account.json
index 3d41bd9..10c0585 100644
--- a/app/tests/fake_service_account.json
+++ b/app/tests/fake_service_account.json
@@ -1,3 +1,3 @@
{
- "client_email": "fake_service_account@fake_service_account.iam.gserviceaccount.com"
-}
\ No newline at end of file
+ "client_email": "fake_service_account@fake_service_account.iam.gserviceaccount.com"
+}
diff --git a/app/tests/orchestration.test.yaml b/app/tests/orchestration.test.yaml
index 4ee1880..ef7ed27 100644
--- a/app/tests/orchestration.test.yaml
+++ b/app/tests/orchestration.test.yaml
@@ -15,7 +15,7 @@ configurations:
gsheet_feeder:
service_account: "app/tests/fake_service_account.json"
cli_feeder:
- urls:
+ urls:
- "url1"
hash_enricher:
algorithm: "SHA-256"
diff --git a/app/tests/shared/db/test_models.py b/app/tests/shared/db/test_models.py
index 35ba368..4da9571 100644
--- a/app/tests/shared/db/test_models.py
+++ b/app/tests/shared/db/test_models.py
@@ -3,4 +3,4 @@ def test_generate_uuid():
assert generate_uuid() != generate_uuid()
assert len(generate_uuid()) == 36
- assert generate_uuid().count("-") == 4
\ No newline at end of file
+ assert generate_uuid().count("-") == 4
diff --git a/app/tests/shared/db/test_worker_crud.py b/app/tests/shared/db/test_worker_crud.py
index 1098cbe..09e9a76 100644
--- a/app/tests/shared/db/test_worker_crud.py
+++ b/app/tests/shared/db/test_worker_crud.py
@@ -1,10 +1,9 @@
-from app.shared.db import models
-from app.shared.db import worker_crud, models
from datetime import datetime
-
+from app.shared.db import models, worker_crud
from app.tests.web.db.test_crud import test_data
+
def test_update_sheet_last_url_archived_at(db_session):
# Create test sheet
@@ -19,7 +18,7 @@ def test_update_sheet_last_url_archived_at(db_session):
db_session.refresh(test_sheet)
assert isinstance(test_sheet.last_url_archived_at, datetime)
assert test_sheet.last_url_archived_at > before
-
+
# Test non-existent sheet
assert worker_crud.update_sheet_last_url_archived_at(db_session, "non-existent-sheet") is False
@@ -73,8 +72,8 @@ def test_create_tag(db_session):
def test_create_task(db_session):
- from app.shared.db import worker_crud
from app.shared import schemas
+ from app.shared.db import worker_crud
task = schemas.ArchiveCreate(
id="archive-id-456-101",
@@ -114,4 +113,4 @@ def test_create_task(db_session):
assert nt.group_id == "spaceship"
assert len(nt.tags) == 0
assert len(nt.urls) == 0
- assert nt.created_at is not None
\ No newline at end of file
+ assert nt.created_at is not None
diff --git a/app/tests/shared/test_business_logic.py b/app/tests/shared/test_business_logic.py
index 225fb11..830fa7b 100644
--- a/app/tests/shared/test_business_logic.py
+++ b/app/tests/shared/test_business_logic.py
@@ -1,7 +1,12 @@
from datetime import datetime, timedelta
from unittest.mock import MagicMock, patch
+
import pytest
-from app.shared.business_logic import get_store_archive_until, get_store_archive_until_or_never
+
+from app.shared.business_logic import (
+ get_store_archive_until,
+ get_store_archive_until_or_never,
+)
class Test_get_store_archive_until:
diff --git a/app/tests/shared/utils/test_misc.py b/app/tests/shared/utils/test_misc.py
index d7595c8..18db28d 100644
--- a/app/tests/shared/utils/test_misc.py
+++ b/app/tests/shared/utils/test_misc.py
@@ -11,7 +11,7 @@ def test_fnv1a_hash_mod():
# Test different modulos
hash1 = fnv1a_hash_mod("test", 5)
- hash2 = fnv1a_hash_mod("test", 10)
+ hash2 = fnv1a_hash_mod("test", 10)
assert 0 <= hash1 < 5
assert 0 <= hash2 < 10
@@ -28,4 +28,4 @@ def test_fnv1a_hash_mod():
assert 0 <= fnv1a_hash_mod("测试", 10) < 10
# Test modulo = 1 edge case
- assert fnv1a_hash_mod("test", 1) == 0
\ No newline at end of file
+ assert fnv1a_hash_mod("test", 1) == 0
diff --git a/app/tests/user-groups.test.broken.yaml b/app/tests/user-groups.test.broken.yaml
index 8bc59c5..9b41741 100644
--- a/app/tests/user-groups.test.broken.yaml
+++ b/app/tests/user-groups.test.broken.yaml
@@ -3,4 +3,4 @@ This is just an invalid yaml for testing
still broken: True
- one
- - two
\ No newline at end of file
+ - two
diff --git a/app/tests/user-groups.test.yaml b/app/tests/user-groups.test.yaml
index 16a3ba7..e9a446f 100644
--- a/app/tests/user-groups.test.yaml
+++ b/app/tests/user-groups.test.yaml
@@ -84,4 +84,4 @@ groups:
# max_archive_lifespan_months: 12
max_monthly_urls: 1
# max_monthly_mbs: 50
- priority: "low"
\ No newline at end of file
+ priority: "low"
diff --git a/app/tests/web/db/test_crud.py b/app/tests/web/db/test_crud.py
index aad9d4c..298a087 100644
--- a/app/tests/web/db/test_crud.py
+++ b/app/tests/web/db/test_crud.py
@@ -3,10 +3,12 @@ from unittest.mock import patch
import pytest
import yaml
+
from app.shared.db import models
from app.shared.settings import Settings
-
from app.web.db import crud
+
+
authors = ["rick@example.com", "morty@example.com", "jerry@example.com"]
@@ -373,6 +375,7 @@ async def test_get_sheets_by_id_hash(async_db_session):
@pytest.mark.asyncio
async def test_delete_stale_sheets(async_db_session):
from datetime import datetime, timedelta
+
from sqlalchemy.sql import select
now = datetime.now()
@@ -435,4 +438,4 @@ async def test_delete_stale_sheets(async_db_session):
# Running again should not delete anything
deleted = await crud.delete_stale_sheets(async_db_session, 7)
- assert len(deleted) == 0
\ No newline at end of file
+ assert len(deleted) == 0
diff --git a/app/tests/web/db/test_user_state.py b/app/tests/web/db/test_user_state.py
index 42c61d1..665bf08 100644
--- a/app/tests/web/db/test_user_state.py
+++ b/app/tests/web/db/test_user_state.py
@@ -1,5 +1,6 @@
from unittest.mock import MagicMock, PropertyMock, patch
+
import pytest
from app.shared.db import models
diff --git a/app/tests/web/endpoints/test_default.py b/app/tests/web/endpoints/test_default.py
index 401a164..e4e34cc 100644
--- a/app/tests/web/endpoints/test_default.py
+++ b/app/tests/web/endpoints/test_default.py
@@ -1,10 +1,12 @@
from unittest.mock import MagicMock
-from fastapi.testclient import TestClient
+
import pytest
+from fastapi.testclient import TestClient
+
from app.shared.schemas import Usage, UsageResponse
from app.shared.user_groups import GroupInfo
-from app.web.config import VERSION
from app.tests.web.db.test_crud import test_data
+from app.web.config import VERSION
def test_endpoint_home(client_with_auth):
diff --git a/app/tests/web/endpoints/test_interoperability.py b/app/tests/web/endpoints/test_interoperability.py
index 31cf8f0..703f69a 100644
--- a/app/tests/web/endpoints/test_interoperability.py
+++ b/app/tests/web/endpoints/test_interoperability.py
@@ -1,5 +1,5 @@
-from datetime import datetime
import json
+from datetime import datetime
from unittest.mock import MagicMock, patch
from app.shared.db import models
diff --git a/app/tests/web/endpoints/test_sheet.py b/app/tests/web/endpoints/test_sheet.py
index 1396d85..9b47228 100644
--- a/app/tests/web/endpoints/test_sheet.py
+++ b/app/tests/web/endpoints/test_sheet.py
@@ -1,5 +1,5 @@
-from datetime import datetime
import json
+from datetime import datetime
from unittest.mock import MagicMock, patch
from fastapi.testclient import TestClient
@@ -45,8 +45,8 @@ def test_create_sheet_endpoint(app_with_auth, db_session):
assert response.json() == {"detail": "User does not have access to this group."}
# switch to jerry who's got less quota/permissions
- from app.web.security import get_user_state
from app.web.db.user_state import UserState
+ from app.web.security import get_user_state
app_with_auth.dependency_overrides[get_user_state] = lambda: UserState(db_session, "jerry@example.com")
client_jerry = TestClient(app_with_auth)
diff --git a/app/tests/web/test_main.py b/app/tests/web/test_main.py
index f77d368..a4ddf1e 100644
--- a/app/tests/web/test_main.py
+++ b/app/tests/web/test_main.py
@@ -1,10 +1,10 @@
import os
-from unittest.mock import patch
-from fastapi.testclient import TestClient
-
import shutil
+from unittest.mock import patch
import pytest
+from fastapi.testclient import TestClient
+
def test_lifespan(app):
with TestClient(app) as client:
@@ -25,7 +25,7 @@ def test_logging_middleware(m1, client_with_auth):
client_with_auth.delete("/url/123")
# creates one empty and one from above
assert len(EXCEPTION_COUNTER.collect()[0].samples) == 2
-
+
def test_serve_local_archive_logic(get_settings):
# create a test file first
@@ -38,7 +38,7 @@ def test_serve_local_archive_logic(get_settings):
get_settings.SERVE_LOCAL_ARCHIVE = "/app/local_archive_test"
from app.web.main import app_factory
app = app_factory(get_settings)
-
+
# test
client = TestClient(app)
r = client.get("/app/local_archive_test/temp.txt")
diff --git a/app/tests/web/test_security.py b/app/tests/web/test_security.py
index 1a6c00b..55a434b 100644
--- a/app/tests/web/test_security.py
+++ b/app/tests/web/test_security.py
@@ -1,8 +1,8 @@
from unittest.mock import Mock, patch
+import pytest
from fastapi import HTTPException
from fastapi.security import HTTPAuthorizationCredentials
-import pytest
from app.web.config import ALLOW_ANY_EMAIL
@@ -108,8 +108,8 @@ async def test_authenticate_user_exception():
def test_get_user_state():
- from app.web.security import get_user_state
from app.web.db.user_state import UserState
+ from app.web.security import get_user_state
mock_session = Mock()
test_email = "test@example.com"
diff --git a/app/tests/worker/test_worker_main.py b/app/tests/worker/test_worker_main.py
index d40c457..9a77528 100644
--- a/app/tests/worker/test_worker_main.py
+++ b/app/tests/worker/test_worker_main.py
@@ -1,13 +1,12 @@
from datetime import datetime
-
from unittest.mock import patch
import pytest
-
-from app.shared.db import models
-from app.shared import schemas
from auto_archiver.core import Media, Metadata
+from app.shared import schemas
+from app.shared.db import models
+
class Test_create_archive_task():
URL = "https://example-live.com"
diff --git a/app/web/__init__.py b/app/web/__init__.py
index a817e9e..98a139a 100644
--- a/app/web/__init__.py
+++ b/app/web/__init__.py
@@ -1,3 +1,4 @@
from app.web.main import app_factory
-app = app_factory
\ No newline at end of file
+
+app = app_factory
diff --git a/app/web/config.py b/app/web/config.py
index 29a6806..b795d88 100644
--- a/app/web/config.py
+++ b/app/web/config.py
@@ -5,7 +5,7 @@ API_DESCRIPTION = """
**Usage notes:**
- The API requires a Bearer token for most operations, which you can obtain by logging in with your Google account.
-- You can use this API to archive single URLs or entire Google Sheets.
+- You can use this API to archive single URLs or entire Google Sheets.
- Once you submit a URL or Sheet for archiving, the API will return a task_id that you can use to check the status of the archiving process. It works asynchronously.
"""
BREAKING_CHANGES = {"minVersion": "0.4.0", "message": "The latest update has breaking changes, please update the extension to the most recent version."}
diff --git a/app/web/db/crud.py b/app/web/db/crud.py
index c16b09a..b33faa2 100644
--- a/app/web/db/crud.py
+++ b/app/web/db/crud.py
@@ -1,18 +1,19 @@
from collections import defaultdict
-from functools import lru_cache
-from sqlalchemy.orm import Session, load_only
-from sqlalchemy import Column, or_, func, select
-from loguru import logger
from datetime import datetime, timedelta
-from sqlalchemy.ext.asyncio import AsyncSession
+from functools import lru_cache
+
from cachetools import LRUCache, cached
from cachetools.keys import hashkey
+from loguru import logger
+from sqlalchemy import Column, func, or_, select
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.orm import Session, load_only
-from app.web.config import ALLOW_ANY_EMAIL
from app.shared.db import models
from app.shared.settings import get_settings
from app.shared.user_groups import UserGroups
from app.shared.utils.misc import fnv1a_hash_mod
+from app.web.config import ALLOW_ANY_EMAIL
from app.web.utils.misc import convert_priority_to_queue_dict
@@ -117,7 +118,7 @@ async def get_group_priority_async(db: AsyncSession, group_id: str) -> dict:
@cached(cache=LRUCache(maxsize=128), key=lambda db, email: hashkey(email))
def get_user_group_names(db: Session, email: str) -> list[str]:
"""
- given an email retrieves the user groups from the DB and then the email-domain groups from a global variable, the email does not need to belong to an existing user.
+ given an email retrieves the user groups from the DB and then the email-domain groups from a global variable, the email does not need to belong to an existing user.
"""
# TODO: the read: [group1, group2] permissions don't currently work
if not email or not len(email) or "@" not in email: return []
@@ -173,7 +174,7 @@ def upsert_user_groups(db: Session):
def display_email_pii(email: str):
return f"'{email[0:3]}...@{email.split('@')[1]}'"
"""
- reads the user_groups yaml file and inserts any new users, groups,
+ reads the user_groups yaml file and inserts any new users, groups,
along with new participation of users in groups
"""
filename = get_settings().USER_GROUPS_FILENAME
@@ -192,6 +193,7 @@ def upsert_user_groups(db: Session):
for group in explicit_groups:
group_domains[group].add(domain)
import json
+
# upsert groups and save a map of groupid -> dbobject
for group_id, g in ug.groups.items():
upsert_group(db, group_id, g.description, g.orchestrator, g.orchestrator_sheet, g.service_account_email, json.loads(g.permissions.model_dump_json()), list(group_domains.get(group_id, [])))
diff --git a/app/web/db/user_state.py b/app/web/db/user_state.py
index 968e1bd..384b0b6 100644
--- a/app/web/db/user_state.py
+++ b/app/web/db/user_state.py
@@ -1,13 +1,14 @@
-from typing import Dict, Set
-import sqlalchemy
-from sqlalchemy.orm import Session
-from sqlalchemy import func
from datetime import datetime
+from typing import Dict, Set
+
+import sqlalchemy
+from sqlalchemy import func
+from sqlalchemy.orm import Session
from app.shared.db import models
-from app.shared.user_groups import GroupInfo, GroupPermissions
from app.shared.schemas import Usage, UsageResponse
+from app.shared.user_groups import GroupInfo, GroupPermissions
from app.web.db import crud
from app.web.utils.misc import convert_priority_to_queue_dict
diff --git a/app/web/endpoints/default.py b/app/web/endpoints/default.py
index 9271992..cd23d13 100644
--- a/app/web/endpoints/default.py
+++ b/app/web/endpoints/default.py
@@ -1,13 +1,15 @@
from typing import Dict
+
from fastapi import APIRouter, Depends, HTTPException
from fastapi.responses import FileResponse, JSONResponse
-from app.web.config import VERSION, BREAKING_CHANGES
from app.shared.schemas import ActiveUser, UsageResponse
+from app.shared.user_groups import GroupInfo
+from app.web.config import BREAKING_CHANGES, VERSION
from app.web.db.user_state import UserState
from app.web.security import get_user_state
-from app.shared.user_groups import GroupInfo
+
default_router = APIRouter()
@@ -42,7 +44,7 @@ def get_user_usage(
if not user.active:
raise HTTPException(status_code=403, detail="User is not active.")
return user.usage()
-
+
@default_router.get('/favicon.ico', include_in_schema=False)
diff --git a/app/web/endpoints/interoperability.py b/app/web/endpoints/interoperability.py
index 06ea175..7892bde 100644
--- a/app/web/endpoints/interoperability.py
+++ b/app/web/endpoints/interoperability.py
@@ -1,19 +1,19 @@
import json
+
+import sqlalchemy
+from auto_archiver.core import Metadata
from fastapi import APIRouter, Depends, HTTPException
from fastapi.responses import JSONResponse
from loguru import logger
-import sqlalchemy
-from auto_archiver.core import Metadata
from sqlalchemy.orm import Session
-from app.shared.aa_utils import get_all_urls
-from app.web.config import ALLOW_ANY_EMAIL
from app.shared import business_logic, schemas
-from app.shared.db import worker_crud
+from app.shared.aa_utils import get_all_urls
+from app.shared.db import models, worker_crud
from app.shared.db.database import get_db_dependency
-from app.web.security import token_api_key_auth
-from app.shared.db import models
from app.shared.log import log_error
+from app.web.config import ALLOW_ANY_EMAIL
+from app.web.security import token_api_key_auth
interoperability_router = APIRouter(prefix="/interop", tags=["Interoperability endpoints."])
diff --git a/app/web/endpoints/sheet.py b/app/web/endpoints/sheet.py
index 7848b5e..d8c089a 100644
--- a/app/web/endpoints/sheet.py
+++ b/app/web/endpoints/sheet.py
@@ -1,16 +1,16 @@
from fastapi import APIRouter, Depends, HTTPException
from fastapi.responses import JSONResponse
-
from sqlalchemy import exc
from sqlalchemy.orm import Session
-from app.web.db.user_state import UserState
from app.shared import schemas
-from app.shared.task_messaging import get_celery
-from app.web.security import get_user_state
-from app.web.db import crud
from app.shared.db.database import get_db_dependency
+from app.shared.task_messaging import get_celery
+from app.web.db import crud
+from app.web.db.user_state import UserState
+from app.web.security import get_user_state
+
sheet_router = APIRouter(prefix="/sheet", tags=["Google Spreadsheet operations"])
@@ -78,4 +78,4 @@ def archive_user_sheet(
group_queue = user.priority_group(sheet.group_id)
task = celery.signature("create_sheet_task", args=[schemas.SubmitSheet(sheet_id=id, author_id=user.email, group_id=sheet.group_id).model_dump_json()]).apply_async(**group_queue)
- return JSONResponse({"id": task.id}, status_code=201)
\ No newline at end of file
+ return JSONResponse({"id": task.id}, status_code=201)
diff --git a/app/web/endpoints/task.py b/app/web/endpoints/task.py
index 610c579..3f2ff94 100644
--- a/app/web/endpoints/task.py
+++ b/app/web/endpoints/task.py
@@ -3,10 +3,10 @@ from fastapi import APIRouter, Depends
from fastapi.encoders import jsonable_encoder
from fastapi.responses import JSONResponse
-from app.shared.task_messaging import get_celery
-from app.web.security import get_token_or_user_auth
from app.shared import schemas
from app.shared.log import log_error
+from app.shared.task_messaging import get_celery
+from app.web.security import get_token_or_user_auth
from app.web.utils.misc import custom_jsonable_encoder
diff --git a/app/web/endpoints/url.py b/app/web/endpoints/url.py
index a7ac4b4..8307c2d 100644
--- a/app/web/endpoints/url.py
+++ b/app/web/endpoints/url.py
@@ -1,22 +1,22 @@
+from datetime import datetime
+from urllib.parse import urlparse
+
from fastapi import APIRouter, Depends, HTTPException
from fastapi.responses import JSONResponse
-from datetime import datetime
from loguru import logger
from sqlalchemy.orm import Session
-from app.web.config import ALLOW_ANY_EMAIL
from app.shared import schemas
+from app.shared.db.database import get_db_dependency
from app.shared.task_messaging import get_celery
-from app.web.security import get_token_or_user_auth, get_user_state
+from app.web.config import ALLOW_ANY_EMAIL
from app.web.db import crud
from app.web.db.user_state import UserState
-from app.shared.db.database import get_db_dependency
-
-from urllib.parse import urlparse
-
+from app.web.security import get_token_or_user_auth, get_user_state
from app.web.utils.misc import convert_priority_to_queue_dict
+
url_router = APIRouter(prefix="/url", tags=["Single URL operations"])
celery = get_celery()
@@ -47,7 +47,7 @@ def archive_url(
else:
archive_create.author_id = archive.author_id or email
group_queue = convert_priority_to_queue_dict("high")
-
+
task = celery.signature("create_archive_task", args=[archive_create.model_dump_json()]).apply_async(**group_queue)
task_response = schemas.Task(id=task.id)
@@ -74,8 +74,8 @@ def search_by_url(
@url_router.delete("/{id}", summary="Delete a single URL archive by id.")
def delete_archive(
- id:str,
- user: UserState = Depends(get_user_state),
+ id:str,
+ user: UserState = Depends(get_user_state),
db: Session = Depends(get_db_dependency)
) -> schemas.DeleteResponse:
logger.info(f"deleting url archive task {id} request by {user.email}")
diff --git a/app/web/events.py b/app/web/events.py
index 625731a..fa15614 100644
--- a/app/web/events.py
+++ b/app/web/events.py
@@ -1,22 +1,32 @@
import asyncio
-from collections import defaultdict
import datetime
import logging
+from collections import defaultdict
+from contextlib import asynccontextmanager
+
import alembic.config
from fastapi import FastAPI
-from contextlib import asynccontextmanager
+from fastapi_mail import FastMail, MessageSchema, MessageType
from fastapi_utils.tasks import repeat_every
from loguru import logger
-from fastapi_mail import FastMail, MessageSchema, MessageType
-from app.shared.db import models
-from app.shared.db.database import get_db, get_db_async, make_engine, wal_checkpoint
from app.shared import schemas
+from app.shared.db import models
+from app.shared.db.database import (
+ get_db,
+ get_db_async,
+ make_engine,
+ wal_checkpoint,
+)
from app.shared.settings import get_settings
from app.shared.task_messaging import get_celery
from app.web.db import crud
from app.web.middleware import increase_exceptions_counter
-from app.web.utils.metrics import measure_regular_metrics, redis_subscribe_worker_exceptions
+from app.web.utils.metrics import (
+ measure_regular_metrics,
+ redis_subscribe_worker_exceptions,
+)
+
celery = get_celery()
@@ -183,4 +193,4 @@ async def delete_stale_sheets():
async def generate_users_export_csv():
#TODO: implement a cronjob that regularly requested user data to a CSV file
# see https://colab.research.google.com/drive/1QDbo3QXHPBdiTuANlA1AWVvN-rqxuCPa?authuser=0#scrollTo=4nPXeSdK8RBT
- pass
\ No newline at end of file
+ pass
diff --git a/app/web/main.py b/app/web/main.py
index ff2266e..69af5c6 100644
--- a/app/web/main.py
+++ b/app/web/main.py
@@ -1,24 +1,23 @@
import os
-from fastapi import FastAPI, Depends
-from fastapi.staticfiles import StaticFiles
+
+from fastapi import Depends, FastAPI
from fastapi.middleware.cors import CORSMiddleware
-from prometheus_fastapi_instrumentator import Instrumentator
+from fastapi.staticfiles import StaticFiles
from loguru import logger
+from prometheus_fastapi_instrumentator import Instrumentator
-from app.web.middleware import logging_middleware
-from app.shared.task_messaging import get_celery
-
-from app.web.security import token_api_key_auth
-from app.web.config import VERSION, API_DESCRIPTION
-from app.web.events import lifespan
from app.shared.settings import get_settings
-
-
+from app.shared.task_messaging import get_celery
+from app.web.config import API_DESCRIPTION, VERSION
from app.web.endpoints.default import default_router
-from app.web.endpoints.url import url_router
+from app.web.endpoints.interoperability import interoperability_router
from app.web.endpoints.sheet import sheet_router
from app.web.endpoints.task import task_router
-from app.web.endpoints.interoperability import interoperability_router
+from app.web.endpoints.url import url_router
+from app.web.events import lifespan
+from app.web.middleware import logging_middleware
+from app.web.security import token_api_key_auth
+
celery = get_celery()
@@ -57,4 +56,4 @@ def app_factory(settings = get_settings()):
logger.warning(f"MOUNTing local archive, use this in development only {settings.SERVE_LOCAL_ARCHIVE}")
app.mount(settings.SERVE_LOCAL_ARCHIVE, StaticFiles(directory=local_dir), name=settings.SERVE_LOCAL_ARCHIVE)
- return app
\ No newline at end of file
+ return app
diff --git a/app/web/middleware.py b/app/web/middleware.py
index 52da626..5ddca4b 100644
--- a/app/web/middleware.py
+++ b/app/web/middleware.py
@@ -1,7 +1,9 @@
import traceback
-from loguru import logger
+
from fastapi import Request
+from loguru import logger
+
from app.shared.log import log_error
from app.web.utils.metrics import EXCEPTION_COUNTER
@@ -25,7 +27,7 @@ async def increase_exceptions_counter(e: Exception, location:str="cronjob"):
last_trace = traceback.extract_tb(e.__traceback__)[-1]
_file, _line, func_name, _text = last_trace
location = func_name
- except Exception as e:
+ except Exception as e:
logger.error(f"Unable to get function name from cronjob exception traceback: {e}")
EXCEPTION_COUNTER.labels(type=e.__class__.__name__, location=location).inc()
- log_error(e)
\ No newline at end of file
+ log_error(e)
diff --git a/app/web/security.py b/app/web/security.py
index 12115af..494e094 100644
--- a/app/web/security.py
+++ b/app/web/security.py
@@ -1,14 +1,17 @@
+import secrets
+
+import requests
+from fastapi import Depends, HTTPException, status
+from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
from loguru import logger
-import requests, secrets
-from fastapi import HTTPException, status, Depends
-from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
from sqlalchemy.orm import Session
-from app.web.config import ALLOW_ANY_EMAIL
-from app.shared.settings import get_settings
from app.shared.db.database import get_db_dependency
+from app.shared.settings import get_settings
+from app.web.config import ALLOW_ANY_EMAIL
from app.web.db.user_state import UserState
+
settings = get_settings()
bearer_security = HTTPBearer()
@@ -80,4 +83,4 @@ def authenticate_user(access_token):
def get_user_state(email:str=Depends(get_user_auth), db:Session=Depends(get_db_dependency)):
- return UserState(db, email)
\ No newline at end of file
+ return UserState(db, email)
diff --git a/app/web/utils/metrics.py b/app/web/utils/metrics.py
index a885b9a..d8026a1 100644
--- a/app/web/utils/metrics.py
+++ b/app/web/utils/metrics.py
@@ -2,12 +2,13 @@ import asyncio
import json
import os
import shutil
+
from prometheus_client import Counter, Gauge
-from app.web.db import crud
from app.shared.db.database import get_db
from app.shared.log import log_error
from app.shared.task_messaging import get_redis
+from app.web.db import crud
# Custom metrics
diff --git a/app/web/utils/misc.py b/app/web/utils/misc.py
index 870a60b..16a6591 100644
--- a/app/web/utils/misc.py
+++ b/app/web/utils/misc.py
@@ -1,4 +1,5 @@
import base64
+
from fastapi.encoders import jsonable_encoder
diff --git a/app/worker/main.py b/app/worker/main.py
index 561245b..7b4826b 100644
--- a/app/worker/main.py
+++ b/app/worker/main.py
@@ -1,21 +1,22 @@
+import datetime
import json
+import traceback
-import traceback, datetime
+from auto_archiver.core.orchestrator import ArchivingOrchestrator
from celery.signals import task_failure
from loguru import logger
from sqlalchemy import exc
-from auto_archiver.core.orchestrator import ArchivingOrchestrator
-from app.shared.db import models
-from app.shared.db.database import get_db
from app.shared import business_logic, schemas
-from app.shared.task_messaging import get_celery, get_redis
-from app.shared.settings import get_settings
-from app.shared.log import log_error
from app.shared.aa_utils import get_all_urls
-from app.shared.db import worker_crud
+from app.shared.db import models, worker_crud
+from app.shared.db.database import get_db
+from app.shared.log import log_error
+from app.shared.settings import get_settings
+from app.shared.task_messaging import get_celery, get_redis
from app.worker.worker_log import setup_celery_logger
+
settings = get_settings()
celery = get_celery("worker")
@@ -26,7 +27,7 @@ USER_GROUPS_FILENAME = settings.USER_GROUPS_FILENAME
setup_celery_logger(celery)
# TODO: these are temporary PATCHES for new aa's functionality
-# logger.add("app/worker/worker_log.log", level="DEBUG")
+# logger.add("app/worker/worker_log.log", level="DEBUG")
logger.remove = lambda x: print(f"logger.remove({x})")
# TODO: after release, as it requires updating past entries with sheet_id where tag is used, drop tags
diff --git a/app/worker/worker_log.py b/app/worker/worker_log.py
index 221d824..022c63d 100644
--- a/app/worker/worker_log.py
+++ b/app/worker/worker_log.py
@@ -1,9 +1,11 @@
-from loguru import logger
-from celery import Celery
import sys
+from celery import Celery
+from loguru import logger
+
from app.shared.task_messaging import get_celery
+
celery = get_celery("worker")
def setup_celery_logger(celery):
@@ -22,7 +24,7 @@ def setup_celery_logger(celery):
if message.strip():
logger.info(message.strip())
# Required to prevent issues with buffered output
- def flush(self): pass
+ def flush(self): pass
def isatty(self): return False
sys.stdout = InterceptHandler()
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 29088c0..9c88a36 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -12,7 +12,7 @@ services:
- ALLOWED_ORIGINS=["http://localhost:8000","http://localhost:8004","http://localhost:8081","chrome-extension://ojcimmjndnlmmlgnjaeojoebaceokpdp"]
- USER_GROUPS_FILENAME=/aa-api/app/user-groups.dev.yaml
- DATABASE_PATH=sqlite:////aa-api/database/auto-archiver.db
-
+
worker:
# command: watchmedo auto-restart --patterns="*.py" --recursive --ignore-directories -- celery -- --app=app.worker.main.celery worker -Q high_priority,low_priority --concurrency=${CONCURRENCY} --max-tasks-per-child=100
diff --git a/docker-compose.yml b/docker-compose.yml
index 31969bc..737c346 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -5,7 +5,7 @@ volumes:
name: "auto-archiver-api"
services:
web:
- build:
+ build:
context: .
dockerfile: web.Dockerfile
restart: always
@@ -29,7 +29,7 @@ services:
retries: 3
worker:
- build:
+ build:
context: .
dockerfile: worker.Dockerfile
restart: always
@@ -68,4 +68,4 @@ services:
test: ["CMD", "redis-cli", "-a", "${REDIS_PASSWORD}", "ping"]
interval: 30s
timeout: 10s
- retries: 3
\ No newline at end of file
+ retries: 3
diff --git a/poetry.lock b/poetry.lock
index ceb824a..9c1ea55 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -616,6 +616,18 @@ files = [
[package.dependencies]
pycparser = "*"
+[[package]]
+name = "cfgv"
+version = "3.4.0"
+description = "Validate configuration and produce human readable error messages."
+optional = false
+python-versions = ">=3.8"
+groups = ["dev"]
+files = [
+ {file = "cfgv-3.4.0-py2.py3-none-any.whl", hash = "sha256:b7265b1f29fd3316bfcd2b330d63d024f2bfd8bcb8b0272f8e19a504856c48f9"},
+ {file = "cfgv-3.4.0.tar.gz", hash = "sha256:e52591d4c5f5dead8e0f673fb16db7949d2cfb3f7da4582893288f0ded8fe560"},
+]
+
[[package]]
name = "charset-normalizer"
version = "3.4.1"
@@ -959,6 +971,18 @@ calendars = ["convertdate (>=2.2.1)", "hijridate"]
fasttext = ["fasttext (>=0.9.1)", "numpy (>=1.19.3,<2)"]
langdetect = ["langdetect (>=1.0.0)"]
+[[package]]
+name = "distlib"
+version = "0.3.9"
+description = "Distribution utilities"
+optional = false
+python-versions = "*"
+groups = ["dev"]
+files = [
+ {file = "distlib-0.3.9-py2.py3-none-any.whl", hash = "sha256:47f8c22fd27c27e25a65601af709b38e4f0a45ea4fc2e710f65755fa8caaaf87"},
+ {file = "distlib-0.3.9.tar.gz", hash = "sha256:a60f20dea646b8a33f3e7772f74dc0b2d0772d2837ee1342a00645c81edf9403"},
+]
+
[[package]]
name = "dnspython"
version = "2.7.0"
@@ -1097,6 +1121,23 @@ future = "*"
[package.extras]
dev = ["Sphinx (==2.1.0)", "future (==0.17.1)", "numpy (==1.16.4)", "pytest (==4.6.1)", "pytest-mock (==1.10.4)", "tox (==3.12.1)"]
+[[package]]
+name = "filelock"
+version = "3.17.0"
+description = "A platform independent file lock."
+optional = false
+python-versions = ">=3.9"
+groups = ["dev"]
+files = [
+ {file = "filelock-3.17.0-py3-none-any.whl", hash = "sha256:533dc2f7ba78dc2f0f531fc6c4940addf7b70a481e269a5a3b93be94ffbe8338"},
+ {file = "filelock-3.17.0.tar.gz", hash = "sha256:ee4e77401ef576ebb38cd7f13b9b28893194acc20a8e68e18730ba9c0e54660e"},
+]
+
+[package.extras]
+docs = ["furo (>=2024.8.6)", "sphinx (>=8.1.3)", "sphinx-autodoc-typehints (>=3)"]
+testing = ["covdefaults (>=2.3)", "coverage (>=7.6.10)", "diff-cover (>=9.2.1)", "pytest (>=8.3.4)", "pytest-asyncio (>=0.25.2)", "pytest-cov (>=6)", "pytest-mock (>=3.14)", "pytest-timeout (>=2.3.1)", "virtualenv (>=20.28.1)"]
+typing = ["typing-extensions (>=4.12.2) ; python_version < \"3.11\""]
+
[[package]]
name = "future"
version = "1.0.0"
@@ -1409,6 +1450,21 @@ http2 = ["h2 (>=3,<5)"]
socks = ["socksio (==1.*)"]
zstd = ["zstandard (>=0.18.0)"]
+[[package]]
+name = "identify"
+version = "2.6.8"
+description = "File identification library for Python"
+optional = false
+python-versions = ">=3.9"
+groups = ["dev"]
+files = [
+ {file = "identify-2.6.8-py2.py3-none-any.whl", hash = "sha256:83657f0f766a3c8d0eaea16d4ef42494b39b34629a4b3192a9d020d349b3e255"},
+ {file = "identify-2.6.8.tar.gz", hash = "sha256:61491417ea2c0c5c670484fd8abbb34de34cdae1e5f39a73ee65e48e4bb663fc"},
+]
+
+[package.extras]
+license = ["ukkonen"]
+
[[package]]
name = "idna"
version = "3.10"
@@ -1724,6 +1780,18 @@ files = [
{file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"},
]
+[[package]]
+name = "nodeenv"
+version = "1.9.1"
+description = "Node.js virtual environment builder"
+optional = false
+python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7"
+groups = ["dev"]
+files = [
+ {file = "nodeenv-1.9.1-py2.py3-none-any.whl", hash = "sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9"},
+ {file = "nodeenv-1.9.1.tar.gz", hash = "sha256:6ec12890a2dab7946721edbfbcd91f3319c6ccc9aec47be7c7e6b7011ee6645f"},
+]
+
[[package]]
name = "numpy"
version = "2.1.3"
@@ -1981,6 +2049,23 @@ tests = ["check-manifest", "coverage (>=7.4.2)", "defusedxml", "markdown2", "ole
typing = ["typing-extensions ; python_version < \"3.10\""]
xmp = ["defusedxml"]
+[[package]]
+name = "platformdirs"
+version = "4.3.6"
+description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`."
+optional = false
+python-versions = ">=3.8"
+groups = ["dev"]
+files = [
+ {file = "platformdirs-4.3.6-py3-none-any.whl", hash = "sha256:73e575e1408ab8103900836b97580d5307456908a03e92031bab39e4554cc3fb"},
+ {file = "platformdirs-4.3.6.tar.gz", hash = "sha256:357fb2acbc885b0419afd3ce3ed34564c13c9b95c89360cd9563f73aa5e2b907"},
+]
+
+[package.extras]
+docs = ["furo (>=2024.8.6)", "proselint (>=0.14)", "sphinx (>=8.0.2)", "sphinx-autodoc-typehints (>=2.4)"]
+test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=8.3.2)", "pytest-cov (>=5)", "pytest-mock (>=3.14)"]
+type = ["mypy (>=1.11.2)"]
+
[[package]]
name = "pluggy"
version = "1.5.0"
@@ -1997,6 +2082,25 @@ files = [
dev = ["pre-commit", "tox"]
testing = ["pytest", "pytest-benchmark"]
+[[package]]
+name = "pre-commit"
+version = "4.1.0"
+description = "A framework for managing and maintaining multi-language pre-commit hooks."
+optional = false
+python-versions = ">=3.9"
+groups = ["dev"]
+files = [
+ {file = "pre_commit-4.1.0-py2.py3-none-any.whl", hash = "sha256:d29e7cb346295bcc1cc75fc3e92e343495e3ea0196c9ec6ba53f49f10ab6ae7b"},
+ {file = "pre_commit-4.1.0.tar.gz", hash = "sha256:ae3f018575a588e30dfddfab9a05448bfbd6b73d78709617b5a2b853549716d4"},
+]
+
+[package.dependencies]
+cfgv = ">=2.0.0"
+identify = ">=1.0.0"
+nodeenv = ">=0.11.1"
+pyyaml = ">=5.1"
+virtualenv = ">=20.10.0"
+
[[package]]
name = "prometheus-client"
version = "0.21.1"
@@ -2557,7 +2661,7 @@ version = "6.0.2"
description = "YAML parser and emitter for Python"
optional = false
python-versions = ">=3.8"
-groups = ["web"]
+groups = ["dev", "web"]
files = [
{file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"},
{file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"},
@@ -3415,6 +3519,27 @@ files = [
{file = "vine-5.1.0.tar.gz", hash = "sha256:8b62e981d35c41049211cf62a0a1242d8c1ee9bd15bb196ce38aefd6799e61e0"},
]
+[[package]]
+name = "virtualenv"
+version = "20.29.2"
+description = "Virtual Python Environment builder"
+optional = false
+python-versions = ">=3.8"
+groups = ["dev"]
+files = [
+ {file = "virtualenv-20.29.2-py3-none-any.whl", hash = "sha256:febddfc3d1ea571bdb1dc0f98d7b45d24def7428214d4fb73cc486c9568cce6a"},
+ {file = "virtualenv-20.29.2.tar.gz", hash = "sha256:fdaabebf6d03b5ba83ae0a02cfe96f48a716f4fae556461d180825866f75b728"},
+]
+
+[package.dependencies]
+distlib = ">=0.3.7,<1"
+filelock = ">=3.12.2,<4"
+platformdirs = ">=3.9.1,<5"
+
+[package.extras]
+docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.2,!=7.3)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=23.6)"]
+test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23.1)", "pytest (>=7.4)", "pytest-env (>=0.8.2)", "pytest-freezer (>=0.4.8) ; platform_python_implementation == \"PyPy\" or platform_python_implementation == \"CPython\" and sys_platform == \"win32\" and python_version >= \"3.13\"", "pytest-mock (>=3.11.1)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)", "setuptools (>=68)", "time-machine (>=2.10) ; platform_python_implementation == \"CPython\""]
+
[[package]]
name = "vk-api"
version = "11.9.9"
@@ -3688,4 +3813,4 @@ test = ["pytest (>=8.1,<9.0)", "pytest-rerunfailures (>=14.0,<15.0)"]
[metadata]
lock-version = "2.1"
python-versions = ">=3.10,<3.13"
-content-hash = "11d734f2ee32206214a7ecb8dc3ec8d19a7b6281ee98b509a5bb8bdb647c674a"
+content-hash = "c4a5c50ac109c9912992ca86d2b5ec712c6bcfc84838bf42f90208b02cc27b3c"
diff --git a/pyproject.toml b/pyproject.toml
index ea1c87d..df7b789 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -52,4 +52,4 @@ pytest = ">=8.3.4,<9.0.0"
httpx = ">=0.28.1,<0.29.0"
coverage = ">=7.6.11,<8.0.0"
pytest-asyncio = ">=0.25.3,<0.26.0"
-
+pre-commit = "^4.1.0"
diff --git a/user-groups.example.yaml b/user-groups.example.yaml
index 32f99b4..22a919d 100644
--- a/user-groups.example.yaml
+++ b/user-groups.example.yaml
@@ -59,4 +59,3 @@ groups:
permissions:
read: ["default"]
read_public: true
-
\ No newline at end of file
diff --git a/worker.Dockerfile b/worker.Dockerfile
index 4e24f87..9154c32 100644
--- a/worker.Dockerfile
+++ b/worker.Dockerfile
@@ -30,4 +30,4 @@ COPY alembic.ini ./
COPY ./app/ ./app/
COPY user-groups.* ./app/
-ENTRYPOINT ["./poetry-venv/bin/poetry", "run"]
\ No newline at end of file
+ENTRYPOINT ["./poetry-venv/bin/poetry", "run"]
From 229db7dd5c88a2af886bbd5cc701f80b865f78f2 Mon Sep 17 00:00:00 2001
From: Michael Plunkett <5885605+michplunkett@users.noreply.github.com>
Date: Wed, 26 Feb 2025 14:37:49 -0600
Subject: [PATCH 08/40] =?UTF-8?q?Add=20test=20CI=20Action=20with=20multipl?=
=?UTF-8?q?e=20=F0=9F=90=8D=20versions=20(#57)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
.github/workflows/ci.yml | 2 +-
.github/workflows/format-and-fail.yml | 2 +-
.github/workflows/test.yml | 40 +++++++++++++++++++++++++++
app/__init__.py | 0
pytest.ini | 2 ++
5 files changed, 44 insertions(+), 2 deletions(-)
create mode 100644 .github/workflows/test.yml
create mode 100644 app/__init__.py
create mode 100644 pytest.ini
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 4135ac2..6b8c556 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -6,7 +6,7 @@ on:
branches: [ main, dev ]
jobs:
- test:
+ test-with-coverage:
runs-on: ubuntu-latest
services:
diff --git a/.github/workflows/format-and-fail.yml b/.github/workflows/format-and-fail.yml
index f1c01f8..c203aa9 100644
--- a/.github/workflows/format-and-fail.yml
+++ b/.github/workflows/format-and-fail.yml
@@ -10,7 +10,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- - uses: actions/setup-python@v4
+ - uses: actions/setup-python@v5
with:
python-version: "3.11"
- uses: pre-commit/action@v3.0.0
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
new file mode 100644
index 0000000..1ad37f3
--- /dev/null
+++ b/.github/workflows/test.yml
@@ -0,0 +1,40 @@
+name: Run Tests
+on:
+ push:
+ branches: [ main, dev ]
+ pull_request:
+ branches: [ main, dev ]
+
+jobs:
+ test:
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ python-version: ['3.10', '3.11', '3.12']
+
+ services:
+ redis:
+ image: redis:6-alpine
+ ports:
+ - 6379:6379
+
+ steps:
+ - name: Set up Python
+ uses: actions/setup-python@v5
+ with:
+ python-version: ${{ matrix.python-version }}
+
+ - name: Checkout code
+ uses: actions/checkout@v4
+
+ - name: Install Poetry
+ run: pipx install poetry
+
+ - name: Install dependencies
+ run: poetry install --no-interaction --with dev
+
+ - name: Set dev environment variable
+ run: echo "ENVIRONMENT_FILE=.env.test" >> $GITHUB_ENV
+
+ - name: Run tests
+ run: poetry run pytest app/tests
diff --git a/app/__init__.py b/app/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/pytest.ini b/pytest.ini
new file mode 100644
index 0000000..a635c5c
--- /dev/null
+++ b/pytest.ini
@@ -0,0 +1,2 @@
+[pytest]
+pythonpath = .
From d575b6f9af5e72ad2f6042dbea5161603f61f756 Mon Sep 17 00:00:00 2001
From: Michael Plunkett <5885605+michplunkett@users.noreply.github.com>
Date: Thu, 27 Feb 2025 12:35:23 -0600
Subject: [PATCH 09/40] Format and lint the `tests` directory (#58)
---
.pre-commit-config.yaml | 7 +-
app/tests/conftest.py | 148 +++-
app/tests/shared/db/test_models.py | 11 +-
app/tests/shared/db/test_worker_crud.py | 51 +-
app/tests/shared/test_business_logic.py | 14 +-
app/tests/web/db/test_crud.py | 714 ++++++++++++++----
app/tests/web/db/test_user_state.py | 613 ++++++++++++---
app/tests/web/endpoints/test_default.py | 80 +-
.../web/endpoints/test_interoperability.py | 131 +++-
app/tests/web/endpoints/test_sheet.py | 202 +++--
app/tests/web/endpoints/test_task.py | 14 +-
app/tests/web/endpoints/test_url.py | 223 ++++--
app/tests/web/test_main.py | 24 +-
app/tests/web/test_security.py | 150 ++--
app/tests/worker/test_worker_main.py | 97 ++-
15 files changed, 1894 insertions(+), 585 deletions(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 73765d5..5e1d2a8 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -5,7 +5,7 @@ repos:
- id: nbqa-ruff
args:
- --fix
- - --target-version=py311
+ - --target-version=py310
- --ignore=E721,E722
- --line-length=80
- id: nbqa-black
@@ -62,18 +62,17 @@ repos:
- --line-length=80
# - repo: https://github.com/astral-sh/ruff-pre-commit
-# rev: v0.4.10
+# rev: v0.9.7
# hooks:
# - id: ruff
# types_or: [python,pyi]
# args:
# - --fix
-# - --target-version=py311
# - --select=B,C,E,F,W,B9
# - --line-length=80
# - --ignore=E203,E402,E501,E261
# - id: ruff-format
# types_or: [ python,pyi]
# args:
-# - --target-version=py311
+# - --target-version=py310
# - --line-length=80
diff --git a/app/tests/conftest.py b/app/tests/conftest.py
index f7da39e..997ea62 100644
--- a/app/tests/conftest.py
+++ b/app/tests/conftest.py
@@ -1,4 +1,6 @@
import os
+from datetime import datetime
+from http import HTTPStatus
from typing import AsyncGenerator
from unittest.mock import patch
@@ -7,15 +9,31 @@ import pytest_asyncio
from fastapi.testclient import TestClient
from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession
+from app.shared.db import models
+from app.shared.db.database import (
+ make_async_engine,
+ make_async_session_local,
+ make_engine,
+ make_session_local,
+)
from app.shared.settings import Settings
from app.web.config import ALLOW_ANY_EMAIL
+from app.web.db import crud
+from app.web.db.crud import get_user_group_names
from app.web.db.user_state import UserState
+from app.web.main import app_factory
+from app.web.security import (
+ get_token_or_user_auth,
+ get_user_auth,
+ get_user_state,
+ token_api_key_auth,
+)
@pytest.fixture(autouse=True)
def mock_logger_add():
"""Fixture to mock loguru.logger.add for all tests."""
- with patch('loguru.logger.add') as mock_add:
+ with patch("loguru.logger.add") as mock_add:
yield mock_add # This makes the mock available to tests
@@ -26,23 +44,22 @@ def get_settings():
@pytest.fixture(autouse=True)
def mock_settings():
- with patch('app.shared.settings.Settings', return_value=Settings(_env_file=".env.test")) as mock_settings:
+ with patch(
+ "app.shared.settings.Settings",
+ return_value=Settings(_env_file=".env.test"),
+ ) as mock_settings:
yield mock_settings
@pytest.fixture()
def test_db(get_settings: Settings):
- from app.shared.db import models
- from app.shared.db.database import make_engine
- from app.web.db.crud import get_user_group_names
-
get_user_group_names.cache_clear()
make_engine.cache_clear()
engine = make_engine(get_settings.DATABASE_PATH)
fs = get_settings.DATABASE_PATH.replace("sqlite:///", "")
if not os.path.exists(fs):
- open(fs, 'w').close()
+ open(fs, "w").close()
models.Base.metadata.create_all(engine)
@@ -59,7 +76,6 @@ def test_db(get_settings: Settings):
@pytest.fixture()
def db_session(test_db):
- from app.shared.db.database import make_session_local
session_local = make_session_local(test_db)
with session_local() as session:
yield session
@@ -67,18 +83,12 @@ def db_session(test_db):
@pytest_asyncio.fixture()
async def async_test_db(get_settings: Settings):
- import asyncio
-
- from app.shared.db import models
- from app.shared.db.database import make_async_engine
- from app.web.db.crud import get_user_group_names
-
get_user_group_names.cache_clear()
engine = await make_async_engine(get_settings.ASYNC_DATABASE_PATH)
fs = get_settings.ASYNC_DATABASE_PATH.replace("sqlite+aiosqlite:///", "")
if not os.path.exists(fs):
- open(fs, 'w').close()
+ open(fs, "w").close()
async def create_all():
async with engine.begin() as conn:
@@ -102,8 +112,9 @@ async def async_test_db(get_settings: Settings):
@pytest_asyncio.fixture()
-async def async_db_session(async_test_db: AsyncEngine) -> AsyncGenerator[AsyncSession, None]:
- from app.shared.db.database import make_async_session_local
+async def async_db_session(
+ async_test_db: AsyncEngine,
+) -> AsyncGenerator[AsyncSession, None]:
session_local = await make_async_session_local(async_test_db)
async with session_local() as session:
yield session
@@ -111,8 +122,6 @@ async def async_db_session(async_test_db: AsyncEngine) -> AsyncGenerator[AsyncSe
@pytest.fixture()
def app(db_session):
- from app.web.db import crud
- from app.web.main import app_factory
app = app_factory()
crud.upsert_user_groups(db_session)
return app
@@ -126,14 +135,13 @@ def client(app):
@pytest.fixture()
def app_with_auth(app, db_session):
- from app.web.security import (
- get_token_or_user_auth,
- get_user_auth,
- get_user_state,
+ app.dependency_overrides[get_token_or_user_auth] = (
+ lambda: "rick@example.com"
)
- app.dependency_overrides[get_token_or_user_auth] = lambda: "rick@example.com"
app.dependency_overrides[get_user_auth] = lambda: "morty@example.com"
- app.dependency_overrides[get_user_state] = lambda: UserState(db_session, "MORTY@example.com")
+ app.dependency_overrides[get_user_state] = lambda: UserState(
+ db_session, "MORTY@example.com"
+ )
return app
@@ -145,7 +153,6 @@ def client_with_auth(app_with_auth):
@pytest.fixture()
def app_with_token(app):
- from app.web.security import get_token_or_user_auth, token_api_key_auth
app.dependency_overrides[token_api_key_auth] = lambda: ALLOW_ANY_EMAIL
app.dependency_overrides[get_token_or_user_auth] = lambda: ALLOW_ANY_EMAIL
return app
@@ -162,6 +169,93 @@ def test_no_auth():
# reusable code to ensure a method/endpoint combination is unauthorized
def no_auth(http_method, endpoint):
response = http_method(endpoint)
- assert response.status_code == 403
+ assert response.status_code == HTTPStatus.FORBIDDEN
assert response.json() == {"detail": "Not authenticated"}
+
return no_auth
+
+
+@pytest.fixture()
+def test_data(db_session):
+ author_emails = [
+ "rick@example.com",
+ "morty@example.com",
+ "jerry@example.com",
+ ]
+
+ # creates 3 users
+ for email in author_emails:
+ db_session.add(models.User(email=email))
+ db_session.commit()
+ assert db_session.query(models.User).count() == 3
+
+ # creates 100 archives for 3 users over 2 months with repeating URLs
+ for i in range(100):
+ author = author_emails[i % 3]
+ archive = models.Archive(
+ id=f"archive-id-456-{i}",
+ url=f"https://example-{i % 3}.com",
+ result={},
+ public=author == "jerry@example.com",
+ author_id=author,
+ group_id="spaceship"
+ if author == "morty@example.com" and i % 2 == 0
+ else None,
+ created_at=datetime(2021, (i % 2) + 1, (i % 25) + 1),
+ )
+ if i % 5 == 0:
+ archive.tags.append(models.Tag(id=f"tag-{i}"))
+ if i % 10 == 0:
+ archive.tags.append(models.Tag(id=f"tag-second-{i}"))
+ if i % 4 == 0:
+ archive.tags.append(models.Tag(id=f"tag-third-{i}"))
+ for j in range(10):
+ archive.urls.append(
+ models.ArchiveUrl(
+ url=f"https://example-{i}.com/{j}", key=f"media_{j}"
+ )
+ )
+ db_session.add(archive)
+
+ # creates a sheet for each user
+ for i, email in enumerate(
+ ["rick@example.com", "morty@example.com", "jerry@example.com"]
+ ):
+ db_session.add(
+ models.Sheet(
+ id=f"sheet-{i}",
+ name=f"sheet-{i}",
+ author_id=email,
+ group_id=None,
+ frequency="daily",
+ )
+ )
+ if email == "rick@example.com":
+ db_session.add(
+ models.Sheet(
+ id=f"sheet-{i}-2",
+ name=f"sheet-{i}-2",
+ author_id=email,
+ group_id="spaceship",
+ frequency="hourly",
+ )
+ )
+
+ db_session.commit()
+
+ assert db_session.query(models.Archive).count() == 100
+ assert db_session.query(models.Tag).count() == 20 + 10 + 25
+ assert db_session.query(models.ArchiveUrl).count() == 1000
+ assert (
+ db_session.query(models.ArchiveUrl)
+ .filter(models.ArchiveUrl.archive_id == "archive-id-456-0")
+ .count()
+ == 10
+ )
+
+ # setup groups
+ assert db_session.query(models.Group).count() == 0
+
+ crud.upsert_user_groups(db_session)
+ assert db_session.query(models.Group).count() == 4
+ assert db_session.query(models.User).count() == 3
diff --git a/app/tests/shared/db/test_models.py b/app/tests/shared/db/test_models.py
index 4da9571..537532b 100644
--- a/app/tests/shared/db/test_models.py
+++ b/app/tests/shared/db/test_models.py
@@ -1,6 +1,7 @@
-def test_generate_uuid():
- from app.shared.db.models import generate_uuid
+from app.shared.db.models import generate_uuid
- assert generate_uuid() != generate_uuid()
- assert len(generate_uuid()) == 36
- assert generate_uuid().count("-") == 4
+
+def test_generate_uuid():
+ assert generate_uuid() != generate_uuid()
+ assert len(generate_uuid()) == 36
+ assert generate_uuid().count("-") == 4
diff --git a/app/tests/shared/db/test_worker_crud.py b/app/tests/shared/db/test_worker_crud.py
index 09e9a76..2258781 100644
--- a/app/tests/shared/db/test_worker_crud.py
+++ b/app/tests/shared/db/test_worker_crud.py
@@ -1,11 +1,10 @@
from datetime import datetime
+from app.shared import schemas
from app.shared.db import models, worker_crud
-from app.tests.web.db.test_crud import test_data
def test_update_sheet_last_url_archived_at(db_session):
-
# Create test sheet
test_sheet = models.Sheet(id="sheet-123")
db_session.add(test_sheet)
@@ -14,17 +13,24 @@ def test_update_sheet_last_url_archived_at(db_session):
# Test updating existing sheet
assert isinstance(test_sheet.last_url_archived_at, datetime)
before = test_sheet.last_url_archived_at
- assert worker_crud.update_sheet_last_url_archived_at(db_session, "sheet-123") is True
+ assert (
+ worker_crud.update_sheet_last_url_archived_at(db_session, "sheet-123")
+ is True
+ )
db_session.refresh(test_sheet)
assert isinstance(test_sheet.last_url_archived_at, datetime)
assert test_sheet.last_url_archived_at > before
# Test non-existent sheet
- assert worker_crud.update_sheet_last_url_archived_at(db_session, "non-existent-sheet") is False
+ assert (
+ worker_crud.update_sheet_last_url_archived_at(
+ db_session, "non-existent-sheet"
+ )
+ is False
+ )
+
def test_get_group(test_data, db_session):
- from app.shared.db import worker_crud
-
assert worker_crud.get_group(db_session, "spaceship") is not None
assert worker_crud.get_group(db_session, "interdimensional") is not None
assert worker_crud.get_group(db_session, "animated-characters") is not None
@@ -32,24 +38,24 @@ def test_get_group(test_data, db_session):
def test_create_or_get_user(test_data, db_session):
- from app.shared.db import worker_crud
-
assert db_session.query(models.User).count() == 3
# already exists
- assert (u1 := worker_crud.create_or_get_user(db_session, "rick@example.com")) is not None
+ assert (
+ u1 := worker_crud.create_or_get_user(db_session, "rick@example.com")
+ ) is not None
assert u1.email == "rick@example.com"
# new user
- assert (u2 := worker_crud.create_or_get_user(db_session, "beth@example.com")) is not None
+ assert (
+ u2 := worker_crud.create_or_get_user(db_session, "beth@example.com")
+ ) is not None
assert u2.email == "beth@example.com"
assert db_session.query(models.User).count() == 4
def test_create_tag(db_session):
- from app.shared.db import worker_crud
-
assert db_session.query(models.Tag).count() == 0
# create first
@@ -57,7 +63,10 @@ def test_create_tag(db_session):
assert create_tag is not None
assert create_tag.id == "tag-101"
assert db_session.query(models.Tag).count() == 1
- assert db_session.query(models.Tag).filter(models.Tag.id == "tag-101").first() == create_tag
+ assert (
+ db_session.query(models.Tag).filter(models.Tag.id == "tag-101").first()
+ == create_tag
+ )
# same id does not add new db entry
existing_tag = worker_crud.create_tag(db_session, "tag-101")
@@ -72,9 +81,6 @@ def test_create_tag(db_session):
def test_create_task(db_session):
- from app.shared import schemas
- from app.shared.db import worker_crud
-
task = schemas.ArchiveCreate(
id="archive-id-456-101",
url="https://example-0.com",
@@ -83,17 +89,22 @@ def test_create_task(db_session):
author_id="rick@example.com",
group_id="spaceship",
tags=[],
- urls=[]
+ urls=[],
)
# with tags and urls
- nt = worker_crud.create_archive(db_session, task, [models.Tag(id="tag-101")], [models.ArchiveUrl(url="https://example-0.com/0", key="media_0")])
+ nt = worker_crud.create_archive(
+ db_session,
+ task,
+ [models.Tag(id="tag-101")],
+ [models.ArchiveUrl(url="https://example-0.com/0", key="media_0")],
+ )
assert nt is not None
assert nt.id == "archive-id-456-101"
assert nt.url == "https://example-0.com"
assert nt.author_id == "rick@example.com"
- assert nt.public == False
+ assert nt.public is False
assert nt.group_id == "spaceship"
assert len(nt.tags) == 1
assert nt.tags[0].id == "tag-101"
@@ -109,7 +120,7 @@ def test_create_task(db_session):
assert nt.id == "archive-id-456-102"
assert nt.url == "https://example-0.com"
assert nt.author_id == "rick@example.com"
- assert nt.public == False
+ assert nt.public is False
assert nt.group_id == "spaceship"
assert len(nt.tags) == 0
assert len(nt.urls) == 0
diff --git a/app/tests/shared/test_business_logic.py b/app/tests/shared/test_business_logic.py
index 830fa7b..e10d402 100644
--- a/app/tests/shared/test_business_logic.py
+++ b/app/tests/shared/test_business_logic.py
@@ -9,7 +9,7 @@ from app.shared.business_logic import (
)
-class Test_get_store_archive_until:
+class TestGetStoreArchiveUntil:
GROUP_ID = "test-group"
def test_group_not_found(self, db_session):
@@ -17,7 +17,10 @@ class Test_get_store_archive_until:
get_store_archive_until(db_session, self.GROUP_ID)
assert str(exc.value) == f"Group {self.GROUP_ID} not found."
- @patch("app.shared.db.worker_crud.get_group", return_value=MagicMock(permissions=None))
+ @patch(
+ "app.shared.db.worker_crud.get_group",
+ return_value=MagicMock(permissions=None),
+ )
def test_group_no_permissions(self, db_session):
with pytest.raises(AssertionError) as exc:
get_store_archive_until(db_session, self.GROUP_ID)
@@ -48,14 +51,17 @@ class Test_get_store_archive_until:
mock_get_group.assert_called_once_with(db_session, self.GROUP_ID)
-class Test_get_store_archive_until_or_never:
+class TestGetStoreArchiveUntilOrNever:
GROUP_ID = "test-group"
def test_group_not_found(self, db_session):
result = get_store_archive_until_or_never(db_session, self.GROUP_ID)
assert result is None
- @patch("app.shared.db.worker_crud.get_group", return_value=MagicMock(permissions=None))
+ @patch(
+ "app.shared.db.worker_crud.get_group",
+ return_value=MagicMock(permissions=None),
+ )
def test_group_no_permissions(self, db_session):
result = get_store_archive_until_or_never(db_session, self.GROUP_ID)
assert result is None
diff --git a/app/tests/web/db/test_crud.py b/app/tests/web/db/test_crud.py
index 298a087..676aa24 100644
--- a/app/tests/web/db/test_crud.py
+++ b/app/tests/web/db/test_crud.py
@@ -2,125 +2,379 @@ from datetime import datetime, timedelta
from unittest.mock import patch
import pytest
+import sqlalchemy
import yaml
+from sqlalchemy import false, true
+from sqlalchemy.sql import select
from app.shared.db import models
from app.shared.settings import Settings
+from app.web.config import ALLOW_ANY_EMAIL
from app.web.db import crud
-authors = ["rick@example.com", "morty@example.com", "jerry@example.com"]
-
-
-@pytest.fixture()
-def test_data(db_session):
-
- # creates 3 users
- for email in authors:
- db_session.add(models.User(email=email))
- db_session.commit()
- assert db_session.query(models.User).count() == 3
-
- # creates 100 archives for 3 users over 2 months with repeating URLs
- for i in range(100):
- author = authors[i % 3]
- archive = models.Archive(
- id=f"archive-id-456-{i}",
- url=f"https://example-{i%3}.com",
- result={},
- public=author == "jerry@example.com",
- author_id=author,
- group_id="spaceship" if author == "morty@example.com" and i % 2 == 0 else None,
- created_at=datetime(2021, (i % 2) + 1, (i % 25) + 1)
- )
- if i % 5 == 0:
- archive.tags.append(models.Tag(id=f"tag-{i}"))
- if i % 10 == 0:
- archive.tags.append(models.Tag(id=f"tag-second-{i}"))
- if i % 4 == 0:
- archive.tags.append(models.Tag(id=f"tag-third-{i}"))
- for j in range(10):
- archive.urls.append(models.ArchiveUrl(url=f"https://example-{i}.com/{j}", key=f"media_{j}"))
- db_session.add(archive)
-
- # creates a sheet for each user
- for i, email in enumerate(authors):
- db_session.add(models.Sheet(id=f"sheet-{i}", name=f"sheet-{i}", author_id=email, group_id=None, frequency="daily"))
- if email == "rick@example.com":
- db_session.add(models.Sheet(id=f"sheet-{i}-2", name=f"sheet-{i}-2", author_id=email, group_id="spaceship", frequency="hourly"))
-
- db_session.commit()
-
- assert db_session.query(models.Archive).count() == 100
- assert db_session.query(models.Tag).count() == 20 + 10 + 25
- assert db_session.query(models.ArchiveUrl).count() == 1000
- assert db_session.query(models.ArchiveUrl).filter(models.ArchiveUrl.archive_id == "archive-id-456-0").count() == 10
-
- # setup groups
- assert db_session.query(models.Group).count() == 0
- from app.web.db import crud
- crud.upsert_user_groups(db_session)
- assert db_session.query(models.Group).count() == 4
- assert db_session.query(models.User).count() == 3
-
-
def test_search_archives_by_url(test_data, db_session):
- from app.web.config import ALLOW_ANY_EMAIL
-
- # rick's archives are private
- assert len(crud.search_archives_by_url(db_session, "https://example-0.com", "rick@example.com", True, False)) == 34
- assert len(crud.search_archives_by_url(db_session, "https://example-0.com", "rick@example.com", [], False)) == 34
- assert len(crud.search_archives_by_url(db_session, "https://example-0.com", "rick@example.com", [], True)) == 34
- assert len(crud.search_archives_by_url(db_session, "https://example-0.com", ALLOW_ANY_EMAIL, [], False)) == 34
- assert len(crud.search_archives_by_url(db_session, "https://example-0.com", ALLOW_ANY_EMAIL, True, False)) == 34
- assert len(crud.search_archives_by_url(db_session, "https://example-0.com", "morty@example.com", [], False)) == 0
- assert len(crud.search_archives_by_url(db_session, "https://example-0.com", "morty@example.com", [], True)) == 0
+ # Rick's archives are private
+ assert (
+ len(
+ crud.search_archives_by_url(
+ db_session,
+ "https://example-0.com",
+ "rick@example.com",
+ True,
+ False,
+ )
+ )
+ == 34
+ )
+ assert (
+ len(
+ crud.search_archives_by_url(
+ db_session,
+ "https://example-0.com",
+ "rick@example.com",
+ [],
+ False,
+ )
+ )
+ == 34
+ )
+ assert (
+ len(
+ crud.search_archives_by_url(
+ db_session,
+ "https://example-0.com",
+ "rick@example.com",
+ [],
+ True,
+ )
+ )
+ == 34
+ )
+ assert (
+ len(
+ crud.search_archives_by_url(
+ db_session, "https://example-0.com", ALLOW_ANY_EMAIL, [], False
+ )
+ )
+ == 34
+ )
+ assert (
+ len(
+ crud.search_archives_by_url(
+ db_session,
+ "https://example-0.com",
+ ALLOW_ANY_EMAIL,
+ True,
+ False,
+ )
+ )
+ == 34
+ )
+ assert (
+ len(
+ crud.search_archives_by_url(
+ db_session,
+ "https://example-0.com",
+ "morty@example.com",
+ [],
+ False,
+ )
+ )
+ == 0
+ )
+ assert (
+ len(
+ crud.search_archives_by_url(
+ db_session,
+ "https://example-0.com",
+ "morty@example.com",
+ [],
+ True,
+ )
+ )
+ == 0
+ )
# morty's archives are public but half are in spaceship group
- assert len(crud.search_archives_by_url(db_session, "https://example-1.com", "rick@example.com", ["spaceship"], False)) == 16
- assert len(crud.search_archives_by_url(db_session, "https://example-1.com", "rick@example.com", True, False)) == 16
- assert len(crud.search_archives_by_url(db_session, "https://example-1.com", "jerry@example.com", True, True)) == 16
+ assert (
+ len(
+ crud.search_archives_by_url(
+ db_session,
+ "https://example-1.com",
+ "rick@example.com",
+ ["spaceship"],
+ False,
+ )
+ )
+ == 16
+ )
+ assert (
+ len(
+ crud.search_archives_by_url(
+ db_session,
+ "https://example-1.com",
+ "rick@example.com",
+ True,
+ False,
+ )
+ )
+ == 16
+ )
+ assert (
+ len(
+ crud.search_archives_by_url(
+ db_session,
+ "https://example-1.com",
+ "jerry@example.com",
+ True,
+ True,
+ )
+ )
+ == 16
+ )
- # jerry's archives are public
- assert len(crud.search_archives_by_url(db_session, "https://example-2.com", "jerry@example.com", [], True)) == 33
- assert len(crud.search_archives_by_url(db_session, "https://example-2.com", "rick@example.com", [], True)) == 33
+ # Jerry's archives are public
+ assert (
+ len(
+ crud.search_archives_by_url(
+ db_session,
+ "https://example-2.com",
+ "jerry@example.com",
+ [],
+ True,
+ )
+ )
+ == 33
+ )
+ assert (
+ len(
+ crud.search_archives_by_url(
+ db_session,
+ "https://example-2.com",
+ "rick@example.com",
+ [],
+ True,
+ )
+ )
+ == 33
+ )
# fuzzy search
- assert len(crud.search_archives_by_url(db_session, "https://example", ALLOW_ANY_EMAIL, False, False)) == 100
- assert len(crud.search_archives_by_url(db_session, "https://EXAMPLE", ALLOW_ANY_EMAIL, False, False)) == 100
- assert len(crud.search_archives_by_url(db_session, "2.com", ALLOW_ANY_EMAIL, False, False)) == 33
+ assert (
+ len(
+ crud.search_archives_by_url(
+ db_session, "https://example", ALLOW_ANY_EMAIL, False, False
+ )
+ )
+ == 100
+ )
+ assert (
+ len(
+ crud.search_archives_by_url(
+ db_session, "https://EXAMPLE", ALLOW_ANY_EMAIL, False, False
+ )
+ )
+ == 100
+ )
+ assert (
+ len(
+ crud.search_archives_by_url(
+ db_session, "2.com", ALLOW_ANY_EMAIL, False, False
+ )
+ )
+ == 33
+ )
# absolute search
- assert len(crud.search_archives_by_url(db_session, "example-2.com", ALLOW_ANY_EMAIL, [], False, absolute_search=True)) == 0
- assert len(crud.search_archives_by_url(db_session, "https://example-2.com", ALLOW_ANY_EMAIL, [], False, absolute_search=True)) == 33
+ assert (
+ len(
+ crud.search_archives_by_url(
+ db_session,
+ "example-2.com",
+ ALLOW_ANY_EMAIL,
+ [],
+ False,
+ absolute_search=True,
+ )
+ )
+ == 0
+ )
+ assert (
+ len(
+ crud.search_archives_by_url(
+ db_session,
+ "https://example-2.com",
+ ALLOW_ANY_EMAIL,
+ [],
+ False,
+ absolute_search=True,
+ )
+ )
+ == 33
+ )
# archived_after
- assert len(crud.search_archives_by_url(db_session, "https://example", ALLOW_ANY_EMAIL, True, True, archived_after=datetime(2010, 1, 1))) == 100
- assert len(crud.search_archives_by_url(db_session, "https://example", ALLOW_ANY_EMAIL, False, False, archived_after=datetime(2021, 1, 15))) == 70
- assert len(crud.search_archives_by_url(db_session, "https://example", ALLOW_ANY_EMAIL, False, False, archived_after=datetime(2031, 1, 1))) == 0
+ assert (
+ len(
+ crud.search_archives_by_url(
+ db_session,
+ "https://example",
+ ALLOW_ANY_EMAIL,
+ True,
+ True,
+ archived_after=datetime(2010, 1, 1),
+ )
+ )
+ == 100
+ )
+ assert (
+ len(
+ crud.search_archives_by_url(
+ db_session,
+ "https://example",
+ ALLOW_ANY_EMAIL,
+ False,
+ False,
+ archived_after=datetime(2021, 1, 15),
+ )
+ )
+ == 70
+ )
+ assert (
+ len(
+ crud.search_archives_by_url(
+ db_session,
+ "https://example",
+ ALLOW_ANY_EMAIL,
+ False,
+ False,
+ archived_after=datetime(2031, 1, 1),
+ )
+ )
+ == 0
+ )
# archived before
- assert len(crud.search_archives_by_url(db_session, "https://example", ALLOW_ANY_EMAIL, False, False, archived_before=datetime(2010, 1, 1))) == 0
- assert len(crud.search_archives_by_url(db_session, "https://example", ALLOW_ANY_EMAIL, False, False, archived_before=datetime(2021, 1, 15))) == 28
- assert len(crud.search_archives_by_url(db_session, "https://example", ALLOW_ANY_EMAIL, False, False, archived_before=datetime(2031, 1, 1))) == 100
+ assert (
+ len(
+ crud.search_archives_by_url(
+ db_session,
+ "https://example",
+ ALLOW_ANY_EMAIL,
+ False,
+ False,
+ archived_before=datetime(2010, 1, 1),
+ )
+ )
+ == 0
+ )
+ assert (
+ len(
+ crud.search_archives_by_url(
+ db_session,
+ "https://example",
+ ALLOW_ANY_EMAIL,
+ False,
+ False,
+ archived_before=datetime(2021, 1, 15),
+ )
+ )
+ == 28
+ )
+ assert (
+ len(
+ crud.search_archives_by_url(
+ db_session,
+ "https://example",
+ ALLOW_ANY_EMAIL,
+ False,
+ False,
+ archived_before=datetime(2031, 1, 1),
+ )
+ )
+ == 100
+ )
# archived before and after
- assert len(crud.search_archives_by_url(db_session, "https://example", ALLOW_ANY_EMAIL, False, False, archived_after=datetime(2001, 1, 1), archived_before=datetime(2031, 1, 11))) == 100
- assert len(crud.search_archives_by_url(db_session, "https://example", ALLOW_ANY_EMAIL, False, False, archived_after=datetime(2021, 1, 14), archived_before=datetime(2021, 1, 16))) == 2
+ assert (
+ len(
+ crud.search_archives_by_url(
+ db_session,
+ "https://example",
+ ALLOW_ANY_EMAIL,
+ False,
+ False,
+ archived_after=datetime(2001, 1, 1),
+ archived_before=datetime(2031, 1, 11),
+ )
+ )
+ == 100
+ )
+ assert (
+ len(
+ crud.search_archives_by_url(
+ db_session,
+ "https://example",
+ ALLOW_ANY_EMAIL,
+ False,
+ False,
+ archived_after=datetime(2021, 1, 14),
+ archived_before=datetime(2021, 1, 16),
+ )
+ )
+ == 2
+ )
# limit
- assert len(crud.search_archives_by_url(db_session, "https://example", ALLOW_ANY_EMAIL, False, False, limit=10)) == 10
- assert len(crud.search_archives_by_url(db_session, "https://example", ALLOW_ANY_EMAIL, False, False, limit=-1)) == 1
+ assert (
+ len(
+ crud.search_archives_by_url(
+ db_session,
+ "https://example",
+ ALLOW_ANY_EMAIL,
+ False,
+ False,
+ limit=10,
+ )
+ )
+ == 10
+ )
+ assert (
+ len(
+ crud.search_archives_by_url(
+ db_session,
+ "https://example",
+ ALLOW_ANY_EMAIL,
+ False,
+ False,
+ limit=-1,
+ )
+ )
+ == 1
+ )
# skip
- assert len(crud.search_archives_by_url(db_session, "https://example", ALLOW_ANY_EMAIL, False, False, skip=10)) == 90
+ assert (
+ len(
+ crud.search_archives_by_url(
+ db_session,
+ "https://example",
+ ALLOW_ANY_EMAIL,
+ False,
+ False,
+ skip=10,
+ )
+ )
+ == 90
+ )
def test_search_archives_by_email(test_data, db_session):
- from app.web.config import ALLOW_ANY_EMAIL
-
# lower/upper case
- assert len(crud.search_archives_by_email(db_session, "rick@example.com")) == 34
+ assert (
+ len(crud.search_archives_by_email(db_session, "rick@example.com")) == 34
+ )
# ALLOW_ANY_EMAIL is not a user
assert len(crud.search_archives_by_email(db_session, ALLOW_ANY_EMAIL)) == 0
@@ -138,45 +392,108 @@ def test_search_archives_by_email(test_data, db_session):
@patch("app.web.db.crud.DATABASE_QUERY_LIMIT", new=25)
def test_max_query_limit(test_data, db_session):
- from app.web.config import ALLOW_ANY_EMAIL
+ assert (
+ len(
+ crud.search_archives_by_url(
+ db_session, "https://example", ALLOW_ANY_EMAIL, [], False
+ )
+ )
+ == 25
+ )
+ assert (
+ len(
+ crud.search_archives_by_url(
+ db_session,
+ "https://example",
+ ALLOW_ANY_EMAIL,
+ True,
+ True,
+ limit=1000,
+ )
+ )
+ == 25
+ )
- assert len(crud.search_archives_by_url(db_session, "https://example", ALLOW_ANY_EMAIL, [], False)) == 25
- assert len(crud.search_archives_by_url(db_session, "https://example", ALLOW_ANY_EMAIL, True, True, limit=1000)) == 25
-
- assert len(crud.search_archives_by_email(db_session, "rick@example.com")) == 25
- assert len(crud.search_archives_by_email(db_session, "rick@example.com", limit=1000)) == 25
+ assert (
+ len(crud.search_archives_by_email(db_session, "rick@example.com")) == 25
+ )
+ assert (
+ len(
+ crud.search_archives_by_email(
+ db_session, "rick@example.com", limit=1000
+ )
+ )
+ == 25
+ )
def test_soft_delete(test_data, db_session):
# none deleted yet
- db_session.query(models.Archive).filter(models.Archive.id == "archive-id-456-0").first() is not None
- assert db_session.query(models.Archive).filter(models.Archive.deleted == True).count() == 0
+ assert (
+ db_session.query(models.Archive)
+ .filter(models.Archive.id == "archive-id-456-0")
+ .first()
+ is not None
+ )
+ assert (
+ db_session.query(models.Archive)
+ .filter(models.Archive.deleted.is_(true()))
+ .count()
+ == 0
+ )
# delete
- assert crud.soft_delete_archive(db_session, "archive-id-456-0", "rick@example.com") == True
+ assert (
+ crud.soft_delete_archive(
+ db_session, "archive-id-456-0", "rick@example.com"
+ )
+ is True
+ )
# ensure soft delete
- assert db_session.query(models.Archive).filter(models.Archive.deleted == True).count() == 1
- db_session.query(models.Archive).filter(models.Archive.id == "archive-id-456-0").first() is None
+ assert (
+ db_session.query(models.Archive)
+ .filter(models.Archive.deleted.is_(true()))
+ .count()
+ == 1
+ )
+ assert (
+ db_session.query(models.Archive)
+ .filter(models.Archive.id == "archive-id-456-0")
+ .filter(models.Archive.deleted.is_(false()))
+ .first()
+ is None
+ )
# already deleted
- assert crud.soft_delete_archive(db_session, "archive-id-456-0", "rick@example.com") == False
+ assert (
+ crud.soft_delete_archive(
+ db_session, "archive-id-456-0", "rick@example.com"
+ )
+ is False
+ )
def test_count_archives(test_data, db_session):
assert crud.count_archives(db_session) == 100
- db_session.query(models.Archive).filter(models.Archive.id == "archive-id-456-0").delete()
+ db_session.query(models.Archive).filter(
+ models.Archive.id == "archive-id-456-0"
+ ).delete()
db_session.commit()
assert crud.count_archives(db_session) == 99
def test_count_archive_urls(test_data, db_session):
assert crud.count_archive_urls(db_session) == 1000
- db_session.query(models.ArchiveUrl).filter(models.ArchiveUrl.url == "https://example-0.com/0").delete()
+ db_session.query(models.ArchiveUrl).filter(
+ models.ArchiveUrl.url == "https://example-0.com/0"
+ ).delete()
db_session.commit()
assert crud.count_archive_urls(db_session) == 999
- db_session.query(models.Archive).filter(models.Archive.id == "archive-id-456-0").delete()
+ db_session.query(models.Archive).filter(
+ models.Archive.id == "archive-id-456-0"
+ ).delete()
db_session.commit()
# no Cascade is enabled
assert crud.count_archives(db_session) == 99
@@ -185,16 +502,23 @@ def test_count_archive_urls(test_data, db_session):
def test_count_users(test_data, db_session):
assert crud.count_users(db_session) == 3
- db_session.query(models.User).filter(models.User.email == "rick@example.com").delete()
+ db_session.query(models.User).filter(
+ models.User.email == "rick@example.com"
+ ).delete()
db_session.commit()
assert crud.count_users(db_session) == 2
def test_count_by_users_since(test_data, db_session):
- from app.web.db import crud
-
# 100y window
- assert len(cu := crud.count_by_user_since(db_session, 60 * 60 * 24 * 31 * 12 * 100)) == 3
+ assert (
+ len(
+ cu := crud.count_by_user_since(
+ db_session, 60 * 60 * 24 * 31 * 12 * 100
+ )
+ )
+ == 3
+ )
assert cu[0].total == 34
assert cu[1].total == 33
assert cu[2].total == 33
@@ -203,9 +527,18 @@ def test_count_by_users_since(test_data, db_session):
def test_upsert_group(test_data, db_session):
assert db_session.query(models.Group).count() == 4
- repeatable_params = ["desc 1", "orch.yaml", "sheet.yaml", "service_account_email@example.com", {"read": ["all"]}, ["example.com"]]
+ repeatable_params = [
+ "desc 1",
+ "orch.yaml",
+ "sheet.yaml",
+ "service_account_email@example.com",
+ {"read": ["all"]},
+ ["example.com"],
+ ]
- assert (g1 := crud.upsert_group(db_session, "spaceship", *repeatable_params)) is not None
+ assert (
+ g1 := crud.upsert_group(db_session, "spaceship", *repeatable_params)
+ ) is not None
assert g1.id == "spaceship"
assert g1.description == "desc 1"
assert g1.orchestrator == "orch.yaml"
@@ -214,14 +547,25 @@ def test_upsert_group(test_data, db_session):
assert g1.permissions == {"read": ["all"]}
assert g1.domains == ["example.com"]
assert len(g1.users) == 2
- assert [u.email for u in g1.users] == ["rick@example.com", "morty@example.com"]
+ assert [u.email for u in g1.users] == [
+ "rick@example.com",
+ "morty@example.com",
+ ]
- assert (g2 := crud.upsert_group(db_session, "interdimensional", *repeatable_params)) is not None
+ assert (
+ g2 := crud.upsert_group(
+ db_session, "interdimensional", *repeatable_params
+ )
+ ) is not None
assert g2.id == "interdimensional"
assert len(g2.users) == 1
assert [u.email for u in g2.users] == ["rick@example.com"]
- assert (g3 := crud.upsert_group(db_session, "this-is-a-new-group", *repeatable_params)) is not None
+ assert (
+ g3 := crud.upsert_group(
+ db_session, "this-is-a-new-group", *repeatable_params
+ )
+ ) is not None
assert g3.id == "this-is-a-new-group"
assert len(g3.users) == 0
@@ -229,29 +573,38 @@ def test_upsert_group(test_data, db_session):
def test_upsert_user_groups(db_session):
- @patch('app.web.db.crud.get_settings', new=lambda: bad_setings)
+ @patch("app.web.db.crud.get_settings", new=lambda: bad_settings)
def test_missing_yaml(db_session):
with pytest.raises(FileNotFoundError):
crud.upsert_user_groups(db_session)
- @patch('app.web.db.crud.get_settings', new=lambda: bad_setings)
+ @patch("app.web.db.crud.get_settings", new=lambda: bad_settings)
def test_broken_yaml(db_session):
with pytest.raises(yaml.YAMLError):
crud.upsert_user_groups(db_session)
- bad_setings = Settings(_env_file=".env.test")
+ bad_settings = Settings(_env_file=".env.test")
- bad_setings.USER_GROUPS_FILENAME = "app/tests/user-groups.test.missing.yaml"
+ bad_settings.USER_GROUPS_FILENAME = (
+ "app/tests/user-groups.test.missing.yaml"
+ )
test_missing_yaml(db_session)
- bad_setings.USER_GROUPS_FILENAME = "app/tests/user-groups.test.broken.yaml"
+ bad_settings.USER_GROUPS_FILENAME = "app/tests/user-groups.test.broken.yaml"
test_broken_yaml(db_session)
def test_create_sheet(db_session):
assert db_session.query(models.Sheet).count() == 0
- s = crud.create_sheet(db_session, "sheet-id-123", "sheet name", "email@example.com", "group-id", "hourly")
+ s = crud.create_sheet(
+ db_session,
+ "sheet-id-123",
+ "sheet name",
+ "email@example.com",
+ "group-id",
+ "hourly",
+ )
assert s is not None
assert s.id == "sheet-id-123"
assert s.name == "sheet name"
@@ -261,19 +614,35 @@ def test_create_sheet(db_session):
assert db_session.query(models.Sheet).count() == 1
- # duplicate id
- import sqlalchemy
with pytest.raises(sqlalchemy.exc.IntegrityError):
- crud.create_sheet(db_session, "sheet-id-123", "I thought this was another sheet", "email", "group-id", "hourly")
+ crud.create_sheet(
+ db_session,
+ "sheet-id-123",
+ "I thought this was another sheet",
+ "email",
+ "group-id",
+ "hourly",
+ )
def test_get_user_sheet(test_data, db_session):
assert crud.get_user_sheet(db_session, "", "sheet-0") is None
- assert crud.get_user_sheet(db_session, "morty@example.com", "sheet-0") is None
+ assert (
+ crud.get_user_sheet(db_session, "morty@example.com", "sheet-0") is None
+ )
- assert crud.get_user_sheet(db_session, "rick@example.com", "sheet-0") is not None
- assert crud.get_user_sheet(db_session, "rick@example.com", "sheet-0-2") is not None
- assert crud.get_user_sheet(db_session, "morty@example.com", "sheet-1") is not None
+ assert (
+ crud.get_user_sheet(db_session, "rick@example.com", "sheet-0")
+ is not None
+ )
+ assert (
+ crud.get_user_sheet(db_session, "rick@example.com", "sheet-0-2")
+ is not None
+ )
+ assert (
+ crud.get_user_sheet(db_session, "morty@example.com", "sheet-1")
+ is not None
+ )
def test_get_user_sheets(test_data, db_session):
@@ -285,9 +654,9 @@ def test_get_user_sheets(test_data, db_session):
def test_delete_sheet(test_data, db_session):
- assert crud.delete_sheet(db_session, "sheet-0", "") == False
- assert crud.delete_sheet(db_session, "sheet-0", "rick@example.com") == True
- assert crud.delete_sheet(db_session, "sheet-0", "rick@example.com") == False
+ assert crud.delete_sheet(db_session, "sheet-0", "") is False
+ assert crud.delete_sheet(db_session, "sheet-0", "rick@example.com") is True
+ assert crud.delete_sheet(db_session, "sheet-0", "rick@example.com") is False
@pytest.mark.asyncio
@@ -299,21 +668,21 @@ async def test_find_by_store_until(async_db_session):
url="https://example-expired-1.com",
result={},
author_id="rick@example.com",
- store_until=now - timedelta(days=1)
+ store_until=now - timedelta(days=1),
)
archive2 = models.Archive(
id="archive-expired-2",
url="https://example-expired-2.com",
result={},
author_id="rick@example.com",
- store_until=now - timedelta(hours=1)
+ store_until=now - timedelta(hours=1),
)
archive3 = models.Archive(
id="archive-active",
url="https://example-active.com",
result={},
author_id="rick@example.com",
- store_until=now + timedelta(days=1)
+ store_until=now + timedelta(days=1),
)
async_db_session.add_all([archive1, archive2, archive3])
await async_db_session.commit()
@@ -323,11 +692,15 @@ async def test_find_by_store_until(async_db_session):
assert len(list(expired)) == 2
# Should find 1 archive expired before 2 hours ago
- expired = await crud.find_by_store_until(async_db_session, now - timedelta(hours=2))
+ expired = await crud.find_by_store_until(
+ async_db_session, now - timedelta(hours=2)
+ )
assert len(list(expired)) == 1
# Should find no archives expired before 2 days ago
- expired = await crud.find_by_store_until(async_db_session, now - timedelta(days=2))
+ expired = await crud.find_by_store_until(
+ async_db_session, now - timedelta(days=2)
+ )
assert len(list(expired)) == 0
# Should not find deleted archives
@@ -339,45 +712,78 @@ async def test_find_by_store_until(async_db_session):
@pytest.mark.asyncio
async def test_get_sheets_by_id_hash(async_db_session):
+ author_emails = ["rick@example.com", "morty@example.com", "jerry@example.com"]
+
# Add test data
- authors = ["rick@example.com", "morty@example.com", "jerry@example.com"]
sheets = [
- models.Sheet(id="sheet-0", name="sheet-0", author_id=authors[0], group_id=None, frequency="daily"),
- models.Sheet(id="sheet-0-2", name="sheet-0-2", author_id=authors[0], group_id="spaceship", frequency="hourly"),
- models.Sheet(id="sheet-1", name="sheet-1", author_id=authors[1], group_id=None, frequency="daily"),
- models.Sheet(id="sheet-2", name="sheet-2", author_id=authors[2], group_id=None, frequency="daily")
+ models.Sheet(
+ id="sheet-0",
+ name="sheet-0",
+ author_id=author_emails[0],
+ group_id=None,
+ frequency="daily",
+ ),
+ models.Sheet(
+ id="sheet-0-2",
+ name="sheet-0-2",
+ author_id=author_emails[0],
+ group_id="spaceship",
+ frequency="hourly",
+ ),
+ models.Sheet(
+ id="sheet-1",
+ name="sheet-1",
+ author_id=author_emails[1],
+ group_id=None,
+ frequency="daily",
+ ),
+ models.Sheet(
+ id="sheet-2",
+ name="sheet-2",
+ author_id=author_emails[2],
+ group_id=None,
+ frequency="daily",
+ ),
]
async_db_session.add_all(sheets)
await async_db_session.commit()
with patch("app.web.db.crud.fnv1a_hash_mod", return_value=1):
# Test retrieving hourly sheets
- hourly_sheets = await crud.get_sheets_by_id_hash(async_db_session, "hourly", 4, 1)
+ hourly_sheets = await crud.get_sheets_by_id_hash(
+ async_db_session, "hourly", 4, 1
+ )
assert len(hourly_sheets) == 1
assert hourly_sheets[0].id == "sheet-0-2"
assert hourly_sheets[0].frequency == "hourly"
# Test retrieving daily sheets
- daily_sheets = await crud.get_sheets_by_id_hash(async_db_session, "daily", 4, 1)
+ daily_sheets = await crud.get_sheets_by_id_hash(
+ async_db_session, "daily", 4, 1
+ )
assert len(daily_sheets) == 3
assert all(sheet.frequency == "daily" for sheet in daily_sheets)
- assert {sheet.id for sheet in daily_sheets} == {"sheet-0", "sheet-1", "sheet-2"}
+ assert {sheet.id for sheet in daily_sheets} == {
+ "sheet-0",
+ "sheet-1",
+ "sheet-2",
+ }
# Test with non-matching hash
- no_sheets = await crud.get_sheets_by_id_hash(async_db_session, "daily", 4, 3)
+ no_sheets = await crud.get_sheets_by_id_hash(
+ async_db_session, "daily", 4, 3
+ )
assert len(no_sheets) == 0
# Test with non-existent frequency
- weekly_sheets = await crud.get_sheets_by_id_hash(async_db_session, "weekly", 4, 1)
+ weekly_sheets = await crud.get_sheets_by_id_hash(
+ async_db_session, "weekly", 4, 1
+ )
assert len(weekly_sheets) == 0
@pytest.mark.asyncio
async def test_delete_stale_sheets(async_db_session):
- from datetime import datetime, timedelta
-
- from sqlalchemy.sql import select
-
now = datetime.now()
active_date = now - timedelta(days=5)
stale_date = now - timedelta(days=15)
@@ -389,29 +795,29 @@ async def test_delete_stale_sheets(async_db_session):
name="Active Sheet 1",
author_id="rick@example.com",
frequency="daily",
- last_url_archived_at=active_date
+ last_url_archived_at=active_date,
),
models.Sheet(
id="sheet-active-2",
name="Active Sheet 2",
author_id="morty@example.com",
frequency="hourly",
- last_url_archived_at=active_date
+ last_url_archived_at=active_date,
),
models.Sheet(
id="sheet-stale-1",
name="Stale Sheet 1",
author_id="rick@example.com",
frequency="daily",
- last_url_archived_at=stale_date
+ last_url_archived_at=stale_date,
),
models.Sheet(
id="sheet-stale-2",
name="Stale Sheet 2",
author_id="morty@example.com",
frequency="daily",
- last_url_archived_at=stale_date
- )
+ last_url_archived_at=stale_date,
+ ),
]
async_db_session.add_all(sheets)
await async_db_session.commit()
diff --git a/app/tests/web/db/test_user_state.py b/app/tests/web/db/test_user_state.py
index 665bf08..5d18cea 100644
--- a/app/tests/web/db/test_user_state.py
+++ b/app/tests/web/db/test_user_state.py
@@ -1,4 +1,3 @@
-
from unittest.mock import MagicMock, PropertyMock, patch
import pytest
@@ -6,6 +5,7 @@ import pytest
from app.shared.db import models
from app.shared.user_groups import GroupInfo, GroupPermissions
from app.web.db.user_state import UserState
+from app.web.utils.misc import convert_priority_to_queue_dict
def fresh_user_state():
@@ -21,39 +21,73 @@ def user_state():
def user_state_with_groups(user_state):
user_groups = [
models.Group(id="no-permissions", permissions={}),
- models.Group(id="group1", description="this is g1", service_account_email="sa1@example.com", permissions={"read": ["group1", "no-permissions"], "read_public": True, "archive_url": True, "archive_sheet": True, "max_archive_lifespan_months": 24, "max_monthly_urls": 100, "max_monthly_mbs": 1000, "priority": "high"}),
- models.Group(id="group2", description="this is g2", service_account_email="sa2@example.com", permissions={"read": ["all"], "read_public": True, "archive_url": False, "archive_sheet": False, "max_archive_lifespan_months": -1, "max_monthly_urls": -1, "max_monthly_mbs": -1, "priority": "low", "sheet_frequency": {"daily"}}),
+ models.Group(
+ id="group1",
+ description="this is g1",
+ service_account_email="sa1@example.com",
+ permissions={
+ "read": ["group1", "no-permissions"],
+ "read_public": True,
+ "archive_url": True,
+ "archive_sheet": True,
+ "max_archive_lifespan_months": 24,
+ "max_monthly_urls": 100,
+ "max_monthly_mbs": 1000,
+ "priority": "high",
+ },
+ ),
+ models.Group(
+ id="group2",
+ description="this is g2",
+ service_account_email="sa2@example.com",
+ permissions={
+ "read": ["all"],
+ "read_public": True,
+ "archive_url": False,
+ "archive_sheet": False,
+ "max_archive_lifespan_months": -1,
+ "max_monthly_urls": -1,
+ "max_monthly_mbs": -1,
+ "priority": "low",
+ "sheet_frequency": {"daily"},
+ },
+ ),
]
- with patch.object(UserState, 'user_groups', new_callable=PropertyMock, return_value=user_groups):
+ with patch.object(
+ UserState,
+ "user_groups",
+ new_callable=PropertyMock,
+ return_value=user_groups,
+ ):
yield user_state
def test_permissions(user_state_with_groups):
permissions = user_state_with_groups.permissions
- assert permissions["all"].read == True
- assert permissions["all"].read_public == True
- assert permissions["all"].archive_url == True
- assert permissions["all"].archive_sheet == True
+ assert permissions["all"].read is True
+ assert permissions["all"].read_public is True
+ assert permissions["all"].archive_url is True
+ assert permissions["all"].archive_sheet is True
assert permissions["all"].max_archive_lifespan_months == -1
assert permissions["all"].max_monthly_urls == -1
assert permissions["all"].max_monthly_mbs == -1
assert permissions["all"].priority == "high"
- assert permissions["group1"].read == set(["group1", "no-permissions"])
- assert permissions["group1"].read_public == True
- assert permissions["group1"].archive_url == True
- assert permissions["group1"].archive_sheet == True
+ assert permissions["group1"].read == {"group1", "no-permissions"}
+ assert permissions["group1"].read_public is True
+ assert permissions["group1"].archive_url is True
+ assert permissions["group1"].archive_sheet is True
assert permissions["group1"].max_archive_lifespan_months == 24
assert permissions["group1"].max_monthly_urls == 100
assert permissions["group1"].max_monthly_mbs == 1000
assert permissions["group1"].priority == "high"
- assert permissions["group2"].read == set(["all"])
- assert permissions["group2"].read_public == True
- assert permissions["group2"].archive_url == False
- assert permissions["group2"].archive_sheet == False
+ assert permissions["group2"].read == {"all"}
+ assert permissions["group2"].read_public is True
+ assert permissions["group2"].archive_url is False
+ assert permissions["group2"].archive_sheet is False
assert permissions["group2"].max_archive_lifespan_months == -1
assert permissions["group2"].max_monthly_urls == -1
assert permissions["group2"].max_monthly_mbs == -1
@@ -63,13 +97,19 @@ def test_permissions(user_state_with_groups):
def test_user_groups_names(user_state):
- with patch('app.web.db.crud.get_user_group_names', return_value=["group1", "group2"]) as mock:
+ with patch(
+ "app.web.db.crud.get_user_group_names",
+ return_value=["group1", "group2"],
+ ) as mock:
assert user_state.user_groups_names == ["group1", "group2", "default"]
mock.assert_called_once_with(None, "test@example.com")
def test_user_groups(user_state):
- with patch('app.web.db.crud.get_user_groups_by_name', return_value=[MagicMock(), MagicMock()]) as mock:
+ with patch(
+ "app.web.db.crud.get_user_groups_by_name",
+ return_value=[MagicMock(), MagicMock()],
+ ) as mock:
user_state._user_groups_names = ["group1", "group2"]
assert len(user_state.user_groups) == 2
mock.assert_called_once_with(None, ["group1", "group2"])
@@ -78,85 +118,166 @@ def test_user_groups(user_state):
def test_read():
us = fresh_user_state()
- with patch.object(UserState, 'user_groups', new_callable=PropertyMock, return_value=[models.Group(id="no-permissions", permissions={})]) as mock:
+ with patch.object(
+ UserState,
+ "user_groups",
+ new_callable=PropertyMock,
+ return_value=[models.Group(id="no-permissions", permissions={})],
+ ) as mock:
assert not hasattr(us, "_read")
assert us.read == set()
assert us._read == set()
mock.assert_called_once()
us = fresh_user_state()
- with patch.object(UserState, 'user_groups', new_callable=PropertyMock, return_value=[models.Group(id="group1", permissions={"read": ["group1", "no-permissions"]})]):
- assert us.read == set(["group1", "no-permissions"])
+ with patch.object(
+ UserState,
+ "user_groups",
+ new_callable=PropertyMock,
+ return_value=[
+ models.Group(
+ id="group1", permissions={"read": ["group1", "no-permissions"]}
+ )
+ ],
+ ):
+ assert us.read == {"group1", "no-permissions"}
us = fresh_user_state()
- with patch.object(UserState, 'user_groups', new_callable=PropertyMock, return_value=[models.Group(id="group1", permissions={"read": ["all"]})]):
- assert us.read == True
+ with patch.object(
+ UserState,
+ "user_groups",
+ new_callable=PropertyMock,
+ return_value=[models.Group(id="group1", permissions={"read": ["all"]})],
+ ):
+ assert us.read is True
def test_read_public():
us = fresh_user_state()
- with patch.object(UserState, 'user_groups', new_callable=PropertyMock, return_value=[models.Group(id="no-permissions", permissions={})]) as mock:
+ with patch.object(
+ UserState,
+ "user_groups",
+ new_callable=PropertyMock,
+ return_value=[models.Group(id="no-permissions", permissions={})],
+ ) as mock:
assert not hasattr(us, "_read_public")
- assert us.read_public == False
- assert us._read_public == False
+ assert us.read_public is False
+ assert us._read_public is False
mock.assert_called_once()
# no new calls
- assert us.read_public == False
+ assert us.read_public is False
mock.assert_called_once()
us = fresh_user_state()
- with patch.object(UserState, 'user_groups', new_callable=PropertyMock, return_value=[models.Group(id="group1", permissions={"read_public": True})]):
- assert us.read_public == True
+ with patch.object(
+ UserState,
+ "user_groups",
+ new_callable=PropertyMock,
+ return_value=[
+ models.Group(id="group1", permissions={"read_public": True})
+ ],
+ ):
+ assert us.read_public is True
us = fresh_user_state()
- with patch.object(UserState, 'user_groups', new_callable=PropertyMock, return_value=[models.Group(id="group1", permissions={"read_public": False})]):
- assert us.read_public == False
+ with patch.object(
+ UserState,
+ "user_groups",
+ new_callable=PropertyMock,
+ return_value=[
+ models.Group(id="group1", permissions={"read_public": False})
+ ],
+ ):
+ assert us.read_public is False
def test_archive_url():
us = fresh_user_state()
- with patch.object(UserState, 'user_groups', new_callable=PropertyMock, return_value=[models.Group(id="no-permissions", permissions={})]) as mock:
+ with patch.object(
+ UserState,
+ "user_groups",
+ new_callable=PropertyMock,
+ return_value=[models.Group(id="no-permissions", permissions={})],
+ ) as mock:
assert not hasattr(us, "_archive_url")
- assert us.archive_url == False
- assert us._archive_url == False
+ assert us.archive_url is False
+ assert us._archive_url is False
mock.assert_called_once()
# no new calls
- assert us.archive_url == False
+ assert us.archive_url is False
mock.assert_called_once()
us = fresh_user_state()
- with patch.object(UserState, 'user_groups', new_callable=PropertyMock, return_value=[models.Group(id="group1", permissions={"archive_url": False})]):
- assert us.archive_url == False
+ with patch.object(
+ UserState,
+ "user_groups",
+ new_callable=PropertyMock,
+ return_value=[
+ models.Group(id="group1", permissions={"archive_url": False})
+ ],
+ ):
+ assert us.archive_url is False
us = fresh_user_state()
- with patch.object(UserState, 'user_groups', new_callable=PropertyMock, return_value=[models.Group(id="group1", permissions={"archive_url": True})]):
- assert us.archive_url == True
+ with patch.object(
+ UserState,
+ "user_groups",
+ new_callable=PropertyMock,
+ return_value=[
+ models.Group(id="group1", permissions={"archive_url": True})
+ ],
+ ):
+ assert us.archive_url is True
def test_archive_sheet():
us = fresh_user_state()
- with patch.object(UserState, 'user_groups', new_callable=PropertyMock, return_value=[models.Group(id="no-permissions", permissions={})]) as mock:
+ with patch.object(
+ UserState,
+ "user_groups",
+ new_callable=PropertyMock,
+ return_value=[models.Group(id="no-permissions", permissions={})],
+ ) as mock:
assert not hasattr(us, "_archive_sheet")
- assert us.archive_sheet == False
- assert us._archive_sheet == False
+ assert us.archive_sheet is False
+ assert us._archive_sheet is False
mock.assert_called_once()
# no new calls
- assert us.archive_sheet == False
+ assert us.archive_sheet is False
mock.assert_called_once()
us = fresh_user_state()
- with patch.object(UserState, 'user_groups', new_callable=PropertyMock, return_value=[models.Group(id="group1", permissions={"archive_sheet": False})]):
- assert us.archive_sheet == False
+ with patch.object(
+ UserState,
+ "user_groups",
+ new_callable=PropertyMock,
+ return_value=[
+ models.Group(id="group1", permissions={"archive_sheet": False})
+ ],
+ ):
+ assert us.archive_sheet is False
us = fresh_user_state()
- with patch.object(UserState, 'user_groups', new_callable=PropertyMock, return_value=[models.Group(id="group1", permissions={"archive_sheet": True})]):
- assert us.archive_sheet == True
+ with patch.object(
+ UserState,
+ "user_groups",
+ new_callable=PropertyMock,
+ return_value=[
+ models.Group(id="group1", permissions={"archive_sheet": True})
+ ],
+ ):
+ assert us.archive_sheet is True
def test_sheet_frequency():
us = fresh_user_state()
- with patch.object(UserState, 'user_groups', new_callable=PropertyMock, return_value=[models.Group(id="no-permissions", permissions={})]) as mock:
+ with patch.object(
+ UserState,
+ "user_groups",
+ new_callable=PropertyMock,
+ return_value=[models.Group(id="no-permissions", permissions={})],
+ ) as mock:
assert not hasattr(us, "_sheet_frequency")
assert us.sheet_frequency == set()
assert us._sheet_frequency == set()
@@ -166,18 +287,42 @@ def test_sheet_frequency():
mock.assert_called_once()
us = fresh_user_state()
- with patch.object(UserState, 'user_groups', new_callable=PropertyMock, return_value=[models.Group(id="group1", permissions={"sheet_frequency": ["daily", "hourly"]})]):
+ with patch.object(
+ UserState,
+ "user_groups",
+ new_callable=PropertyMock,
+ return_value=[
+ models.Group(
+ id="group1",
+ permissions={"sheet_frequency": ["daily", "hourly"]},
+ )
+ ],
+ ):
assert us.sheet_frequency == {"daily", "hourly"}
us = fresh_user_state()
- with patch.object(UserState, 'user_groups', new_callable=PropertyMock, return_value=[models.Group(id="group1", permissions={"sheet_frequency": []})]):
+ with patch.object(
+ UserState,
+ "user_groups",
+ new_callable=PropertyMock,
+ return_value=[
+ models.Group(id="group1", permissions={"sheet_frequency": []})
+ ],
+ ):
assert us.sheet_frequency == set()
def test_max_archive_lifespan_months():
us = fresh_user_state()
- default = GroupPermissions.model_fields["max_archive_lifespan_months"].default
- with patch.object(UserState, 'user_groups', new_callable=PropertyMock, return_value=[models.Group(id="no-permissions", permissions={})]) as mock:
+ default = GroupPermissions.model_fields[
+ "max_archive_lifespan_months"
+ ].default
+ with patch.object(
+ UserState,
+ "user_groups",
+ new_callable=PropertyMock,
+ return_value=[models.Group(id="no-permissions", permissions={})],
+ ) as mock:
assert not hasattr(us, "_max_archive_lifespan_months")
assert us.max_archive_lifespan_months == default
assert us._max_archive_lifespan_months == default
@@ -187,18 +332,44 @@ def test_max_archive_lifespan_months():
mock.assert_called_once()
us = fresh_user_state()
- with patch.object(UserState, 'user_groups', new_callable=PropertyMock, return_value=[models.Group(id="group1", permissions={"max_archive_lifespan_months": 24})]):
+ with patch.object(
+ UserState,
+ "user_groups",
+ new_callable=PropertyMock,
+ return_value=[
+ models.Group(
+ id="group1", permissions={"max_archive_lifespan_months": 24}
+ )
+ ],
+ ):
assert us.max_archive_lifespan_months == 24
us = fresh_user_state()
- with patch.object(UserState, 'user_groups', new_callable=PropertyMock, return_value=[models.Group(id="group1", permissions={"max_archive_lifespan_months": 150}), models.Group(id="group2", permissions={"max_archive_lifespan_months": -1})]):
+ with patch.object(
+ UserState,
+ "user_groups",
+ new_callable=PropertyMock,
+ return_value=[
+ models.Group(
+ id="group1", permissions={"max_archive_lifespan_months": 150}
+ ),
+ models.Group(
+ id="group2", permissions={"max_archive_lifespan_months": -1}
+ ),
+ ],
+ ):
assert us.max_archive_lifespan_months == -1
def test_max_monthly_urls():
us = fresh_user_state()
default = GroupPermissions.model_fields["max_monthly_urls"].default
- with patch.object(UserState, 'user_groups', new_callable=PropertyMock, return_value=[models.Group(id="no-permissions", permissions={})]) as mock:
+ with patch.object(
+ UserState,
+ "user_groups",
+ new_callable=PropertyMock,
+ return_value=[models.Group(id="no-permissions", permissions={})],
+ ) as mock:
assert not hasattr(us, "_max_monthly_urls")
assert us.max_monthly_urls == default
assert us._max_monthly_urls == default
@@ -208,18 +379,38 @@ def test_max_monthly_urls():
mock.assert_called_once()
us = fresh_user_state()
- with patch.object(UserState, 'user_groups', new_callable=PropertyMock, return_value=[models.Group(id="group1", permissions={"max_monthly_urls": 100})]):
+ with patch.object(
+ UserState,
+ "user_groups",
+ new_callable=PropertyMock,
+ return_value=[
+ models.Group(id="group1", permissions={"max_monthly_urls": 100})
+ ],
+ ):
assert us.max_monthly_urls == 100
us = fresh_user_state()
- with patch.object(UserState, 'user_groups', new_callable=PropertyMock, return_value=[models.Group(id="group1", permissions={"max_monthly_urls": 150}), models.Group(id="group2", permissions={"max_monthly_urls": -1})]):
+ with patch.object(
+ UserState,
+ "user_groups",
+ new_callable=PropertyMock,
+ return_value=[
+ models.Group(id="group1", permissions={"max_monthly_urls": 150}),
+ models.Group(id="group2", permissions={"max_monthly_urls": -1}),
+ ],
+ ):
assert us.max_monthly_urls == -1
def test_max_monthly_mbs():
us = fresh_user_state()
default = GroupPermissions.model_fields["max_monthly_mbs"].default
- with patch.object(UserState, 'user_groups', new_callable=PropertyMock, return_value=[models.Group(id="no-permissions", permissions={})]) as mock:
+ with patch.object(
+ UserState,
+ "user_groups",
+ new_callable=PropertyMock,
+ return_value=[models.Group(id="no-permissions", permissions={})],
+ ) as mock:
assert not hasattr(us, "_max_monthly_mbs")
assert us.max_monthly_mbs == default
assert us._max_monthly_mbs == default
@@ -229,17 +420,37 @@ def test_max_monthly_mbs():
mock.assert_called_once()
us = fresh_user_state()
- with patch.object(UserState, 'user_groups', new_callable=PropertyMock, return_value=[models.Group(id="group1", permissions={"max_monthly_mbs": 1000})]):
+ with patch.object(
+ UserState,
+ "user_groups",
+ new_callable=PropertyMock,
+ return_value=[
+ models.Group(id="group1", permissions={"max_monthly_mbs": 1000})
+ ],
+ ):
assert us.max_monthly_mbs == 1000
us = fresh_user_state()
- with patch.object(UserState, 'user_groups', new_callable=PropertyMock, return_value=[models.Group(id="group1", permissions={"max_monthly_mbs": 1500}), models.Group(id="group2", permissions={"max_monthly_mbs": -1})]):
+ with patch.object(
+ UserState,
+ "user_groups",
+ new_callable=PropertyMock,
+ return_value=[
+ models.Group(id="group1", permissions={"max_monthly_mbs": 1500}),
+ models.Group(id="group2", permissions={"max_monthly_mbs": -1}),
+ ],
+ ):
assert us.max_monthly_mbs == -1
def test_priority(user_state):
default = GroupPermissions.model_fields["priority"].default
- with patch.object(UserState, 'user_groups', new_callable=PropertyMock, return_value=[models.Group(id="no-permissions", permissions={})]) as mock:
+ with patch.object(
+ UserState,
+ "user_groups",
+ new_callable=PropertyMock,
+ return_value=[models.Group(id="no-permissions", permissions={})],
+ ) as mock:
assert not hasattr(user_state, "_priority")
assert user_state.priority == default
assert user_state._priority == default
@@ -249,11 +460,26 @@ def test_priority(user_state):
mock.assert_called_once()
us = fresh_user_state()
- with patch.object(UserState, 'user_groups', new_callable=PropertyMock, return_value=[models.Group(id="group1", permissions={"priority": "high"})]):
+ with patch.object(
+ UserState,
+ "user_groups",
+ new_callable=PropertyMock,
+ return_value=[
+ models.Group(id="group1", permissions={"priority": "high"})
+ ],
+ ):
assert us.priority == "high"
us = fresh_user_state()
- with patch.object(UserState, 'user_groups', new_callable=PropertyMock, return_value=[models.Group(id="group1", permissions={"priority": "low"}), models.Group(id="group2", permissions={"priority": "medium"})]):
+ with patch.object(
+ UserState,
+ "user_groups",
+ new_callable=PropertyMock,
+ return_value=[
+ models.Group(id="group1", permissions={"priority": "low"}),
+ models.Group(id="group2", permissions={"priority": "medium"}),
+ ],
+ ):
assert us.priority == "low"
@@ -263,21 +489,45 @@ def test_active():
(True, False, False, False, True),
(False, True, False, False, True),
(False, False, True, False, True),
- (False, False, False, True, True)
+ (False, False, False, True, True),
]:
us = fresh_user_state()
- with patch.object(UserState, 'read', new_callable=PropertyMock, return_value=read), \
- patch.object(UserState, 'read_public', new_callable=PropertyMock, return_value=read_public), \
- patch.object(UserState, 'archive_url', new_callable=PropertyMock, return_value=archive_url), \
- patch.object(UserState, 'archive_sheet', new_callable=PropertyMock, return_value=archive_sheet):
+ with (
+ patch.object(
+ UserState, "read", new_callable=PropertyMock, return_value=read
+ ),
+ patch.object(
+ UserState,
+ "read_public",
+ new_callable=PropertyMock,
+ return_value=read_public,
+ ),
+ patch.object(
+ UserState,
+ "archive_url",
+ new_callable=PropertyMock,
+ return_value=archive_url,
+ ),
+ patch.object(
+ UserState,
+ "archive_sheet",
+ new_callable=PropertyMock,
+ return_value=archive_sheet,
+ ),
+ ):
assert us.active == is_active
def test_in_group(user_state):
- with patch.object(UserState, 'user_groups_names', new_callable=PropertyMock, return_value=["group1", "group2"]):
- assert user_state.in_group("group1") == True
- assert user_state.in_group("group2") == True
- assert user_state.in_group("group3") == False
+ with patch.object(
+ UserState,
+ "user_groups_names",
+ new_callable=PropertyMock,
+ return_value=["group1", "group2"],
+ ):
+ assert user_state.in_group("group1") is True
+ assert user_state.in_group("group2") is True
+ assert user_state.in_group("group3") is False
def test_usage(db_session):
@@ -295,10 +545,34 @@ def test_usage(db_session):
]
megabytes = int(sum(bytes) / 1024 / 1024)
- with patch.object(db_session, 'query', side_effect=[
- MagicMock(filter=MagicMock(return_value=MagicMock(group_by=MagicMock(return_value=MagicMock(all=MagicMock(return_value=user_sheets)))))),
- MagicMock(filter=MagicMock(return_value=MagicMock(group_by=MagicMock(return_value=MagicMock(all=MagicMock(return_value=urls_by_group))))))
- ]):
+ with patch.object(
+ db_session,
+ "query",
+ side_effect=[
+ MagicMock(
+ filter=MagicMock(
+ return_value=MagicMock(
+ group_by=MagicMock(
+ return_value=MagicMock(
+ all=MagicMock(return_value=user_sheets)
+ )
+ )
+ )
+ )
+ ),
+ MagicMock(
+ filter=MagicMock(
+ return_value=MagicMock(
+ group_by=MagicMock(
+ return_value=MagicMock(
+ all=MagicMock(return_value=urls_by_group)
+ )
+ )
+ )
+ )
+ ),
+ ],
+ ):
usage_response = user_state.usage()
assert usage_response.monthly_urls == 155
@@ -306,11 +580,15 @@ def test_usage(db_session):
assert usage_response.total_sheets == 115
assert usage_response.groups["group1"].monthly_urls == 50
- assert usage_response.groups["group1"].monthly_mbs == int(bytes[0] / 1024 / 1024)
+ assert usage_response.groups["group1"].monthly_mbs == int(
+ bytes[0] / 1024 / 1024
+ )
assert usage_response.groups["group1"].total_sheets == 5
assert usage_response.groups["group2"].monthly_urls == 100
- assert usage_response.groups["group2"].monthly_mbs == int(bytes[1] / 1024 / 1024)
+ assert usage_response.groups["group2"].monthly_mbs == int(
+ bytes[1] / 1024 / 1024
+ )
assert usage_response.groups["group2"].total_sheets == 10
assert usage_response.groups["group3"].monthly_urls == 0
@@ -318,7 +596,9 @@ def test_usage(db_session):
assert usage_response.groups["group3"].total_sheets == 100
assert usage_response.groups["group4"].monthly_urls == 5
- assert usage_response.groups["group4"].monthly_mbs == int(bytes[2] / 1024 / 1024)
+ assert usage_response.groups["group4"].monthly_mbs == int(
+ bytes[2] / 1024 / 1024
+ )
assert usage_response.groups["group4"].total_sheets == 0
@@ -334,8 +614,23 @@ def test_has_quota_monthly_sheets(db_session):
]
for permissions, count, expected in test_cases:
- with patch.object(UserState, 'permissions', new_callable=PropertyMock, return_value=permissions):
- with patch.object(us.db, 'query', return_value=MagicMock(filter=MagicMock(return_value=MagicMock(count=MagicMock(return_value=count))))):
+ with patch.object(
+ UserState,
+ "permissions",
+ new_callable=PropertyMock,
+ return_value=permissions,
+ ):
+ with patch.object(
+ us.db,
+ "query",
+ return_value=MagicMock(
+ filter=MagicMock(
+ return_value=MagicMock(
+ count=MagicMock(return_value=count)
+ )
+ )
+ ),
+ ):
assert us.has_quota_monthly_sheets("group1") == expected
@@ -350,8 +645,23 @@ def test_has_quota_max_monthly_urls(db_session):
]
for permissions, count, expected in test_cases:
- with patch.object(UserState, 'permissions', new_callable=PropertyMock, return_value=permissions):
- with patch.object(us.db, 'query', return_value=MagicMock(filter=MagicMock(return_value=MagicMock(count=MagicMock(return_value=count))))):
+ with patch.object(
+ UserState,
+ "permissions",
+ new_callable=PropertyMock,
+ return_value=permissions,
+ ):
+ with patch.object(
+ us.db,
+ "query",
+ return_value=MagicMock(
+ filter=MagicMock(
+ return_value=MagicMock(
+ count=MagicMock(return_value=count)
+ )
+ )
+ ),
+ ):
assert us.has_quota_max_monthly_urls("group1") == expected
test_cases = [
(-1, 1000, True),
@@ -361,8 +671,23 @@ def test_has_quota_max_monthly_urls(db_session):
]
for max_urls, count, expected in test_cases:
- with patch.object(UserState, 'max_monthly_urls', new_callable=PropertyMock, return_value=max_urls):
- with patch.object(us.db, 'query', return_value=MagicMock(filter=MagicMock(return_value=MagicMock(count=MagicMock(return_value=count))))):
+ with patch.object(
+ UserState,
+ "max_monthly_urls",
+ new_callable=PropertyMock,
+ return_value=max_urls,
+ ):
+ with patch.object(
+ us.db,
+ "query",
+ return_value=MagicMock(
+ filter=MagicMock(
+ return_value=MagicMock(
+ count=MagicMock(return_value=count)
+ )
+ )
+ ),
+ ):
assert us.has_quota_max_monthly_urls("") == expected
@@ -377,8 +702,29 @@ def test_has_quota_max_monthly_mbs(db_session):
]
for permissions, mbs, expected in test_cases:
- with patch.object(UserState, 'permissions', new_callable=PropertyMock, return_value=permissions):
- with patch.object(us.db, 'query', return_value=MagicMock(filter=MagicMock(return_value=MagicMock(with_entities=MagicMock(return_value=MagicMock(scalar=MagicMock(return_value=mbs * 1024 * 1024))))))):
+ with patch.object(
+ UserState,
+ "permissions",
+ new_callable=PropertyMock,
+ return_value=permissions,
+ ):
+ with patch.object(
+ us.db,
+ "query",
+ return_value=MagicMock(
+ filter=MagicMock(
+ return_value=MagicMock(
+ with_entities=MagicMock(
+ return_value=MagicMock(
+ scalar=MagicMock(
+ return_value=mbs * 1024 * 1024
+ )
+ )
+ )
+ )
+ )
+ ),
+ ):
assert us.has_quota_max_monthly_mbs("group1") == expected
test_cases = [
@@ -389,8 +735,29 @@ def test_has_quota_max_monthly_mbs(db_session):
]
for max_mbs, mbs, expected in test_cases:
- with patch.object(UserState, 'max_monthly_mbs', new_callable=PropertyMock, return_value=max_mbs):
- with patch.object(us.db, 'query', return_value=MagicMock(filter=MagicMock(return_value=MagicMock(with_entities=MagicMock(return_value=MagicMock(scalar=MagicMock(return_value=mbs * 1024 * 1024))))))):
+ with patch.object(
+ UserState,
+ "max_monthly_mbs",
+ new_callable=PropertyMock,
+ return_value=max_mbs,
+ ):
+ with patch.object(
+ us.db,
+ "query",
+ return_value=MagicMock(
+ filter=MagicMock(
+ return_value=MagicMock(
+ with_entities=MagicMock(
+ return_value=MagicMock(
+ scalar=MagicMock(
+ return_value=mbs * 1024 * 1024
+ )
+ )
+ )
+ )
+ )
+ ),
+ ):
assert us.has_quota_max_monthly_mbs("") == expected
@@ -400,10 +767,15 @@ def test_can_manually_trigger(user_state):
"group2": GroupInfo(manually_trigger_sheet=False),
}
- with patch.object(UserState, 'permissions', new_callable=PropertyMock, return_value=permissions):
- assert user_state.can_manually_trigger("group1") == True
- assert user_state.can_manually_trigger("group2") == False
- assert user_state.can_manually_trigger("group3") == False
+ with patch.object(
+ UserState,
+ "permissions",
+ new_callable=PropertyMock,
+ return_value=permissions,
+ ):
+ assert user_state.can_manually_trigger("group1") is True
+ assert user_state.can_manually_trigger("group2") is False
+ assert user_state.can_manually_trigger("group3") is False
def test_is_sheet_frequency_allowed(user_state):
@@ -412,23 +784,44 @@ def test_is_sheet_frequency_allowed(user_state):
"group2": GroupInfo(sheet_frequency={"daily"}),
}
- with patch.object(UserState, 'permissions', new_callable=PropertyMock, return_value=permissions):
- assert user_state.is_sheet_frequency_allowed("group1", "daily") == True
- assert user_state.is_sheet_frequency_allowed("group1", "hourly") == True
- assert user_state.is_sheet_frequency_allowed("group1", "weekly") == False
- assert user_state.is_sheet_frequency_allowed("group2", "hourly") == False
- assert user_state.is_sheet_frequency_allowed("group2", "daily") == True
- assert user_state.is_sheet_frequency_allowed("group3", "daily") == False
+ with patch.object(
+ UserState,
+ "permissions",
+ new_callable=PropertyMock,
+ return_value=permissions,
+ ):
+ assert user_state.is_sheet_frequency_allowed("group1", "daily") is True
+ assert user_state.is_sheet_frequency_allowed("group1", "hourly") is True
+ assert (
+ user_state.is_sheet_frequency_allowed("group1", "weekly") is False
+ )
+ assert (
+ user_state.is_sheet_frequency_allowed("group2", "hourly") is False
+ )
+ assert user_state.is_sheet_frequency_allowed("group2", "daily") is True
+ assert user_state.is_sheet_frequency_allowed("group3", "daily") is False
def test_priority_group(user_state):
- from app.web.utils.misc import convert_priority_to_queue_dict
- with patch.object(UserState, 'user_groups', new_callable=PropertyMock, return_value=[
- models.Group(id="group1", permissions={"priority": "high"}),
- models.Group(id="group2", permissions={"priority": "medium"}),
- models.Group(id="group3", permissions={"priority": "low"}),
- ]):
- assert user_state.priority_group("group1") == convert_priority_to_queue_dict("high")
- assert user_state.priority_group("group2") == convert_priority_to_queue_dict("medium")
- assert user_state.priority_group("group3") == convert_priority_to_queue_dict("low")
- assert user_state.priority_group("group4") == convert_priority_to_queue_dict("low")
+ with patch.object(
+ UserState,
+ "user_groups",
+ new_callable=PropertyMock,
+ return_value=[
+ models.Group(id="group1", permissions={"priority": "high"}),
+ models.Group(id="group2", permissions={"priority": "medium"}),
+ models.Group(id="group3", permissions={"priority": "low"}),
+ ],
+ ):
+ assert user_state.priority_group(
+ "group1"
+ ) == convert_priority_to_queue_dict("high")
+ assert user_state.priority_group(
+ "group2"
+ ) == convert_priority_to_queue_dict("medium")
+ assert user_state.priority_group(
+ "group3"
+ ) == convert_priority_to_queue_dict("low")
+ assert user_state.priority_group(
+ "group4"
+ ) == convert_priority_to_queue_dict("low")
diff --git a/app/tests/web/endpoints/test_default.py b/app/tests/web/endpoints/test_default.py
index e4e34cc..970e705 100644
--- a/app/tests/web/endpoints/test_default.py
+++ b/app/tests/web/endpoints/test_default.py
@@ -1,17 +1,20 @@
+from http import HTTPStatus
from unittest.mock import MagicMock
import pytest
from fastapi.testclient import TestClient
+from loguru import logger
from app.shared.schemas import Usage, UsageResponse
from app.shared.user_groups import GroupInfo
-from app.tests.web.db.test_crud import test_data
from app.web.config import VERSION
+from app.web.security import get_user_state
+from app.web.utils.metrics import measure_regular_metrics
def test_endpoint_home(client_with_auth):
r = client_with_auth.get("/")
- assert r.status_code == 200
+ assert r.status_code == HTTPStatus.OK
j = r.json()
assert "version" in j and j["version"] == VERSION
assert "breakingChanges" in j
@@ -20,7 +23,7 @@ def test_endpoint_home(client_with_auth):
def test_endpoint_health(client_with_auth):
r = client_with_auth.get("/health")
- assert r.status_code == 200
+ assert r.status_code == HTTPStatus.OK
assert r.json() == {"status": "ok"}
@@ -31,32 +34,31 @@ def test_endpoint_active_no_auth(client, test_no_auth):
def test_endpoint_active(app):
m_user_state = MagicMock()
- from app.web.security import get_user_state
app.dependency_overrides[get_user_state] = lambda: m_user_state
# inactive user
m_user_state.active = False
client = TestClient(app)
r = client.get("/user/active")
- assert r.status_code == 200
+ assert r.status_code == HTTPStatus.OK
assert r.json() == {"active": False}
# active user
m_user_state.active = True
client = TestClient(app)
r = client.get("/user/active")
- assert r.status_code == 200
+ assert r.status_code == HTTPStatus.OK
assert r.json() == {"active": True}
def test_no_serve_local_archive_by_default(client_with_auth):
r = client_with_auth.get("/app/local_archive_test/temp.txt")
- assert r.status_code == 404
+ assert r.status_code == HTTPStatus.NOT_FOUND
def test_favicon(client_with_auth):
r = client_with_auth.get("/favicon.ico")
- assert r.status_code == 200
+ assert r.status_code == HTTPStatus.OK
assert r.headers["content-type"] == "image/vnd.microsoft.icon"
@@ -72,8 +74,10 @@ def test_endpoint_test_prometheus_no_user_auth(client_with_auth, test_no_auth):
async def test_prometheus_metrics(test_data, client_with_token, get_settings):
# before metrics calculation
r = client_with_token.get("/metrics")
- assert r.status_code == 200
- assert r.headers["content-type"] == "text/plain; version=0.0.4; charset=utf-8"
+ assert r.status_code == HTTPStatus.OK
+ assert (
+ r.headers["content-type"] == "text/plain; version=0.0.4; charset=utf-8"
+ )
assert "disk_utilization" in r.text
assert "database_metrics" in r.text
assert "exceptions" in r.text
@@ -81,8 +85,9 @@ async def test_prometheus_metrics(test_data, client_with_token, get_settings):
assert 'disk_utilization{type="used"}' not in r.text
# after metrics calculation
- from app.web.utils.metrics import measure_regular_metrics
- await measure_regular_metrics(get_settings.DATABASE_PATH, 60 * 60 * 24 * 31 * 12 * 100)
+ await measure_regular_metrics(
+ get_settings.DATABASE_PATH, 60 * 60 * 24 * 31 * 12 * 100
+ )
r2 = client_with_token.get("/metrics")
assert 'disk_utilization{type="used"}' in r2.text
assert 'disk_utilization{type="free"}' in r2.text
@@ -90,20 +95,37 @@ async def test_prometheus_metrics(test_data, client_with_token, get_settings):
assert 'database_metrics{query="count_archives"} 100.0' in r2.text
assert 'database_metrics{query="count_archive_urls"} 1000.0' in r2.text
assert 'database_metrics{query="count_users"} 3.0' in r2.text
- assert 'database_metrics_counter_total{query="count_by_user",user="rick@example.com"} 34.0' in r2.text
- assert 'database_metrics_counter_total{query="count_by_user",user="morty@example.com"} 33.0' in r2.text
- assert 'database_metrics_counter_total{query="count_by_user",user="jerry@example.com"} 33.0' in r2.text
+ assert (
+ 'database_metrics_counter_total{query="count_by_user",user="rick@example.com"} 34.0'
+ in r2.text
+ )
+ assert (
+ 'database_metrics_counter_total{query="count_by_user",user="morty@example.com"} 33.0'
+ in r2.text
+ )
+ assert (
+ 'database_metrics_counter_total{query="count_by_user",user="jerry@example.com"} 33.0'
+ in r2.text
+ )
# 30s window, should not change the gauges nor the total in the counters
- from app.web.utils.metrics import measure_regular_metrics
await measure_regular_metrics(get_settings.DATABASE_PATH, 30)
r3 = client_with_token.get("/metrics")
assert 'database_metrics{query="count_archives"} 100.0' in r3.text
assert 'database_metrics{query="count_archive_urls"} 1000.0' in r3.text
assert 'database_metrics{query="count_users"} 3.0' in r3.text
- assert 'database_metrics_counter_total{query="count_by_user",user="rick@example.com"} 34.0' in r3.text
- assert 'database_metrics_counter_total{query="count_by_user",user="morty@example.com"} 33.0' in r3.text
- assert 'database_metrics_counter_total{query="count_by_user",user="jerry@example.com"} 33.0' in r3.text
+ assert (
+ 'database_metrics_counter_total{query="count_by_user",user="rick@example.com"} 34.0'
+ in r3.text
+ )
+ assert (
+ 'database_metrics_counter_total{query="count_by_user",user="morty@example.com"} 33.0'
+ in r3.text
+ )
+ assert (
+ 'database_metrics_counter_total{query="count_by_user",user="jerry@example.com"} 33.0'
+ in r3.text
+ )
def test_endpoint_get_user_permissions_no_user_auth(client, test_no_auth):
@@ -111,14 +133,12 @@ def test_endpoint_get_user_permissions_no_user_auth(client, test_no_auth):
def test_endpoint_get_user_permissions(app):
- from app.web.security import get_user_state
-
m_user_state = MagicMock()
rv = {
"all": GroupInfo(read=True),
"group1": GroupInfo(archive_url=True),
}
- from loguru import logger
+
logger.info(rv)
m_user_state.permissions = rv
@@ -126,13 +146,13 @@ def test_endpoint_get_user_permissions(app):
client = TestClient(app)
r = client.get("/user/permissions")
- assert r.status_code == 200
+ assert r.status_code == HTTPStatus.OK
response = r.json()
assert response.keys() == {"all", "group1"}
assert response["all"]["read"]
assert response["group1"]["read"] == []
assert response["group1"]["archive_url"]
- assert response["all"]["archive_url"] == False
+ assert response["all"]["archive_url"] is False
def test_endpoint_get_user_usage_no_user_auth(client, test_no_auth):
@@ -140,8 +160,6 @@ def test_endpoint_get_user_usage_no_user_auth(client, test_no_auth):
def test_endpoint_get_user_usage_inactive(app):
- from app.web.security import get_user_state
-
m_user_state = MagicMock()
m_user_state.active = False
@@ -149,13 +167,11 @@ def test_endpoint_get_user_usage_inactive(app):
client = TestClient(app)
r = client.get("/user/usage")
- assert r.status_code == 403
+ assert r.status_code == HTTPStatus.FORBIDDEN
assert r.json() == {"detail": "User is not active."}
def test_endpoint_get_user_usage_active(app):
- from app.web.security import get_user_state
-
m_user_state = MagicMock()
m_user_state.active = True
mock_usage = UsageResponse(
@@ -164,8 +180,8 @@ def test_endpoint_get_user_usage_active(app):
total_sheets=3,
groups={
"group1": Usage(monthly_urls=4, monthly_mbs=5, total_sheets=6),
- "group2": Usage(monthly_urls=7, monthly_mbs=8, total_sheets=9)
- }
+ "group2": Usage(monthly_urls=7, monthly_mbs=8, total_sheets=9),
+ },
)
m_user_state.usage.return_value = mock_usage
@@ -173,5 +189,5 @@ def test_endpoint_get_user_usage_active(app):
client = TestClient(app)
r = client.get("/user/usage")
- assert r.status_code == 200
+ assert r.status_code == HTTPStatus.OK
assert UsageResponse(**r.json()) == mock_usage
diff --git a/app/tests/web/endpoints/test_interoperability.py b/app/tests/web/endpoints/test_interoperability.py
index 703f69a..d102116 100644
--- a/app/tests/web/endpoints/test_interoperability.py
+++ b/app/tests/web/endpoints/test_interoperability.py
@@ -1,10 +1,9 @@
import json
from datetime import datetime
+from http import HTTPStatus
from unittest.mock import MagicMock, patch
from app.shared.db import models
-from app.web.config import ALLOW_ANY_EMAIL
-from app.web.db import crud
def test_submit_manual_archive_unauthenticated(client, test_no_auth):
@@ -15,46 +14,134 @@ def test_submit_manual_archive_not_user_auth(client_with_auth, test_no_auth):
test_no_auth(client_with_auth.post, "/interop/submit-archive")
-@patch("app.web.endpoints.interoperability.business_logic", return_value=MagicMock(get_store_archive_until=MagicMock(return_value=datetime)))
+@patch(
+ "app.web.endpoints.interoperability.business_logic",
+ return_value=MagicMock(
+ get_store_archive_until=MagicMock(return_value=datetime)
+ ),
+)
def test_submit_manual_archive(m1, client_with_token, db_session):
# normal workflow
- aa_metadata = json.dumps({"status": "test: success", "metadata": {"url": "http://example.com"}, "media": [{"filename": "fn1", "urls": ["http://example.s3.com"]}]})
- r = client_with_token.post("/interop/submit-archive", json={"result": aa_metadata, "public": True, "author_id": "jerry@gmail.com", "group_id": "spaceship", "tags": ["test"], "url": "http://example.com"})
- assert r.status_code == 201
+ aa_metadata = json.dumps(
+ {
+ "status": "test: success",
+ "metadata": {"url": "http://example.com"},
+ "media": [{"filename": "fn1", "urls": ["http://example.s3.com"]}],
+ }
+ )
+ r = client_with_token.post(
+ "/interop/submit-archive",
+ json={
+ "result": aa_metadata,
+ "public": True,
+ "author_id": "jerry@gmail.com",
+ "group_id": "spaceship",
+ "tags": ["test"],
+ "url": "http://example.com",
+ },
+ )
+ assert r.status_code == HTTPStatus.CREATED
assert "id" in r.json()
- inserted = db_session.query(models.Archive).filter(models.Archive.id == r.json()["id"]).first()
+ inserted = (
+ db_session.query(models.Archive)
+ .filter(models.Archive.id == r.json()["id"])
+ .first()
+ )
assert inserted.url == "http://example.com"
assert inserted.group_id == "spaceship"
assert inserted.author_id == "jerry@gmail.com"
assert sorted([t.id for t in inserted.tags]) == sorted(["test", "manual"])
assert inserted.public
- assert type(inserted.result) == dict
+ assert isinstance(inserted.result, dict)
assert [u.url for u in inserted.urls] == ["http://example.s3.com"]
- assert type(inserted.store_until) == datetime
+ assert isinstance(inserted.store_until, datetime)
# cannot have the same URL twice
- aa_metadata = json.dumps({"status": "test: success", "metadata": {"url": "http://example.com"}, "media": [{"filename": "fn1", "urls": ["http://example.com", "http://example.com"]}]})
- r = client_with_token.post("/interop/submit-archive", json={"result": aa_metadata, "public": False, "author_id": "jerry@gmail.com", "tags": ["test"], "url": "http://example.com"})
- assert r.status_code == 422
- assert r.json() == {"detail": "Cannot insert into DB due to integrity error, likely duplicate urls."}
+ aa_metadata = json.dumps(
+ {
+ "status": "test: success",
+ "metadata": {"url": "http://example.com"},
+ "media": [
+ {
+ "filename": "fn1",
+ "urls": ["http://example.com", "http://example.com"],
+ }
+ ],
+ }
+ )
+ r = client_with_token.post(
+ "/interop/submit-archive",
+ json={
+ "result": aa_metadata,
+ "public": False,
+ "author_id": "jerry@gmail.com",
+ "tags": ["test"],
+ "url": "http://example.com",
+ },
+ )
+ assert r.status_code == HTTPStatus.UNPROCESSABLE_ENTITY
+ assert r.json() == {
+ "detail": "Cannot insert into DB due to integrity error, likely duplicate urls."
+ }
# test with invalid JSON
def test_submit_manual_archive_invalid_json(client_with_token):
- r = client_with_token.post("/interop/submit-archive", json={"result": "invalid json", "public": False, "author_id": "jer", "tags": ["test"], "url": "http://example.com"})
- assert r.status_code == 422
+ r = client_with_token.post(
+ "/interop/submit-archive",
+ json={
+ "result": "invalid json",
+ "public": False,
+ "author_id": "jer",
+ "tags": ["test"],
+ "url": "http://example.com",
+ },
+ )
+ assert r.status_code == HTTPStatus.UNPROCESSABLE_ENTITY
assert r.json() == {"detail": "Invalid JSON in result field."}
-@patch("app.web.endpoints.interoperability.business_logic.get_store_archive_until", side_effect=AssertionError("AssertionError"))
-def test_submit_manual_archive_no_store_until(m_sau, client_with_token, db_session):
- aa_metadata = json.dumps({"status": "test: success", "metadata": {"url": "http://example.com"}, "media": [{"filename": "fn1", "urls": ["http://example.s3.com"]}]})
- r = client_with_token.post("/interop/submit-archive", json={"result": aa_metadata, "public": True, "author_id": "jerry@gmail.com", "group_id": "spaceship", "tags": ["test"], "url": "http://example.com"})
- assert r.status_code == 201
+@patch(
+ "app.web.endpoints.interoperability.business_logic.get_store_archive_until",
+ side_effect=AssertionError("AssertionError"),
+)
+def test_submit_manual_archive_no_store_until(
+ m_sau, client_with_token, db_session
+):
+ aa_metadata = json.dumps(
+ {
+ "status": "test: success",
+ "metadata": {"url": "http://example.com"},
+ "media": [{"filename": "fn1", "urls": ["http://example.s3.com"]}],
+ }
+ )
+ r = client_with_token.post(
+ "/interop/submit-archive",
+ json={
+ "result": aa_metadata,
+ "public": True,
+ "author_id": "jerry@gmail.com",
+ "group_id": "spaceship",
+ "tags": ["test"],
+ "url": "http://example.com",
+ },
+ )
+ assert r.status_code == HTTPStatus.CREATED
assert len(r.json()["id"]) == 36
- res = db_session.query(models.Archive).filter(models.Archive.id == r.json()["id"]).first()
+ res = (
+ db_session.query(models.Archive)
+ .filter(models.Archive.id == r.json()["id"])
+ .first()
+ )
assert res.store_until is None
# testing that store_until = None is not comparable with datetime, and will always return False
- res = db_session.query(models.Archive).filter(models.Archive.id == r.json()["id"], models.Archive.store_until < datetime.now()).first()
+ res = (
+ db_session.query(models.Archive)
+ .filter(
+ models.Archive.id == r.json()["id"],
+ models.Archive.store_until < datetime.now(),
+ )
+ .first()
+ )
assert res is None
diff --git a/app/tests/web/endpoints/test_sheet.py b/app/tests/web/endpoints/test_sheet.py
index 9b47228..c318496 100644
--- a/app/tests/web/endpoints/test_sheet.py
+++ b/app/tests/web/endpoints/test_sheet.py
@@ -1,10 +1,13 @@
-import json
from datetime import datetime
+from http import HTTPStatus
from unittest.mock import MagicMock, patch
from fastapi.testclient import TestClient
+from app.shared.db import models
from app.shared.schemas import TaskResult
+from app.web.db.user_state import UserState
+from app.web.security import get_user_state
def test_endpoints_no_auth(client, test_no_auth):
@@ -20,34 +23,38 @@ def test_create_sheet_endpoint(app_with_auth, db_session):
"id": "123-sheet-id",
"name": "Test Sheet",
"group_id": "spaceship",
- "frequency": "daily"
+ "frequency": "daily",
}
# with good data
response = client_with_auth.post("/sheet/create", json=good_data)
- assert response.status_code == 201
+ assert response.status_code == HTTPStatus.CREATED
j = response.json()
assert datetime.fromisoformat(j.pop("created_at"))
assert datetime.fromisoformat(j.pop("last_url_archived_at"))
- assert j.pop("author_id") == 'morty@example.com'
+ assert j.pop("author_id") == "morty@example.com"
assert j == good_data
# already exists
response = client_with_auth.post("/sheet/create", json=good_data)
- assert response.status_code == 400
- assert response.json() == {"detail": "Sheet with this ID is already being archived."}
+ assert response.status_code == HTTPStatus.BAD_REQUEST
+ assert response.json() == {
+ "detail": "Sheet with this ID is already being archived."
+ }
# bad group
bad_data = good_data.copy()
bad_data["group_id"] = "not a group"
response = client_with_auth.post("/sheet/create", json=bad_data)
- assert response.status_code == 403
- assert response.json() == {"detail": "User does not have access to this group."}
+ assert response.status_code == HTTPStatus.FORBIDDEN
+ assert response.json() == {
+ "detail": "User does not have access to this group."
+ }
# switch to jerry who's got less quota/permissions
- from app.web.db.user_state import UserState
- from app.web.security import get_user_state
- app_with_auth.dependency_overrides[get_user_state] = lambda: UserState(db_session, "jerry@example.com")
+ app_with_auth.dependency_overrides[get_user_state] = lambda: UserState(
+ db_session, "jerry@example.com"
+ )
client_jerry = TestClient(app_with_auth)
# frequency not allowed
@@ -56,39 +63,62 @@ def test_create_sheet_endpoint(app_with_auth, db_session):
jerry_data["frequency"] = "hourly"
jerry_data["id"] = "jerry-sheet-id"
response = client_jerry.post("/sheet/create", json=jerry_data)
- assert response.status_code == 422
- assert response.json() == {"detail": "Invalid frequency selected for this group."}
+ assert response.status_code == HTTPStatus.UNPROCESSABLE_ENTITY
+ assert response.json() == {
+ "detail": "Invalid frequency selected for this group."
+ }
jerry_data["frequency"] = "daily"
# success for the first sheet, bad quota on second
response = client_jerry.post("/sheet/create", json=jerry_data)
- assert response.status_code == 201
+ assert response.status_code == HTTPStatus.CREATED
response = client_jerry.post("/sheet/create", json=jerry_data)
- assert response.status_code == 429
- assert response.json() == {"detail": "User has reached their sheet quota for this group."}
+ assert response.status_code == HTTPStatus.TOO_MANY_REQUESTS
+ assert response.json() == {
+ "detail": "User has reached their sheet quota for this group."
+ }
def test_get_user_sheets_endpoint(client_with_auth, db_session):
# no data
response = client_with_auth.get("/sheet/mine")
- assert response.status_code == 200
+ assert response.status_code == HTTPStatus.OK
assert response.json() == []
# with data
- from app.shared.db import models
db_session.add(
- models.Sheet(id="123", name="Test Sheet 1", author_id="morty@example.com", group_id="spaceship", frequency="hourly")
+ models.Sheet(
+ id="123",
+ name="Test Sheet 1",
+ author_id="morty@example.com",
+ group_id="spaceship",
+ frequency="hourly",
+ )
)
db_session.commit()
- db_session.add_all([
- models.Sheet(id="456", name="Test Sheet 2", author_id="morty@example.com", group_id="interdimensional", frequency="daily"),
- models.Sheet(id="789", name="Test Sheet 3", author_id="rick@example.com", group_id="interdimensional", frequency="hourly"),
- ])
+ db_session.add_all(
+ [
+ models.Sheet(
+ id="456",
+ name="Test Sheet 2",
+ author_id="morty@example.com",
+ group_id="interdimensional",
+ frequency="daily",
+ ),
+ models.Sheet(
+ id="789",
+ name="Test Sheet 3",
+ author_id="rick@example.com",
+ group_id="interdimensional",
+ frequency="hourly",
+ ),
+ ]
+ )
db_session.commit()
response = client_with_auth.get("/sheet/mine")
- assert response.status_code == 200
+ assert response.status_code == HTTPStatus.OK
r = response.json()
assert isinstance(r, list)
assert len(r) == 2
@@ -97,65 +127,84 @@ def test_get_user_sheets_endpoint(client_with_auth, db_session):
assert datetime.fromisoformat(r[1].pop("created_at"))
assert datetime.fromisoformat(r[1].pop("last_url_archived_at"))
assert r[0] == {
- 'id': '123',
- 'author_id': 'morty@example.com',
- 'frequency': 'hourly',
- 'group_id': 'spaceship',
- 'name': 'Test Sheet 1',
+ "id": "123",
+ "author_id": "morty@example.com",
+ "frequency": "hourly",
+ "group_id": "spaceship",
+ "name": "Test Sheet 1",
}
assert r[1] == {
- 'id': '456',
- 'author_id': 'morty@example.com',
- 'frequency': 'daily',
- 'group_id': 'interdimensional',
- 'name': 'Test Sheet 2',
+ "id": "456",
+ "author_id": "morty@example.com",
+ "frequency": "daily",
+ "group_id": "interdimensional",
+ "name": "Test Sheet 2",
}
def test_delete_sheet_endpoint(client_with_auth, db_session):
# missing sheet
response = client_with_auth.delete("/sheet/123-sheet-id")
- assert response.status_code == 200
- assert response.json() == {
- "id": "123-sheet-id",
- "deleted": False
- }
+ assert response.status_code == HTTPStatus.OK
+ assert response.json() == {"id": "123-sheet-id", "deleted": False}
# add sheets for deletion
- from app.shared.db import models
- db_session.add_all([
- models.Sheet(id="123-sheet-id", name="Test Sheet 1", author_id="morty@example.com", group_id="interdimensional", frequency="daily"),
- models.Sheet(id="456-sheet-id", name="Test Sheet 2", author_id="rick@example.com", group_id="spaceship", frequency="hourly"),
- ])
+ db_session.add_all(
+ [
+ models.Sheet(
+ id="123-sheet-id",
+ name="Test Sheet 1",
+ author_id="morty@example.com",
+ group_id="interdimensional",
+ frequency="daily",
+ ),
+ models.Sheet(
+ id="456-sheet-id",
+ name="Test Sheet 2",
+ author_id="rick@example.com",
+ group_id="spaceship",
+ frequency="hourly",
+ ),
+ ]
+ )
db_session.commit()
# morty can delete his
response = client_with_auth.delete("/sheet/123-sheet-id")
- assert response.status_code == 200
+ assert response.status_code == HTTPStatus.OK
assert response.json() == {"id": "123-sheet-id", "deleted": True}
# but only once
response = client_with_auth.delete("/sheet/123-sheet-id")
- assert response.status_code == 200
+ assert response.status_code == HTTPStatus.OK
assert response.json() == {"id": "123-sheet-id", "deleted": False}
- # and not rick's
+ # and not Rick's
response = client_with_auth.delete("/sheet/456-sheet-id")
- assert response.status_code == 200
+ assert response.status_code == HTTPStatus.OK
assert response.json() == {"id": "456-sheet-id", "deleted": False}
class TestArchiveUserSheetEndpoint:
@patch("app.web.endpoints.sheet.celery", return_value=MagicMock())
def test_normal_flow(self, m_celery, client_with_auth, db_session):
- from app.shared.db import models
- db_session.add(models.Sheet(id="123-sheet-id", name="Test Sheet 1", author_id="morty@example.com", group_id="spaceship", frequency="hourly"))
+ db_session.add(
+ models.Sheet(
+ id="123-sheet-id",
+ name="Test Sheet 1",
+ author_id="morty@example.com",
+ group_id="spaceship",
+ frequency="hourly",
+ )
+ )
db_session.commit()
m_signature = MagicMock()
- m_signature.apply_async.return_value = TaskResult(id="123-taskid", status="PENDING", result="")
+ m_signature.apply_async.return_value = TaskResult(
+ id="123-taskid", status="PENDING", result=""
+ )
m_celery.signature.return_value = m_signature
r = client_with_auth.post("/sheet/123-sheet-id/archive")
- assert r.status_code == 201
+ assert r.status_code == HTTPStatus.CREATED
assert r.json() == {"id": "123-taskid"}
m_celery.signature.assert_called_once()
m_signature.apply_async.assert_called_once()
@@ -165,29 +214,54 @@ class TestArchiveUserSheetEndpoint:
def test_missing_data(self, client_with_auth):
r = client_with_auth.post("/sheet/123-sheet-id/archive")
- assert r.status_code == 403
+ assert r.status_code == HTTPStatus.FORBIDDEN
assert r.json() == {"detail": "No access to this sheet."}
def test_no_access(self, client_with_auth, db_session):
- from app.shared.db import models
- db_session.add(models.Sheet(id="123-sheet-id", name="Test Sheet 1", author_id="rick@example.com", group_id="spaceship", frequency="hourly"))
+ db_session.add(
+ models.Sheet(
+ id="123-sheet-id",
+ name="Test Sheet 1",
+ author_id="rick@example.com",
+ group_id="spaceship",
+ frequency="hourly",
+ )
+ )
db_session.commit()
r = client_with_auth.post("/sheet/123-sheet-id/archive")
- assert r.status_code == 403
+ assert r.status_code == HTTPStatus.FORBIDDEN
assert r.json() == {"detail": "No access to this sheet."}
def test_user_not_in_group(self, client_with_auth, db_session):
- from app.shared.db import models
- db_session.add(models.Sheet(id="123-sheet-id", name="Test Sheet 1", author_id="morty@example.com", group_id="interdimensional", frequency="hourly"))
+ db_session.add(
+ models.Sheet(
+ id="123-sheet-id",
+ name="Test Sheet 1",
+ author_id="morty@example.com",
+ group_id="interdimensional",
+ frequency="hourly",
+ )
+ )
db_session.commit()
r = client_with_auth.post("/sheet/123-sheet-id/archive")
- assert r.status_code == 403
- assert r.json() == {"detail": "User does not have access to this group."}
+ assert r.status_code == HTTPStatus.FORBIDDEN
+ assert r.json() == {
+ "detail": "User does not have access to this group."
+ }
def test_user_cannot_manually_trigger(self, client_with_auth, db_session):
- from app.shared.db import models
- db_session.add(models.Sheet(id="123-sheet-id", name="Test Sheet 1", author_id="morty@example.com", group_id="default", frequency="hourly"))
+ db_session.add(
+ models.Sheet(
+ id="123-sheet-id",
+ name="Test Sheet 1",
+ author_id="morty@example.com",
+ group_id="default",
+ frequency="hourly",
+ )
+ )
db_session.commit()
r = client_with_auth.post("/sheet/123-sheet-id/archive")
- assert r.status_code == 429
- assert r.json() == {"detail": "User cannot manually trigger sheet archiving in this group."}
+ assert r.status_code == HTTPStatus.TOO_MANY_REQUESTS
+ assert r.json() == {
+ "detail": "User cannot manually trigger sheet archiving in this group."
+ }
diff --git a/app/tests/web/endpoints/test_task.py b/app/tests/web/endpoints/test_task.py
index 937ad46..038babf 100644
--- a/app/tests/web/endpoints/test_task.py
+++ b/app/tests/web/endpoints/test_task.py
@@ -1,3 +1,4 @@
+from http import HTTPStatus
from unittest.mock import patch
@@ -12,27 +13,26 @@ def test_get_status_success(mock_async_result, client_with_auth):
response = client_with_auth.get("/task/test-task-id")
- assert response.status_code == 200
+ assert response.status_code == HTTPStatus.OK
assert response.json() == {
"id": "test-task-id",
"status": "SUCCESS",
- "result": {"data": "some result"}
+ "result": {"data": "some result"},
}
@patch("app.web.endpoints.task.AsyncResult")
def test_get_status_failure(mock_async_result, client_with_auth):
-
mock_async_result.return_value.status = "FAILURE"
mock_async_result.return_value.result = Exception("Some error")
response = client_with_auth.get("/task/test-task-id")
- assert response.status_code == 200
+ assert response.status_code == HTTPStatus.OK
assert response.json() == {
"id": "test-task-id",
"status": "FAILURE",
- "result": {"error": "Some error"}
+ "result": {"error": "Some error"},
}
@@ -43,9 +43,9 @@ def test_get_status_pending(mock_async_result, client_with_auth):
response = client_with_auth.get("/task/test-task-id")
- assert response.status_code == 200
+ assert response.status_code == HTTPStatus.OK
assert response.json() == {
"id": "test-task-id",
"status": "PENDING",
- "result": None
+ "result": None,
}
diff --git a/app/tests/web/endpoints/test_url.py b/app/tests/web/endpoints/test_url.py
index 1b6ee85..cd64262 100644
--- a/app/tests/web/endpoints/test_url.py
+++ b/app/tests/web/endpoints/test_url.py
@@ -1,6 +1,9 @@
import json
+from http import HTTPStatus
from unittest.mock import MagicMock, patch
+from app.shared import schemas
+from app.shared.db import worker_crud
from app.shared.schemas import ArchiveCreate, TaskResult
from app.web.config import ALLOW_ANY_EMAIL
@@ -13,7 +16,9 @@ def test_archive_url_unauthenticated(client, test_no_auth):
@patch("app.web.endpoints.url.celery", return_value=MagicMock())
def test_archive_url(m_celery, m2, client_with_auth):
m_signature = MagicMock()
- m_signature.apply_async.return_value = TaskResult(id="123-456-789", status="PENDING", result="")
+ m_signature.apply_async.return_value = TaskResult(
+ id="123-456-789", status="PENDING", result=""
+ )
m_celery.signature.return_value = m_signature
m_user_state = MagicMock()
@@ -21,62 +26,98 @@ def test_archive_url(m_celery, m2, client_with_auth):
# url is too short
response = client_with_auth.post("/url/archive", json={"url": "bad"})
- assert response.status_code == 422
- assert response.json()["detail"][0]["msg"] == 'String should have at least 5 characters'
+ assert response.status_code == HTTPStatus.UNPROCESSABLE_ENTITY
+ assert (
+ response.json()["detail"][0]["msg"]
+ == "String should have at least 5 characters"
+ )
m_celery.signature.assert_not_called()
# url is invalid
- response = client_with_auth.post("/url/archive", json={"url": "example.com"})
- assert response.status_code == 400
+ response = client_with_auth.post(
+ "/url/archive", json={"url": "example.com"}
+ )
+ assert response.status_code == HTTPStatus.BAD_REQUEST
assert response.json()["detail"] == "Invalid URL received."
# valid request
m_user_state.has_quota_max_monthly_urls.return_value = True
m_user_state.has_quota_max_monthly_mbs.return_value = True
- response = client_with_auth.post("/url/archive", json={"url": "https://example.com"})
- assert response.status_code == 201
- assert response.json() == {'id': '123-456-789'}
+ response = client_with_auth.post(
+ "/url/archive", json={"url": "https://example.com"}
+ )
+ assert response.status_code == HTTPStatus.CREATED
+ assert response.json() == {"id": "123-456-789"}
m_celery.signature.assert_called_once()
m_signature.apply_async.assert_called_once()
called_val = m_celery.signature.call_args
assert called_val[0][0] == "create_archive_task"
- assert json.loads(called_val[1]['args'][0]) == {"id": None, "url": "https://example.com", "result": None, "public": False, "author_id": "rick@example.com", "group_id": "default", "tags": None, "sheet_id": None, "store_until": None, "urls": None}
+ assert json.loads(called_val[1]["args"][0]) == {
+ "id": None,
+ "url": "https://example.com",
+ "result": None,
+ "public": False,
+ "author_id": "rick@example.com",
+ "group_id": "default",
+ "tags": None,
+ "sheet_id": None,
+ "store_until": None,
+ "urls": None,
+ }
m_user_state.has_quota_max_monthly_urls.assert_called_once()
m_user_state.has_quota_max_monthly_mbs.assert_called_once()
m_user_state.in_group.assert_called_once_with("default")
# user is not in group
m_user_state.in_group.return_value = False
- response = client_with_auth.post("/url/archive", json={"url": "https://example.com", "group_id": "new-group"})
- assert response.status_code == 403
- assert response.json()["detail"] == "User does not have access to this group."
+ response = client_with_auth.post(
+ "/url/archive",
+ json={"url": "https://example.com", "group_id": "new-group"},
+ )
+ assert response.status_code == HTTPStatus.FORBIDDEN
+ assert (
+ response.json()["detail"] == "User does not have access to this group."
+ )
m_user_state.in_group.assert_called_with("new-group")
# user is in group
m_user_state.in_group.return_value = True
- response = client_with_auth.post("/url/archive", json={"url": "https://example.com", "group_id": "spaceship"})
- assert response.status_code == 201
- assert response.json() == {'id': '123-456-789'}
+ response = client_with_auth.post(
+ "/url/archive",
+ json={"url": "https://example.com", "group_id": "spaceship"},
+ )
+ assert response.status_code == HTTPStatus.CREATED
+ assert response.json() == {"id": "123-456-789"}
assert m_celery.signature.call_count == 2
assert m_signature.apply_async.call_count == 2
called_val = m_celery.signature.call_args
- assert json.loads(called_val[1]['args'][0])["group_id"] == "spaceship"
+ assert json.loads(called_val[1]["args"][0])["group_id"] == "spaceship"
m_user_state.in_group.assert_called_with("spaceship")
# user is over monthly URL quota
m_user_state.has_quota_max_monthly_urls.return_value = False
m_user_state.has_quota_max_monthly_mbs.return_value = True
- response = client_with_auth.post("/url/archive", json={"url": "https://example.com", "group_id": "spaceship"})
- assert response.status_code == 429
- assert response.json()["detail"] == "User has reached their monthly URL quota."
+ response = client_with_auth.post(
+ "/url/archive",
+ json={"url": "https://example.com", "group_id": "spaceship"},
+ )
+ assert response.status_code == HTTPStatus.TOO_MANY_REQUESTS
+ assert (
+ response.json()["detail"] == "User has reached their monthly URL quota."
+ )
m_user_state.has_quota_max_monthly_urls.assert_called_with("spaceship")
# user is over monthly MB quota
m_user_state.has_quota_max_monthly_urls.return_value = True
m_user_state.has_quota_max_monthly_mbs.return_value = False
- response = client_with_auth.post("/url/archive", json={"url": "https://example.com", "group_id": "spacesuit"})
- assert response.status_code == 429
- assert response.json()["detail"] == "User has reached their monthly MB quota."
+ response = client_with_auth.post(
+ "/url/archive",
+ json={"url": "https://example.com", "group_id": "spacesuit"},
+ )
+ assert response.status_code == HTTPStatus.TOO_MANY_REQUESTS
+ assert (
+ response.json()["detail"] == "User has reached their monthly MB quota."
+ )
m_user_state.has_quota_max_monthly_mbs.assert_called_with("spacesuit")
assert m_celery.signature.call_count == 2
assert m_signature.apply_async.call_count == 2
@@ -89,40 +130,77 @@ def test_archive_url_quotas(m1, client_with_auth):
# misses on monthly URLs quota
m_user_state.has_quota_max_monthly_urls.return_value = False
- response = client_with_auth.post("/url/archive", json={"url": "https://example.com"})
- assert response.status_code == 429
- assert response.json()["detail"] == "User has reached their monthly URL quota."
+ response = client_with_auth.post(
+ "/url/archive", json={"url": "https://example.com"}
+ )
+ assert response.status_code == HTTPStatus.TOO_MANY_REQUESTS
+ assert (
+ response.json()["detail"] == "User has reached their monthly URL quota."
+ )
m_user_state.has_quota_max_monthly_urls.assert_called_once()
# misses on monthly MBs quota
m_user_state.has_quota_max_monthly_urls.return_value = True
m_user_state.has_quota_max_monthly_mbs.return_value = False
- response = client_with_auth.post("/url/archive", json={"url": "https://example.com"})
- assert response.status_code == 429
- assert response.json()["detail"] == "User has reached their monthly MB quota."
+ response = client_with_auth.post(
+ "/url/archive", json={"url": "https://example.com"}
+ )
+ assert response.status_code == HTTPStatus.TOO_MANY_REQUESTS
+ assert (
+ response.json()["detail"] == "User has reached their monthly MB quota."
+ )
m_user_state.has_quota_max_monthly_mbs.assert_called_once()
@patch("app.web.endpoints.url.celery", return_value=MagicMock())
def test_archive_url_with_api_token(m_celery, client_with_token):
m_signature = MagicMock()
- m_signature.apply_async.return_value = TaskResult(id="123-456-789", status="PENDING", result="")
+ m_signature.apply_async.return_value = TaskResult(
+ id="123-456-789", status="PENDING", result=""
+ )
m_celery.signature.return_value = m_signature
- response = client_with_token.post("/url/archive", json={"url": "https://example.com", "author_id": "someone@example.com"})
- assert response.status_code == 201
- assert response.json() == {'id': '123-456-789'}
+ response = client_with_token.post(
+ "/url/archive",
+ json={"url": "https://example.com", "author_id": "someone@example.com"},
+ )
+ assert response.status_code == HTTPStatus.CREATED
+ assert response.json() == {"id": "123-456-789"}
m_celery.signature.assert_called_once()
m_signature.apply_async.assert_called_once()
called_val = m_celery.signature.call_args
assert called_val[0][0] == "create_archive_task"
- assert json.loads(called_val[1]['args'][0]) == {"id": None, "url": "https://example.com", "result": None, "public": False, "author_id": "someone@example.com", "group_id": "default", "tags": None, "sheet_id": None, "store_until": None, "urls": None}
+ assert json.loads(called_val[1]["args"][0]) == {
+ "id": None,
+ "url": "https://example.com",
+ "result": None,
+ "public": False,
+ "author_id": "someone@example.com",
+ "group_id": "default",
+ "tags": None,
+ "sheet_id": None,
+ "store_until": None,
+ "urls": None,
+ }
# missing id should use ALLOW_ANY_EMAIL
- response = client_with_token.post("/url/archive", json={"url": "https://example.com", "author_id": None})
- assert response.status_code == 201
+ response = client_with_token.post(
+ "/url/archive", json={"url": "https://example.com", "author_id": None}
+ )
+ assert response.status_code == HTTPStatus.CREATED
called_val = m_celery.signature.call_args
assert called_val[0][0] == "create_archive_task"
- assert json.loads(called_val[1]['args'][0]) == {"id": None, "url": "https://example.com", "result": None, "public": False, "author_id": ALLOW_ANY_EMAIL, "group_id": "default", "tags": None, "sheet_id": None, "store_until": None, "urls": None}
+ assert json.loads(called_val[1]["args"][0]) == {
+ "id": None,
+ "url": "https://example.com",
+ "result": None,
+ "public": False,
+ "author_id": ALLOW_ANY_EMAIL,
+ "group_id": "default",
+ "tags": None,
+ "sheet_id": None,
+ "store_until": None,
+ "urls": None,
+ }
def test_search_by_url_unauthenticated(client, test_no_auth):
@@ -132,46 +210,67 @@ def test_search_by_url_unauthenticated(client, test_no_auth):
def test_search_by_url(client_with_auth, client_with_token, db_session):
# tests the search endpoint, including through some db data for the endpoint params
response = client_with_auth.get("/url/search")
- assert response.status_code == 422
+ assert response.status_code == HTTPStatus.UNPROCESSABLE_ENTITY
assert response.json()["detail"][0]["msg"] == "Field required"
response = client_with_auth.get("/url/search?url=https://example.com")
- assert response.status_code == 200
+ assert response.status_code == HTTPStatus.OK
assert response.json() == []
- from app.shared import schemas
- from app.shared.db import worker_crud
for i in range(11):
- worker_crud.create_archive(db_session, ArchiveCreate(id=f"url-456-{i}", url="https://example.com" if i < 10 else "https://something-else.com", result={}, public=True, author_id="rick@example.com"), [], [])
+ worker_crud.create_archive(
+ db_session,
+ ArchiveCreate(
+ id=f"url-456-{i}",
+ url="https://example.com"
+ if i < 10
+ else "https://something-else.com",
+ result={},
+ public=True,
+ author_id="rick@example.com",
+ ),
+ [],
+ [],
+ )
# NB: this insertion is too fast for the ordering to be correct as they are within the same second
response = client_with_auth.get("/url/search?url=https://example.com")
- assert response.status_code == 200
+ assert response.status_code == HTTPStatus.OK
assert len(j := response.json()) == 10
assert "url-456-0" in [i["id"] for i in j]
assert "url-456-9" in [i["id"] for i in j]
assert "url-456-10" not in [i["id"] for i in j]
assert j[0].keys() == schemas.ArchiveResult.model_fields.keys()
- response = client_with_auth.get("/url/search?url=https://example.com&limit=5")
- assert response.status_code == 200
+ response = client_with_auth.get(
+ "/url/search?url=https://example.com&limit=5"
+ )
+ assert response.status_code == HTTPStatus.OK
assert len(response.json()) == 5
- response = client_with_auth.get("/url/search?url=https://example.com&skip=5&limit=2")
- assert response.status_code == 200
+ response = client_with_auth.get(
+ "/url/search?url=https://example.com&skip=5&limit=2"
+ )
+ assert response.status_code == HTTPStatus.OK
assert len(response.json()) == 2
- response = client_with_auth.get("/url/search?url=https://example.com&archived_before=2010-01-01")
- assert response.status_code == 200
+ response = client_with_auth.get(
+ "/url/search?url=https://example.com&archived_before=2010-01-01"
+ )
+ assert response.status_code == HTTPStatus.OK
assert len(response.json()) == 0
- response = client_with_auth.get("/url/search?url=https://example.com&archived_after=2010-01-01")
- assert response.status_code == 200
+ response = client_with_auth.get(
+ "/url/search?url=https://example.com&archived_after=2010-01-01"
+ )
+ assert response.status_code == HTTPStatus.OK
assert len(response.json()) == 10
# API token will also work
- response = client_with_token.get("/url/search?url=https://example.com&archived_after=2010-01-01")
- assert response.status_code == 200
+ response = client_with_token.get(
+ "/url/search?url=https://example.com&archived_after=2010-01-01"
+ )
+ assert response.status_code == HTTPStatus.OK
assert len(response.json()) == 10
@@ -181,7 +280,7 @@ def test_search_no_read_access(mock_user_state, client_with_auth):
mock_user_state.return_value.read_public = False
response = client_with_auth.get("/url/search?url=https://example.com")
- assert response.status_code == 403
+ assert response.status_code == HTTPStatus.FORBIDDEN
assert response.json() == {"detail": "User does not have read access."}
@@ -191,12 +290,22 @@ def test_delete_task_unauthenticated(client, test_no_auth):
def test_delete_task(client_with_auth, db_session):
response = client_with_auth.delete("/url/delete-123-456-789")
- assert response.status_code == 200
+ assert response.status_code == HTTPStatus.OK
assert response.json() == {"id": "delete-123-456-789", "deleted": False}
- from app.shared.db import worker_crud
- worker_crud.create_archive(db_session, ArchiveCreate(id="delete-123-456-789", url="https://example.com", result={}, public=True, author_id="morty@example.com"), [], [])
+ worker_crud.create_archive(
+ db_session,
+ ArchiveCreate(
+ id="delete-123-456-789",
+ url="https://example.com",
+ result={},
+ public=True,
+ author_id="morty@example.com",
+ ),
+ [],
+ [],
+ )
response = client_with_auth.delete("/url/delete-123-456-789")
- assert response.status_code == 200
+ assert response.status_code == HTTPStatus.OK
assert response.json() == {"id": "delete-123-456-789", "deleted": True}
diff --git a/app/tests/web/test_main.py b/app/tests/web/test_main.py
index a4ddf1e..0dbd5fa 100644
--- a/app/tests/web/test_main.py
+++ b/app/tests/web/test_main.py
@@ -1,25 +1,32 @@
import os
import shutil
+from http import HTTPStatus
from unittest.mock import patch
+import alembic.config
import pytest
from fastapi.testclient import TestClient
+from app.web.utils.metrics import EXCEPTION_COUNTER
+
def test_lifespan(app):
with TestClient(app) as client:
r = client.get("/health")
- assert r.status_code == 200
+ assert r.status_code == HTTPStatus.OK
assert r.json() == {"status": "ok"}
-def test_alembic(db_session):
- import alembic.config
- alembic.config.main(argv=['--raiseerr', 'upgrade', 'head'])
- alembic.config.main(argv=['--raiseerr', 'downgrade', 'base'])
-@patch("app.web.endpoints.url.crud.soft_delete_archive", side_effect=Exception('mocked error'))
+def test_alembic(db_session):
+ alembic.config.main(argv=["--raiseerr", "upgrade", "head"])
+ alembic.config.main(argv=["--raiseerr", "downgrade", "base"])
+
+
+@patch(
+ "app.web.endpoints.url.crud.soft_delete_archive",
+ side_effect=Exception("mocked error"),
+)
def test_logging_middleware(m1, client_with_auth):
- from app.web.utils.metrics import EXCEPTION_COUNTER
assert len(EXCEPTION_COUNTER.collect()[0].samples) == 0
with pytest.raises(Exception, match="mocked error"):
client_with_auth.delete("/url/123")
@@ -37,12 +44,13 @@ def test_serve_local_archive_logic(get_settings):
# modify the settings
get_settings.SERVE_LOCAL_ARCHIVE = "/app/local_archive_test"
from app.web.main import app_factory
+
app = app_factory(get_settings)
# test
client = TestClient(app)
r = client.get("/app/local_archive_test/temp.txt")
- assert r.status_code == 200
+ assert r.status_code == HTTPStatus.OK
assert r.text == "test"
finally:
# cleanup
diff --git a/app/tests/web/test_security.py b/app/tests/web/test_security.py
index 55a434b..07943e5 100644
--- a/app/tests/web/test_security.py
+++ b/app/tests/web/test_security.py
@@ -1,3 +1,4 @@
+from http import HTTPStatus
from unittest.mock import Mock, patch
import pytest
@@ -5,112 +6,177 @@ from fastapi import HTTPException
from fastapi.security import HTTPAuthorizationCredentials
from app.web.config import ALLOW_ANY_EMAIL
+from app.web.db.user_state import UserState
+from app.web.security import (
+ authenticate_user,
+ get_token_or_user_auth,
+ get_user_auth,
+ get_user_state,
+ secure_compare,
+ token_api_key_auth,
+)
def test_secure_compare():
- from app.web.security import secure_compare
-
assert secure_compare("test", "test")
assert not secure_compare("test", "test2")
@pytest.mark.asyncio
async def test_get_token_or_user_auth_with_api():
- from app.web.security import get_token_or_user_auth
- mock_api = HTTPAuthorizationCredentials(scheme="lorem", credentials="this_is_the_test_api_token")
+ mock_api = HTTPAuthorizationCredentials(
+ scheme="lorem", credentials="this_is_the_test_api_token"
+ )
assert await get_token_or_user_auth(mock_api) == ALLOW_ANY_EMAIL
@pytest.mark.asyncio
async def test_get_token_or_user_auth_with_user():
- from app.web.security import get_token_or_user_auth
- bad_user = HTTPAuthorizationCredentials(scheme="ipsum", credentials="invalid")
- e: pytest.ExceptionInfo = None
+ bad_user = HTTPAuthorizationCredentials(
+ scheme="ipsum", credentials="invalid"
+ )
with pytest.raises(HTTPException) as e:
await get_token_or_user_auth(bad_user)
- assert e.value.status_code == 401
+ assert e.value.status_code == HTTPStatus.UNAUTHORIZED
assert e.value.detail == "invalid access_token"
-@patch("app.web.security.authenticate_user", return_value=(True, "summer@example.com"))
+@patch(
+ "app.web.security.authenticate_user",
+ return_value=(True, "summer@example.com"),
+)
@pytest.mark.asyncio
async def test_get_user_auth(m1):
- from app.web.security import get_user_auth
- good_user = HTTPAuthorizationCredentials(scheme="ipsum", credentials="valid-and-good")
+ good_user = HTTPAuthorizationCredentials(
+ scheme="ipsum", credentials="valid-and-good"
+ )
assert await get_user_auth(good_user) == "summer@example.com"
@patch("app.web.security.secure_compare", return_value=False)
@pytest.mark.asyncio
async def test_token_api_key_auth_exception(m1):
- from app.web.security import token_api_key_auth
-
- e: pytest.ExceptionInfo = None
with pytest.raises(HTTPException) as e:
- await token_api_key_auth(HTTPAuthorizationCredentials(scheme="ipsum", credentials="does-not-matter"), auto_error=True)
- assert e.value.status_code == 401
+ await token_api_key_auth(
+ HTTPAuthorizationCredentials(
+ scheme="ipsum", credentials="does-not-matter"
+ ),
+ auto_error=True,
+ )
+ assert e.value.status_code == HTTPStatus.UNAUTHORIZED
assert e.value.detail == "Wrong auth credentials"
@pytest.mark.asyncio
async def test_authenticate_user():
- from app.web.security import authenticate_user
-
assert authenticate_user("test") == (False, "invalid access_token")
assert authenticate_user(123) == (False, "invalid access_token")
with patch("app.web.security.requests.get") as mock_get:
# bad response from oauth2
- mock_get.return_value.status_code = 403
- assert authenticate_user("this-will-call-requests") == (False, "invalid token")
+ mock_get.return_value.status_code = HTTPStatus.FORBIDDEN
+ assert authenticate_user("this-will-call-requests") == (
+ False,
+ "invalid token",
+ )
assert mock_get.call_count == 1
# 200 but invalid json
- mock_get.return_value.status_code = 200
- assert authenticate_user("this-will-call-requests") == (False, "token does not belong to valid APP_ID")
+ mock_get.return_value.status_code = HTTPStatus.OK
+ assert authenticate_user("this-will-call-requests") == (
+ False,
+ "token does not belong to valid APP_ID",
+ )
assert mock_get.call_count == 2
# 200 but invalid azp and aud
- mock_get.return_value.json.return_value = {"email": "summer@example.com", "azp": "not_an_app"}
- assert authenticate_user("this-will-call-requests") == (False, "token does not belong to valid APP_ID")
+ mock_get.return_value.json.return_value = {
+ "email": "summer@example.com",
+ "azp": "not_an_app",
+ }
+ assert authenticate_user("this-will-call-requests") == (
+ False,
+ "token does not belong to valid APP_ID",
+ )
- mock_get.return_value.json.return_value = {"email": "summer@example.com", "aud": "not_an_app"}
- assert authenticate_user("this-will-call-requests") == (False, "token does not belong to valid APP_ID")
+ mock_get.return_value.json.return_value = {
+ "email": "summer@example.com",
+ "aud": "not_an_app",
+ }
+ assert authenticate_user("this-will-call-requests") == (
+ False,
+ "token does not belong to valid APP_ID",
+ )
- mock_get.return_value.json.return_value = {"email": "summer@example.com", "azp": "not_an_app", "aud": "not_an_app"}
- assert authenticate_user("this-will-call-requests") == (False, "token does not belong to valid APP_ID")
+ mock_get.return_value.json.return_value = {
+ "email": "summer@example.com",
+ "azp": "not_an_app",
+ "aud": "not_an_app",
+ }
+ assert authenticate_user("this-will-call-requests") == (
+ False,
+ "token does not belong to valid APP_ID",
+ )
# blocked email
- mock_get.return_value.json.return_value = {"email": "blocked@example.com", "azp": "test_app_id_1", "aud": "not_an_app"}
- assert authenticate_user("this-will-call-requests") == (False, "email 'blocked@example.com' not allowed")
+ mock_get.return_value.json.return_value = {
+ "email": "blocked@example.com",
+ "azp": "test_app_id_1",
+ "aud": "not_an_app",
+ }
+ assert authenticate_user("this-will-call-requests") == (
+ False,
+ "email 'blocked@example.com' not allowed",
+ )
# not verified
- mock_get.return_value.json.return_value = {"email": "summer@example.com", "azp": "not_an_app", "aud": "test_app_id_1"}
- assert authenticate_user("this-will-call-requests") == (False, "email 'summer@example.com' not verified")
+ mock_get.return_value.json.return_value = {
+ "email": "summer@example.com",
+ "azp": "not_an_app",
+ "aud": "test_app_id_1",
+ }
+ assert authenticate_user("this-will-call-requests") == (
+ False,
+ "email 'summer@example.com' not verified",
+ )
# token expired
- mock_get.return_value.json.return_value = {"email": "summer@example.com", "azp": "test_app_id_2", "email_verified": "true"}
- assert authenticate_user("this-will-call-requests") == (False, "Token expired")
+ mock_get.return_value.json.return_value = {
+ "email": "summer@example.com",
+ "azp": "test_app_id_2",
+ "email_verified": "true",
+ }
+ assert authenticate_user("this-will-call-requests") == (
+ False,
+ "Token expired",
+ )
# 200 and valid azp and aup and verified
- mock_get.return_value.json.return_value = {"email": "summer@example.com", "azp": "test_app_id_2", "email_verified": "true", "expires_in": 100}
- assert authenticate_user("this-will-call-requests") == (True, "summer@example.com")
+ mock_get.return_value.json.return_value = {
+ "email": "summer@example.com",
+ "azp": "test_app_id_2",
+ "email_verified": "true",
+ "expires_in": 100,
+ }
+ assert authenticate_user("this-will-call-requests") == (
+ True,
+ "summer@example.com",
+ )
assert mock_get.call_count == 9
@pytest.mark.asyncio
async def test_authenticate_user_exception():
- from app.web.security import authenticate_user
with patch("app.web.security.requests.get") as mock_get:
- mock_get.return_value.status_code = 200
+ mock_get.return_value.status_code = HTTPStatus.OK
mock_get.return_value.json.side_effect = Exception("mocked error")
- assert authenticate_user("this-will-call-requests") == (False, "exception occurred")
+ assert authenticate_user("this-will-call-requests") == (
+ False,
+ "exception occurred",
+ )
def test_get_user_state():
- from app.web.db.user_state import UserState
- from app.web.security import get_user_state
-
mock_session = Mock()
test_email = "test@example.com"
diff --git a/app/tests/worker/test_worker_main.py b/app/tests/worker/test_worker_main.py
index 9a77528..e67aa88 100644
--- a/app/tests/worker/test_worker_main.py
+++ b/app/tests/worker/test_worker_main.py
@@ -6,23 +6,41 @@ from auto_archiver.core import Media, Metadata
from app.shared import schemas
from app.shared.db import models
+from app.worker.main import create_archive_task, create_sheet_task, get_all_urls
-class Test_create_archive_task():
+class TestCreateArchiveTask:
URL = "https://example-live.com"
- archive = schemas.ArchiveCreate(url=URL, tags=["tag-celery"], public=True, author_id="rick@example.com", group_id="interstellar")
+ archive = schemas.ArchiveCreate(
+ url=URL,
+ tags=["tag-celery"],
+ public=True,
+ author_id="rick@example.com",
+ group_id="interstellar",
+ )
@patch("app.worker.main.ArchivingOrchestrator")
@patch("app.worker.main.get_all_urls", return_value=[])
@patch("app.worker.main.insert_result_into_db")
@patch("app.worker.main.get_store_until", return_value=datetime.now())
- @patch("app.worker.main.get_orchestrator_args", return_value=["arg1", "arg2"])
+ @patch(
+ "app.worker.main.get_orchestrator_args", return_value=["arg1", "arg2"]
+ )
@patch("celery.app.task.Task.request")
- def test_success(self, m_req, m_args, m_store, m_insert, m_urls, m_orchestrator, db_session):
- from app.worker.main import create_archive_task
-
+ def test_success(
+ self,
+ m_req,
+ m_args,
+ m_store,
+ m_insert,
+ m_urls,
+ m_orchestrator,
+ db_session,
+ ):
m_req.id = "this-just-in"
- m_orchestrator.return_value.feed.return_value = iter([Metadata().set_url(self.URL).success()])
+ m_orchestrator.return_value.feed.return_value = iter(
+ [Metadata().set_url(self.URL).success()]
+ )
task = create_archive_task(self.archive.model_dump_json())
@@ -38,15 +56,15 @@ class Test_create_archive_task():
assert len(task["media"]) == 0
def test_raise_invalid(self):
- from app.worker.main import create_archive_task
- with pytest.raises(Exception):
+ with pytest.raises(Exception) as _:
create_archive_task(self.archive.model_dump_json())
@patch("app.worker.main.ArchivingOrchestrator")
@patch("app.worker.main.get_orchestrator_args")
def test_raise_db_error(self, m_args, m_orchestrator):
- from app.worker.main import create_archive_task
- m_orchestrator.return_value.feed.side_effect = Exception("Orchestrator failed")
+ m_orchestrator.return_value.feed.side_effect = Exception(
+ "Orchestrator failed"
+ )
with pytest.raises(Exception) as e:
create_archive_task(self.archive.model_dump_json())
@@ -58,7 +76,6 @@ class Test_create_archive_task():
@patch("app.worker.main.insert_result_into_db", return_value=None)
@patch("app.worker.main.get_orchestrator_args")
def test_raise_empty_result(self, m_args, m_insert, m_orchestrator):
- from app.worker.main import create_archive_task
m_orchestrator.return_value.feed.return_value = iter([None])
with pytest.raises(Exception) as e:
@@ -67,61 +84,83 @@ class Test_create_archive_task():
m_orchestrator.return_value.feed.assert_called_once()
-class Test_create_sheet_task():
+class TestCreateSheetTask:
URL = "https://example-live.com"
- sheet = schemas.SubmitSheet(sheet_id="123", author_id="rick@example.com", group_id="interstellar", tags=["spaceship"])
+ sheet = schemas.SubmitSheet(
+ sheet_id="123",
+ author_id="rick@example.com",
+ group_id="interstellar",
+ tags=["spaceship"],
+ )
@patch("app.worker.main.get_all_urls", return_value=[])
@patch("app.worker.main.ArchivingOrchestrator")
@patch("app.worker.main.models.generate_uuid", return_value="constant-uuid")
@patch("app.worker.main.get_store_until", return_value=datetime.now())
@patch("app.worker.main.get_orchestrator_args")
- def test_success(self, m_args, m_store, m_uuid, m_orchestrator, m_urls, db_session):
- from app.worker.main import create_sheet_task
-
- assert db_session.query(models.Archive).filter(models.Archive.url == self.URL).count() == 0
+ def test_success(
+ self, m_args, m_store, m_uuid, m_orchestrator, m_urls, db_session
+ ):
+ assert (
+ db_session.query(models.Archive)
+ .filter(models.Archive.url == self.URL)
+ .count()
+ == 0
+ )
mock_metadata = Metadata().set_url(self.URL).success()
mock_metadata.add_media(Media("fn1.txt", urls=["outcome1.com"]))
- m_orchestrator.return_value.feed.return_value = iter([False, mock_metadata, mock_metadata])
+ m_orchestrator.return_value.feed.return_value = iter(
+ [False, mock_metadata, mock_metadata]
+ )
res = create_sheet_task(self.sheet.model_dump_json())
- m_args.assert_called_once_with("interstellar", True, ["--gsheet_feeder.sheet_id", "123"])
+ m_args.assert_called_once_with(
+ "interstellar", True, ["--gsheet_feeder.sheet_id", "123"]
+ )
m_orchestrator.return_value.setup.assert_called_once()
m_orchestrator.return_value.feed.assert_called_once()
m_store.assert_called_with("interstellar")
- m_store.call_count == 2
- m_uuid.call_count == 2
- assert type(res) == dict
+ assert m_store.call_count == 2
+ assert m_uuid.call_count == 2
+ assert isinstance(res, dict)
assert res["stats"]["archived"] == 1
assert res["stats"]["failed"] == 1
assert len(res["stats"]["errors"]) == 1
assert res["sheet_id"] == "123"
assert res["success"]
- assert type(res["time"]) == datetime
+ assert isinstance(res["time"], datetime)
# query created archive entry
- inserted = db_session.query(models.Archive).filter(models.Archive.url == self.URL).one()
+ inserted = (
+ db_session.query(models.Archive)
+ .filter(models.Archive.url == self.URL)
+ .one()
+ )
assert inserted is not None
assert inserted.url == self.URL
assert len(inserted.tags) == 1
assert inserted.tags[0].id == "spaceship"
assert inserted.group_id == "interstellar"
assert inserted.author_id == "rick@example.com"
- assert inserted.public == False
+ assert inserted.public is False
def test_get_all_urls(db_session):
- from app.worker.main import get_all_urls
-
meta = Metadata().set_url("https://example.com")
m1 = meta.add_media(Media("fn1.txt", urls=["outcome1.com"]))
m2 = meta.add_media(Media("fn2.txt", urls=["outcome2.com"]))
m3 = meta.add_media(Media("fn3.txt", urls=["outcome3.com"]))
m1.set("screenshot", Media("screenshot.png", urls=["screenshot.com"]))
- m2.set("thumbnails", [Media("thumb1.png", urls=["thumb1.com"]), Media("thumb2.png", urls=["thumb2.com"])])
+ m2.set(
+ "thumbnails",
+ [
+ Media("thumb1.png", urls=["thumb1.com"]),
+ Media("thumb2.png", urls=["thumb2.com"]),
+ ],
+ )
m3.set("ssl_data", Media("ssl_data.txt", urls=["ssl_data.com"]).to_dict())
m3.set("bad_data", {"bad": "dict is ignored"})
From 9529784fa288924817775f6db7ce6feff536a1c5 Mon Sep 17 00:00:00 2001
From: Michael Plunkett <5885605+michplunkett@users.noreply.github.com>
Date: Fri, 28 Feb 2025 11:46:39 -0600
Subject: [PATCH 10/40] Format and lint `migrations` directory (#59)
---
.pre-commit-config.yaml | 6 +--
app/migrations/env.py | 7 ++-
...7ed0_create_archives_store_until_column.py | 18 ++++---
...24ec4b1_rename_sheets_last_archived_col.py | 21 +++++---
...dd_new_service_account_email_column_to_.py | 13 +++--
...21d2c96d8_add_sheet_id_to_archive_table.py | 34 +++++++------
...45b_modify_archive_url_to_have_uuid_id_.py | 8 +--
...vacuum_database_if_there_s_enough_space.py | 9 ++--
.../a23aaf3ae930_drop_active_column.py | 25 +++++++---
...a012ec405b8_add_columns_to_groups_table.py | 49 ++++++++++++-------
10 files changed, 120 insertions(+), 70 deletions(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 5e1d2a8..edb6bcf 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,6 +1,6 @@
repos:
- repo: https://github.com/nbQA-dev/nbQA
- rev: 1.8.5
+ rev: 1.9.1
hooks:
- id: nbqa-ruff
args:
@@ -17,7 +17,7 @@ repos:
- --profile=black
- repo: https://github.com/pre-commit/pre-commit-hooks
- rev: v4.4.0
+ rev: v5.0.0
hooks:
- id: trailing-whitespace
- id: check-docstring-first
@@ -49,7 +49,7 @@ repos:
- id: python-no-log-warn
- repo: https://github.com/PyCQA/isort
- rev: 5.12.0
+ rev: 6.0.1
hooks:
- id: isort
name: Run isort to sort imports
diff --git a/app/migrations/env.py b/app/migrations/env.py
index 1f579ab..54c1f48 100644
--- a/app/migrations/env.py
+++ b/app/migrations/env.py
@@ -9,11 +9,14 @@ from app.shared.settings import get_settings
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
config = context.config
-config.set_main_option('sqlalchemy.url', get_settings().DATABASE_PATH)
+config.set_main_option("sqlalchemy.url", get_settings().DATABASE_PATH)
# Interpret the config file for Python logging.
# This line sets up loggers basically.
if config.config_file_name is not None:
- fileConfig(config.config_file_name, disable_existing_loggers=False) # disable_existing_loggers prevents loguru disabling
+ # disable_existing_loggers prevents loguru disabling
+ fileConfig(
+ config.config_file_name, disable_existing_loggers=False
+ )
# add your model's MetaData object here
# for 'autogenerate' support
diff --git a/app/migrations/versions/02b2f6d17ed0_create_archives_store_until_column.py b/app/migrations/versions/02b2f6d17ed0_create_archives_store_until_column.py
index 8642f2b..1bb5695 100644
--- a/app/migrations/versions/02b2f6d17ed0_create_archives_store_until_column.py
+++ b/app/migrations/versions/02b2f6d17ed0_create_archives_store_until_column.py
@@ -5,13 +5,14 @@ Revises: 1636724ec4b1
Create Date: 2025-02-08 15:22:20.392522
"""
+
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
-revision = '02b2f6d17ed0'
-down_revision = '1636724ec4b1'
+revision = "02b2f6d17ed0"
+down_revision = "1636724ec4b1"
branch_labels = None
depends_on = None
STORE_UNTIL_COL = "store_until"
@@ -20,15 +21,20 @@ STORE_UNTIL_COL = "store_until"
def upgrade() -> None:
conn = op.get_bind()
inspector = sa.inspect(conn)
- columns = [col['name'] for col in inspector.get_columns('archives')]
+ columns = [col["name"] for col in inspector.get_columns("archives")]
if STORE_UNTIL_COL not in columns:
- op.add_column('archives', sa.Column(STORE_UNTIL_COL, sa.DateTime(), nullable=True, default=None))
+ op.add_column(
+ "archives",
+ sa.Column(
+ STORE_UNTIL_COL, sa.DateTime(), nullable=True, default=None
+ ),
+ )
def downgrade() -> None:
conn = op.get_bind()
inspector = sa.inspect(conn)
- columns = [col['name'] for col in inspector.get_columns('archives')]
+ columns = [col["name"] for col in inspector.get_columns("archives")]
if STORE_UNTIL_COL in columns:
- op.drop_column('archives', STORE_UNTIL_COL)
+ op.drop_column("archives", STORE_UNTIL_COL)
diff --git a/app/migrations/versions/1636724ec4b1_rename_sheets_last_archived_col.py b/app/migrations/versions/1636724ec4b1_rename_sheets_last_archived_col.py
index 324f75a..ad972fc 100644
--- a/app/migrations/versions/1636724ec4b1_rename_sheets_last_archived_col.py
+++ b/app/migrations/versions/1636724ec4b1_rename_sheets_last_archived_col.py
@@ -5,13 +5,14 @@ Revises: a23aaf3ae930
Create Date: 2025-02-05 19:19:01.984396
"""
+
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
-revision = '1636724ec4b1'
-down_revision = 'a23aaf3ae930'
+revision = "1636724ec4b1"
+down_revision = "a23aaf3ae930"
branch_labels = None
depends_on = None
@@ -19,14 +20,18 @@ depends_on = None
def upgrade() -> None:
conn = op.get_bind()
inspector = sa.inspect(conn)
- columns = [col['name'] for col in inspector.get_columns('sheets')]
- if 'last_archived_at' in columns:
- op.alter_column('sheets', 'last_archived_at', new_column_name='last_url_archived_at')
+ columns = [col["name"] for col in inspector.get_columns("sheets")]
+ if "last_archived_at" in columns:
+ op.alter_column(
+ "sheets", "last_archived_at", new_column_name="last_url_archived_at"
+ )
def downgrade() -> None:
conn = op.get_bind()
inspector = sa.inspect(conn)
- columns = [col['name'] for col in inspector.get_columns('sheets')]
- if 'last_url_archived_at' in columns:
- op.alter_column('sheets', 'last_url_archived_at', new_column_name='last_archived_at')
+ columns = [col["name"] for col in inspector.get_columns("sheets")]
+ if "last_url_archived_at" in columns:
+ op.alter_column(
+ "sheets", "last_url_archived_at", new_column_name="last_archived_at"
+ )
diff --git a/app/migrations/versions/63ac79df4ad0_add_new_service_account_email_column_to_.py b/app/migrations/versions/63ac79df4ad0_add_new_service_account_email_column_to_.py
index 572905d..0ea0b11 100644
--- a/app/migrations/versions/63ac79df4ad0_add_new_service_account_email_column_to_.py
+++ b/app/migrations/versions/63ac79df4ad0_add_new_service_account_email_column_to_.py
@@ -5,13 +5,14 @@ Revises: 02b2f6d17ed0
Create Date: 2025-02-11 21:53:23.293274
"""
+
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
-revision = '63ac79df4ad0'
-down_revision = '02b2f6d17ed0'
+revision = "63ac79df4ad0"
+down_revision = "02b2f6d17ed0"
branch_labels = None
depends_on = None
@@ -22,15 +23,17 @@ TABLE = "groups"
def upgrade() -> None:
conn = op.get_bind()
inspector = sa.inspect(conn)
- columns = [col['name'] for col in inspector.get_columns(TABLE)]
+ columns = [col["name"] for col in inspector.get_columns(TABLE)]
if NEW_COL not in columns:
- op.add_column(TABLE, sa.Column(NEW_COL, sa.String, nullable=True, default=None))
+ op.add_column(
+ TABLE, sa.Column(NEW_COL, sa.String, nullable=True, default=None)
+ )
def downgrade() -> None:
conn = op.get_bind()
inspector = sa.inspect(conn)
- columns = [col['name'] for col in inspector.get_columns(TABLE)]
+ columns = [col["name"] for col in inspector.get_columns(TABLE)]
if NEW_COL in columns:
op.drop_column(TABLE, NEW_COL)
diff --git a/app/migrations/versions/89121d2c96d8_add_sheet_id_to_archive_table.py b/app/migrations/versions/89121d2c96d8_add_sheet_id_to_archive_table.py
index 892e853..e34e06d 100644
--- a/app/migrations/versions/89121d2c96d8_add_sheet_id_to_archive_table.py
+++ b/app/migrations/versions/89121d2c96d8_add_sheet_id_to_archive_table.py
@@ -5,14 +5,14 @@ Revises: fa012ec405b8
Create Date: 2024-11-04 11:12:30.237299
"""
+
import sqlalchemy as sa
from alembic import op
-from sqlalchemy.engine.reflection import Inspector
# revision identifiers, used by Alembic.
-revision = '89121d2c96d8'
-down_revision = 'fa012ec405b8'
+revision = "89121d2c96d8"
+down_revision = "fa012ec405b8"
branch_labels = None
depends_on = None
@@ -20,23 +20,27 @@ depends_on = None
def upgrade() -> None:
conn = op.get_bind()
inspector = sa.inspect(conn)
- columns = [col['name'] for col in inspector.get_columns('archives')]
+ columns = [col["name"] for col in inspector.get_columns("archives")]
- if 'sheet_id' not in columns:
- with op.batch_alter_table('archives') as batch_op:
- batch_op.add_column(sa.Column('sheet_id', sa.String(), nullable=True, default=None))
- batch_op.create_foreign_key('fk_sheet_id', 'sheets', ['sheet_id'], ['id'])
+ if "sheet_id" not in columns:
+ with op.batch_alter_table("archives") as batch_op:
+ batch_op.add_column(
+ sa.Column("sheet_id", sa.String(), nullable=True, default=None)
+ )
+ batch_op.create_foreign_key(
+ "fk_sheet_id", "sheets", ["sheet_id"], ["id"]
+ )
def downgrade() -> None:
conn = op.get_bind()
inspector = sa.inspect(conn)
- foreign_keys = [fk['name'] for fk in inspector.get_foreign_keys('archives')]
- columns = [col['name'] for col in inspector.get_columns('archives')]
+ foreign_keys = [fk["name"] for fk in inspector.get_foreign_keys("archives")]
+ columns = [col["name"] for col in inspector.get_columns("archives")]
- with op.batch_alter_table('archives') as batch_op:
- if 'fk_sheet_id' in foreign_keys:
- batch_op.drop_constraint('fk_sheet_id', type_='foreignkey')
+ with op.batch_alter_table("archives") as batch_op:
+ if "fk_sheet_id" in foreign_keys:
+ batch_op.drop_constraint("fk_sheet_id", type_="foreignkey")
- if 'sheet_id' in columns:
- batch_op.drop_column('sheet_id')
+ if "sheet_id" in columns:
+ batch_op.drop_column("sheet_id")
diff --git a/app/migrations/versions/9369a264945b_modify_archive_url_to_have_uuid_id_.py b/app/migrations/versions/9369a264945b_modify_archive_url_to_have_uuid_id_.py
index bdfe474..1f7f348 100644
--- a/app/migrations/versions/9369a264945b_modify_archive_url_to_have_uuid_id_.py
+++ b/app/migrations/versions/9369a264945b_modify_archive_url_to_have_uuid_id_.py
@@ -5,11 +5,12 @@ Revises:
Create Date: 2023-12-20 17:24:59.320691
"""
+
from alembic import op
# revision identifiers, used by Alembic.
-revision = '9369a264945b'
+revision = "9369a264945b"
down_revision = None
branch_labels = None
depends_on = None
@@ -19,10 +20,11 @@ def upgrade() -> None:
# since the primary key constraint is not named, we have to recreate it first
with op.batch_alter_table("archive_urls") as batch_op:
batch_op.create_primary_key("pk_url", ["url"])
- batch_op.drop_constraint("pk_url", type_='primary')
+ batch_op.drop_constraint("pk_url", type_="primary")
batch_op.create_primary_key("pk_url_archive_id", ["url", "archive_id"])
+
def downgrade() -> None:
with op.batch_alter_table("archive_urls") as batch_op:
- batch_op.drop_constraint("pk_url_archive_id", type_='primary')
+ batch_op.drop_constraint("pk_url_archive_id", type_="primary")
batch_op.create_primary_key("url", ["url"])
diff --git a/app/migrations/versions/93a611e4c066_vacuum_database_if_there_s_enough_space.py b/app/migrations/versions/93a611e4c066_vacuum_database_if_there_s_enough_space.py
index 6b8d8d2..b099f59 100644
--- a/app/migrations/versions/93a611e4c066_vacuum_database_if_there_s_enough_space.py
+++ b/app/migrations/versions/93a611e4c066_vacuum_database_if_there_s_enough_space.py
@@ -5,12 +5,13 @@ Revises: 9369a264945b
Create Date: 2023-12-20 18:33:27.132566
"""
+
from alembic import op
# revision identifiers, used by Alembic.
-revision = '93a611e4c066'
-down_revision = '9369a264945b'
+revision = "93a611e4c066"
+down_revision = "9369a264945b"
branch_labels = None
depends_on = None
@@ -20,7 +21,9 @@ def upgrade() -> None:
with op.get_context().autocommit_block():
op.execute("VACUUM")
except Exception as e:
- print("Unable to run vacuum, maybe there's not enough disk space. it should be 2x the size of the database")
+ print(
+ "Unable to run vacuum, maybe there's not enough disk space. it should be 2x the size of the database"
+ )
print(e)
diff --git a/app/migrations/versions/a23aaf3ae930_drop_active_column.py b/app/migrations/versions/a23aaf3ae930_drop_active_column.py
index ebc85e7..aa8f97b 100644
--- a/app/migrations/versions/a23aaf3ae930_drop_active_column.py
+++ b/app/migrations/versions/a23aaf3ae930_drop_active_column.py
@@ -5,13 +5,14 @@ Revises: 89121d2c96d8
Create Date: 2025-02-04 12:19:20.753570
"""
+
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
-revision = 'a23aaf3ae930'
-down_revision = '89121d2c96d8'
+revision = "a23aaf3ae930"
+down_revision = "89121d2c96d8"
branch_labels = None
depends_on = None
@@ -19,16 +20,24 @@ depends_on = None
def upgrade() -> None:
conn = op.get_bind()
inspector = sa.inspect(conn)
- columns = [col['name'] for col in inspector.get_columns('users')]
+ columns = [col["name"] for col in inspector.get_columns("users")]
- if 'is_active' in columns:
- op.drop_column('users', 'is_active')
+ if "is_active" in columns:
+ op.drop_column("users", "is_active")
def downgrade() -> None:
conn = op.get_bind()
inspector = sa.inspect(conn)
- columns = [col['name'] for col in inspector.get_columns('users')]
+ columns = [col["name"] for col in inspector.get_columns("users")]
- if 'is_active' not in columns:
- op.add_column('users', sa.Column('is_active', sa.Boolean(), nullable=False, server_default=sa.false()))
+ if "is_active" not in columns:
+ op.add_column(
+ "users",
+ sa.Column(
+ "is_active",
+ sa.Boolean(),
+ nullable=False,
+ server_default=sa.false(),
+ ),
+ )
diff --git a/app/migrations/versions/fa012ec405b8_add_columns_to_groups_table.py b/app/migrations/versions/fa012ec405b8_add_columns_to_groups_table.py
index c3169c3..f5cbcaa 100644
--- a/app/migrations/versions/fa012ec405b8_add_columns_to_groups_table.py
+++ b/app/migrations/versions/fa012ec405b8_add_columns_to_groups_table.py
@@ -5,14 +5,14 @@ Revises: 93a611e4c066
Create Date: 2024-10-31 09:36:50.360710
"""
+
import sqlalchemy as sa
from alembic import op
-from sqlalchemy.engine.reflection import Inspector
# revision identifiers, used by Alembic.
-revision = 'fa012ec405b8'
-down_revision = '93a611e4c066'
+revision = "fa012ec405b8"
+down_revision = "93a611e4c066"
branch_labels = None
depends_on = None
@@ -20,26 +20,41 @@ depends_on = None
def upgrade() -> None:
conn = op.get_bind()
inspector = sa.inspect(conn)
- columns = [col['name'] for col in inspector.get_columns('groups')]
+ columns = [col["name"] for col in inspector.get_columns("groups")]
- if 'description' not in columns:
- op.add_column('groups', sa.Column('description', sa.String(), nullable=True))
- if 'orchestrator' not in columns:
- op.add_column('groups', sa.Column('orchestrator', sa.String(), nullable=True))
- if 'orchestrator_sheet' not in columns:
- op.add_column('groups', sa.Column('orchestrator_sheet', sa.String(), nullable=True))
- if 'permissions' not in columns:
- op.add_column('groups', sa.Column('permissions', sa.JSON(), nullable=True))
- if 'domains' not in columns:
- op.add_column('groups', sa.Column('domains', sa.JSON(), nullable=True))
+ if "description" not in columns:
+ op.add_column(
+ "groups", sa.Column("description", sa.String(), nullable=True)
+ )
+ if "orchestrator" not in columns:
+ op.add_column(
+ "groups", sa.Column("orchestrator", sa.String(), nullable=True)
+ )
+ if "orchestrator_sheet" not in columns:
+ op.add_column(
+ "groups",
+ sa.Column("orchestrator_sheet", sa.String(), nullable=True),
+ )
+ if "permissions" not in columns:
+ op.add_column(
+ "groups", sa.Column("permissions", sa.JSON(), nullable=True)
+ )
+ if "domains" not in columns:
+ op.add_column("groups", sa.Column("domains", sa.JSON(), nullable=True))
def downgrade() -> None:
conn = op.get_bind()
inspector = sa.inspect(conn)
- columns = [col['name'] for col in inspector.get_columns('groups')]
+ columns = [col["name"] for col in inspector.get_columns("groups")]
- column_names = ['description', 'orchestrator', 'orchestrator_sheet', 'permissions', 'domains']
+ column_names = [
+ "description",
+ "orchestrator",
+ "orchestrator_sheet",
+ "permissions",
+ "domains",
+ ]
for column_name in column_names:
if column_name in columns:
- op.drop_column('groups', column_name)
+ op.drop_column("groups", column_name)
From 85cec9fbb94ed31b5840108666dc86a69310e388 Mon Sep 17 00:00:00 2001
From: Michael Plunkett <5885605+michplunkett@users.noreply.github.com>
Date: Mon, 3 Mar 2025 08:10:23 -0600
Subject: [PATCH 11/40] Consolidate and organize config files (#61)
---
.coveragerc | 3 ---
docker-compose.yml | 4 ++--
web.Dockerfile => docker/web/Dockerfile | 8 ++++----
worker.Dockerfile => docker/worker/Dockerfile | 8 ++++----
pyproject.toml | 7 +++++++
pytest.ini | 2 --
6 files changed, 17 insertions(+), 15 deletions(-)
delete mode 100644 .coveragerc
rename web.Dockerfile => docker/web/Dockerfile (76%)
rename worker.Dockerfile => docker/worker/Dockerfile (86%)
delete mode 100644 pytest.ini
diff --git a/.coveragerc b/.coveragerc
deleted file mode 100644
index f6cc4e8..0000000
--- a/.coveragerc
+++ /dev/null
@@ -1,3 +0,0 @@
-[run]
-omit =
- app/migrations/*
diff --git a/docker-compose.yml b/docker-compose.yml
index 737c346..635f7cd 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -7,7 +7,7 @@ services:
web:
build:
context: .
- dockerfile: web.Dockerfile
+ dockerfile: docker/web/Dockerfile
restart: always
env_file: .env.prod
environment:
@@ -31,7 +31,7 @@ services:
worker:
build:
context: .
- dockerfile: worker.Dockerfile
+ dockerfile: docker/worker/Dockerfile
restart: always
env_file: .env.prod
command: celery --app=app.worker.main.celery worker -Q high_priority,low_priority --concurrency=${CONCURRENCY} --max-tasks-per-child=100
diff --git a/web.Dockerfile b/docker/web/Dockerfile
similarity index 76%
rename from web.Dockerfile
rename to docker/web/Dockerfile
index d142877..cfd60d6 100644
--- a/web.Dockerfile
+++ b/docker/web/Dockerfile
@@ -10,13 +10,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
&& rm -rf /var/lib/apt/lists/*
RUN pip install --no-cache-dir poetry
-COPY pyproject.toml poetry.lock README.md .
+COPY ../../pyproject.toml ../../poetry.lock ../../README.md ./
RUN poetry install --with web --no-interaction --no-ansi --no-cache
# Copy the application code and configurations
-COPY alembic.ini ./
-COPY ./app/ ./app/
-COPY user-groups.* ./app/
+COPY ../../alembic.ini ./
+COPY ../../app ./app/
+COPY ../../user-groups.* ./app/
# Run the FastAPI app with Uvicorn
ENTRYPOINT ["poetry", "run"]
diff --git a/worker.Dockerfile b/docker/worker/Dockerfile
similarity index 86%
rename from worker.Dockerfile
rename to docker/worker/Dockerfile
index 9154c32..23c1e45 100644
--- a/worker.Dockerfile
+++ b/docker/worker/Dockerfile
@@ -20,14 +20,14 @@ RUN apt update -y && \
python3 -m venv ./poetry-venv && \
./poetry-venv/bin/python -m pip install --upgrade pip && \
./poetry-venv/bin/python -m pip install "poetry>=2.0.0,<3.0.0"
-COPY pyproject.toml poetry.lock ./
+COPY ../../pyproject.toml ../../poetry.lock ./
RUN ./poetry-venv/bin/poetry install --without dev --no-root --no-cache
# install dependencies
# copy source code and .env files over
-COPY alembic.ini ./
-COPY ./app/ ./app/
-COPY user-groups.* ./app/
+COPY ../../alembic.ini ./
+COPY ../../app ./app/
+COPY ../../user-groups.* ./app/
ENTRYPOINT ["./poetry-venv/bin/poetry", "run"]
diff --git a/pyproject.toml b/pyproject.toml
index df7b789..75f6e63 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -31,6 +31,13 @@ dependencies = [
"requests (>=2.25.1)",
"pyopenssl (>=23.3.0)",
]
+
+[tool.pytest.ini_options]
+pythonpath = "."
+
+[tool.coverage.run]
+omit = ["app/migrations/*"]
+
[tool.poetry.group.worker.dependencies]
watchdog = ">=6.0.0,<7.0.0"
setuptools = "^75.8.0"
diff --git a/pytest.ini b/pytest.ini
deleted file mode 100644
index a635c5c..0000000
--- a/pytest.ini
+++ /dev/null
@@ -1,2 +0,0 @@
-[pytest]
-pythonpath = .
From b5571fc61fae667a7973125ae04121876e83764b Mon Sep 17 00:00:00 2001
From: Michael Plunkett <5885605+michplunkett@users.noreply.github.com>
Date: Mon, 3 Mar 2025 11:59:04 -0600
Subject: [PATCH 12/40] Format and lint `workers` directory (#60)
---
app/worker/main.py | 81 ++++++++++++++++++++++++++++++----------
app/worker/worker_log.py | 45 ++++++++++++----------
2 files changed, 87 insertions(+), 39 deletions(-)
diff --git a/app/worker/main.py b/app/worker/main.py
index 7b4826b..f7b2915 100644
--- a/app/worker/main.py
+++ b/app/worker/main.py
@@ -30,8 +30,16 @@ setup_celery_logger(celery)
# logger.add("app/worker/worker_log.log", level="DEBUG")
logger.remove = lambda x: print(f"logger.remove({x})")
-# TODO: after release, as it requires updating past entries with sheet_id where tag is used, drop tags
-@celery.task(name="create_archive_task", bind=True, autoretry_for=(Exception,), retry_backoff=True, retry_kwargs={'max_retries': 1})
+
+# TODO: after release, as it requires updating past entries with sheet_id where tag
+# is used, drop tags
+@celery.task(
+ name="create_archive_task",
+ bind=True,
+ autoretry_for=(Exception,),
+ retry_backoff=True,
+ retry_kwargs={"max_retries": 1},
+)
def create_archive_task(self, archive_json: str):
archive = schemas.ArchiveCreate.model_validate_json(archive_json)
@@ -42,9 +50,9 @@ def create_archive_task(self, archive_json: str):
orchestrator.setup(args)
result = next(orchestrator.feed())
except SystemExit as e:
- log_error(e, f"create_archive_task: SystemExit from AA")
+ log_error(e, "create_archive_task: SystemExit from AA")
except Exception as e:
- log_error(e, f"create_archive_task")
+ log_error(e, "create_archive_task")
raise e
assert result, f"UNABLE TO archive: {archive.url}"
@@ -61,10 +69,14 @@ def create_archive_task(self, archive_json: str):
@celery.task(name="create_sheet_task", bind=True)
def create_sheet_task(self, sheet_json: str):
sheet = schemas.SubmitSheet.model_validate_json(sheet_json)
- queue_name = (create_sheet_task.request.delivery_info or {}).get('routing_key', 'unknown')
+ queue_name = (create_sheet_task.request.delivery_info or {}).get(
+ "routing_key", "unknown"
+ )
logger.info(f"[queue={queue_name}] SHEET START {sheet=}")
- args = get_orchestrator_args(sheet.group_id, True, ["--gsheet_feeder.sheet_id", sheet.sheet_id])
+ args = get_orchestrator_args(
+ sheet.group_id, True, ["--gsheet_feeder.sheet_id", sheet.sheet_id]
+ )
orchestrator = ArchivingOrchestrator()
orchestrator.setup(args)
@@ -82,7 +94,7 @@ def create_sheet_task(self, sheet_json: str):
result=json.loads(result.to_json()),
sheet_id=sheet.sheet_id,
urls=get_all_urls(result),
- store_until=get_store_until(sheet.group_id)
+ store_until=get_store_until(sheet.group_id),
)
insert_result_into_db(archive)
stats["archived"] += 1
@@ -95,25 +107,39 @@ def create_sheet_task(self, sheet_json: str):
stats["errors"].append(str(e))
except SystemExit as e:
- log_error(e, f"create_sheet_task: SystemExit from AA")
+ log_error(e, "create_sheet_task: SystemExit from AA")
if stats["archived"] > 0:
with get_db() as session:
- worker_crud.update_sheet_last_url_archived_at(session, sheet.sheet_id)
+ worker_crud.update_sheet_last_url_archived_at(
+ session, sheet.sheet_id
+ )
logger.info(f"SHEET DONE {sheet=}")
# TODO: is this used anywhere? maybe drop it
- return schemas.CelerySheetTask(success=True, sheet_id=sheet.sheet_id, time=datetime.datetime.now().isoformat(), stats=stats).model_dump()
+ return schemas.CelerySheetTask(
+ success=True,
+ sheet_id=sheet.sheet_id,
+ time=datetime.datetime.now().isoformat(),
+ stats=stats,
+ ).model_dump()
-def get_orchestrator_args(group_id: str, orchestrator_for_sheet: bool, cli_args: list = []) -> list:
+def get_orchestrator_args(
+ group_id: str, orchestrator_for_sheet: bool, cli_args: list = None
+) -> list:
+ if cli_args is None:
+ cli_args = []
+
aa_configs = []
with get_db() as session:
group = worker_crud.get_group(session, group_id)
if orchestrator_for_sheet:
orchestrator_fn = group.orchestrator_sheet
else:
- orchestrator_fn = worker_crud.get_group(session, group_id).orchestrator
+ orchestrator_fn = worker_crud.get_group(
+ session, group_id
+ ).orchestrator
assert orchestrator_fn, f"no orchestrator found for {group_id}"
aa_configs.extend(["--config", orchestrator_fn])
aa_configs.extend(cli_args)
@@ -123,7 +149,9 @@ def get_orchestrator_args(group_id: str, orchestrator_for_sheet: bool, cli_args:
def insert_result_into_db(archive: schemas.ArchiveCreate) -> str:
with get_db() as session:
db_archive = worker_crud.store_archived_url(session, archive)
- logger.debug(f"[ARCHIVE STORED] {db_archive.author_id} {db_archive.url}")
+ logger.debug(
+ f"[ARCHIVE STORED] {db_archive.author_id} {db_archive.url}"
+ )
return db_archive.id
@@ -132,13 +160,22 @@ def get_store_until(group_id: str) -> datetime.datetime:
return business_logic.get_store_archive_until(session, group_id)
-def redis_publish_exception(exception, task_name, traceback: str = ""):
+def redis_publish_exception(exception, task_name, trace_back: str = ""):
REDIS_EXCEPTIONS_CHANNEL = settings.REDIS_EXCEPTIONS_CHANNEL
try:
- exception_data = {"task": task_name, "type": exception.__class__.__name__, "exception": exception, "traceback": traceback}
- Redis.publish(REDIS_EXCEPTIONS_CHANNEL, json.dumps(exception_data, default=str))
+ exception_data = {
+ "task": task_name,
+ "type": exception.__class__.__name__,
+ "exception": exception,
+ "traceback": trace_back,
+ }
+ Redis.publish(
+ REDIS_EXCEPTIONS_CHANNEL, json.dumps(exception_data, default=str)
+ )
except Exception as e:
- log_error(e, f"[CRITICAL] Could not publish to {REDIS_EXCEPTIONS_CHANNEL}")
+ log_error(
+ e, f"[CRITICAL] Could not publish to {REDIS_EXCEPTIONS_CHANNEL}"
+ )
@task_failure.connect(sender=create_sheet_task)
@@ -146,6 +183,10 @@ def redis_publish_exception(exception, task_name, traceback: str = ""):
def task_failure_notifier(sender, **kwargs):
# automatically capture exceptions in the worker tasks
logger.warning(f"⚠️ worker task failed: {sender.name}")
- traceback_msg = "\n".join(traceback.format_list(traceback.extract_tb(kwargs['traceback'])))
- log_error(kwargs['exception'], traceback_msg, f"task_failure: {sender.name}")
- redis_publish_exception(kwargs['exception'], sender.name, traceback_msg)
+ traceback_msg = "\n".join(
+ traceback.format_list(traceback.extract_tb(kwargs["traceback"]))
+ )
+ log_error(
+ kwargs["exception"], traceback_msg, f"task_failure: {sender.name}"
+ )
+ redis_publish_exception(kwargs["exception"], sender.name, traceback_msg)
diff --git a/app/worker/worker_log.py b/app/worker/worker_log.py
index 022c63d..c1c8dcc 100644
--- a/app/worker/worker_log.py
+++ b/app/worker/worker_log.py
@@ -1,6 +1,5 @@
import sys
-from celery import Celery
from loguru import logger
from app.shared.task_messaging import get_celery
@@ -8,24 +7,32 @@ from app.shared.task_messaging import get_celery
celery = get_celery("worker")
-def setup_celery_logger(celery):
- # Remove Celery's default handlers to prevent duplicate logs
- celery_logger = celery.log.get_default_logger()
- for handler in celery_logger.handlers[:]:
- celery_logger.removeHandler(handler)
- # Set up Loguru logging
- logger.add("logs/celery_logs.log", retention="30 days", level="DEBUG")
- logger.add("logs/celery_error_logs.log", retention="30 days", level="ERROR")
+def setup_celery_logger(c):
+ # Remove Celery's default handlers to prevent duplicate logs
+ celery_logger = c.log.get_default_logger()
+ for handler in celery_logger.handlers[:]:
+ celery_logger.removeHandler(handler)
- # Redirect Celery logs to Loguru
- class InterceptHandler:
- def write(self, message):
- if message.strip():
- logger.info(message.strip())
- # Required to prevent issues with buffered output
- def flush(self): pass
- def isatty(self): return False
+ # Set up Loguru logging
+ logger.add("logs/celery_logs.log", retention="30 days", level="DEBUG")
+ logger.add("logs/celery_error_logs.log", retention="30 days", level="ERROR")
- sys.stdout = InterceptHandler()
- sys.stderr = InterceptHandler()
+ # Redirect Celery logs to Loguru
+ class InterceptHandler:
+ @staticmethod
+ def write(message):
+ if message.strip():
+ logger.info(message.strip())
+
+ # Required to prevent issues with buffered output
+ @staticmethod
+ def flush():
+ pass
+
+ @staticmethod
+ def isatty():
+ return False
+
+ sys.stdout = InterceptHandler()
+ sys.stderr = InterceptHandler()
From a9ca410d08f326737199fdf73e3e357880c180f7 Mon Sep 17 00:00:00 2001
From: Michael Plunkett <5885605+michplunkett@users.noreply.github.com>
Date: Mon, 3 Mar 2025 13:20:11 -0600
Subject: [PATCH 13/40] Move `alembic.ini` to `migrations` directory (#66)
---
alembic.ini => app/migrations/alembic.ini | 6 +-
app/tests/web/test_main.py | 20 +++-
app/web/events.py | 114 +++++++++++++++++-----
docker/web/Dockerfile | 1 -
docker/worker/Dockerfile | 1 -
5 files changed, 110 insertions(+), 32 deletions(-)
rename alembic.ini => app/migrations/alembic.ini (95%)
diff --git a/alembic.ini b/app/migrations/alembic.ini
similarity index 95%
rename from alembic.ini
rename to app/migrations/alembic.ini
index 30d7030..fa1578b 100644
--- a/alembic.ini
+++ b/app/migrations/alembic.ini
@@ -2,7 +2,7 @@
[alembic]
# path to migration scripts
-script_location = app/migrations
+script_location = ./app/migrations
# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
# Uncomment the line below if you want the files to be prepended with date and time
@@ -10,10 +10,6 @@ script_location = app/migrations
# for all available tokens
# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
-# sys.path path, will be prepended to sys.path if present.
-# defaults to the current working directory.
-prepend_sys_path = .
-
# timezone to use when rendering the date within the migration file
# as well as the filename.
# If specified, requires the python-dateutil library that can be
diff --git a/app/tests/web/test_main.py b/app/tests/web/test_main.py
index 0dbd5fa..eb985a8 100644
--- a/app/tests/web/test_main.py
+++ b/app/tests/web/test_main.py
@@ -18,8 +18,24 @@ def test_lifespan(app):
def test_alembic(db_session):
- alembic.config.main(argv=["--raiseerr", "upgrade", "head"])
- alembic.config.main(argv=["--raiseerr", "downgrade", "base"])
+ alembic.config.main(
+ argv=[
+ "-c",
+ "./app/migrations/alembic.ini",
+ "--raiseerr",
+ "upgrade",
+ "head",
+ ]
+ )
+ alembic.config.main(
+ argv=[
+ "-c",
+ "./app/migrations/alembic.ini",
+ "--raiseerr",
+ "downgrade",
+ "base",
+ ]
+ )
@patch(
diff --git a/app/web/events.py b/app/web/events.py
index fa15614..17a24b9 100644
--- a/app/web/events.py
+++ b/app/web/events.py
@@ -38,9 +38,22 @@ async def lifespan(app: FastAPI):
# STARTUP
engine = make_engine(get_settings().DATABASE_PATH)
models.Base.metadata.create_all(bind=engine)
- alembic.config.main(prog="alembic", argv=['--raiseerr', 'upgrade', 'head'])
+ alembic.config.main(
+ prog="alembic",
+ argv=[
+ "-c",
+ "./app/migrations/alembic.ini",
+ "--raiseerr",
+ "upgrade",
+ "head",
+ ],
+ )
logging.getLogger("uvicorn.access").disabled = True # loguru
- asyncio.create_task(redis_subscribe_worker_exceptions(get_settings().REDIS_EXCEPTIONS_CHANNEL))
+ asyncio.create_task(
+ redis_subscribe_worker_exceptions(
+ get_settings().REDIS_EXCEPTIONS_CHANNEL
+ )
+ )
asyncio.create_task(repeat_measure_regular_metrics())
with get_db() as db:
crud.upsert_user_groups(db)
@@ -71,41 +84,74 @@ async def lifespan(app: FastAPI):
# CRON JOBS
-@repeat_every(seconds=get_settings().REPEAT_COUNT_METRICS_SECONDS, on_exception=increase_exceptions_counter)
+@repeat_every(
+ seconds=get_settings().REPEAT_COUNT_METRICS_SECONDS,
+ on_exception=increase_exceptions_counter,
+)
async def repeat_measure_regular_metrics():
- await measure_regular_metrics(get_settings().DATABASE_PATH, get_settings().REPEAT_COUNT_METRICS_SECONDS)
+ await measure_regular_metrics(
+ get_settings().DATABASE_PATH,
+ get_settings().REPEAT_COUNT_METRICS_SECONDS,
+ )
-@repeat_every(seconds=60, wait_first=120, on_exception=increase_exceptions_counter)
+@repeat_every(
+ seconds=60, wait_first=120, on_exception=increase_exceptions_counter
+)
async def archive_hourly_sheets_cronjob():
await archive_sheets_cronjob("hourly", 60, datetime.datetime.now().minute)
-@repeat_every(seconds=3600, wait_first=120, on_exception=increase_exceptions_counter)
+@repeat_every(
+ seconds=3600, wait_first=120, on_exception=increase_exceptions_counter
+)
async def archive_daily_sheets_cronjob():
await archive_sheets_cronjob("daily", 24, datetime.datetime.now().hour)
-async def archive_sheets_cronjob(frequency: str, interval: int, current_time_unit: int):
+async def archive_sheets_cronjob(
+ frequency: str, interval: int, current_time_unit: int
+):
triggered_jobs = []
async with get_db_async() as db:
- sheets = await crud.get_sheets_by_id_hash(db, frequency, interval, current_time_unit)
+ sheets = await crud.get_sheets_by_id_hash(
+ db, frequency, str(interval), current_time_unit
+ )
for s in sheets:
group_queue = await crud.get_group_priority_async(db, s.group_id)
- task = celery.signature("create_sheet_task", args=[schemas.SubmitSheet(sheet_id=s.id, author_id=s.author_id, group_id=s.group_id).model_dump_json()]).apply_async(**group_queue)
+ task = celery.signature(
+ "create_sheet_task",
+ args=[
+ schemas.SubmitSheet(
+ sheet_id=s.id,
+ author_id=s.author_id,
+ group_id=s.group_id,
+ ).model_dump_json()
+ ],
+ ).apply_async(**group_queue)
triggered_jobs.append({"sheet_id": s.id, "task_id": task.id})
- logger.debug(f"[CRON {frequency.upper()}:{current_time_unit}] Triggered {len(triggered_jobs)} sheet tasks: {triggered_jobs}")
+ logger.debug(
+ f"[CRON {frequency.upper()}:{current_time_unit}] Triggered {len(triggered_jobs)} sheet tasks: {triggered_jobs}"
+ )
# TODO: on exception should logerror but also prometheus counter
-DELETE_WINDOW = get_settings().DELETE_SCHEDULED_ARCHIVES_CHECK_EVERY_N_DAYS * 24 * 60 * 60
+DELETE_WINDOW = (
+ get_settings().DELETE_SCHEDULED_ARCHIVES_CHECK_EVERY_N_DAYS * 24 * 60 * 60
+)
-@repeat_every(seconds=DELETE_WINDOW, wait_first=180, on_exception=increase_exceptions_counter)
+@repeat_every(
+ seconds=DELETE_WINDOW,
+ wait_first=180,
+ on_exception=increase_exceptions_counter,
+)
async def notify_about_expired_archives():
- notify_from = datetime.datetime.now() + datetime.timedelta(days=get_settings().DELETE_SCHEDULED_ARCHIVES_CHECK_EVERY_N_DAYS)
+ notify_from = datetime.datetime.now() + datetime.timedelta(
+ days=get_settings().DELETE_SCHEDULED_ARCHIVES_CHECK_EVERY_N_DAYS
+ )
async with get_db_async() as db:
scheduled_deletions = await crud.find_by_store_until(db, notify_from)
@@ -117,7 +163,12 @@ async def notify_about_expired_archives():
fastmail = FastMail(get_settings().MAIL_CONFIG)
# notify users
for email in user_archives:
- list_of_archives = "\n".join([f'{a.url}, {a.id}, {a.store_until.isoformat()}
' for a in user_archives[email]])
+ list_of_archives = "\n".join(
+ [
+ f"{a.url}, {a.id}, {a.store_until.isoformat()}
"
+ for a in user_archives[email]
+ ]
+ )
# TODO: how can users download them in bulk?
message = MessageSchema(
subject="Auto Archiver: Archives Scheduled for Deletion",
@@ -137,16 +188,23 @@ async def notify_about_expired_archives():