From 88be84127a2bf8dc5c7a2b0b57c681fef34a0a59 Mon Sep 17 00:00:00 2001 From: msramalho <19508417+msramalho@users.noreply.github.com> Date: Fri, 26 May 2023 13:15:36 +0100 Subject: [PATCH] adds /metrics --- src/Pipfile | 1 + src/Pipfile.lock | 134 +++++++++++++++++++++++-------------------- src/main.py | 5 ++ src/requirements.txt | 23 ++++---- src/security.py | 2 +- 5 files changed, 90 insertions(+), 75 deletions(-) diff --git a/src/Pipfile b/src/Pipfile index d31f2d0..b45cb36 100644 --- a/src/Pipfile +++ b/src/Pipfile @@ -20,6 +20,7 @@ sqlalchemy = "*" auto-archiver = "*" alembic = "*" fastapi-utils = "*" +prometheus-fastapi-instrumentator = "*" [dev-packages] watchdog = "*" diff --git a/src/Pipfile.lock b/src/Pipfile.lock index aa0d229..57d6949 100644 --- a/src/Pipfile.lock +++ b/src/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "2db7ae3621bf6c9c8a256e8d34ae667eafaf4e4203c450ca1aac415f5a7cac2f" + "sha256": "99873b0aa525e1ec13479dcae7a1a8f3f5a18a454b478beccb88eb798911ce47" }, "pipfile-spec": 6, "requires": { @@ -197,11 +197,11 @@ }, "auto-archiver": { "hashes": [ - "sha256:530eeec3a03fd38410cf864b641df38c61505afc8c402d730ba5aca2ff67ffa8", - "sha256:54a1e202f8c1fdfb46f8295a07e1e9e121bec2b6f81c85c53134bbb80a07501a" + "sha256:b2aa8b6b98fe11a78ba25fbaf6c4d43e5de9f442546a100c1fa47c16218ce6a0", + "sha256:c24e7f696e13532f0beb45d544f8e0cb1c4cce3521acacece35cb2d961c6a2ad" ], "index": "pypi", - "version": "==0.5.18" + "version": "==0.5.20" }, "beautifulsoup4": { "hashes": [ @@ -228,19 +228,19 @@ }, "boto3": { "hashes": [ - "sha256:5b61a82f0c1cd006bd109ddf27c93d9b010c4c188fc583ee257ff6f3bb89970d", - "sha256:fe19d287bc8ede385e1b9136f135ee8f93eab81404ad1445b1a70cabfe3f7087" + "sha256:152def2fcc9854dcc42383d2b53e2ed2c9ccb5ff6cc0f3ada20f1ab54418ede4", + "sha256:a5d6fdcaec863bc7ad2f8133ff9a926d6f06468b83b5fb631cd90bd33b709c45" ], "markers": "python_version >= '3.7'", - "version": "==1.26.139" + "version": "==1.26.141" }, "botocore": { "hashes": [ - "sha256:acc62710bdf11e47f4f26fb290a9082ff00377d7e93a16e1f080f9c789898114", - "sha256:b164af929eb2f1507833718de9eb8811e3adc6943b464c1869e95ac87f3bab88" + "sha256:b01d156c42765f3f437959e01a8c7f3cb0e29b24aa0b8f373498133408b2e3c7", + "sha256:e86e1633f98838317b9e1b5c874c4d85339b77f6b7e55c2a4d83913f6166f9ad" ], "markers": "python_version >= '3.7'", - "version": "==1.29.139" + "version": "==1.29.141" }, "brotli": { "hashes": [ @@ -725,19 +725,19 @@ }, "google-api-python-client": { "hashes": [ - "sha256:0f320190ab9d5bd2fdb0cb894e8e53bb5e17d4888ee8dc4d26ba65ce378409e2", - "sha256:3ca4e93821f4e9ac29b91ab0d9df168b42c8ad0fb8bff65b8c2ccb2d462b0464" + "sha256:29b52232b159be72a79890b6d9f703cf6d8ebbec0ef6371c5670c1abeca5a9f9", + "sha256:bbea5869877c822d12d318943833d988497b3a18b9ca2386967118074db676f3" ], "markers": "python_version >= '3.7'", - "version": "==2.86.0" + "version": "==2.87.0" }, "google-auth": { "hashes": [ - "sha256:55a395cdfd3f3dd3f649131d41f97c17b4ed8a2aac1be3502090c716314e8a37", - "sha256:d7a3249027e7f464fbbfd7ee8319a08ad09d2eea51578575c4bd360ffa049ccb" + "sha256:be617bfaf77774008e9d177573f782e109188c8a64ae6e744285df5cea3e7df6", + "sha256:f39d528077ac540793dd3c22a8706178f157642a67d874db25c640b7fead277e" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", - "version": "==2.18.1" + "markers": "python_version >= '3.6'", + "version": "==2.19.0" }, "google-auth-httplib2": { "hashes": [ @@ -825,7 +825,7 @@ "sha256:f82d4d717d8ef19188687aa32b8363e96062911e63ba22a0cff7802a8e58e5f1", "sha256:fc3a569657468b6f3fb60587e48356fe512c1754ca05a564f11366ac9e306526" ], - "markers": "python_version >= '3' and (platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32'))))))", + "markers": "python_version >= '3' and platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32')))))", "version": "==2.0.2" }, "gspread": { @@ -1259,6 +1259,14 @@ ], "version": "==0.8.0" }, + "prometheus-fastapi-instrumentator": { + "hashes": [ + "sha256:6f66a951a4801667f7311d161f3aebfe0cd202391d0f067fbbe169792e2d987b", + "sha256:f1ddd0b8ead75e71d055bdf4cb7e995ec6a6ca63543245e7bbc5ca9b14c45191" + ], + "index": "pypi", + "version": "==6.0.0" + }, "protobuf": { "hashes": [ "sha256:2036a3a1e7fc27f973fa0a7888dce712393af644f4695385f117886abc792e39", @@ -1355,45 +1363,45 @@ }, "pydantic": { "hashes": [ - "sha256:01aea3a42c13f2602b7ecbbea484a98169fb568ebd9e247593ea05f01b884b2e", - "sha256:0cd181f1d0b1d00e2b705f1bf1ac7799a2d938cce3376b8007df62b29be3c2c6", - "sha256:10a86d8c8db68086f1e30a530f7d5f83eb0685e632e411dbbcf2d5c0150e8dcd", - "sha256:193924c563fae6ddcb71d3f06fa153866423ac1b793a47936656e806b64e24ca", - "sha256:464855a7ff7f2cc2cf537ecc421291b9132aa9c79aef44e917ad711b4a93163b", - "sha256:516f1ed9bc2406a0467dd777afc636c7091d71f214d5e413d64fef45174cfc7a", - "sha256:6434b49c0b03a51021ade5c4daa7d70c98f7a79e95b551201fff682fc1661245", - "sha256:64d34ab766fa056df49013bb6e79921a0265204c071984e75a09cbceacbbdd5d", - "sha256:670bb4683ad1e48b0ecb06f0cfe2178dcf74ff27921cdf1606e527d2617a81ee", - "sha256:68792151e174a4aa9e9fc1b4e653e65a354a2fa0fed169f7b3d09902ad2cb6f1", - "sha256:701daea9ffe9d26f97b52f1d157e0d4121644f0fcf80b443248434958fd03dc3", - "sha256:7d45fc99d64af9aaf7e308054a0067fdcd87ffe974f2442312372dfa66e1001d", - "sha256:80b1fab4deb08a8292d15e43a6edccdffa5377a36a4597bb545b93e79c5ff0a5", - "sha256:82dffb306dd20bd5268fd6379bc4bfe75242a9c2b79fec58e1041fbbdb1f7914", - "sha256:8c7f51861d73e8b9ddcb9916ae7ac39fb52761d9ea0df41128e81e2ba42886cd", - "sha256:950ce33857841f9a337ce07ddf46bc84e1c4946d2a3bba18f8280297157a3fd1", - "sha256:976cae77ba6a49d80f461fd8bba183ff7ba79f44aa5cfa82f1346b5626542f8e", - "sha256:9f6f0fd68d73257ad6685419478c5aece46432f4bdd8d32c7345f1986496171e", - "sha256:a7cd2251439988b413cb0a985c4ed82b6c6aac382dbaff53ae03c4b23a70e80a", - "sha256:abfb7d4a7cd5cc4e1d1887c43503a7c5dd608eadf8bc615413fc498d3e4645cd", - "sha256:ae150a63564929c675d7f2303008d88426a0add46efd76c3fc797cd71cb1b46f", - "sha256:b0f85904f73161817b80781cc150f8b906d521fa11e3cdabae19a581c3606209", - "sha256:b4a849d10f211389502059c33332e91327bc154acc1845f375a99eca3afa802d", - "sha256:c15582f9055fbc1bfe50266a19771bbbef33dd28c45e78afbe1996fd70966c2a", - "sha256:c230c0d8a322276d6e7b88c3f7ce885f9ed16e0910354510e0bae84d54991143", - "sha256:cc1dde4e50a5fc1336ee0581c1612215bc64ed6d28d2c7c6f25d2fe3e7c3e918", - "sha256:cf135c46099ff3f919d2150a948ce94b9ce545598ef2c6c7bf55dca98a304b52", - "sha256:cfc83c0678b6ba51b0532bea66860617c4cd4251ecf76e9846fa5a9f3454e97e", - "sha256:d2a5ebb48958754d386195fe9e9c5106f11275867051bf017a8059410e9abf1f", - "sha256:d71e69699498b020ea198468e2480a2f1e7433e32a3a99760058c6520e2bea7e", - "sha256:d75ae19d2a3dbb146b6f324031c24f8a3f52ff5d6a9f22f0683694b3afcb16fb", - "sha256:dfe2507b8ef209da71b6fb5f4e597b50c5a34b78d7e857c4f8f3115effaef5fe", - "sha256:e0cfe895a504c060e5d36b287ee696e2fdad02d89e0d895f83037245218a87fe", - "sha256:e79e999e539872e903767c417c897e729e015872040e56b96e67968c3b918b2d", - "sha256:ecbbc51391248116c0a055899e6c3e7ffbb11fb5e2a4cd6f2d0b93272118a209", - "sha256:f4a2b50e2b03d5776e7f21af73e2070e1b5c0d0df255a827e7c632962f8315af" + "sha256:052d8654cb65174d6f9490cc9b9a200083a82cf5c3c5d3985db765757eb3b375", + "sha256:0c6fafa0965b539d7aab0a673a046466d23b86e4b0e8019d25fd53f4df62c277", + "sha256:1243d28e9b05003a89d72e7915fdb26ffd1d39bdd39b00b7dbe4afae4b557f9d", + "sha256:12f7b0bf8553e310e530e9f3a2f5734c68699f42218bf3568ef49cd9b0e44df4", + "sha256:1410275520dfa70effadf4c21811d755e7ef9bb1f1d077a21958153a92c8d9ca", + "sha256:16f8c3e33af1e9bb16c7a91fc7d5fa9fe27298e9f299cff6cb744d89d573d62c", + "sha256:17aef11cc1b997f9d574b91909fed40761e13fac438d72b81f902226a69dac01", + "sha256:191ba419b605f897ede9892f6c56fb182f40a15d309ef0142212200a10af4c18", + "sha256:1952526ba40b220b912cdc43c1c32bcf4a58e3f192fa313ee665916b26befb68", + "sha256:1ced8375969673929809d7f36ad322934c35de4af3b5e5b09ec967c21f9f7887", + "sha256:2e4148e635994d57d834be1182a44bdb07dd867fa3c2d1b37002000646cc5459", + "sha256:34d327c81e68a1ecb52fe9c8d50c8a9b3e90d3c8ad991bfc8f953fb477d42fb4", + "sha256:35db5301b82e8661fa9c505c800d0990bc14e9f36f98932bb1d248c0ac5cada5", + "sha256:3e59417ba8a17265e632af99cc5f35ec309de5980c440c255ab1ca3ae96a3e0e", + "sha256:42aa0c4b5c3025483240a25b09f3c09a189481ddda2ea3a831a9d25f444e03c1", + "sha256:666bdf6066bf6dbc107b30d034615d2627e2121506c555f73f90b54a463d1f33", + "sha256:66a703d1983c675a6e0fed8953b0971c44dba48a929a2000a493c3772eb61a5a", + "sha256:6a82d6cda82258efca32b40040228ecf43a548671cb174a1e81477195ed3ed56", + "sha256:6f2e754d5566f050954727c77f094e01793bcb5725b663bf628fa6743a5a9108", + "sha256:7456eb22ed9aaa24ff3e7b4757da20d9e5ce2a81018c1b3ebd81a0b88a18f3b2", + "sha256:7b1f6cb446470b7ddf86c2e57cd119a24959af2b01e552f60705910663af09a4", + "sha256:7d5b8641c24886d764a74ec541d2fc2c7fb19f6da2a4001e6d580ba4a38f7878", + "sha256:84d80219c3f8d4cad44575e18404099c76851bc924ce5ab1c4c8bb5e2a2227d0", + "sha256:88f195f582851e8db960b4a94c3e3ad25692c1c1539e2552f3df7a9e972ef60e", + "sha256:93e6bcfccbd831894a6a434b0aeb1947f9e70b7468f274154d03d71fabb1d7c6", + "sha256:93e766b4a8226e0708ef243e843105bf124e21331694367f95f4e3b4a92bbb3f", + "sha256:ab523c31e22943713d80d8d342d23b6f6ac4b792a1e54064a8d0cf78fd64e800", + "sha256:bb14388ec45a7a0dc429e87def6396f9e73c8c77818c927b6a60706603d5f2ea", + "sha256:c0ab53b609c11dfc0c060d94335993cc2b95b2150e25583bec37a49b2d6c6c3f", + "sha256:c33b60054b2136aef8cf190cd4c52a3daa20b2263917c49adad20eaf381e823b", + "sha256:ceb6a23bf1ba4b837d0cfe378329ad3f351b5897c8d4914ce95b85fba96da5a1", + "sha256:d532bf00f381bd6bc62cabc7d1372096b75a33bc197a312b03f5838b4fb84edd", + "sha256:df7800cb1984d8f6e249351139667a8c50a379009271ee6236138a22a0c0f319", + "sha256:e82d4566fcd527eae8b244fa952d99f2ca3172b7e97add0b43e2d97ee77f81ab", + "sha256:f90c1e29f447557e9e26afb1c4dbf8768a10cc676e3781b6a577841ade126b85", + "sha256:f9613fadad06b4f3bc5db2653ce2f22e0de84a7c6c293909b48f6ed37b83c61f" ], "markers": "python_version >= '3.7'", - "version": "==1.10.7" + "version": "==1.10.8" }, "pygments": { "hashes": [ @@ -1790,7 +1798,7 @@ "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.10.2" }, "tornado": { @@ -1807,7 +1815,7 @@ "sha256:c2de14066c4a38b4ecbbcd55c5cc4b5340eb04f1c5e81da7451ef555859c833f", "sha256:c367ab6c0393d71171123ca5515c61ff62fe09024fa6bf299cd1339dc9456829" ], - "markers": "python_version >= '3.5.2'", + "markers": "python_full_version >= '3.5.2'", "version": "==6.3.2" }, "tqdm": { @@ -1836,18 +1844,18 @@ }, "typing-extensions": { "hashes": [ - "sha256:558bc0c4145f01e6405f4a5fdbd82050bd221b119f4bf72a961a1cfd471349d6", - "sha256:6bac751f4789b135c43228e72de18637e9a6c29d12777023a703fd1a6858469f" + "sha256:06006244c70ac8ee83fa8282cb188f697b8db25bc8b4df07be1873c43897060c", + "sha256:3a8b36f13dd5fdc5d1b16fe317f5668545de77fa0b8e02006381fd49d731ab98" ], "markers": "python_version >= '3.7'", - "version": "==4.6.1" + "version": "==4.6.2" }, "typing-inspect": { "hashes": [ - "sha256:5fbf9c1e65d4fa01e701fe12a5bca6c6e08a4ffd5bc60bfac028253a447c5188", - "sha256:8b1ff0c400943b6145df8119c41c244ca8207f1f10c9c057aeed1560e4806e3d" + "sha256:9ee6fc59062311ef8547596ab6b955e1b8aa46242d854bfc78f4f6b0eff35f9f", + "sha256:b23fc42ff6f6ef6954e4852c1fb512cdd18dbea03134f91f856a95ccc9461f78" ], - "version": "==0.8.0" + "version": "==0.9.0" }, "tzlocal": { "hashes": [ diff --git a/src/main.py b/src/main.py index 86a975d..420ff0a 100644 --- a/src/main.py +++ b/src/main.py @@ -10,6 +10,7 @@ from dotenv import load_dotenv import traceback, os, logging from loguru import logger import sqlalchemy +from prometheus_fastapi_instrumentator import Instrumentator from worker import create_archive_task, create_sheet_task, celery, insert_result_into_db @@ -36,6 +37,10 @@ app.add_middleware( allow_methods=["*"], allow_headers=["*"], ) + +# prometheus exposed in /metrics with authentication +Instrumentator().instrument(app).expose(app, dependencies=[Depends(get_basic_auth)]) + app.mount("/static", StaticFiles(directory="static"), name="static") def get_db(): diff --git a/src/requirements.txt b/src/requirements.txt index 74bb63a..65ce931 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -11,12 +11,12 @@ async-generator==1.10 ; python_version >= '3.5' async-timeout==4.0.2 ; python_version >= '3.6' attrs==23.1.0 ; python_version >= '3.7' authlib==0.15.6 -auto-archiver==0.5.18 +auto-archiver==0.5.20 beautifulsoup4==4.12.2 ; python_full_version >= '3.6.0' billiard==3.6.4.0 blinker==1.6.2 ; python_version >= '3.7' -boto3==1.26.139 ; python_version >= '3.7' -botocore==1.29.139 ; python_version >= '3.7' +boto3==1.26.141 ; python_version >= '3.7' +botocore==1.29.141 ; python_version >= '3.7' brotli==1.0.9 ; platform_python_implementation == 'CPython' bs4==0.0.1 cachetools==5.3.0 ; python_version ~= '3.7' @@ -39,12 +39,12 @@ flower==0.9.7 frozenlist==1.3.3 ; python_version >= '3.7' future==0.18.3 ; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' google-api-core==2.11.0 ; python_version >= '3.7' -google-api-python-client==2.86.0 ; python_version >= '3.7' -google-auth==2.18.1 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' +google-api-python-client==2.87.0 ; python_version >= '3.7' +google-auth==2.19.0 ; python_version >= '3.6' google-auth-httplib2==0.1.0 google-auth-oauthlib==1.0.0 ; python_version >= '3.6' googleapis-common-protos==1.59.0 ; python_version >= '3.7' -greenlet==2.0.2 ; python_version >= '3' and (platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32')))))) +greenlet==2.0.2 ; python_version >= '3' and platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32'))))) gspread==5.9.0 ; python_version not in '3.0, 3.1, 3.2, 3.3' and python_version >= '3.6' h11==0.14.0 ; python_version >= '3.7' httpcore==0.17.2 ; python_version >= '3.7' @@ -75,6 +75,7 @@ outcome==1.2.0 ; python_version >= '3.7' packaging==23.1 ; python_version >= '3.7' pluggy==0.13.1 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' prometheus-client==0.8.0 +prometheus-fastapi-instrumentator==6.0.0 protobuf==4.23.1 ; python_version >= '3.7' py==1.11.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' pyaes==1.6.1 @@ -82,7 +83,7 @@ pyasn1==0.5.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2 pyasn1-modules==0.3.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' pycparser==2.21 pycryptodomex==3.18.0 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' -pydantic==1.10.7 ; python_version >= '3.7' +pydantic==1.10.8 ; python_version >= '3.7' pygments==2.15.1 ; python_version >= '3.7' pyparsing==3.0.9 ; python_version >= '3.1' pysocks==1.7.1 @@ -112,13 +113,13 @@ starlette==0.27.0 ; python_version >= '3.7' telethon==1.28.5 ; python_version >= '3.5' text-unidecode==1.3 tiktok-downloader==0.3.5 -toml==0.10.2 ; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2' -tornado==6.3.2 ; python_version >= '3.5.2' +toml==0.10.2 ; python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3' +tornado==6.3.2 ; python_full_version >= '3.5.2' tqdm==4.65.0 ; python_version >= '3.7' trio==0.22.0 ; python_version >= '3.7' trio-websocket==0.10.2 ; python_version >= '3.7' -typing-extensions==4.6.1 ; python_version >= '3.7' -typing-inspect==0.8.0 +typing-extensions==4.6.2 ; python_version >= '3.7' +typing-inspect==0.9.0 tzlocal==5.0.1 ; python_version >= '3.7' uritemplate==4.1.1 ; python_version >= '3.6' urllib3==1.26.16 ; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' diff --git a/src/security.py b/src/security.py index c24ad00..c63baa2 100644 --- a/src/security.py +++ b/src/security.py @@ -66,6 +66,6 @@ async def get_basic_auth(credentials: HTTPBasicCredentials = Depends(basic_secur if is_correct_password: return True raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, - detail="Wrong static file access credentials", + detail="Wrong auth credentials", headers={"WWW-Authenticate": "Basic"} )