diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..4a5b7f0 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,3 @@ +[MAIN] + +ignore-patterns=(.*tests.*.py, __manifest__.py) \ No newline at end of file diff --git a/README.md b/README.md index 1bd6ddd..c52c464 100644 --- a/README.md +++ b/README.md @@ -218,7 +218,7 @@ configurations: ## Running on Google Sheets Feeder (gsheet_feeder) The `--gsheet_feeder.sheet` property is the name of the Google Sheet to check for URLs. This sheet must have been shared with the Google Service account used by `gspread`. -This sheet must also have specific columns (case-insensitive) in the `header` as specified in [Gsheet.configs](src/auto_archiver/utils/gsheet.py). The default names of these columns and their purpose is: +This sheet must also have specific columns (case-insensitive) in the `header` as specified in [gsheet_feeder.__manifest__.py](src/auto_archiver/modules/gsheet_feeder/__manifest__.py). The default names of these columns and their purpose is: Inputs: diff --git a/poetry.lock b/poetry.lock index 39780ad..8fb48ec 100644 --- a/poetry.lock +++ b/poetry.lock @@ -64,14 +64,14 @@ typing-extensions = {version = ">=4.0.0", markers = "python_version < \"3.11\""} [[package]] name = "attrs" -version = "24.3.0" +version = "25.1.0" description = "Classes Without Boilerplate" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "attrs-24.3.0-py3-none-any.whl", hash = "sha256:ac96cd038792094f438ad1f6ff80837353805ac950cd2aa0e0625ef19850c308"}, - {file = "attrs-24.3.0.tar.gz", hash = "sha256:8f5c07333d543103541ba7be0e2ce16eeee8130cb0b3f9238ab904ce1e85baff"}, + {file = "attrs-25.1.0-py3-none-any.whl", hash = "sha256:c75a69e28a550a7e93789579c22aa26b0f5b83b75dc4e08fe092980051e1090a"}, + {file = "attrs-25.1.0.tar.gz", hash = "sha256:1c97078a80c814273a76b2a298a932eb681c87415c11dee0a6921de7f1b02c3e"}, ] [package.extras] @@ -152,18 +152,18 @@ lxml = ["lxml"] [[package]] name = "boto3" -version = "1.36.3" +version = "1.36.6" description = "The AWS SDK for Python" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "boto3-1.36.3-py3-none-any.whl", hash = "sha256:f9843a5d06f501d66ada06f5a5417f671823af2cf319e36ceefa1bafaaaaa953"}, - {file = "boto3-1.36.3.tar.gz", hash = "sha256:53a5307f6a3526ee2f8590e3c45efa504a3ea4532c1bfe4926c0c19bf188d141"}, + {file = "boto3-1.36.6-py3-none-any.whl", hash = "sha256:6d473f0f340d02b4e9ad5b8e68786a09728101a8b950231b89ebdaf72b6dca21"}, + {file = "boto3-1.36.6.tar.gz", hash = "sha256:b36feae061dc0793cf311468956a0a9e99215ce38bc99a1a4e55a5b105f16297"}, ] [package.dependencies] -botocore = ">=1.36.3,<1.37.0" +botocore = ">=1.36.6,<1.37.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.11.0,<0.12.0" @@ -172,14 +172,14 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.36.3" +version = "1.36.6" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "botocore-1.36.3-py3-none-any.whl", hash = "sha256:536ab828e6f90dbb000e3702ac45fd76642113ae2db1b7b1373ad24104e89255"}, - {file = "botocore-1.36.3.tar.gz", hash = "sha256:775b835e979da5c96548ed1a0b798101a145aec3cd46541d62e27dda5a94d7f8"}, + {file = "botocore-1.36.6-py3-none-any.whl", hash = "sha256:f77bbbb03fb420e260174650fb5c0cc142ec20a96967734eed2b0ef24334ef34"}, + {file = "botocore-1.36.6.tar.gz", hash = "sha256:4864c53d638da191a34daf3ede3ff1371a3719d952cc0c6bd24ce2836a38dd77"}, ] [package.dependencies] @@ -798,14 +798,14 @@ uritemplate = ">=3.0.1,<5" [[package]] name = "google-auth" -version = "2.37.0" +version = "2.38.0" description = "Google Authentication Library" optional = false python-versions = ">=3.7" groups = ["main"] files = [ - {file = "google_auth-2.37.0-py2.py3-none-any.whl", hash = "sha256:42664f18290a6be591be5329a96fe30184be1a1badb7292a7f686a9659de9ca0"}, - {file = "google_auth-2.37.0.tar.gz", hash = "sha256:0054623abf1f9c83492c63d3f47e77f0a544caa3d40b2d98e099a611c2dd5d00"}, + {file = "google_auth-2.38.0-py2.py3-none-any.whl", hash = "sha256:e7dae6694313f434a2727bf2906f27ad259bae090d7aa896590d86feec3d9d4a"}, + {file = "google_auth-2.38.0.tar.gz", hash = "sha256:8285113607d3b80a3f1543b75962447ba8a09fe85783432a784fdeef6ac094c4"}, ] [package.dependencies] @@ -958,13 +958,14 @@ files = [ [[package]] name = "instaloader" -version = "4.14" +version = "4.14.1" description = "Download pictures (or videos) along with their captions and other metadata from Instagram." optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "instaloader-4.14.tar.gz", hash = "sha256:754425eb17af44ce4bb6056e4eacd044a518d13b5efc11b9d80eb229bb96c652"}, + {file = "instaloader-4.14.1-py3-none-any.whl", hash = "sha256:43356f696231621ea5a93354f9a4578124fe131940ee9aa1e83c20f57e18f26d"}, + {file = "instaloader-4.14.1.tar.gz", hash = "sha256:a41a7372a18fb096b3ed545469479884de9cf768e12020c0e0e67c488d9d599c"}, ] [package.dependencies] @@ -1024,7 +1025,7 @@ version = "0.7.3" description = "Python logging made (stupidly) simple" optional = false python-versions = "<4.0,>=3.5" -groups = ["main"] +groups = ["main", "dev"] files = [ {file = "loguru-0.7.3-py3-none-any.whl", hash = "sha256:31a33c10c8e1e10422bfd431aeb5d351c7cf7fa671e3c4df004162264b28220c"}, {file = "loguru-0.7.3.tar.gz", hash = "sha256:19480589e77d47b8d85b2c827ad95d49bf31b0dcde16593892eb51dd18706eb6"}, @@ -1043,7 +1044,7 @@ version = "3.0.0" description = "Python port of markdown-it. Markdown parsing, done right!" optional = false python-versions = ">=3.8" -groups = ["docs"] +groups = ["main", "docs"] files = [ {file = "markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb"}, {file = "markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1"}, @@ -1135,14 +1136,14 @@ files = [ [[package]] name = "marshmallow" -version = "3.25.1" +version = "3.26.0" description = "A lightweight library for converting complex datatypes to and from native Python datatypes." optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "marshmallow-3.25.1-py3-none-any.whl", hash = "sha256:ec5d00d873ce473b7f2ffcb7104286a376c354cab0c2fa12f5573dab03e87210"}, - {file = "marshmallow-3.25.1.tar.gz", hash = "sha256:f4debda3bb11153d81ac34b0d582bf23053055ee11e791b54b4b35493468040a"}, + {file = "marshmallow-3.26.0-py3-none-any.whl", hash = "sha256:1287bca04e6a5f4094822ac153c03da5e214a0a60bcd557b140f3e66991b8ca1"}, + {file = "marshmallow-3.26.0.tar.gz", hash = "sha256:eb36762a1cc76d7abf831e18a3a1b26d3d481bbc74581b8e532a3d3a8115e1cb"}, ] [package.dependencies] @@ -1179,7 +1180,7 @@ version = "0.1.2" description = "Markdown URL utilities" optional = false python-versions = ">=3.7" -groups = ["docs"] +groups = ["main", "docs"] files = [ {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, @@ -1658,7 +1659,7 @@ version = "2.19.1" description = "Pygments is a syntax highlighting package written in Python." optional = false python-versions = ">=3.8" -groups = ["docs"] +groups = ["main", "docs"] files = [ {file = "pygments-2.19.1-py3-none-any.whl", hash = "sha256:9ea1544ad55cecf4b8242fab6dd35a93bbce657034b0611ee383099054ab6d8c"}, {file = "pygments-2.19.1.tar.gz", hash = "sha256:61c16d2a8576dc0649d9f39e089b5f02bcd27fba10d8fb4dcc28173f7a45151f"}, @@ -1749,6 +1750,24 @@ tomli = {version = ">=1", markers = "python_version < \"3.11\""} [package.extras] dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] +[[package]] +name = "pytest-loguru" +version = "0.4.0" +description = "Pytest Loguru" +optional = false +python-versions = ">=3.8" +groups = ["dev"] +files = [ + {file = "pytest_loguru-0.4.0-py3-none-any.whl", hash = "sha256:3cc7b9c6b22cb158209ccbabf0d678dacd3f3c7497d6f46f1c338c13bee1ac77"}, + {file = "pytest_loguru-0.4.0.tar.gz", hash = "sha256:0d9e4e72ae9bfd92f774c666e7353766af11b0b78edd59c290e89be116050f03"}, +] + +[package.dependencies] +loguru = "*" + +[package.extras] +test = ["pytest", "pytest-cov"] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -1817,7 +1836,7 @@ version = "6.0.2" description = "YAML parser and emitter for Python" optional = false python-versions = ">=3.8" -groups = ["main", "docs"] +groups = ["docs"] files = [ {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"}, {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"}, @@ -2035,6 +2054,41 @@ files = [ [package.dependencies] six = ">=1.7.0" +[[package]] +name = "rich" +version = "13.9.4" +description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" +optional = false +python-versions = ">=3.8.0" +groups = ["main"] +files = [ + {file = "rich-13.9.4-py3-none-any.whl", hash = "sha256:6049d5e6ec054bf2779ab3358186963bac2ea89175919d699e378b99738c2a90"}, + {file = "rich-13.9.4.tar.gz", hash = "sha256:439594978a49a09530cff7ebc4b5c7103ef57baf48d5ea3184f21d9a2befa098"}, +] + +[package.dependencies] +markdown-it-py = ">=2.2.0" +pygments = ">=2.13.0,<3.0.0" +typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.11\""} + +[package.extras] +jupyter = ["ipywidgets (>=7.5.1,<9)"] + +[[package]] +name = "rich-argparse" +version = "1.6.0" +description = "Rich help formatters for argparse and optparse" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "rich_argparse-1.6.0-py3-none-any.whl", hash = "sha256:fbe70a1d821b3f2fa8958cddf0cae131870a6e9faa04ab52b409cb1eda809bd7"}, + {file = "rich_argparse-1.6.0.tar.gz", hash = "sha256:092083c30da186f25bcdff8b1d47fdfb571288510fb051e0488a72cc3128de13"}, +] + +[package.dependencies] +rich = ">=11.0.0" + [[package]] name = "rsa" version = "4.9" @@ -2050,16 +2104,92 @@ files = [ [package.dependencies] pyasn1 = ">=0.1.3" +[[package]] +name = "ruamel-yaml" +version = "0.18.10" +description = "ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "ruamel.yaml-0.18.10-py3-none-any.whl", hash = "sha256:30f22513ab2301b3d2b577adc121c6471f28734d3d9728581245f1e76468b4f1"}, + {file = "ruamel.yaml-0.18.10.tar.gz", hash = "sha256:20c86ab29ac2153f80a428e1254a8adf686d3383df04490514ca3b79a362db58"}, +] + +[package.dependencies] +"ruamel.yaml.clib" = {version = ">=0.2.7", markers = "platform_python_implementation == \"CPython\" and python_version < \"3.13\""} + +[package.extras] +docs = ["mercurial (>5.7)", "ryd"] +jinja2 = ["ruamel.yaml.jinja2 (>=0.2)"] + +[[package]] +name = "ruamel-yaml-clib" +version = "0.2.12" +description = "C version of reader, parser and emitter for ruamel.yaml derived from libyaml" +optional = false +python-versions = ">=3.9" +groups = ["main"] +markers = "platform_python_implementation == \"CPython\"" +files = [ + {file = "ruamel.yaml.clib-0.2.12-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:11f891336688faf5156a36293a9c362bdc7c88f03a8a027c2c1d8e0bcde998e5"}, + {file = "ruamel.yaml.clib-0.2.12-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:a606ef75a60ecf3d924613892cc603b154178ee25abb3055db5062da811fd969"}, + {file = "ruamel.yaml.clib-0.2.12-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd5415dded15c3822597455bc02bcd66e81ef8b7a48cb71a33628fc9fdde39df"}, + {file = "ruamel.yaml.clib-0.2.12-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f66efbc1caa63c088dead1c4170d148eabc9b80d95fb75b6c92ac0aad2437d76"}, + {file = "ruamel.yaml.clib-0.2.12-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:22353049ba4181685023b25b5b51a574bce33e7f51c759371a7422dcae5402a6"}, + {file = "ruamel.yaml.clib-0.2.12-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:932205970b9f9991b34f55136be327501903f7c66830e9760a8ffb15b07f05cd"}, + {file = "ruamel.yaml.clib-0.2.12-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:a52d48f4e7bf9005e8f0a89209bf9a73f7190ddf0489eee5eb51377385f59f2a"}, + {file = "ruamel.yaml.clib-0.2.12-cp310-cp310-win32.whl", hash = "sha256:3eac5a91891ceb88138c113f9db04f3cebdae277f5d44eaa3651a4f573e6a5da"}, + {file = "ruamel.yaml.clib-0.2.12-cp310-cp310-win_amd64.whl", hash = "sha256:ab007f2f5a87bd08ab1499bdf96f3d5c6ad4dcfa364884cb4549aa0154b13a28"}, + {file = "ruamel.yaml.clib-0.2.12-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:4a6679521a58256a90b0d89e03992c15144c5f3858f40d7c18886023d7943db6"}, + {file = "ruamel.yaml.clib-0.2.12-cp311-cp311-manylinux2014_aarch64.whl", hash = "sha256:d84318609196d6bd6da0edfa25cedfbabd8dbde5140a0a23af29ad4b8f91fb1e"}, + {file = "ruamel.yaml.clib-0.2.12-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bb43a269eb827806502c7c8efb7ae7e9e9d0573257a46e8e952f4d4caba4f31e"}, + {file = "ruamel.yaml.clib-0.2.12-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:811ea1594b8a0fb466172c384267a4e5e367298af6b228931f273b111f17ef52"}, + {file = "ruamel.yaml.clib-0.2.12-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:cf12567a7b565cbf65d438dec6cfbe2917d3c1bdddfce84a9930b7d35ea59642"}, + {file = "ruamel.yaml.clib-0.2.12-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:7dd5adc8b930b12c8fc5b99e2d535a09889941aa0d0bd06f4749e9a9397c71d2"}, + {file = "ruamel.yaml.clib-0.2.12-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1492a6051dab8d912fc2adeef0e8c72216b24d57bd896ea607cb90bb0c4981d3"}, + {file = "ruamel.yaml.clib-0.2.12-cp311-cp311-win32.whl", hash = "sha256:bd0a08f0bab19093c54e18a14a10b4322e1eacc5217056f3c063bd2f59853ce4"}, + {file = "ruamel.yaml.clib-0.2.12-cp311-cp311-win_amd64.whl", hash = "sha256:a274fb2cb086c7a3dea4322ec27f4cb5cc4b6298adb583ab0e211a4682f241eb"}, + {file = "ruamel.yaml.clib-0.2.12-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:20b0f8dc160ba83b6dcc0e256846e1a02d044e13f7ea74a3d1d56ede4e48c632"}, + {file = "ruamel.yaml.clib-0.2.12-cp312-cp312-manylinux2014_aarch64.whl", hash = "sha256:943f32bc9dedb3abff9879edc134901df92cfce2c3d5c9348f172f62eb2d771d"}, + {file = "ruamel.yaml.clib-0.2.12-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95c3829bb364fdb8e0332c9931ecf57d9be3519241323c5274bd82f709cebc0c"}, + {file = "ruamel.yaml.clib-0.2.12-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:749c16fcc4a2b09f28843cda5a193e0283e47454b63ec4b81eaa2242f50e4ccd"}, + {file = "ruamel.yaml.clib-0.2.12-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bf165fef1f223beae7333275156ab2022cffe255dcc51c27f066b4370da81e31"}, + {file = "ruamel.yaml.clib-0.2.12-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:32621c177bbf782ca5a18ba4d7af0f1082a3f6e517ac2a18b3974d4edf349680"}, + {file = "ruamel.yaml.clib-0.2.12-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b82a7c94a498853aa0b272fd5bc67f29008da798d4f93a2f9f289feb8426a58d"}, + {file = "ruamel.yaml.clib-0.2.12-cp312-cp312-win32.whl", hash = "sha256:e8c4ebfcfd57177b572e2040777b8abc537cdef58a2120e830124946aa9b42c5"}, + {file = "ruamel.yaml.clib-0.2.12-cp312-cp312-win_amd64.whl", hash = "sha256:0467c5965282c62203273b838ae77c0d29d7638c8a4e3a1c8bdd3602c10904e4"}, + {file = "ruamel.yaml.clib-0.2.12-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:4c8c5d82f50bb53986a5e02d1b3092b03622c02c2eb78e29bec33fd9593bae1a"}, + {file = "ruamel.yaml.clib-0.2.12-cp313-cp313-manylinux2014_aarch64.whl", hash = "sha256:e7e3736715fbf53e9be2a79eb4db68e4ed857017344d697e8b9749444ae57475"}, + {file = "ruamel.yaml.clib-0.2.12-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0b7e75b4965e1d4690e93021adfcecccbca7d61c7bddd8e22406ef2ff20d74ef"}, + {file = "ruamel.yaml.clib-0.2.12-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:96777d473c05ee3e5e3c3e999f5d23c6f4ec5b0c38c098b3a5229085f74236c6"}, + {file = "ruamel.yaml.clib-0.2.12-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:3bc2a80e6420ca8b7d3590791e2dfc709c88ab9152c00eeb511c9875ce5778bf"}, + {file = "ruamel.yaml.clib-0.2.12-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:e188d2699864c11c36cdfdada94d781fd5d6b0071cd9c427bceb08ad3d7c70e1"}, + {file = "ruamel.yaml.clib-0.2.12-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4f6f3eac23941b32afccc23081e1f50612bdbe4e982012ef4f5797986828cd01"}, + {file = "ruamel.yaml.clib-0.2.12-cp313-cp313-win32.whl", hash = "sha256:6442cb36270b3afb1b4951f060eccca1ce49f3d087ca1ca4563a6eb479cb3de6"}, + {file = "ruamel.yaml.clib-0.2.12-cp313-cp313-win_amd64.whl", hash = "sha256:e5b8daf27af0b90da7bb903a876477a9e6d7270be6146906b276605997c7e9a3"}, + {file = "ruamel.yaml.clib-0.2.12-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:fc4b630cd3fa2cf7fce38afa91d7cfe844a9f75d7f0f36393fa98815e911d987"}, + {file = "ruamel.yaml.clib-0.2.12-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:bc5f1e1c28e966d61d2519f2a3d451ba989f9ea0f2307de7bc45baa526de9e45"}, + {file = "ruamel.yaml.clib-0.2.12-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5a0e060aace4c24dcaf71023bbd7d42674e3b230f7e7b97317baf1e953e5b519"}, + {file = "ruamel.yaml.clib-0.2.12-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e2f1c3765db32be59d18ab3953f43ab62a761327aafc1594a2a1fbe038b8b8a7"}, + {file = "ruamel.yaml.clib-0.2.12-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:d85252669dc32f98ebcd5d36768f5d4faeaeaa2d655ac0473be490ecdae3c285"}, + {file = "ruamel.yaml.clib-0.2.12-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:e143ada795c341b56de9418c58d028989093ee611aa27ffb9b7f609c00d813ed"}, + {file = "ruamel.yaml.clib-0.2.12-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:2c59aa6170b990d8d2719323e628aaf36f3bfbc1c26279c0eeeb24d05d2d11c7"}, + {file = "ruamel.yaml.clib-0.2.12-cp39-cp39-win32.whl", hash = "sha256:beffaed67936fbbeffd10966a4eb53c402fafd3d6833770516bf7314bc6ffa12"}, + {file = "ruamel.yaml.clib-0.2.12-cp39-cp39-win_amd64.whl", hash = "sha256:040ae85536960525ea62868b642bdb0c2cc6021c9f9d507810c0c604e66f5a7b"}, + {file = "ruamel.yaml.clib-0.2.12.tar.gz", hash = "sha256:6c8fbb13ec503f99a91901ab46e0b07ae7941cd527393187039aec586fdfd36f"}, +] + [[package]] name = "s3transfer" -version = "0.11.1" +version = "0.11.2" description = "An Amazon S3 Transfer Manager" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "s3transfer-0.11.1-py3-none-any.whl", hash = "sha256:8fa0aa48177be1f3425176dfe1ab85dcd3d962df603c3dbfc585e6bf857ef0ff"}, - {file = "s3transfer-0.11.1.tar.gz", hash = "sha256:3f25c900a367c8b7f7d8f9c34edc87e300bde424f779dc9f0a8ae4f9df9264f6"}, + {file = "s3transfer-0.11.2-py3-none-any.whl", hash = "sha256:be6ecb39fadd986ef1701097771f87e4d2f821f27f6071c872143884d2950fbc"}, + {file = "s3transfer-0.11.2.tar.gz", hash = "sha256:3b39185cb72f5acc77db1a58b6e25b977f28d20496b6e58d6813d75f464d632f"}, ] [package.dependencies] @@ -2070,14 +2200,14 @@ crt = ["botocore[crt] (>=1.36.0,<2.0a.0)"] [[package]] name = "selenium" -version = "4.28.0" +version = "4.28.1" description = "Official Python bindings for Selenium WebDriver" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "selenium-4.28.0-py3-none-any.whl", hash = "sha256:3d6a2e8e1b850a1078884ea19f4e011ecdc12263434d87a0b78769836fb82dd8"}, - {file = "selenium-4.28.0.tar.gz", hash = "sha256:a9fae6eef48d470a1b0c6e45185d96f0dafb025e8da4b346cc41e4da3ac54fa0"}, + {file = "selenium-4.28.1-py3-none-any.whl", hash = "sha256:4238847e45e24e4472cfcf3554427512c7aab9443396435b1623ef406fff1cc1"}, + {file = "selenium-4.28.1.tar.gz", hash = "sha256:0072d08670d7ec32db901bd0107695a330cecac9f196e3afb3fa8163026e022a"}, ] [package.dependencies] @@ -2386,14 +2516,14 @@ test = ["pytest"] [[package]] name = "starlette" -version = "0.45.2" +version = "0.45.3" description = "The little ASGI library that shines." optional = false python-versions = ">=3.9" groups = ["docs"] files = [ - {file = "starlette-0.45.2-py3-none-any.whl", hash = "sha256:4daec3356fb0cb1e723a5235e5beaf375d2259af27532958e2d79df549dad9da"}, - {file = "starlette-0.45.2.tar.gz", hash = "sha256:bba1831d15ae5212b22feab2f218bab6ed3cd0fc2dc1d4442443bb1ee52260e0"}, + {file = "starlette-0.45.3-py3-none-any.whl", hash = "sha256:dfb6d332576f136ec740296c7e8bb8c8a7125044e7c6da30744718880cdd059d"}, + {file = "starlette-0.45.3.tar.gz", hash = "sha256:2cbcba2a75806f8a41c722141486f37c28e30a0921c5f6fe4346cb0dcee1302f"}, ] [package.dependencies] @@ -2920,7 +3050,7 @@ version = "1.2.0" description = "A small Python utility to set file creation time on Windows" optional = false python-versions = ">=3.5" -groups = ["main"] +groups = ["main", "dev"] markers = "sys_platform == \"win32\"" files = [ {file = "win32_setctime-1.2.0-py3-none-any.whl", hash = "sha256:95d644c4e708aba81dc3704a116d8cbc974d70b3bdb8be1d150e36be6e9d1390"}, @@ -2947,14 +3077,14 @@ h11 = ">=0.9.0,<1" [[package]] name = "yt-dlp" -version = "2025.1.12" +version = "2025.1.26" description = "A feature-rich command-line audio/video downloader" optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "yt_dlp-2025.1.12-py3-none-any.whl", hash = "sha256:f7ea19afb64f8e457a1b9598ddb67f8deaa313bf1d57abd5612db9272ab10795"}, - {file = "yt_dlp-2025.1.12.tar.gz", hash = "sha256:8e7e246e2a5a2cff0a9c13db46844a37a547680702012058c94ec18fce0ca25a"}, + {file = "yt_dlp-2025.1.26-py3-none-any.whl", hash = "sha256:3e76bd896b9f96601021ca192ca0fbdd195e3c3dcc28302a3a34c9bc4979da7b"}, + {file = "yt_dlp-2025.1.26.tar.gz", hash = "sha256:1c9738266921ad43c568ad01ac3362fb7c7af549276fbec92bd72f140da16240"}, ] [package.extras] @@ -2970,4 +3100,4 @@ test = ["pytest (>=8.1,<9.0)", "pytest-rerunfailures (>=14.0,<15.0)"] [metadata] lock-version = "2.1" python-versions = ">=3.10,<3.13" -content-hash = "4873baccbe879f3e277bbe4354823ee6a494b1d362939f991dfca46ee9c6a906" +content-hash = "9ca114395e73af8982abbccc25b385bbca62e50ba7cca8239e52e5c1227cb4b0" diff --git a/pyproject.toml b/pyproject.toml index d1f3bd4..f1be273 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,6 @@ dependencies = [ "pdqhash (>=0.0.0)", "pillow (>=0.0.0)", "python-slugify (>=0.0.0)", - "pyyaml (>=0.0.0)", "dateparser (>=0.0.0)", "python-twitter-v2 (>=0.0.0)", "instaloader (>=0.0.0)", @@ -47,7 +46,7 @@ dependencies = [ "cryptography (>=41.0.0,<42.0.0)", "boto3 (>=1.28.0,<2.0.0)", "dataclasses-json (>=0.0.0)", - "yt-dlp (==2025.1.12)", + "yt-dlp (>=2025.1.26,<2026.0.0)", "numpy (==2.1.3)", "vk-url-scraper (>=0.0.0)", "requests[socks] (>=0.0.0)", @@ -57,11 +56,14 @@ dependencies = [ "retrying (>=0.0.0)", "tsp-client (>=0.0.0)", "certvalidator (>=0.0.0)", + "rich-argparse (>=1.6.0,<2.0.0)", + "ruamel-yaml (>=0.18.10,<0.19.0)", ] [tool.poetry.group.dev.dependencies] pytest = "^8.3.4" autopep8 = "^2.3.1" +pytest-loguru = "^0.4.0" [tool.poetry.group.docs.dependencies] sphinx = "^8.1.3" diff --git a/scripts/create_update_gdrive_oauth_token.py b/scripts/create_update_gdrive_oauth_token.py index ec8a120..eb6fdbe 100644 --- a/scripts/create_update_gdrive_oauth_token.py +++ b/scripts/create_update_gdrive_oauth_token.py @@ -12,7 +12,7 @@ from googleapiclient.errors import HttpError # Code below from https://developers.google.com/drive/api/quickstart/python # Example invocation: py scripts/create_update_gdrive_oauth_token.py -c secrets/credentials.json -t secrets/gd-token.json -SCOPES = ['https://www.googleapis.com/auth/drive'] +SCOPES = ["https://www.googleapis.com/auth/drive.file"] @click.command( @@ -23,7 +23,7 @@ SCOPES = ['https://www.googleapis.com/auth/drive'] "-c", type=click.Path(exists=True), help="path to the credentials.json file downloaded from https://console.cloud.google.com/apis/credentials", - required=True + required=True, ) @click.option( "--token", @@ -31,59 +31,62 @@ SCOPES = ['https://www.googleapis.com/auth/drive'] type=click.Path(exists=False), default="gd-token.json", help="file where to place the OAuth token, defaults to gd-token.json which you must then move to where your orchestration file points to, defaults to gd-token.json", - required=True + required=True, ) def main(credentials, token): # The file token.json stores the user's access and refresh tokens, and is # created automatically when the authorization flow completes for the first time. creds = None if os.path.exists(token): - with open(token, 'r') as stream: + with open(token, "r") as stream: creds_json = json.load(stream) # creds = Credentials.from_authorized_user_file(creds_json, SCOPES) - creds_json['refresh_token'] = creds_json.get("refresh_token", "") + creds_json["refresh_token"] = creds_json.get("refresh_token", "") creds = Credentials.from_authorized_user_info(creds_json, SCOPES) # If there are no (valid) credentials available, let the user log in. if not creds or not creds.valid: if creds and creds.expired and creds.refresh_token: - print('Requesting new token') + print("Requesting new token") creds.refresh(Request()) else: - print('First run through so putting up login dialog') + print("First run through so putting up login dialog") # credentials.json downloaded from https://console.cloud.google.com/apis/credentials flow = InstalledAppFlow.from_client_secrets_file(credentials, SCOPES) creds = flow.run_local_server(port=55192) # Save the credentials for the next run - with open(token, 'w') as token: - print('Saving new token') + with open(token, "w") as token: + print("Saving new token") token.write(creds.to_json()) else: - print('Token valid') + print("Token valid") try: - service = build('drive', 'v3', credentials=creds) + service = build("drive", "v3", credentials=creds) # About the user results = service.about().get(fields="*").execute() - emailAddress = results['user']['emailAddress'] + emailAddress = results["user"]["emailAddress"] print(emailAddress) # Call the Drive v3 API and return some files - results = service.files().list( - pageSize=10, fields="nextPageToken, files(id, name)").execute() - items = results.get('files', []) + results = ( + service.files() + .list(pageSize=10, fields="nextPageToken, files(id, name)") + .execute() + ) + items = results.get("files", []) if not items: - print('No files found.') + print("No files found.") return - print('Files:') + print("Files:") for item in items: - print(u'{0} ({1})'.format(item['name'], item['id'])) + print("{0} ({1})".format(item["name"], item["id"])) except HttpError as error: - print(f'An error occurred: {error}') + print(f"An error occurred: {error}") -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/scripts/telegram_setup.py b/scripts/telegram_setup.py new file mode 100644 index 0000000..e6fa43c --- /dev/null +++ b/scripts/telegram_setup.py @@ -0,0 +1,29 @@ +""" +This script is used to create a new session file for the Telegram client. +To do this you must first create a Telegram application at https://my.telegram.org/apps +And store your id and hash in the environment variables TELEGRAM_API_ID and TELEGRAM_API_HASH. +Create a .env file, or add the following to your environment : +``` +export TELEGRAM_API_ID=[YOUR_ID_HERE] +export TELEGRAM_API_HASH=[YOUR_HASH_HERE] +``` +Then run this script to create a new session file. + +You will need to provide your phone number and a 2FA code the first time you run this script. +""" + + +import os +from telethon.sync import TelegramClient +from loguru import logger + + +# Create a +API_ID = os.getenv("TELEGRAM_API_ID") +API_HASH = os.getenv("TELEGRAM_API_HASH") +SESSION_FILE = "secrets/anon-insta" + +os.makedirs("secrets", exist_ok=True) +with TelegramClient(SESSION_FILE, API_ID, API_HASH) as client: + logger.success(f"New session file created: {SESSION_FILE}.session") + diff --git a/src/auto_archiver/__init__.py b/src/auto_archiver/__init__.py deleted file mode 100644 index e9fe79f..0000000 --- a/src/auto_archiver/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -from . import archivers, databases, enrichers, feeders, formatters, storages, utils, core - -# need to manually specify due to cyclical deps -from .core.orchestrator import ArchivingOrchestrator -from .core.config import Config -# making accessible directly -from .core.metadata import Metadata diff --git a/src/auto_archiver/__main__.py b/src/auto_archiver/__main__.py index 1254ec4..0023a59 100644 --- a/src/auto_archiver/__main__.py +++ b/src/auto_archiver/__main__.py @@ -1,13 +1,9 @@ """ Entry point for the auto_archiver package. """ -from . import Config -from . import ArchivingOrchestrator +from auto_archiver.core.orchestrator import ArchivingOrchestrator +import sys def main(): - config = Config() - config.parse() - orchestrator = ArchivingOrchestrator(config) - for r in orchestrator.feed(): pass - + ArchivingOrchestrator().run(sys.argv[1:]) if __name__ == "__main__": main() diff --git a/src/auto_archiver/archivers/__init__.py b/src/auto_archiver/archivers/__init__.py deleted file mode 100644 index 5733290..0000000 --- a/src/auto_archiver/archivers/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -""" -Archivers are responsible for retrieving the content from various external platforms. -They act as specialized modules, each tailored to interact with a specific platform, -service, or data source. The archivers collectively enable the tool to comprehensively -collect and preserve a variety of content types, such as posts, images, videos and metadata. - -""" -from .archiver import Archiver -from .telethon_archiver import TelethonArchiver -from .twitter_api_archiver import TwitterApiArchiver -from .instagram_archiver import InstagramArchiver -from .instagram_tbot_archiver import InstagramTbotArchiver -from .telegram_archiver import TelegramArchiver -from .vk_archiver import VkArchiver -from .generic_archiver.generic_archiver import GenericArchiver as YoutubeDLArchiver -from .instagram_api_archiver import InstagramAPIArchiver diff --git a/src/auto_archiver/archivers/generic_archiver/__init__.py b/src/auto_archiver/archivers/generic_archiver/__init__.py deleted file mode 100644 index 0788ae0..0000000 --- a/src/auto_archiver/archivers/generic_archiver/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .generic_archiver import GenericArchiver \ No newline at end of file diff --git a/src/auto_archiver/archivers/instagram_tbot_archiver.py b/src/auto_archiver/archivers/instagram_tbot_archiver.py deleted file mode 100644 index 01b1614..0000000 --- a/src/auto_archiver/archivers/instagram_tbot_archiver.py +++ /dev/null @@ -1,104 +0,0 @@ -""" -InstagramTbotArchiver Module - -This module provides functionality to archive Instagram content (posts, stories, etc.) using a Telegram bot (`instagram_load_bot`). -It interacts with the Telegram API via the Telethon library to send Instagram URLs to the bot, which retrieves the -relevant media and metadata. The fetched content is saved as `Media` objects in a temporary directory and returned as a -`Metadata` object. -""" - -import shutil -from telethon.sync import TelegramClient -from loguru import logger -import time, os -from sqlite3 import OperationalError -from . import Archiver -from ..core import Metadata, Media, ArchivingContext -from ..utils import random_str - - -class InstagramTbotArchiver(Archiver): - """ - calls a telegram bot to fetch instagram posts/stories... and gets available media from it - https://github.com/adw0rd/instagrapi - https://t.me/instagram_load_bot - """ - name = "instagram_tbot_archiver" - - def __init__(self, config: dict) -> None: - super().__init__(config) - self.assert_valid_string("api_id") - self.assert_valid_string("api_hash") - self.timeout = int(self.timeout) - - @staticmethod - def configs() -> dict: - return { - "api_id": {"default": None, "help": "telegram API_ID value, go to https://my.telegram.org/apps"}, - "api_hash": {"default": None, "help": "telegram API_HASH value, go to https://my.telegram.org/apps"}, - "session_file": {"default": "secrets/anon-insta", "help": "optional, records the telegram login session for future usage, '.session' will be appended to the provided value."}, - "timeout": {"default": 45, "help": "timeout to fetch the instagram content in seconds."}, - } - - def setup(self) -> None: - """ - 1. makes a copy of session_file that is removed in cleanup - 2. checks if the session file is valid - """ - logger.info(f"SETUP {self.name} checking login...") - - # make a copy of the session that is used exclusively with this archiver instance - new_session_file = os.path.join("secrets/", f"instabot-{time.strftime('%Y-%m-%d')}{random_str(8)}.session") - shutil.copy(self.session_file + ".session", new_session_file) - self.session_file = new_session_file.replace(".session", "") - - try: - self.client = TelegramClient(self.session_file, self.api_id, self.api_hash) - except OperationalError as e: - logger.error(f"Unable to access the {self.session_file} session, please make sure you don't use the same session file here and in telethon_archiver. if you do then disable at least one of the archivers for the 1st time you setup telethon session: {e}") - - with self.client.start(): - logger.success(f"SETUP {self.name} login works.") - - def cleanup(self) -> None: - logger.info(f"CLEANUP {self.name}.") - session_file_name = self.session_file + ".session" - if os.path.exists(session_file_name): - os.remove(session_file_name) - - def download(self, item: Metadata) -> Metadata: - url = item.get_url() - if not "instagram.com" in url: return False - - result = Metadata() - tmp_dir = ArchivingContext.get_tmp_dir() - with self.client.start(): - chat = self.client.get_entity("instagram_load_bot") - since_id = self.client.send_message(entity=chat, message=url).id - - attempts = 0 - seen_media = [] - message = "" - time.sleep(3) - # media is added before text by the bot so it can be used as a stop-logic mechanism - while attempts < (self.timeout - 3) and (not message or not len(seen_media)): - attempts += 1 - time.sleep(1) - for post in self.client.iter_messages(chat, min_id=since_id): - since_id = max(since_id, post.id) - if post.media and post.id not in seen_media: - filename_dest = os.path.join(tmp_dir, f'{chat.id}_{post.id}') - media = self.client.download_media(post.media, filename_dest) - if media: - result.add_media(Media(media)) - seen_media.append(post.id) - if post.message: message += post.message - - if "You must enter a URL to a post" in message: - logger.debug(f"invalid link {url=} for {self.name}: {message}") - return False - - if message: - result.set_content(message).set_title(message[:128]) - - return result.success("insta-via-bot") diff --git a/src/auto_archiver/archivers/youtubedl_archiver.py b/src/auto_archiver/archivers/youtubedl_archiver.py deleted file mode 100644 index 8b61974..0000000 --- a/src/auto_archiver/archivers/youtubedl_archiver.py +++ /dev/null @@ -1,2 +0,0 @@ -# temporary hack, as we implement module -from .generic_archiver.generic_archiver import GenericArchiver as YoutubeDLArchiver diff --git a/src/auto_archiver/core/__init__.py b/src/auto_archiver/core/__init__.py index b78df83..ae4c41c 100644 --- a/src/auto_archiver/core/__init__.py +++ b/src/auto_archiver/core/__init__.py @@ -3,9 +3,15 @@ """ from .metadata import Metadata from .media import Media -from .step import Step -from .context import ArchivingContext +from .module import BaseModule # cannot import ArchivingOrchestrator/Config to avoid circular dep # from .orchestrator import ArchivingOrchestrator -# from .config import Config \ No newline at end of file +# from .config import Config + +from .database import Database +from .enricher import Enricher +from .feeder import Feeder +from .storage import Storage +from .extractor import Extractor +from .formatter import Formatter \ No newline at end of file diff --git a/src/auto_archiver/core/base_module.py b/src/auto_archiver/core/base_module.py new file mode 100644 index 0000000..ece4719 --- /dev/null +++ b/src/auto_archiver/core/base_module.py @@ -0,0 +1,146 @@ + +from urllib.parse import urlparse +from typing import Mapping, Any +from abc import ABC +from copy import deepcopy, copy +from tempfile import TemporaryDirectory +from auto_archiver.utils import url as UrlUtil + +from loguru import logger + +class BaseModule(ABC): + + """ + Base module class. All modules should inherit from this class. + + The exact methods a class implements will depend on the type of module it is, + however modules can have a .setup() method to run any setup code + (e.g. logging in to a site, spinning up a browser etc.) + + See BaseModule.MODULE_TYPES for the types of modules you can create, noting that + a subclass can be of multiple types. For example, a module that extracts data from + a website and stores it in a database would be both an 'extractor' and a 'database' module. + + Each module is a python package, and should have a __manifest__.py file in the + same directory as the module file. The __manifest__.py specifies the module information + like name, author, version, dependencies etc. See BaseModule._DEFAULT_MANIFEST for the + default manifest structure. + + """ + + MODULE_TYPES = [ + 'feeder', + 'extractor', + 'enricher', + 'database', + 'storage', + 'formatter' + ] + + _DEFAULT_MANIFEST = { + 'name': '', # the display name of the module + 'author': 'Bellingcat', # creator of the module, leave this as Bellingcat or set your own name! + 'type': [], # the type of the module, can be one or more of BaseModule.MODULE_TYPES + 'requires_setup': True, # whether or not this module requires additional setup such as setting API Keys or installing additional softare + 'description': '', # a description of the module + 'dependencies': {}, # external dependencies, e.g. python packages or binaries, in dictionary format + 'entry_point': '', # the entry point for the module, in the format 'module_name::ClassName'. This can be left blank to use the default entry point of module_name::ModuleName + 'version': '1.0', # the version of the module + 'configs': {} # any configuration options this module has, these will be exposed to the user in the config file or via the command line +} + + config: Mapping[str, Any] + authentication: Mapping[str, Mapping[str, str]] + name: str + + # this is set by the orchestrator prior to archiving + tmp_dir: TemporaryDirectory = None + + @property + def storages(self) -> list: + return self.config.get('storages', []) + + def config_setup(self, config: dict): + + authentication = config.get('authentication', {}) + # extract out concatenated sites + for key, val in copy(authentication).items(): + if "," in key: + for site in key.split(","): + authentication[site] = val + del authentication[key] + + # this is important. Each instance is given its own deepcopied config, so modules cannot + # change values to affect other modules + config = deepcopy(config) + authentication = deepcopy(config.pop('authentication', {})) + + self.authentication = authentication + self.config = config + for key, val in config.get(self.name, {}).items(): + setattr(self, key, val) + + def setup(self): + # For any additional setup required by modules, e.g. autehntication + pass + + def auth_for_site(self, site: str, extract_cookies=True) -> Mapping[str, Any]: + """ + Returns the authentication information for a given site. This is used to authenticate + with a site before extracting data. The site should be the domain of the site, e.g. 'twitter.com' + + extract_cookies: bool - whether or not to extract cookies from the given browser and return the + cookie jar (disabling can speed up) processing if you don't actually need the cookies jar + + Currently, the dict can have keys of the following types: + - username: str - the username to use for login + - password: str - the password to use for login + - api_key: str - the API key to use for login + - api_secret: str - the API secret to use for login + - cookie: str - a cookie string to use for login (specific to this site) + - cookies_jar: YoutubeDLCookieJar | http.cookiejar.MozillaCookieJar - a cookie jar compatible with requests (e.g. `requests.get(cookies=cookie_jar)`) + """ + # TODO: think about if/how we can deal with sites that have multiple domains (main one is x.com/twitter.com) + # for now the user must enter them both, like "x.com,twitter.com" in their config. Maybe we just hard-code? + + site = UrlUtil.domain_for_url(site) + # add the 'www' version of the site to the list of sites to check + authdict = {} + + + for to_try in [site, f"www.{site}"]: + if to_try in self.authentication: + authdict.update(self.authentication[to_try]) + break + + # do a fuzzy string match just to print a warning - don't use it since it's insecure + if not authdict: + for key in self.authentication.keys(): + if key in site or site in key: + logger.debug(f"Could not find exact authentication information for site '{site}'. \ + did find information for '{key}' which is close, is this what you meant? \ + If so, edit your authentication settings to make sure it exactly matches.") + + def get_ytdlp_cookiejar(args): + import yt_dlp + from yt_dlp import parse_options + logger.debug(f"Extracting cookies from settings: {args[1]}") + # parse_options returns a named tuple as follows, we only need the ydl_options part + # collections.namedtuple('ParsedOptions', ('parser', 'options', 'urls', 'ydl_opts')) + ytdlp_opts = getattr(parse_options(args), 'ydl_opts') + return yt_dlp.YoutubeDL(ytdlp_opts).cookiejar + + # get the cookies jar, prefer the browser cookies than the file + if 'cookies_from_browser' in self.authentication: + authdict['cookies_from_browser'] = self.authentication['cookies_from_browser'] + if extract_cookies: + authdict['cookies_jar'] = get_ytdlp_cookiejar(['--cookies-from-browser', self.authentication['cookies_from_browser']]) + elif 'cookies_file' in self.authentication: + authdict['cookies_file'] = self.authentication['cookies_file'] + if extract_cookies: + authdict['cookies_jar'] = get_ytdlp_cookiejar(['--cookies', self.authentication['cookies_file']]) + + return authdict + + def repr(self): + return f"Module<'{self.display_name}' (config: {self.config[self.name]})>" \ No newline at end of file diff --git a/src/auto_archiver/core/config.py b/src/auto_archiver/core/config.py index c6a2209..9bb080f 100644 --- a/src/auto_archiver/core/config.py +++ b/src/auto_archiver/core/config.py @@ -5,125 +5,159 @@ flexible setup in various environments. """ -import importlib import argparse -import yaml -from dataclasses import dataclass, field -from typing import List -from collections import defaultdict +from ruamel.yaml import YAML, CommentedMap, add_representer + from loguru import logger -from ..archivers import Archiver -from ..feeders import Feeder -from ..databases import Database -from ..formatters import Formatter -from ..storages import Storage -from ..enrichers import Enricher -from . import Step -from ..utils import update_nested_dict +from copy import deepcopy +from .module import BaseModule +from typing import Any, List, Type, Tuple -@dataclass -class Config: - configurable_parents = [ - Feeder, - Enricher, - Archiver, - Database, - Storage, - Formatter - # Util - ] - feeder: Feeder - formatter: Formatter - archivers: List[Archiver] = field(default_factory=[]) - enrichers: List[Enricher] = field(default_factory=[]) - storages: List[Storage] = field(default_factory=[]) - databases: List[Database] = field(default_factory=[]) +_yaml: YAML = YAML() - def __init__(self) -> None: - self.defaults = {} - self.cli_ops = {} - self.config = {} +EMPTY_CONFIG = _yaml.load(""" +# Auto Archiver Configuration +# Steps are the modules that will be run in the order they are defined - def parse(self, use_cli=True, yaml_config_filename: str = None, overwrite_configs: str = {}): +steps:""" + "".join([f"\n {module}s: []" for module in BaseModule.MODULE_TYPES]) + \ +""" + +# Global configuration + +# Authentication +# a dictionary of authentication information that can be used by extractors to login to website. +# you can use a comma separated list for multiple domains on the same line (common usecase: x.com,twitter.com) +# Common login 'types' are username/password, cookie, api key/token. +# There are two special keys for using cookies, they are: cookies_file and cookies_from_browser. +# Some Examples: +# facebook.com: +# username: "my_username" +# password: "my_password" +# or for a site that uses an API key: +# twitter.com,x.com: +# api_key +# api_secret +# youtube.com: +# cookie: "login_cookie=value ; other_cookie=123" # multiple 'key=value' pairs should be separated by ; + +authentication: {} + +# These are the global configurations that are used by the modules + +logging: + level: INFO +""") +# note: 'logging' is explicitly added above in order to better format the config file + +class DefaultValidatingParser(argparse.ArgumentParser): + + def error(self, message): """ - if yaml_config_filename is provided, the --config argument is ignored, - useful for library usage when the config values are preloaded - overwrite_configs is a dict that overwrites the yaml file contents + Override of error to format a nicer looking error message using logger """ - # 1. parse CLI values - if use_cli: - parser = argparse.ArgumentParser( - # prog = "auto-archiver", - description="Auto Archiver is a CLI tool to archive media/metadata from online URLs; it can read URLs from many sources (Google Sheets, Command Line, ...); and write results to many destinations too (CSV, Google Sheets, MongoDB, ...)!", - epilog="Check the code at https://github.com/bellingcat/auto-archiver" - ) + logger.error("Problem with configuration file (tip: use --help to see the available options):") + logger.error(message) + self.exit(2) - parser.add_argument('--config', action='store', dest='config', help='the filename of the YAML configuration file (defaults to \'config.yaml\')', default='orchestration.yaml') - parser.add_argument('--version', action='version', version=importlib.metadata.version('auto_archiver')) + def parse_known_args(self, args=None, namespace=None): + """ + Override of parse_known_args to also check the 'defaults' values - which are passed in from the config file + """ + for action in self._actions: + if not namespace or action.dest not in namespace: + # for actions that are required and already have a default value, remove the 'required' check + if action.required and action.default is not None: + action.required = False - # Iterate over all step subclasses to gather default configs and CLI arguments - for configurable in self.configurable_parents: - child: Step - for child in configurable.__subclasses__(): - assert child.configs() is not None and type(child.configs()) == dict, f"class '{child.name}' should have a configs method returning a dict." - for config, details in child.configs().items(): - assert "." not in child.name, f"class prop name cannot contain dots('.'): {child.name}" - assert "." not in config, f"config property cannot contain dots('.'): {config}" - config_path = f"{child.name}.{config}" + if action.default is not None: + try: + self._check_value(action, action.default) + except argparse.ArgumentError as e: + logger.error(f"You have an invalid setting in your configuration file ({action.dest}):") + logger.error(e) + exit() - if use_cli: - try: - parser.add_argument(f'--{config_path}', action='store', dest=config_path, help=f"{details['help']} (defaults to {details['default']})", choices=details.get("choices", None)) - except argparse.ArgumentError: - # captures cases when a Step is used in 2 flows, eg: wayback enricher vs wayback archiver - pass + return super().parse_known_args(args, namespace) - self.defaults[config_path] = details["default"] - if "cli_set" in details: - self.cli_ops[config_path] = details["cli_set"] - if use_cli: - args = parser.parse_args() - yaml_config_filename = yaml_config_filename or getattr(args, "config") - else: args = {} +def to_dot_notation(yaml_conf: CommentedMap | dict) -> dict: + dotdict = {} - # 2. read YAML config file (or use provided value) - self.yaml_config = self.read_yaml(yaml_config_filename) - update_nested_dict(self.yaml_config, overwrite_configs) + def process_subdict(subdict, prefix=""): + for key, value in subdict.items(): + if is_dict_type(value): + process_subdict(value, f"{prefix}{key}.") + else: + dotdict[f"{prefix}{key}"] = value - # 3. CONFIGS: decide value with priority: CLI >> config.yaml >> default - self.config = defaultdict(dict) - for config_path, default in self.defaults.items(): - child, config = tuple(config_path.split(".")) - val = getattr(args, config_path, None) - if val is not None and config_path in self.cli_ops: - val = self.cli_ops[config_path](val, default) - if val is None: - val = self.yaml_config.get("configurations", {}).get(child, {}).get(config, default) - self.config[child][config] = val - self.config = dict(self.config) + process_subdict(yaml_conf) + return dotdict - # 4. STEPS: read steps and validate they exist - steps = self.yaml_config.get("steps", {}) - assert "archivers" in steps, "your configuration steps are missing the archivers property" - assert "storages" in steps, "your configuration steps are missing the storages property" +def from_dot_notation(dotdict: dict) -> dict: + normal_dict = {} - self.feeder = Feeder.init(steps.get("feeder", "cli_feeder"), self.config) - self.formatter = Formatter.init(steps.get("formatter", "mute_formatter"), self.config) - self.enrichers = [Enricher.init(e, self.config) for e in steps.get("enrichers", [])] - self.archivers = [Archiver.init(e, self.config) for e in (steps.get("archivers") or [])] - self.databases = [Database.init(e, self.config) for e in steps.get("databases", [])] - self.storages = [Storage.init(e, self.config) for e in steps.get("storages", [])] + def add_part(key, value, current_dict): + if "." in key: + key_parts = key.split(".") + current_dict.setdefault(key_parts[0], {}) + add_part(".".join(key_parts[1:]), value, current_dict[key_parts[0]]) + else: + current_dict[key] = value - logger.info(f"FEEDER: {self.feeder.name}") - logger.info(f"ENRICHERS: {[x.name for x in self.enrichers]}") - logger.info(f"ARCHIVERS: {[x.name for x in self.archivers]}") - logger.info(f"DATABASES: {[x.name for x in self.databases]}") - logger.info(f"STORAGES: {[x.name for x in self.storages]}") - logger.info(f"FORMATTER: {self.formatter.name}") + for key, value in dotdict.items(): + add_part(key, value, normal_dict) - def read_yaml(self, yaml_filename: str) -> dict: + return normal_dict + + +def is_list_type(value): + return isinstance(value, list) or isinstance(value, tuple) or isinstance(value, set) + +def is_dict_type(value): + return isinstance(value, dict) or isinstance(value, CommentedMap) + +def merge_dicts(dotdict: dict, yaml_dict: CommentedMap) -> CommentedMap: + yaml_dict: CommentedMap = deepcopy(yaml_dict) + + # first deal with lists, since 'update' replaces lists from a in b, but we want to extend + def update_dict(subdict, yaml_subdict): + for key, value in subdict.items(): + if not yaml_subdict.get(key): + yaml_subdict[key] = value + continue + + if is_dict_type(value): + update_dict(value, yaml_subdict[key]) + elif is_list_type(value): + yaml_subdict[key].extend(s for s in value if s not in yaml_subdict[key]) + else: + yaml_subdict[key] = value + + update_dict(from_dot_notation(dotdict), yaml_dict) + + return yaml_dict + +def read_yaml(yaml_filename: str) -> CommentedMap: + config = None + try: with open(yaml_filename, "r", encoding="utf-8") as inf: - return yaml.safe_load(inf) + config = _yaml.load(inf) + except FileNotFoundError: + pass + + if not config: + config = EMPTY_CONFIG + + return config + +# TODO: make this tidier/find a way to notify of which keys should not be stored + + +def store_yaml(config: CommentedMap, yaml_filename: str) -> None: + config_to_save = deepcopy(config) + + config_to_save.pop('urls', None) + with open(yaml_filename, "w", encoding="utf-8") as outf: + _yaml.dump(config_to_save, outf) \ No newline at end of file diff --git a/src/auto_archiver/core/context.py b/src/auto_archiver/core/context.py deleted file mode 100644 index 9a21b5c..0000000 --- a/src/auto_archiver/core/context.py +++ /dev/null @@ -1,64 +0,0 @@ -""" ArchivingContext provides a global context for managing configurations and temporary data during the archiving process. - -This singleton class allows for: -- Storing and retrieving key-value pairs that are accessible throughout the application lifecycle. -- Marking certain values to persist across resets using `keep_on_reset`. -- Managing temporary directories and other shared data used during the archiving process. - -### Key Features: -- Creates a single global instance. -- Reset functionality allows for clearing configurations, with options for partial or full resets. -- Custom getters and setters for commonly used context values like temporary directories. - -""" - -class ArchivingContext: - """ - Singleton context class for managing global configurations and temporary data. - - ArchivingContext._get_instance() to retrieve it if needed - otherwise just - ArchivingContext.set(key, value) - and - ArchivingContext.get(key, default) - - When reset is called, all values are cleared EXCEPT if they were .set(keep_on_reset=True) - reset(full_reset=True) will recreate everything including the keep_on_reset status - """ - _instance = None - - def __init__(self): - self.configs = {} - self.keep_on_reset = set() - - @staticmethod - def get_instance(): - if ArchivingContext._instance is None: - ArchivingContext._instance = ArchivingContext() - return ArchivingContext._instance - - @staticmethod - def set(key, value, keep_on_reset: bool = False): - ac = ArchivingContext.get_instance() - ac.configs[key] = value - if keep_on_reset: ac.keep_on_reset.add(key) - - @staticmethod - def get(key: str, default=None): - return ArchivingContext.get_instance().configs.get(key, default) - - @staticmethod - def reset(full_reset: bool = False): - ac = ArchivingContext.get_instance() - if full_reset: ac.keep_on_reset = set() - ac.configs = {k: v for k, v in ac.configs.items() if k in ac.keep_on_reset} - - # ---- custom getters/setters for widely used context values - - @staticmethod - def set_tmp_dir(tmp_dir: str): - ArchivingContext.get_instance().configs["tmp_dir"] = tmp_dir - - @staticmethod - def get_tmp_dir() -> str: - return ArchivingContext.get_instance().configs.get("tmp_dir") diff --git a/src/auto_archiver/databases/database.py b/src/auto_archiver/core/database.py similarity index 66% rename from src/auto_archiver/databases/database.py rename to src/auto_archiver/core/database.py index 30cba7e..0eb5d81 100644 --- a/src/auto_archiver/databases/database.py +++ b/src/auto_archiver/core/database.py @@ -1,22 +1,10 @@ from __future__ import annotations -from dataclasses import dataclass -from abc import abstractmethod, ABC +from abc import abstractmethod from typing import Union -from ..core import Metadata, Step +from auto_archiver.core import Metadata, BaseModule - -@dataclass -class Database(Step, ABC): - name = "database" - - def __init__(self, config: dict) -> None: - # without this STEP.__init__ is not called - super().__init__(config) - - def init(name: str, config: dict) -> Database: - # only for typing... - return Step.init(name, config, Database) +class Database(BaseModule): def started(self, item: Metadata) -> None: """signals the DB that the given item archival has started""" diff --git a/src/auto_archiver/core/enricher.py b/src/auto_archiver/core/enricher.py new file mode 100644 index 0000000..0e50fa9 --- /dev/null +++ b/src/auto_archiver/core/enricher.py @@ -0,0 +1,19 @@ +""" +Enrichers are modular components that enhance archived content by adding +context, metadata, or additional processing. + +These add additional information to the context, such as screenshots, hashes, and metadata. +They are designed to work within the archiving pipeline, operating on `Metadata` objects after +the archiving step and before storage or formatting. + +Enrichers are optional but highly useful for making the archived data more powerful. +""" +from __future__ import annotations +from abc import abstractmethod +from auto_archiver.core import Metadata, BaseModule + +class Enricher(BaseModule): + """Base classes and utilities for enrichers in the Auto-Archiver system.""" + + @abstractmethod + def enrich(self, to_enrich: Metadata) -> None: pass diff --git a/src/auto_archiver/archivers/archiver.py b/src/auto_archiver/core/extractor.py similarity index 65% rename from src/auto_archiver/archivers/archiver.py rename to src/auto_archiver/core/extractor.py index 7ec699e..2792184 100644 --- a/src/auto_archiver/archivers/archiver.py +++ b/src/auto_archiver/core/extractor.py @@ -1,7 +1,7 @@ -""" The `archiver` module defines the base functionality for implementing archivers in the media archiving framework. - This class provides common utility methods and a standard interface for archivers. +""" The `extractor` module defines the base functionality for implementing extractors in the media archiving framework. + This class provides common utility methods and a standard interface for extractors. - Factory method to initialize an archiver instance based on its name. + Factory method to initialize an extractor instance based on its name. """ @@ -11,48 +11,44 @@ from abc import abstractmethod from dataclasses import dataclass import mimetypes import os -import mimetypes, requests +import mimetypes +import requests from loguru import logger from retrying import retry +import re -from ..core import Metadata, Step, ArchivingContext +from auto_archiver.core import Metadata, BaseModule -@dataclass -class Archiver(Step): +class Extractor(BaseModule): """ - Base class for implementing archivers in the media archiving framework. + Base class for implementing extractors in the media archiving framework. Subclasses must implement the `download` method to define platform-specific behavior. """ - name = "archiver" - - def __init__(self, config: dict) -> None: - # without this STEP.__init__ is not called - super().__init__(config) - - def init(name: str, config: dict) -> Archiver: - # only for typing... - return Step.init(name, config, Archiver) - - def setup(self) -> None: - # used when archivers need to login or do other one-time setup - pass + valid_url: re.Pattern = None def cleanup(self) -> None: - # called when archivers are done, or upon errors, cleanup any resources + # called when extractors are done, or upon errors, cleanup any resources pass def sanitize_url(self, url: str) -> str: # used to clean unnecessary URL parameters OR unfurl redirect links return url + def match_link(self, url: str) -> re.Match: + return self.valid_url.match(url) + def suitable(self, url: str) -> bool: """ - Returns True if this archiver can handle the given URL - + Returns True if this extractor can handle the given URL + Should be overridden by subclasses + """ + if self.valid_url: + return self.match_link(url) is not None + return True def _guess_file_type(self, path: str) -> str: @@ -74,7 +70,7 @@ class Archiver(Step): to_filename = url.split('/')[-1].split('?')[0] if len(to_filename) > 64: to_filename = to_filename[-64:] - to_filename = os.path.join(ArchivingContext.get_tmp_dir(), to_filename) + to_filename = os.path.join(self.tmp_dir, to_filename) if verbose: logger.debug(f"downloading {url[0:50]=} {to_filename=}") headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36' @@ -84,8 +80,8 @@ class Archiver(Step): d.raise_for_status() # get mimetype from the response headers - if not Path(to_filename).suffix: - content_type = d.headers.get('Content-Type') + if not mimetypes.guess_type(to_filename)[0]: + content_type = d.headers.get('Content-Type') or self._guess_file_type(url) extension = mimetypes.guess_extension(content_type) if extension: to_filename += extension @@ -94,10 +90,16 @@ class Archiver(Step): for chunk in d.iter_content(chunk_size=8192): f.write(chunk) return to_filename - + except requests.RequestException as e: logger.warning(f"Failed to fetch the Media URL: {e}") @abstractmethod - def download(self, item: Metadata) -> Metadata: - pass + def download(self, item: Metadata) -> Metadata | False: + """ + Downloads the media from the given URL and returns a Metadata object with the downloaded media. + + If the URL is not supported or the download fails, this method should return False. + + """ + pass \ No newline at end of file diff --git a/src/auto_archiver/core/feeder.py b/src/auto_archiver/core/feeder.py new file mode 100644 index 0000000..352cfd9 --- /dev/null +++ b/src/auto_archiver/core/feeder.py @@ -0,0 +1,9 @@ +from __future__ import annotations +from abc import abstractmethod +from auto_archiver.core import Metadata +from auto_archiver.core import BaseModule + +class Feeder(BaseModule): + + @abstractmethod + def __iter__(self) -> Metadata: return None \ No newline at end of file diff --git a/src/auto_archiver/core/formatter.py b/src/auto_archiver/core/formatter.py new file mode 100644 index 0000000..cf27cb3 --- /dev/null +++ b/src/auto_archiver/core/formatter.py @@ -0,0 +1,9 @@ +from __future__ import annotations +from abc import abstractmethod +from auto_archiver.core import Metadata, Media, BaseModule + + +class Formatter(BaseModule): + + @abstractmethod + def format(self, item: Metadata) -> Media: return None \ No newline at end of file diff --git a/src/auto_archiver/core/media.py b/src/auto_archiver/core/media.py index d204a6e..b6820ab 100644 --- a/src/auto_archiver/core/media.py +++ b/src/auto_archiver/core/media.py @@ -11,11 +11,6 @@ from dataclasses import dataclass, field from dataclasses_json import dataclass_json, config import mimetypes -import ffmpeg -from ffmpeg._run import Error - -from .context import ArchivingContext - from loguru import logger @@ -39,12 +34,11 @@ class Media: _mimetype: str = None # eg: image/jpeg _stored: bool = field(default=False, repr=False, metadata=config(exclude=lambda _: True)) # always exclude - def store(self: Media, override_storages: List = None, url: str = "url-not-available", metadata: Any = None): + def store(self: Media, metadata: Any, url: str = "url-not-available", storages: List[Any] = None) -> None: # 'Any' typing for metadata to avoid circular imports. Stores the media # into the provided/available storages [Storage] repeats the process for # its properties, in case they have inner media themselves for now it # only goes down 1 level but it's easy to make it recursive if needed. - storages = override_storages or ArchivingContext.get("storages") if not len(storages): logger.warning(f"No storages found in local context or provided directly for {self.filename}.") return @@ -69,8 +63,9 @@ class Media: for inner_media in prop_media.all_inner_media(include_self=True): yield inner_media - def is_stored(self) -> bool: - return len(self.urls) > 0 and len(self.urls) == len(ArchivingContext.get("storages")) + def is_stored(self, in_storage) -> bool: + # checks if the media is already stored in the given storage + return len(self.urls) > 0 and len(self.urls) == len(in_storage.config["steps"]["storages"]) def set(self, key: str, value: Any) -> Media: self.properties[key] = value @@ -106,6 +101,12 @@ class Media: return self.mimetype.startswith("image") def is_valid_video(self) -> bool: + # Note: this is intentional, to only import ffmpeg here - when the method is called + # this speeds up loading the module. We check that 'ffmpeg' is available on startup + # when we load each manifest file + import ffmpeg + from ffmpeg._run import Error + # checks for video streams with ffmpeg, or min file size for a video # self.is_video() should be used together with this method try: diff --git a/src/auto_archiver/core/metadata.py b/src/auto_archiver/core/metadata.py index 04683dd..a8d2ad4 100644 --- a/src/auto_archiver/core/metadata.py +++ b/src/auto_archiver/core/metadata.py @@ -20,8 +20,6 @@ from dateutil.parser import parse as parse_dt from loguru import logger from .media import Media -from .context import ArchivingContext - @dataclass_json # annotation order matters @dataclass @@ -32,6 +30,7 @@ class Metadata: def __post_init__(self): self.set("_processed_at", datetime.datetime.now(datetime.timezone.utc)) + self._context = {} def merge(self: Metadata, right: Metadata, overwrite_left=True) -> Metadata: """ @@ -45,6 +44,7 @@ class Metadata: if overwrite_left: if right.status and len(right.status): self.status = right.status + self._context.update(right._context) for k, v in right.metadata.items(): assert k not in self.metadata or type(v) == type(self.get(k)) if type(v) not in [dict, list, set] or k not in self.metadata: @@ -57,12 +57,11 @@ class Metadata: return right.merge(self) return self - def store(self: Metadata, override_storages: List = None): + def store(self, storages=[]): # calls .store for all contained media. storages [Storage] self.remove_duplicate_media_by_hash() - storages = override_storages or ArchivingContext.get("storages") for media in self.media: - media.store(override_storages=storages, url=self.get_url(), metadata=self) + media.store(url=self.get_url(), metadata=self, storages=storages) def set(self, key: str, val: Any) -> Metadata: self.metadata[key] = val @@ -206,3 +205,10 @@ class Metadata: if len(r.media) > len(most_complete.media): most_complete = r elif len(r.media) == len(most_complete.media) and len(r.metadata) > len(most_complete.metadata): most_complete = r return most_complete + + def set_context(self, key: str, val: Any) -> Metadata: + self._context[key] = val + return self + + def get_context(self, key: str, default: Any = None) -> Any: + return self._context.get(key, default) \ No newline at end of file diff --git a/src/auto_archiver/core/module.py b/src/auto_archiver/core/module.py new file mode 100644 index 0000000..c81e26a --- /dev/null +++ b/src/auto_archiver/core/module.py @@ -0,0 +1,249 @@ +""" +Defines the Step abstract base class, which acts as a blueprint for steps in the archiving pipeline +by handling user configuration, validating the steps properties, and implementing dynamic instantiation. + +""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import List +import shutil +import ast +import copy +import sys +from importlib.util import find_spec +import os +from os.path import join +from loguru import logger +import auto_archiver +from .base_module import BaseModule + +_LAZY_LOADED_MODULES = {} + +MANIFEST_FILE = "__manifest__.py" + + +def setup_paths(paths: list[str]) -> None: + """ + Sets up the paths for the modules to be loaded from + + This is necessary for the modules to be imported correctly + + """ + for path in paths: + # check path exists, if it doesn't, log a warning + if not os.path.exists(path): + logger.warning(f"Path '{path}' does not exist. Skipping...") + continue + + # see odoo/module/module.py -> initialize_sys_path + if path not in auto_archiver.modules.__path__: + auto_archiver.modules.__path__.append(path) + + # sort based on the length of the path, so that the longest path is last in the list + auto_archiver.modules.__path__ = sorted(auto_archiver.modules.__path__, key=len, reverse=True) + +def get_module(module_name: str, config: dict) -> BaseModule: + """ + Gets and sets up a module using the provided config + + This will actually load and instantiate the module, and load all its dependencies (i.e. not lazy) + + """ + return get_module_lazy(module_name).load(config) + +def get_module_lazy(module_name: str, suppress_warnings: bool = False) -> LazyBaseModule: + """ + Lazily loads a module, returning a LazyBaseModule + + This has all the information about the module, but does not load the module itself or its dependencies + + To load an actual module, call .setup() on a lazy module + + """ + if module_name in _LAZY_LOADED_MODULES: + return _LAZY_LOADED_MODULES[module_name] + + available = available_modules(limit_to_modules=[module_name], suppress_warnings=suppress_warnings) + if not available: + raise IndexError(f"Module '{module_name}' not found. Are you sure it's installed/exists?") + return available[0] + +def available_modules(with_manifest: bool=False, limit_to_modules: List[str]= [], suppress_warnings: bool = False) -> List[LazyBaseModule]: + + # search through all valid 'modules' paths. Default is 'modules' in the current directory + + # see odoo/modules/module.py -> get_modules + def is_really_module(module_path): + if os.path.isfile(join(module_path, MANIFEST_FILE)): + return True + + all_modules = [] + + for module_folder in auto_archiver.modules.__path__: + # walk through each module in module_folder and check if it has a valid manifest + try: + possible_modules = os.listdir(module_folder) + except FileNotFoundError: + logger.warning(f"Module folder {module_folder} does not exist") + continue + + for possible_module in possible_modules: + if limit_to_modules and possible_module not in limit_to_modules: + continue + + possible_module_path = join(module_folder, possible_module) + if not is_really_module(possible_module_path): + continue + if _LAZY_LOADED_MODULES.get(possible_module): + continue + lazy_module = LazyBaseModule(possible_module, possible_module_path) + + _LAZY_LOADED_MODULES[possible_module] = lazy_module + + all_modules.append(lazy_module) + + if not suppress_warnings: + for module in limit_to_modules: + if not any(module == m.name for m in all_modules): + logger.warning(f"Module '{module}' not found. Are you sure it's installed?") + + return all_modules + +@dataclass +class LazyBaseModule: + + """ + A lazy module class, which only loads the manifest and does not load the module itself. + + This is useful for getting information about a module without actually loading it. + + """ + name: str + type: list + description: str + path: str + + _manifest: dict = None + _instance: BaseModule = None + _entry_point: str = None + + def __init__(self, module_name, path): + self.name = module_name + self.path = path + + @property + def entry_point(self): + if not self._entry_point and not self.manifest['entry_point']: + # try to create the entry point from the module name + self._entry_point = f"{self.name}::{self.name.replace('_', ' ').title().replace(' ', '')}" + return self._entry_point + + @property + def dependencies(self) -> dict: + return self.manifest['dependencies'] + + @property + def configs(self) -> dict: + return self.manifest['configs'] + + @property + def requires_setup(self) -> bool: + return self.manifest['requires_setup'] + + @property + def display_name(self) -> str: + return self.manifest['name'] + + @property + def manifest(self) -> dict: + if self._manifest: + return self._manifest + # print(f"Loading manifest for module {module_path}") + # load the manifest file + manifest = copy.deepcopy(BaseModule._DEFAULT_MANIFEST) + + with open(join(self.path, MANIFEST_FILE)) as f: + try: + manifest.update(ast.literal_eval(f.read())) + except (ValueError, TypeError, SyntaxError, MemoryError, RecursionError) as e: + logger.error(f"Error loading manifest from file {self.path}/{MANIFEST_FILE}: {e}") + + self._manifest = manifest + self.type = manifest['type'] + self._entry_point = manifest['entry_point'] + self.description = manifest['description'] + self.version = manifest['version'] + + return manifest + + def load(self, config) -> BaseModule: + + if self._instance: + return self._instance + + # check external dependencies are installed + def check_deps(deps, check): + for dep in deps: + if not len(dep): + # clear out any empty strings that a user may have erroneously added + continue + if not check(dep): + logger.error(f"Module '{self.name}' requires external dependency '{dep}' which is not available/setup. Have you installed the required dependencies for the '{self.name}' module? See the README for more information.") + exit(1) + + def check_python_dep(dep): + # first check if it's a module: + try: + m = get_module_lazy(dep, suppress_warnings=True) + try: + # we must now load this module and set it up with the config + m.load(config) + return True + except: + logger.error(f"Unable to setup module '{dep}' for use in module '{self.name}'") + return False + except IndexError: + # not a module, continue + pass + + return find_spec(dep) + + check_deps(self.dependencies.get('python', []), check_python_dep) + check_deps(self.dependencies.get('bin', []), lambda dep: shutil.which(dep)) + + + logger.debug(f"Loading module '{self.display_name}'...") + + for qualname in [self.name, f'auto_archiver.modules.{self.name}']: + try: + # first import the whole module, to make sure it's working properly + __import__(qualname) + break + except ImportError: + pass + + # then import the file for the entry point + file_name, class_name = self.entry_point.split('::') + sub_qualname = f'{qualname}.{file_name}' + + __import__(f'{qualname}.{file_name}', fromlist=[self.entry_point]) + # finally, get the class instance + instance: BaseModule = getattr(sys.modules[sub_qualname], class_name)() + if not getattr(instance, 'name', None): + instance.name = self.name + + if not getattr(instance, 'display_name', None): + instance.display_name = self.display_name + + self._instance = instance + + # merge the default config with the user config + default_config = dict((k, v['default']) for k, v in self.configs.items() if v.get('default')) + config[self.name] = default_config | config.get(self.name, {}) + instance.config_setup(config) + instance.setup() + return instance + + def __repr__(self): + return f"Module<'{self.display_name}' ({self.name})>" \ No newline at end of file diff --git a/src/auto_archiver/core/orchestrator.py b/src/auto_archiver/core/orchestrator.py index 3290070..bb5f9e3 100644 --- a/src/auto_archiver/core/orchestrator.py +++ b/src/auto_archiver/core/orchestrator.py @@ -5,48 +5,358 @@ """ from __future__ import annotations -from typing import Generator, Union, List +from typing import Generator, Union, List, Type from urllib.parse import urlparse from ipaddress import ip_address +import argparse +import os +import sys +import json +from tempfile import TemporaryDirectory +import traceback -from .context import ArchivingContext +from rich_argparse import RichHelpFormatter -from ..archivers import Archiver -from ..feeders import Feeder -from ..formatters import Formatter -from ..storages import Storage -from ..enrichers import Enricher -from ..databases import Database -from .metadata import Metadata -import tempfile, traceback +from .metadata import Metadata, Media +from auto_archiver.version import __version__ +from .config import _yaml, read_yaml, store_yaml, to_dot_notation, merge_dicts, EMPTY_CONFIG, DefaultValidatingParser +from .module import available_modules, LazyBaseModule, get_module, setup_paths +from . import validators, Feeder, Extractor, Database, Storage, Formatter, Enricher +from .module import BaseModule + from loguru import logger +DEFAULT_CONFIG_FILE = "orchestration.yaml" + +class JsonParseAction(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + try: + setattr(namespace, self.dest, json.loads(values)) + except json.JSONDecodeError as e: + raise argparse.ArgumentTypeError(f"Invalid JSON input for argument '{self.dest}': {e}") + + +class AuthenticationJsonParseAction(JsonParseAction): + def __call__(self, parser, namespace, values, option_string=None): + super().__call__(parser, namespace, values, option_string) + auth_dict = getattr(namespace, self.dest) + if isinstance(auth_dict, str): + # if it's a string + try: + with open(auth_dict, 'r') as f: + try: + auth_dict = json.load(f) + except json.JSONDecodeError: + # maybe it's yaml, try that + auth_dict = _yaml.load(f) + except: + pass + + if not isinstance(auth_dict, dict): + raise argparse.ArgumentTypeError("Authentication must be a dictionary of site names and their authentication methods") + for site, auth in auth_dict.items(): + if not isinstance(site, str) or not isinstance(auth, dict): + raise argparse.ArgumentTypeError("Authentication must be a dictionary of site names and their authentication methods") + setattr(namespace, self.dest, auth_dict) +class UniqueAppendAction(argparse.Action): + def __call__(self, parser, namespace, values, option_string=None): + if not hasattr(namespace, self.dest): + setattr(namespace, self.dest, []) + for value in values: + if value not in getattr(namespace, self.dest): + getattr(namespace, self.dest).append(value) + class ArchivingOrchestrator: - def __init__(self, config) -> None: - self.feeder: Feeder = config.feeder - self.formatter: Formatter = config.formatter - self.enrichers: List[Enricher] = config.enrichers - self.archivers: List[Archiver] = config.archivers - self.databases: List[Database] = config.databases - self.storages: List[Storage] = config.storages - ArchivingContext.set("storages", self.storages, keep_on_reset=True) - try: - for a in self.all_archivers_for_setup(): a.setup() - except (KeyboardInterrupt, Exception) as e: - logger.error(f"Error during setup of archivers: {e}\n{traceback.format_exc()}") - self.cleanup() + feeders: List[Type[Feeder]] + extractors: List[Type[Extractor]] + enrichers: List[Type[Enricher]] + databases: List[Type[Database]] + storages: List[Type[Storage]] + formatters: List[Type[Formatter]] + + def setup_basic_parser(self): + parser = argparse.ArgumentParser( + prog="auto-archiver", + add_help=False, + description=""" + Auto Archiver is a CLI tool to archive media/metadata from online URLs; + it can read URLs from many sources (Google Sheets, Command Line, ...); and write results to many destinations too (CSV, Google Sheets, MongoDB, ...)! + """, + epilog="Check the code at https://github.com/bellingcat/auto-archiver", + formatter_class=RichHelpFormatter, + ) + parser.add_argument('--help', '-h', action='store_true', dest='help', help='show a full help message and exit') + parser.add_argument('--version', action='version', version=__version__) + parser.add_argument('--config', action='store', dest="config_file", help='the filename of the YAML configuration file (defaults to \'config.yaml\')', default=DEFAULT_CONFIG_FILE) + parser.add_argument('--mode', action='store', dest='mode', type=str, choices=['simple', 'full'], help='the mode to run the archiver in', default='simple') + # override the default 'help' so we can inject all the configs and show those + parser.add_argument('-s', '--store', dest='store', default=False, help='Store the created config in the config file', action=argparse.BooleanOptionalAction) + parser.add_argument('--module_paths', dest='module_paths', nargs='+', default=[], help='additional paths to search for modules', action=UniqueAppendAction) + self.basic_parser = parser + return parser + + def setup_complete_parser(self, basic_config: dict, yaml_config: dict, unused_args: list[str]) -> None: + parser = DefaultValidatingParser( + add_help=False, + ) + self.add_additional_args(parser) + + # check what mode we're in + # if we have a config file, use that to decide which modules to load + # if simple, we'll load just the modules that has requires_setup = False + # if full, we'll load all modules + # TODO: BUG** - basic_config won't have steps in it, since these args aren't added to 'basic_parser' + # but should we add them? Or should we just add them to the 'complete' parser? + if yaml_config != EMPTY_CONFIG: + # only load the modules enabled in config + # TODO: if some steps are empty (e.g. 'feeders' is empty), should we default to the 'simple' ones? Or only if they are ALL empty? + enabled_modules = [] + # first loads the modules from the config file, then from the command line + for config in [yaml_config['steps'], basic_config.__dict__]: + for module_type in BaseModule.MODULE_TYPES: + enabled_modules.extend(config.get(f"{module_type}s", [])) + + # clear out duplicates, but keep the order + enabled_modules = list(dict.fromkeys(enabled_modules)) + avail_modules = available_modules(with_manifest=True, limit_to_modules=enabled_modules, suppress_warnings=True) + self.add_module_args(avail_modules, parser) + elif basic_config.mode == 'simple': + simple_modules = [module for module in available_modules(with_manifest=True) if not module.requires_setup] + self.add_module_args(simple_modules, parser) + + # for simple mode, we use the cli_feeder and any modules that don't require setup + yaml_config['steps']['feeders'] = ['cli_feeder'] + + # add them to the config + for module in simple_modules: + for module_type in module.type: + yaml_config['steps'].setdefault(f"{module_type}s", []).append(module.name) + else: + # load all modules, they're not using the 'simple' mode + self.add_module_args(available_modules(with_manifest=True), parser) + + parser.set_defaults(**to_dot_notation(yaml_config)) + + # reload the parser with the new arguments, now that we have them + parsed, unknown = parser.parse_known_args(unused_args) + + # merge the new config with the old one + self.config = merge_dicts(vars(parsed), yaml_config) + # clean out args from the base_parser that we don't want in the config + for key in vars(basic_config): + self.config.pop(key, None) + + # setup the logging + self.setup_logging() + + if unknown: + logger.warning(f"Ignoring unknown/unused arguments: {unknown}\nPerhaps you don't have this module enabled?") + + if (self.config != yaml_config and basic_config.store) or not os.path.isfile(basic_config.config_file): + logger.info(f"Storing configuration file to {basic_config.config_file}") + store_yaml(self.config, basic_config.config_file) + + return self.config + + def add_additional_args(self, parser: argparse.ArgumentParser = None): + if not parser: + parser = self.parser + + + # allow passing URLs directly on the command line + parser.add_argument('urls', nargs='*', default=[], help='URL(s) to archive, either a single URL or a list of urls, should not come from config.yaml') + + parser.add_argument('--feeders', dest='steps.feeders', nargs='+', default=['cli_feeder'], help='the feeders to use', action=UniqueAppendAction) + parser.add_argument('--enrichers', dest='steps.enrichers', nargs='+', help='the enrichers to use', action=UniqueAppendAction) + parser.add_argument('--extractors', dest='steps.extractors', nargs='+', help='the extractors to use', action=UniqueAppendAction) + parser.add_argument('--databases', dest='steps.databases', nargs='+', help='the databases to use', action=UniqueAppendAction) + parser.add_argument('--storages', dest='steps.storages', nargs='+', help='the storages to use', action=UniqueAppendAction) + parser.add_argument('--formatters', dest='steps.formatters', nargs='+', help='the formatter to use', action=UniqueAppendAction) + + parser.add_argument('--authentication', dest='authentication', help='A dictionary of sites and their authentication methods \ + (token, username etc.) that extractors can use to log into \ + a website. If passing this on the command line, use a JSON string. \ + You may also pass a path to a valid JSON/YAML file which will be parsed.',\ + default={}, + action=AuthenticationJsonParseAction) + # logging arguments + parser.add_argument('--logging.level', action='store', dest='logging.level', choices=['INFO', 'DEBUG', 'ERROR', 'WARNING'], help='the logging level to use', default='INFO', type=str.upper) + parser.add_argument('--logging.file', action='store', dest='logging.file', help='the logging file to write to', default=None) + parser.add_argument('--logging.rotation', action='store', dest='logging.rotation', help='the logging rotation to use', default=None) + + + def add_module_args(self, modules: list[LazyBaseModule] = None, parser: argparse.ArgumentParser = None) -> None: + + if not modules: + modules = available_modules(with_manifest=True) + + module: LazyBaseModule + for module in modules: + + if not module.configs: + # this module has no configs, don't show anything in the help + # (TODO: do we want to show something about this module though, like a description?) + continue + + group = parser.add_argument_group(module.display_name or module.name, f"{module.description[:100]}...") + + for name, kwargs in module.configs.items(): + if not kwargs.get('metavar', None): + # make a nicer metavar, metavar is what's used in the help, e.g. --cli_feeder.urls [METAVAR] + kwargs['metavar'] = name.upper() + + if kwargs.get('required', False): + # required args shouldn't have a 'default' value, remove it + kwargs.pop('default', None) + + kwargs.pop('cli_set', None) + should_store = kwargs.pop('should_store', False) + kwargs['dest'] = f"{module.name}.{kwargs.pop('dest', name)}" + try: + kwargs['type'] = getattr(validators, kwargs.get('type', '__invalid__')) + except AttributeError: + kwargs['type'] = __builtins__.get(kwargs.get('type'), str) + arg = group.add_argument(f"--{module.name}.{name}", **kwargs) + arg.should_store = should_store + + def show_help(self, basic_config: dict): + # for the help message, we want to load *all* possible modules and show the help + # add configs as arg parser arguments + + self.add_additional_args(self.basic_parser) + self.add_module_args(parser=self.basic_parser) + self.basic_parser.print_help() + self.basic_parser.exit() + + def setup_logging(self): + # setup loguru logging + logger.remove(0) # remove the default logger + logging_config = self.config['logging'] + logger.add(sys.stderr, level=logging_config['level']) + if log_file := logging_config['file']: + logger.add(log_file) if not logging_config['rotation'] else logger.add(log_file, rotation=logging_config['rotation']) + + def install_modules(self, modules_by_type): + """ + Traverses all modules in 'steps' and loads them into the orchestrator, storing them in the + orchestrator's attributes (self.feeders, self.extractors etc.). If no modules of a certain type + are loaded, the program will exit with an error message. + """ + + invalid_modules = [] + for module_type in BaseModule.MODULE_TYPES: + + step_items = [] + modules_to_load = modules_by_type[f"{module_type}s"] + assert modules_to_load, f"No {module_type}s were configured. Make sure to set at least one {module_type} in your configuration file or on the command line (using --{module_type}s)" + + def check_steps_ok(): + if not len(step_items): + logger.error(f"NO {module_type.upper()}S LOADED. Please check your configuration and try again.") + if len(modules_to_load): + logger.error(f"Tried to load the following modules, but none were available: {modules_to_load}") + exit() + + if (module_type == 'feeder' or module_type == 'formatter') and len(step_items) > 1: + logger.error(f"Only one {module_type} is allowed, found {len(step_items)} {module_type}s. Please remove one of the following from your configuration file: {modules_to_load}") + exit() + + for module in modules_to_load: + if module == 'cli_feeder': + # pseudo module, don't load it + urls = self.config['urls'] + if not urls: + logger.error("No URLs provided. Please provide at least one URL via the command line, or set up an alternative feeder. Use --help for more information.") + exit() + # cli_feeder is a pseudo module, it just takes the command line args + def feed(self) -> Generator[Metadata]: + for url in urls: + logger.debug(f"Processing URL: '{url}'") + yield Metadata().set_url(url) + + pseudo_module = type('CLIFeeder', (Feeder,), { + 'name': 'cli_feeder', + 'display_name': 'CLI Feeder', + '__iter__': feed + + })() + + + pseudo_module.__iter__ = feed + step_items.append(pseudo_module) + continue + + if module in invalid_modules: + continue + try: + loaded_module: BaseModule = get_module(module, self.config) + except (KeyboardInterrupt, Exception) as e: + logger.error(f"Error during setup of modules: {e}\n{traceback.format_exc()}") + if module_type == 'extractor' and loaded_module.name == module: + loaded_module.cleanup() + exit() + + if not loaded_module: + invalid_modules.append(module) + continue + if loaded_module: + step_items.append(loaded_module) + + check_steps_ok() + setattr(self, f"{module_type}s", step_items) + + def load_config(self, config_file: str) -> dict: + if not os.path.exists(config_file) and config_file != DEFAULT_CONFIG_FILE: + logger.error(f"The configuration file {config_file} was not found. Make sure the file exists and try again, or run without the --config file to use the default settings.") + exit() + + return read_yaml(config_file) + + def run(self, args: list) -> None: + + self.setup_basic_parser() + + # parse the known arguments for now (basically, we want the config file) + basic_config, unused_args = self.basic_parser.parse_known_args(args) + + # setup any custom module paths, so they'll show in the help and for arg parsing + setup_paths(basic_config.module_paths) + + # if help flag was called, then show the help + if basic_config.help: + self.show_help(basic_config) + + yaml_config = self.load_config(basic_config.config_file) + self.setup_complete_parser(basic_config, yaml_config, unused_args) + + logger.info(f"======== Welcome to the AUTO ARCHIVER ({__version__}) ==========") + self.install_modules(self.config['steps']) + + # log out the modules that were loaded + for module_type in BaseModule.MODULE_TYPES: + logger.info(f"{module_type.upper()}S: " + ", ".join(m.display_name for m in getattr(self, f"{module_type}s"))) + + for _ in self.feed(): + pass def cleanup(self)->None: logger.info("Cleaning up") - for a in self.all_archivers_for_setup(): a.cleanup() + for e in self.extractors: + e.cleanup() def feed(self) -> Generator[Metadata]: - for item in self.feeder: - yield self.feed_item(item) + + url_count = 0 + for feeder in self.feeders: + for item in feeder: + yield self.feed_item(item) + url_count += 1 + + logger.success(f"Processed {url_count} URL(s)") self.cleanup() def feed_item(self, item: Metadata) -> Metadata: @@ -55,22 +365,33 @@ class ArchivingOrchestrator: - catches keyboard interruptions to do a clean exit - catches any unexpected error, logs it, and does a clean exit """ + tmp_dir: TemporaryDirectory = None try: - ArchivingContext.reset() - with tempfile.TemporaryDirectory(dir="./") as tmp_dir: - ArchivingContext.set_tmp_dir(tmp_dir) - return self.archive(item) + tmp_dir = TemporaryDirectory(dir="./") + # set tmp_dir on all modules + for m in self.all_modules: + m.tmp_dir = tmp_dir.name + return self.archive(item) except KeyboardInterrupt: # catches keyboard interruptions to do a clean exit logger.warning(f"caught interrupt on {item=}") - for d in self.databases: d.aborted(item) + for d in self.databases: + d.aborted(item) self.cleanup() exit() except Exception as e: logger.error(f'Got unexpected error on item {item}: {e}\n{traceback.format_exc()}') for d in self.databases: - if type(e) == AssertionError: d.failed(item, str(e)) - else: d.failed(item) + if type(e) == AssertionError: + d.failed(item, str(e)) + else: + d.failed(item, reason="unexpected error") + finally: + if tmp_dir: + # remove the tmp_dir from all modules + for m in self.all_modules: + m.tmp_dir = None + tmp_dir.cleanup() def archive(self, result: Metadata) -> Union[Metadata, None]: @@ -83,12 +404,19 @@ class ArchivingOrchestrator: 5. Store all downloaded/generated media 6. Call selected Formatter and store formatted if needed """ + original_url = result.get_url().strip() - self.assert_valid_url(original_url) + try: + self.assert_valid_url(original_url) + except AssertionError as e: + logger.error(f"Error archiving URL {original_url}: {e}") + raise e # 1 - sanitize - each archiver is responsible for cleaning/expanding its own URLs url = original_url - for a in self.archivers: url = a.sanitize_url(url) + for a in self.extractors: + url = a.sanitize_url(url) + result.set_url(url) if original_url != url: result.set("original_url", original_url) @@ -96,8 +424,8 @@ class ArchivingOrchestrator: cached_result = None for d in self.databases: d.started(result) - if (local_result := d.fetch(result)): - cached_result = (cached_result or Metadata()).merge(local_result) + if local_result := d.fetch(result): + cached_result = (cached_result or Metadata()).merge(local_result).merge(result) if cached_result: logger.debug("Found previously archived entry") for d in self.databases: @@ -106,9 +434,9 @@ class ArchivingOrchestrator: logger.error(f"ERROR database {d.name}: {e}: {traceback.format_exc()}") return cached_result - # 3 - call archivers until one succeeds - for a in self.archivers: - logger.info(f"Trying archiver {a.name} for {url}") + # 3 - call extractors until one succeeds + for a in self.extractors: + logger.info(f"Trying extractor {a.name} for {url}") try: result.merge(a.download(result)) if result.is_success(): break @@ -122,11 +450,12 @@ class ArchivingOrchestrator: logger.error(f"ERROR enricher {e.name}: {exc}: {traceback.format_exc()}") # 5 - store all downloaded/generated media - result.store() + result.store(storages=self.storages) # 6 - format and store formatted if needed - if (final_media := self.formatter.format(result)): - final_media.store(url=url, metadata=result) + final_media: Media + if final_media := self.formatters[0].format(result): + final_media.store(url=url, metadata=result, storages=self.storages) result.set_final_media(final_media) if result.is_empty(): @@ -160,5 +489,9 @@ class ArchivingOrchestrator: assert not ip.is_link_local, f"Invalid IP used" assert not ip.is_private, f"Invalid IP used" - def all_archivers_for_setup(self) -> List[Archiver]: - return self.archivers + [e for e in self.enrichers if isinstance(e, Archiver)] \ No newline at end of file + + # Helper Properties + + @property + def all_modules(self) -> List[Type[BaseModule]]: + return self.feeders + self.extractors + self.enrichers + self.databases + self.storages + self.formatters \ No newline at end of file diff --git a/src/auto_archiver/core/step.py b/src/auto_archiver/core/step.py deleted file mode 100644 index 9f294fe..0000000 --- a/src/auto_archiver/core/step.py +++ /dev/null @@ -1,48 +0,0 @@ -""" -Defines the Step abstract base class, which acts as a blueprint for steps in the archiving pipeline -by handling user configuration, validating the steps properties, and implementing dynamic instantiation. - -""" - -from __future__ import annotations -from dataclasses import dataclass -from inspect import ClassFoundException -from typing import Type -from abc import ABC - - -@dataclass -class Step(ABC): - name: str = None - - def __init__(self, config: dict) -> None: - # Initialises each step by reading the relevant entries - # reads the configs into object properties - # self.config = config[self.name] - for k, v in config.get(self.name, {}).items(): - self.__setattr__(k, v) - - @staticmethod - def configs() -> dict: return {} - - def init(name: str, config: dict, child: Type[Step]) -> Step: - """ - Attempts to instantiate a subclass of the provided `child` type - matching the given `name`. - Raises ClassFoundException if no matching subclass is found. - TODO: cannot find subclasses of child.subclasses - """ - for sub in child.__subclasses__(): - if sub.name == name: - return sub(config) - raise ClassFoundException(f"Unable to initialize STEP with {name=}, check your configuration file/step names, and make sure you made the step discoverable by putting it into __init__.py") - - def assert_valid_string(self, prop: str) -> None: - """ - Receives a property name and ensures it exists and is a valid non-empty string, - raising an AssertionError if not. - TODO: replace assertions with custom exceptions - """ - assert hasattr(self, prop), f"property {prop} not found" - s = getattr(self, prop) - assert s is not None and type(s) == str and len(s) > 0, f"invalid property {prop} value '{s}', it should be a valid string" diff --git a/src/auto_archiver/core/storage.py b/src/auto_archiver/core/storage.py new file mode 100644 index 0000000..5dfa39d --- /dev/null +++ b/src/auto_archiver/core/storage.py @@ -0,0 +1,65 @@ +from __future__ import annotations +from abc import abstractmethod +from typing import IO +import os + +from loguru import logger +from slugify import slugify + +from auto_archiver.utils.misc import random_str + +from auto_archiver.core import Media, BaseModule, Metadata +from auto_archiver.modules.hash_enricher.hash_enricher import HashEnricher +from auto_archiver.core.module import get_module +class Storage(BaseModule): + + def store(self, media: Media, url: str, metadata: Metadata=None) -> None: + if media.is_stored(in_storage=self): + logger.debug(f"{media.key} already stored, skipping") + return + self.set_key(media, url, metadata) + self.upload(media, metadata=metadata) + media.add_url(self.get_cdn_url(media)) + + @abstractmethod + def get_cdn_url(self, media: Media) -> str: pass + + @abstractmethod + def uploadf(self, file: IO[bytes], key: str, **kwargs: dict) -> bool: pass + + def upload(self, media: Media, **kwargs) -> bool: + logger.debug(f'[{self.__class__.__name__}] storing file {media.filename} with key {media.key}') + with open(media.filename, 'rb') as f: + return self.uploadf(f, media, **kwargs) + + def set_key(self, media: Media, url, metadata: Metadata) -> None: + """takes the media and optionally item info and generates a key""" + if media.key is not None and len(media.key) > 0: return + folder = metadata.get_context('folder', '') + filename, ext = os.path.splitext(media.filename) + + # Handle path_generator logic + path_generator = self.config.get("path_generator", "url") + if path_generator == "flat": + path = "" + filename = slugify(filename) # Ensure filename is slugified + elif path_generator == "url": + path = slugify(url) + elif path_generator == "random": + path = self.config.get("random_path", random_str(24), True) + else: + raise ValueError(f"Invalid path_generator: {path_generator}") + + # Handle filename_generator logic + filename_generator = self.config.get("filename_generator", "random") + if filename_generator == "random": + filename = random_str(24) + elif filename_generator == "static": + # load the hash_enricher module + he = get_module(HashEnricher, self.config) + hd = he.calculate_hash(media.filename) + filename = hd[:24] + else: + raise ValueError(f"Invalid filename_generator: {filename_generator}") + + media.key = os.path.join(folder, path, f"{filename}{ext}") diff --git a/src/auto_archiver/core/validators.py b/src/auto_archiver/core/validators.py new file mode 100644 index 0000000..b868ddf --- /dev/null +++ b/src/auto_archiver/core/validators.py @@ -0,0 +1,19 @@ +# used as validators for config values. Should raise an exception if the value is invalid. +from pathlib import Path +import argparse + +def example_validator(value): + if "example" not in value: + raise argparse.ArgumentTypeError(f"{value} is not a valid value for this argument") + return value + +def positive_number(value): + if value < 0: + raise argparse.ArgumentTypeError(f"{value} is not a positive number") + return value + + +def valid_file(value): + if not Path(value).is_file(): + raise argparse.ArgumentTypeError(f"File '{value}' does not exist.") + return value \ No newline at end of file diff --git a/src/auto_archiver/databases/__init__.py b/src/auto_archiver/databases/__init__.py deleted file mode 100644 index 4c73896..0000000 --- a/src/auto_archiver/databases/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -""" Databases are used to store the outputs from running the Autp Archiver. - - -""" -from .database import Database -from .gsheet_db import GsheetsDb -from .console_db import ConsoleDb -from .csv_db import CSVDb -from .api_db import AAApiDb -from .atlos_db import AtlosDb \ No newline at end of file diff --git a/src/auto_archiver/databases/api_db.py b/src/auto_archiver/databases/api_db.py deleted file mode 100644 index 4304855..0000000 --- a/src/auto_archiver/databases/api_db.py +++ /dev/null @@ -1,70 +0,0 @@ -from typing import Union -import requests, os -from loguru import logger - -from . import Database -from ..core import Metadata - - -class AAApiDb(Database): - """ - Connects to auto-archiver-api instance - """ - name = "auto_archiver_api_db" - - def __init__(self, config: dict) -> None: - # without this STEP.__init__ is not called - super().__init__(config) - self.allow_rearchive = bool(self.allow_rearchive) - self.store_results = bool(self.store_results) - self.assert_valid_string("api_endpoint") - - @staticmethod - def configs() -> dict: - return { - "api_endpoint": {"default": None, "help": "API endpoint where calls are made to"}, - "api_token": {"default": None, "help": "API Bearer token."}, - "public": {"default": False, "help": "whether the URL should be publicly available via the API"}, - "author_id": {"default": None, "help": "which email to assign as author"}, - "group_id": {"default": None, "help": "which group of users have access to the archive in case public=false as author"}, - "allow_rearchive": {"default": True, "help": "if False then the API database will be queried prior to any archiving operations and stop if the link has already been archived"}, - "store_results": {"default": True, "help": "when set, will send the results to the API database."}, - "tags": {"default": [], "help": "what tags to add to the archived URL", "cli_set": lambda cli_val, cur_val: set(cli_val.split(","))}, - } - def fetch(self, item: Metadata) -> Union[Metadata, bool]: - """ query the database for the existence of this item. - Helps avoid re-archiving the same URL multiple times. - """ - if not self.allow_rearchive: return - - params = {"url": item.get_url(), "limit": 15} - headers = {"Authorization": f"Bearer {self.api_token}", "accept": "application/json"} - response = requests.get(os.path.join(self.api_endpoint, "tasks/search-url"), params=params, headers=headers) - - if response.status_code == 200: - if len(response.json()): - logger.success(f"API returned {len(response.json())} previously archived instance(s)") - fetched_metadata = [Metadata.from_dict(r["result"]) for r in response.json()] - return Metadata.choose_most_complete(fetched_metadata) - else: - logger.error(f"AA API FAIL ({response.status_code}): {response.json()}") - return False - - - def done(self, item: Metadata, cached: bool=False) -> None: - """archival result ready - should be saved to DB""" - if not self.store_results: return - if cached: - logger.debug(f"skipping saving archive of {item.get_url()} to the AA API because it was cached") - return - logger.debug(f"saving archive of {item.get_url()} to the AA API.") - - payload = {'result': item.to_json(), 'public': self.public, 'author_id': self.author_id, 'group_id': self.group_id, 'tags': list(self.tags)} - headers = {"Authorization": f"Bearer {self.api_token}"} - response = requests.post(os.path.join(self.api_endpoint, "submit-archive"), json=payload, headers=headers) - - if response.status_code == 200: - logger.success(f"AA API: {response.json()}") - else: - logger.error(f"AA API FAIL ({response.status_code}): {response.json()}") - diff --git a/src/auto_archiver/enrichers/__init__.py b/src/auto_archiver/enrichers/__init__.py deleted file mode 100644 index 64ce248..0000000 --- a/src/auto_archiver/enrichers/__init__.py +++ /dev/null @@ -1,24 +0,0 @@ -""" -Enrichers are modular components that enhance archived content by adding -context, metadata, or additional processing. - -These add additional information to the context, such as screenshots, hashes, and metadata. -They are designed to work within the archiving pipeline, operating on `Metadata` objects after -the archiving step and before storage or formatting. - -Enrichers are optional but highly useful for making the archived data more powerful. - - -""" -from .enricher import Enricher -from .screenshot_enricher import ScreenshotEnricher -from .wayback_enricher import WaybackArchiverEnricher -from .hash_enricher import HashEnricher -from .thumbnail_enricher import ThumbnailEnricher -from .wacz_enricher import WaczArchiverEnricher -from .whisper_enricher import WhisperEnricher -from .pdq_hash_enricher import PdqHashEnricher -from .metadata_enricher import MetadataEnricher -from .meta_enricher import MetaEnricher -from .ssl_enricher import SSLEnricher -from .timestamping_enricher import TimestampingEnricher \ No newline at end of file diff --git a/src/auto_archiver/enrichers/enricher.py b/src/auto_archiver/enrichers/enricher.py deleted file mode 100644 index f195f23..0000000 --- a/src/auto_archiver/enrichers/enricher.py +++ /dev/null @@ -1,22 +0,0 @@ -""" Base classes and utilities for enrichers in the Auto-Archiver system. -""" -from __future__ import annotations -from dataclasses import dataclass -from abc import abstractmethod, ABC -from ..core import Metadata, Step - -@dataclass -class Enricher(Step, ABC): - name = "enricher" - - def __init__(self, config: dict) -> None: - # without this STEP.__init__ is not called - super().__init__(config) - - - # only for typing... - def init(name: str, config: dict) -> Enricher: - return Step.init(name, config, Enricher) - - @abstractmethod - def enrich(self, to_enrich: Metadata) -> None: pass diff --git a/src/auto_archiver/enrichers/hash_enricher.py b/src/auto_archiver/enrichers/hash_enricher.py deleted file mode 100644 index 69973b7..0000000 --- a/src/auto_archiver/enrichers/hash_enricher.py +++ /dev/null @@ -1,75 +0,0 @@ -""" Hash Enricher for generating cryptographic hashes of media files. - -The `HashEnricher` calculates cryptographic hashes (e.g., SHA-256, SHA3-512) -for media files stored in `Metadata` objects. These hashes are used for -validating content integrity, ensuring data authenticity, and identifying -exact duplicates. The hash is computed by reading the file's bytes in chunks, -making it suitable for handling large files efficiently. - -""" -import hashlib -from loguru import logger - -from . import Enricher -from ..core import Metadata, ArchivingContext - - -class HashEnricher(Enricher): - """ - Calculates hashes for Media instances - """ - name = "hash_enricher" - - def __init__(self, config: dict) -> None: - # without this STEP.__init__ is not called - super().__init__(config) - algos = self.configs()["algorithm"] - algo_choices = algos["choices"] - if not getattr(self, 'algorithm', None): - if not config.get('algorithm'): - logger.warning(f"No hash algorithm selected, defaulting to {algos['default']}") - self.algorithm = algos["default"] - else: - self.algorithm = config["algorithm"] - - assert self.algorithm in algo_choices, f"Invalid hash algorithm selected, must be one of {algo_choices} (you selected {self.algorithm})." - - if not getattr(self, 'chunksize', None): - if config.get('chunksize'): - self.chunksize = config["chunksize"] - else: - self.chunksize = self.configs()["chunksize"]["default"] - - self.chunksize = int(self.chunksize) - assert self.chunksize >= -1, "read length must be non-negative or -1" - - ArchivingContext.set("hash_enricher.algorithm", self.algorithm, keep_on_reset=True) - - @staticmethod - def configs() -> dict: - return { - "algorithm": {"default": "SHA-256", "help": "hash algorithm to use", "choices": ["SHA-256", "SHA3-512"]}, - "chunksize": {"default": int(1.6e7), "help": "number of bytes to use when reading files in chunks (if this value is too large you will run out of RAM), default is 16MB"}, - } - - def enrich(self, to_enrich: Metadata) -> None: - url = to_enrich.get_url() - logger.debug(f"calculating media hashes for {url=} (using {self.algorithm})") - - for i, m in enumerate(to_enrich.media): - if len(hd := self.calculate_hash(m.filename)): - to_enrich.media[i].set("hash", f"{self.algorithm}:{hd}") - - def calculate_hash(self, filename) -> str: - hash = None - if self.algorithm == "SHA-256": - hash = hashlib.sha256() - elif self.algorithm == "SHA3-512": - hash = hashlib.sha3_512() - else: return "" - with open(filename, "rb") as f: - while True: - buf = f.read(self.chunksize) - if not buf: break - hash.update(buf) - return hash.hexdigest() diff --git a/src/auto_archiver/enrichers/screenshot_enricher.py b/src/auto_archiver/enrichers/screenshot_enricher.py deleted file mode 100644 index b2ef096..0000000 --- a/src/auto_archiver/enrichers/screenshot_enricher.py +++ /dev/null @@ -1,51 +0,0 @@ -from loguru import logger -import time, os -import base64 - -from selenium.common.exceptions import TimeoutException - - -from . import Enricher -from ..utils import Webdriver, UrlUtil, random_str -from ..core import Media, Metadata, ArchivingContext - -class ScreenshotEnricher(Enricher): - name = "screenshot_enricher" - - @staticmethod - def configs() -> dict: - return { - "width": {"default": 1280, "help": "width of the screenshots"}, - "height": {"default": 720, "help": "height of the screenshots"}, - "timeout": {"default": 60, "help": "timeout for taking the screenshot"}, - "sleep_before_screenshot": {"default": 4, "help": "seconds to wait for the pages to load before taking screenshot"}, - "http_proxy": {"default": "", "help": "http proxy to use for the webdriver, eg http://proxy-user:password@proxy-ip:port"}, - "save_to_pdf": {"default": False, "help": "save the page as pdf along with the screenshot. PDF saving options can be adjusted with the 'print_options' parameter"}, - "print_options": {"default": {}, "help": "options to pass to the pdf printer"} - } - - def enrich(self, to_enrich: Metadata) -> None: - url = to_enrich.get_url() - - if UrlUtil.is_auth_wall(url): - logger.debug(f"[SKIP] SCREENSHOT since url is behind AUTH WALL: {url=}") - return - - logger.debug(f"Enriching screenshot for {url=}") - with Webdriver(self.width, self.height, self.timeout, 'facebook.com' in url, http_proxy=self.http_proxy, print_options=self.print_options) as driver: - try: - driver.get(url) - time.sleep(int(self.sleep_before_screenshot)) - screenshot_file = os.path.join(ArchivingContext.get_tmp_dir(), f"screenshot_{random_str(8)}.png") - driver.save_screenshot(screenshot_file) - to_enrich.add_media(Media(filename=screenshot_file), id="screenshot") - if self.save_to_pdf: - pdf_file = os.path.join(ArchivingContext.get_tmp_dir(), f"pdf_{random_str(8)}.pdf") - pdf = driver.print_page(driver.print_options) - with open(pdf_file, "wb") as f: - f.write(base64.b64decode(pdf)) - to_enrich.add_media(Media(filename=pdf_file), id="pdf") - except TimeoutException: - logger.info("TimeoutException loading page for screenshot") - except Exception as e: - logger.error(f"Got error while loading webdriver for screenshot enricher: {e}") diff --git a/src/auto_archiver/feeders/__init__.py b/src/auto_archiver/feeders/__init__.py deleted file mode 100644 index 8117672..0000000 --- a/src/auto_archiver/feeders/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -""" Feeders handle the input of media into the Auto Archiver. - -""" -from.feeder import Feeder -from .gsheet_feeder import GsheetsFeeder -from .cli_feeder import CLIFeeder -from .atlos_feeder import AtlosFeeder \ No newline at end of file diff --git a/src/auto_archiver/feeders/cli_feeder.py b/src/auto_archiver/feeders/cli_feeder.py deleted file mode 100644 index b2f0add..0000000 --- a/src/auto_archiver/feeders/cli_feeder.py +++ /dev/null @@ -1,32 +0,0 @@ -from loguru import logger - -from . import Feeder -from ..core import Metadata, ArchivingContext - - -class CLIFeeder(Feeder): - name = "cli_feeder" - - def __init__(self, config: dict) -> None: - # without this STEP.__init__ is not called - super().__init__(config) - if type(self.urls) != list or len(self.urls) == 0: - raise Exception("CLI Feeder did not receive any URL to process") - - @staticmethod - def configs() -> dict: - return { - "urls": { - "default": None, - "help": "URL(s) to archive, either a single URL or a list of urls, should not come from config.yaml", - "cli_set": lambda cli_val, cur_val: list(set(cli_val.split(","))) - }, - } - - def __iter__(self) -> Metadata: - for url in self.urls: - logger.debug(f"Processing {url}") - yield Metadata().set_url(url) - ArchivingContext.set("folder", "cli") - - logger.success(f"Processed {len(self.urls)} URL(s)") diff --git a/src/auto_archiver/feeders/csv_feeder.py b/src/auto_archiver/feeders/csv_feeder.py deleted file mode 100644 index 00bf7d7..0000000 --- a/src/auto_archiver/feeders/csv_feeder.py +++ /dev/null @@ -1,41 +0,0 @@ -from loguru import logger -import csv - -from . import Feeder -from ..core import Metadata, ArchivingContext -from ..utils import url_or_none - -class CSVFeeder(Feeder): - - @staticmethod - def configs() -> dict: - return { - "files": { - "default": None, - "help": "Path to the input file(s) to read the URLs from, comma separated. \ - Input files should be formatted with one URL per line", - "cli_set": lambda cli_val, cur_val: list(set(cli_val.split(","))) - }, - "column": { - "default": None, - "help": "Column number or name to read the URLs from, 0-indexed", - } - } - - - def __iter__(self) -> Metadata: - url_column = self.column or 0 - for file in self.files: - with open(file, "r") as f: - reader = csv.reader(f) - first_row = next(reader) - if not(url_or_none(first_row[url_column])): - # it's a header row, skip it - logger.debug(f"Skipping header row: {first_row}") - for row in reader: - url = row[0] - logger.debug(f"Processing {url}") - yield Metadata().set_url(url) - ArchivingContext.set("folder", "cli") - - logger.success(f"Processed {len(self.urls)} URL(s)") \ No newline at end of file diff --git a/src/auto_archiver/feeders/feeder.py b/src/auto_archiver/feeders/feeder.py deleted file mode 100644 index 4aa263f..0000000 --- a/src/auto_archiver/feeders/feeder.py +++ /dev/null @@ -1,21 +0,0 @@ -from __future__ import annotations -from dataclasses import dataclass -from abc import abstractmethod -from ..core import Metadata -from ..core import Step - - -@dataclass -class Feeder(Step): - name = "feeder" - - def __init__(self, config: dict) -> None: - # without this STEP.__init__ is not called - super().__init__(config) - - def init(name: str, config: dict) -> Feeder: - # only for code typing - return Step.init(name, config, Feeder) - - @abstractmethod - def __iter__(self) -> Metadata: return None \ No newline at end of file diff --git a/src/auto_archiver/feeders/gsheet_feeder.py b/src/auto_archiver/feeders/gsheet_feeder.py deleted file mode 100644 index 1c4fc32..0000000 --- a/src/auto_archiver/feeders/gsheet_feeder.py +++ /dev/null @@ -1,105 +0,0 @@ -""" -GsheetsFeeder: A Google Sheets-based feeder for the Auto Archiver. - -This reads data from Google Sheets and filters rows based on user-defined rules. -The filtered rows are processed into `Metadata` objects. - -### Key properties -- validates the sheet's structure and filters rows based on input configurations. -- Ensures only rows with valid URLs and unprocessed statuses are included. -""" -import gspread, os - -from loguru import logger -from slugify import slugify - -# from . import Enricher -from . import Feeder -from ..core import Metadata, ArchivingContext -from ..utils import Gsheets, GWorksheet - - -class GsheetsFeeder(Gsheets, Feeder): - name = "gsheet_feeder" - - def __init__(self, config: dict) -> None: - # without this STEP.__init__ is not called - super().__init__(config) - self.gsheets_client = gspread.service_account(filename=self.service_account) - - @staticmethod - def configs() -> dict: - return dict( - Gsheets.configs(), - ** { - "allow_worksheets": { - "default": set(), - "help": "(CSV) only worksheets whose name is included in allow are included (overrides worksheet_block), leave empty so all are allowed", - "cli_set": lambda cli_val, cur_val: set(cli_val.split(",")) - }, - "block_worksheets": { - "default": set(), - "help": "(CSV) explicitly block some worksheets from being processed", - "cli_set": lambda cli_val, cur_val: set(cli_val.split(",")) - }, - "use_sheet_names_in_stored_paths": { - "default": True, - "help": "if True the stored files path will include 'workbook_name/worksheet_name/...'", - } - }) - - def __iter__(self) -> Metadata: - sh = self.open_sheet() - for ii, wks in enumerate(sh.worksheets()): - if not self.should_process_sheet(wks.title): - logger.debug(f"SKIPPED worksheet '{wks.title}' due to allow/block rules") - continue - - logger.info(f'Opening worksheet {ii=}: {wks.title=} header={self.header}') - gw = GWorksheet(wks, header_row=self.header, columns=self.columns) - - if len(missing_cols := self.missing_required_columns(gw)): - logger.warning(f"SKIPPED worksheet '{wks.title}' due to missing required column(s) for {missing_cols}") - continue - - for row in range(1 + self.header, gw.count_rows() + 1): - url = gw.get_cell(row, 'url').strip() - if not len(url): continue - - original_status = gw.get_cell(row, 'status') - status = gw.get_cell(row, 'status', fresh=original_status in ['', None]) - # TODO: custom status parser(?) aka should_retry_from_status - if status not in ['', None]: continue - - # All checks done - archival process starts here - m = Metadata().set_url(url) - ArchivingContext.set("gsheet", {"row": row, "worksheet": gw}, keep_on_reset=True) - if gw.get_cell_or_default(row, 'folder', "") is None: - folder = '' - else: - folder = slugify(gw.get_cell_or_default(row, 'folder', "").strip()) - if len(folder): - if self.use_sheet_names_in_stored_paths: - ArchivingContext.set("folder", os.path.join(folder, slugify(self.sheet), slugify(wks.title)), True) - else: - ArchivingContext.set("folder", folder, True) - - yield m - - logger.success(f'Finished worksheet {wks.title}') - - def should_process_sheet(self, sheet_name: str) -> bool: - if len(self.allow_worksheets) and sheet_name not in self.allow_worksheets: - # ALLOW rules exist AND sheet name not explicitly allowed - return False - if len(self.block_worksheets) and sheet_name in self.block_worksheets: - # BLOCK rules exist AND sheet name is blocked - return False - return True - - def missing_required_columns(self, gw: GWorksheet) -> list: - missing = [] - for required_col in ['url', 'status']: - if not gw.col_exists(required_col): - missing.append(required_col) - return missing diff --git a/src/auto_archiver/formatters/__init__.py b/src/auto_archiver/formatters/__init__.py deleted file mode 100644 index af96f15..0000000 --- a/src/auto_archiver/formatters/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -""" Formatters for the output of the content. """ -from .formatter import Formatter -from .html_formatter import HtmlFormatter -from .mute_formatter import MuteFormatter \ No newline at end of file diff --git a/src/auto_archiver/formatters/formatter.py b/src/auto_archiver/formatters/formatter.py deleted file mode 100644 index b10477e..0000000 --- a/src/auto_archiver/formatters/formatter.py +++ /dev/null @@ -1,20 +0,0 @@ -from __future__ import annotations -from dataclasses import dataclass -from abc import abstractmethod -from ..core import Metadata, Media, Step - - -@dataclass -class Formatter(Step): - name = "formatter" - - def __init__(self, config: dict) -> None: - # without this STEP.__init__ is not called - super().__init__(config) - - def init(name: str, config: dict) -> Formatter: - # only for code typing - return Step.init(name, config, Formatter) - - @abstractmethod - def format(self, item: Metadata) -> Media: return None \ No newline at end of file diff --git a/src/auto_archiver/formatters/mute_formatter.py b/src/auto_archiver/formatters/mute_formatter.py deleted file mode 100644 index 19830b1..0000000 --- a/src/auto_archiver/formatters/mute_formatter.py +++ /dev/null @@ -1,16 +0,0 @@ -from __future__ import annotations -from dataclasses import dataclass - -from ..core import Metadata, Media -from . import Formatter - - -@dataclass -class MuteFormatter(Formatter): - name = "mute_formatter" - - def __init__(self, config: dict) -> None: - # without this STEP.__init__ is not called - super().__init__(config) - - def format(self, item: Metadata) -> Media: return None diff --git a/src/auto_archiver/modules/api_db/__init__.py b/src/auto_archiver/modules/api_db/__init__.py new file mode 100644 index 0000000..a4f39a1 --- /dev/null +++ b/src/auto_archiver/modules/api_db/__init__.py @@ -0,0 +1 @@ +from .api_db import AAApiDb \ No newline at end of file diff --git a/src/auto_archiver/modules/api_db/__manifest__.py b/src/auto_archiver/modules/api_db/__manifest__.py new file mode 100644 index 0000000..8359174 --- /dev/null +++ b/src/auto_archiver/modules/api_db/__manifest__.py @@ -0,0 +1,54 @@ +{ + "name": "Auto-Archiver API Database", + "type": ["database"], + "entry_point": "api_db::AAApiDb", + "requires_setup": True, + "dependencies": { + "python": ["requests", "loguru"], + }, + "configs": { + "api_endpoint": { + "required": True, + "help": "API endpoint where calls are made to", + }, + "api_token": {"default": None, + "help": "API Bearer token."}, + "public": { + "default": False, + "type": "bool", + "help": "whether the URL should be publicly available via the API", + }, + "author_id": {"default": None, "help": "which email to assign as author"}, + "group_id": { + "default": None, + "help": "which group of users have access to the archive in case public=false as author", + }, + "use_api_cache": { + "default": True, + "type": "bool", + "help": "if False then the API database will be queried prior to any archiving operations and stop if the link has already been archived", + }, + "store_results": { + "default": True, + "type": "bool", + "help": "when set, will send the results to the API database.", + }, + "tags": { + "default": [], + "help": "what tags to add to the archived URL", + }, + }, + "description": """ + Provides integration with the Auto-Archiver API for querying and storing archival data. + +### Features +- **API Integration**: Supports querying for existing archives and submitting results. +- **Duplicate Prevention**: Avoids redundant archiving when `use_api_cache` is disabled. +- **Configurable**: Supports settings like API endpoint, authentication token, tags, and permissions. +- **Tagging and Metadata**: Adds tags and manages metadata for archives. +- **Optional Storage**: Archives results conditionally based on configuration. + +### Setup +Requires access to an Auto-Archiver API instance and a valid API token. + """, +} diff --git a/src/auto_archiver/modules/api_db/api_db.py b/src/auto_archiver/modules/api_db/api_db.py new file mode 100644 index 0000000..753ff3f --- /dev/null +++ b/src/auto_archiver/modules/api_db/api_db.py @@ -0,0 +1,55 @@ +from typing import Union + +import os +import requests +from loguru import logger + +from auto_archiver.core import Database +from auto_archiver.core import Metadata + + +class AAApiDb(Database): + """Connects to auto-archiver-api instance""" + + def fetch(self, item: Metadata) -> Union[Metadata, bool]: + """ query the database for the existence of this item. + Helps avoid re-archiving the same URL multiple times. + """ + if not self.use_api_cache: return + + params = {"url": item.get_url(), "limit": 15} + headers = {"Authorization": f"Bearer {self.api_token}", "accept": "application/json"} + response = requests.get(os.path.join(self.api_endpoint, "url/search"), params=params, headers=headers) + + if response.status_code == 200: + if len(response.json()): + logger.success(f"API returned {len(response.json())} previously archived instance(s)") + fetched_metadata = [Metadata.from_dict(r["result"]) for r in response.json()] + return Metadata.choose_most_complete(fetched_metadata) + else: + logger.error(f"AA API FAIL ({response.status_code}): {response.json()}") + return False + + def done(self, item: Metadata, cached: bool = False) -> None: + """archival result ready - should be saved to DB""" + if not self.store_results: return + if cached: + logger.debug(f"skipping saving archive of {item.get_url()} to the AA API because it was cached") + return + logger.debug(f"saving archive of {item.get_url()} to the AA API.") + + payload = { + 'author_id': self.author_id, + 'url': item.get_url(), + 'public': self.public, + 'group_id': self.group_id, + 'tags': list(self.tags), + 'result': item.to_json(), + } + headers = {"Authorization": f"Bearer {self.api_token}"} + response = requests.post(os.path.join(self.api_endpoint, "interop/submit-archive"), json=payload, headers=headers) + + if response.status_code == 201: + logger.success(f"AA API: {response.json()}") + else: + logger.error(f"AA API FAIL ({response.status_code}): {response.json()}") diff --git a/src/auto_archiver/modules/atlos_db/__init__.py b/src/auto_archiver/modules/atlos_db/__init__.py new file mode 100644 index 0000000..1552e39 --- /dev/null +++ b/src/auto_archiver/modules/atlos_db/__init__.py @@ -0,0 +1 @@ +from atlos_db import AtlosDb \ No newline at end of file diff --git a/src/auto_archiver/modules/atlos_db/__manifest__.py b/src/auto_archiver/modules/atlos_db/__manifest__.py new file mode 100644 index 0000000..b9cabf2 --- /dev/null +++ b/src/auto_archiver/modules/atlos_db/__manifest__.py @@ -0,0 +1,36 @@ +{ + "name": "Atlos Database", + "type": ["database"], + "entry_point": "atlos_db::AtlosDb", + "requires_setup": True, + "dependencies": + {"python": ["loguru", + ""], + "bin": [""]}, + "configs": { + "api_token": { + "default": None, + "help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/", + }, + "atlos_url": { + "default": "https://platform.atlos.org", + "help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.", + "type": "str" + }, + }, + "description": """ +Handles integration with the Atlos platform for managing archival results. + +### Features +- Outputs archival results to the Atlos API for storage and tracking. +- Updates failure status with error details when archiving fails. +- Processes and formats metadata, including ISO formatting for datetime fields. +- Skips processing for items without an Atlos ID. + +### Setup +Required configs: +- atlos_url: Base URL for the Atlos API. +- api_token: Authentication token for API access. +""" +, +} diff --git a/src/auto_archiver/databases/atlos_db.py b/src/auto_archiver/modules/atlos_db/atlos_db.py similarity index 84% rename from src/auto_archiver/databases/atlos_db.py rename to src/auto_archiver/modules/atlos_db/atlos_db.py index 16c4910..baa9fef 100644 --- a/src/auto_archiver/databases/atlos_db.py +++ b/src/auto_archiver/modules/atlos_db/atlos_db.py @@ -1,13 +1,10 @@ -import os from typing import Union -from loguru import logger -from csv import DictWriter -from dataclasses import asdict -import requests -from . import Database -from ..core import Metadata -from ..utils import get_atlos_config_options +import requests +from loguru import logger + +from auto_archiver.core import Database +from auto_archiver.core import Metadata class AtlosDb(Database): @@ -15,16 +12,6 @@ class AtlosDb(Database): Outputs results to Atlos """ - name = "atlos_db" - - def __init__(self, config: dict) -> None: - # without this STEP.__init__ is not called - super().__init__(config) - - @staticmethod - def configs() -> dict: - return get_atlos_config_options() - def failed(self, item: Metadata, reason: str) -> None: """Update DB accordingly for failure""" # If the item has no Atlos ID, there's nothing for us to do diff --git a/src/auto_archiver/modules/atlos_db/base_configs.py b/src/auto_archiver/modules/atlos_db/base_configs.py new file mode 100644 index 0000000..f672f82 --- /dev/null +++ b/src/auto_archiver/modules/atlos_db/base_configs.py @@ -0,0 +1,13 @@ +def get_atlos_config_options(): + return { + "api_token": { + "default": None, + "help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/", + "type": str + }, + "atlos_url": { + "default": "https://platform.atlos.org", + "help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.", + "type": str + }, + } \ No newline at end of file diff --git a/src/auto_archiver/modules/atlos_feeder/__init__.py b/src/auto_archiver/modules/atlos_feeder/__init__.py new file mode 100644 index 0000000..67b243a --- /dev/null +++ b/src/auto_archiver/modules/atlos_feeder/__init__.py @@ -0,0 +1 @@ +from .atlos_feeder import AtlosFeeder \ No newline at end of file diff --git a/src/auto_archiver/modules/atlos_feeder/__manifest__.py b/src/auto_archiver/modules/atlos_feeder/__manifest__.py new file mode 100644 index 0000000..d59f420 --- /dev/null +++ b/src/auto_archiver/modules/atlos_feeder/__manifest__.py @@ -0,0 +1,34 @@ +{ + "name": "Atlos Feeder", + "type": ["feeder"], + "requires_setup": True, + "dependencies": { + "python": ["loguru", "requests"], + }, + "configs": { + "api_token": { + "type": "str", + "required": True, + "help": "An Atlos API token. For more information, see https://docs.atlos.org/technical/api/", + }, + "atlos_url": { + "default": "https://platform.atlos.org", + "help": "The URL of your Atlos instance (e.g., https://platform.atlos.org), without a trailing slash.", + "type": "str" + }, + }, + "description": """ + AtlosFeeder: A feeder module that integrates with the Atlos API to fetch source material URLs for archival. + + ### Features + - Connects to the Atlos API to retrieve a list of source material URLs. + - Filters source materials based on visibility, processing status, and metadata. + - Converts filtered source materials into `Metadata` objects with the relevant `atlos_id` and URL. + - Iterates through paginated results using a cursor for efficient API interaction. + + ### Notes + - Requires an Atlos API endpoint and a valid API token for authentication. + - Ensures only unprocessed, visible, and ready-to-archive URLs are returned. + - Handles pagination transparently when retrieving data from the Atlos API. + """ +} diff --git a/src/auto_archiver/feeders/atlos_feeder.py b/src/auto_archiver/modules/atlos_feeder/atlos_feeder.py similarity index 70% rename from src/auto_archiver/feeders/atlos_feeder.py rename to src/auto_archiver/modules/atlos_feeder/atlos_feeder.py index d3acc00..8c8f9cb 100644 --- a/src/auto_archiver/feeders/atlos_feeder.py +++ b/src/auto_archiver/modules/atlos_feeder/atlos_feeder.py @@ -1,23 +1,11 @@ -from loguru import logger import requests +from loguru import logger -from . import Feeder -from ..core import Metadata, ArchivingContext -from ..utils import get_atlos_config_options +from auto_archiver.core import Feeder +from auto_archiver.core import Metadata class AtlosFeeder(Feeder): - name = "atlos_feeder" - - def __init__(self, config: dict) -> None: - # without this STEP.__init__ is not called - super().__init__(config) - if type(self.api_token) != str: - raise Exception("Atlos Feeder did not receive an Atlos API token") - - @staticmethod - def configs() -> dict: - return get_atlos_config_options() def __iter__(self) -> Metadata: # Get all the urls from the Atlos API @@ -52,5 +40,3 @@ class AtlosFeeder(Feeder): if len(data["results"]) == 0 or cursor is None: break - - logger.success(f"Processed {count} URL(s)") diff --git a/src/auto_archiver/storages/atlos.py b/src/auto_archiver/modules/atlos_storage/atlos_storage.py similarity index 86% rename from src/auto_archiver/storages/atlos.py rename to src/auto_archiver/modules/atlos_storage/atlos_storage.py index 3b13aa0..f8eef68 100644 --- a/src/auto_archiver/storages/atlos.py +++ b/src/auto_archiver/modules/atlos_storage/atlos_storage.py @@ -1,23 +1,15 @@ -import os -from typing import IO, List, Optional -from loguru import logger -import requests import hashlib +import os +from typing import IO, Optional -from ..core import Media, Metadata -from ..storages import Storage -from ..utils import get_atlos_config_options +import requests +from loguru import logger + +from auto_archiver.core import Media, Metadata +from auto_archiver.core import Storage class AtlosStorage(Storage): - name = "atlos_storage" - - def __init__(self, config: dict) -> None: - super().__init__(config) - - @staticmethod - def configs() -> dict: - return dict(Storage.configs(), **get_atlos_config_options()) def get_cdn_url(self, _media: Media) -> str: # It's not always possible to provide an exact URL, because it's diff --git a/src/auto_archiver/modules/console_db/__init__.py b/src/auto_archiver/modules/console_db/__init__.py new file mode 100644 index 0000000..343f09c --- /dev/null +++ b/src/auto_archiver/modules/console_db/__init__.py @@ -0,0 +1 @@ +from .console_db import ConsoleDb \ No newline at end of file diff --git a/src/auto_archiver/modules/console_db/__manifest__.py b/src/auto_archiver/modules/console_db/__manifest__.py new file mode 100644 index 0000000..a1d0d48 --- /dev/null +++ b/src/auto_archiver/modules/console_db/__manifest__.py @@ -0,0 +1,22 @@ +{ + "name": "Console Database", + "type": ["database"], + "requires_setup": False, + "dependencies": { + "python": ["loguru"], + }, + "description": """ +Provides a simple database implementation that outputs archival results and status updates to the console. + +### Features +- Logs the status of archival tasks directly to the console, including: + - started + - failed (with error details) + - aborted + - done (with optional caching status) +- Useful for debugging or lightweight setups where no external database is required. + +### Setup +No additional configuration is required. +""", +} diff --git a/src/auto_archiver/databases/console_db.py b/src/auto_archiver/modules/console_db/console_db.py similarity index 67% rename from src/auto_archiver/databases/console_db.py rename to src/auto_archiver/modules/console_db/console_db.py index bd45f95..48609b0 100644 --- a/src/auto_archiver/databases/console_db.py +++ b/src/auto_archiver/modules/console_db/console_db.py @@ -1,22 +1,13 @@ from loguru import logger -from . import Database -from ..core import Metadata +from auto_archiver.core import Database +from auto_archiver.core import Metadata class ConsoleDb(Database): """ Outputs results to the console """ - name = "console_db" - - def __init__(self, config: dict) -> None: - # without this STEP.__init__ is not called - super().__init__(config) - - @staticmethod - def configs() -> dict: - return {} def started(self, item: Metadata) -> None: logger.warning(f"STARTED {item}") diff --git a/src/auto_archiver/modules/csv_db/__init__.py b/src/auto_archiver/modules/csv_db/__init__.py new file mode 100644 index 0000000..1092cb2 --- /dev/null +++ b/src/auto_archiver/modules/csv_db/__init__.py @@ -0,0 +1 @@ +from .csv_db import CSVDb \ No newline at end of file diff --git a/src/auto_archiver/modules/csv_db/__manifest__.py b/src/auto_archiver/modules/csv_db/__manifest__.py new file mode 100644 index 0000000..507ce14 --- /dev/null +++ b/src/auto_archiver/modules/csv_db/__manifest__.py @@ -0,0 +1,23 @@ +{ + "name": "CSV Database", + "type": ["database"], + "requires_setup": False, + "dependencies": {"python": ["loguru"] + }, + 'entry_point': 'csv_db::CSVDb', + "configs": { + "csv_file": {"default": "db.csv", "help": "CSV file name"} + }, + "description": """ +Handles exporting archival results to a CSV file. + +### Features +- Saves archival metadata as rows in a CSV file. +- Automatically creates the CSV file with a header if it does not exist. +- Appends new metadata entries to the existing file. + +### Setup +Required config: +- csv_file: Path to the CSV file where results will be stored (default: "db.csv"). +""", +} diff --git a/src/auto_archiver/databases/csv_db.py b/src/auto_archiver/modules/csv_db/csv_db.py similarity index 62% rename from src/auto_archiver/databases/csv_db.py rename to src/auto_archiver/modules/csv_db/csv_db.py index f0d7153..b5985e2 100644 --- a/src/auto_archiver/databases/csv_db.py +++ b/src/auto_archiver/modules/csv_db/csv_db.py @@ -3,26 +3,14 @@ from loguru import logger from csv import DictWriter from dataclasses import asdict -from . import Database -from ..core import Metadata +from auto_archiver.core import Database +from auto_archiver.core import Metadata class CSVDb(Database): """ Outputs results to a CSV file """ - name = "csv_db" - - def __init__(self, config: dict) -> None: - # without this STEP.__init__ is not called - super().__init__(config) - self.assert_valid_string("csv_file") - - @staticmethod - def configs() -> dict: - return { - "csv_file": {"default": "db.csv", "help": "CSV file name"} - } def done(self, item: Metadata, cached: bool=False) -> None: """archival result ready - should be saved to DB""" diff --git a/src/auto_archiver/modules/csv_feeder/__init__.py b/src/auto_archiver/modules/csv_feeder/__init__.py new file mode 100644 index 0000000..161b78d --- /dev/null +++ b/src/auto_archiver/modules/csv_feeder/__init__.py @@ -0,0 +1 @@ +from .csv_feeder import CSVFeeder \ No newline at end of file diff --git a/src/auto_archiver/modules/csv_feeder/__manifest__.py b/src/auto_archiver/modules/csv_feeder/__manifest__.py new file mode 100644 index 0000000..6d4c7bf --- /dev/null +++ b/src/auto_archiver/modules/csv_feeder/__manifest__.py @@ -0,0 +1,37 @@ +{ + "name": "CSV Feeder", + "type": ["feeder"], + "requires_setup": False, + "dependencies": { + "python": ["loguru"], + "bin": [""] + }, + 'requires_setup': True, + 'entry_point': "csv_feeder::CSVFeeder", + "configs": { + "files": { + "default": None, + "help": "Path to the input file(s) to read the URLs from, comma separated. \ + Input files should be formatted with one URL per line", + "required": True, + "type": "valid_file", + "nargs": "+", + }, + "column": { + "default": None, + "help": "Column number or name to read the URLs from, 0-indexed", + } + }, + "description": """ + Reads URLs from CSV files and feeds them into the archiving process. + + ### Features + - Supports reading URLs from multiple input files, specified as a comma-separated list. + - Allows specifying the column number or name to extract URLs from. + - Skips header rows if the first value is not a valid URL. + + ### Setup + - Input files should be formatted with one URL per line, with or without a header row. + - If you have a header row, you can specify the column number or name to read URLs from using the 'column' config option. + """ +} diff --git a/src/auto_archiver/modules/csv_feeder/csv_feeder.py b/src/auto_archiver/modules/csv_feeder/csv_feeder.py new file mode 100644 index 0000000..c3f6eea --- /dev/null +++ b/src/auto_archiver/modules/csv_feeder/csv_feeder.py @@ -0,0 +1,38 @@ +from loguru import logger +import csv + +from auto_archiver.core import Feeder +from auto_archiver.core import Metadata +from auto_archiver.utils import url_or_none + +class CSVFeeder(Feeder): + + column = None + + + def __iter__(self) -> Metadata: + for file in self.files: + with open(file, "r") as f: + reader = csv.reader(f) + first_row = next(reader) + url_column = self.column or 0 + if isinstance(url_column, str): + try: + url_column = first_row.index(url_column) + except ValueError: + logger.error(f"Column {url_column} not found in header row: {first_row}. Did you set the 'column' config correctly?") + return + elif not(url_or_none(first_row[url_column])): + # it's a header row, but we've been given a column number already + logger.debug(f"Skipping header row: {first_row}") + else: + # first row isn't a header row, rewind the file + f.seek(0) + + for row in reader: + if not url_or_none(row[url_column]): + logger.warning(f"Not a valid URL in row: {row}, skipping") + continue + url = row[url_column] + logger.debug(f"Processing {url}") + yield Metadata().set_url(url) \ No newline at end of file diff --git a/src/auto_archiver/modules/gdrive_storage/__init__.py b/src/auto_archiver/modules/gdrive_storage/__init__.py new file mode 100644 index 0000000..2765e4b --- /dev/null +++ b/src/auto_archiver/modules/gdrive_storage/__init__.py @@ -0,0 +1 @@ +from .gdrive_storage import GDriveStorage \ No newline at end of file diff --git a/src/auto_archiver/modules/gdrive_storage/__manifest__.py b/src/auto_archiver/modules/gdrive_storage/__manifest__.py new file mode 100644 index 0000000..632e52b --- /dev/null +++ b/src/auto_archiver/modules/gdrive_storage/__manifest__.py @@ -0,0 +1,99 @@ +{ + "name": "Google Drive Storage", + "type": ["storage"], + "author": "Dave Mateer", + "entry_point": "gdrive_storage::GDriveStorage", + "requires_setup": True, + "dependencies": { + "python": [ + "loguru", + "googleapiclient", + "google", + ], + }, + "configs": { + "path_generator": { + "default": "url", + "help": "how to store the file in terms of directory structure: 'flat' sets to root; 'url' creates a directory based on the provided URL; 'random' creates a random directory.", + "choices": ["flat", "url", "random"], + }, + "filename_generator": { + "default": "static", + "help": "how to name stored files: 'random' creates a random string; 'static' uses a replicable strategy such as a hash.", + "choices": ["random", "static"], + }, + "root_folder_id": {"required": True, + "help": "root google drive folder ID to use as storage, found in URL: 'https://drive.google.com/drive/folders/FOLDER_ID'"}, + "oauth_token": {"default": None, + "help": "JSON filename with Google Drive OAuth token: check auto-archiver repository scripts folder for create_update_gdrive_oauth_token.py. NOTE: storage used will count towards owner of GDrive folder, therefore it is best to use oauth_token_filename over service_account."}, + "service_account": {"default": "secrets/service_account.json", "help": "service account JSON file path, same as used for Google Sheets. NOTE: storage used will count towards the developer account."}, + }, + "description": """ + + GDriveStorage: A storage module for saving archived content to Google Drive. + + Author: Dave Mateer, (And maintained by: ) + Source Documentation: https://davemateer.com/2022/04/28/google-drive-with-python + + ### Features + - Saves media files to Google Drive, organizing them into folders based on the provided path structure. + - Supports OAuth token-based authentication or service account credentials for API access. + - Automatically creates folders in Google Drive if they don't exist. + - Retrieves CDN URLs for stored files, enabling easy sharing and access. + + ### Notes + - Requires setup with either a Google OAuth token or a service account JSON file. + - Files are uploaded to the specified `root_folder_id` and organized by the `media.key` structure. + - Automatically handles Google Drive API token refreshes for long-running jobs. + + ## Overview +This module integrates Google Drive as a storage backend, enabling automatic folder creation and file uploads. It supports authentication via **service accounts** (recommended for automation) or **OAuth tokens** (for user-based authentication). + +## Features +- Saves files to Google Drive, organizing them into structured folders. +- Supports both **service account** and **OAuth token** authentication. +- Automatically creates folders if they don't exist. +- Generates public URLs for easy file sharing. + +## Setup Guide +1. **Enable Google Drive API** + - Create a Google Cloud project at [Google Cloud Console](https://console.cloud.google.com/) + - Enable the **Google Drive API**. + +2. **Set Up a Google Drive Folder** + - Create a folder in **Google Drive** and copy its **folder ID** from the URL. + - Add the **folder ID** to your configuration (`orchestration.yaml`): + ```yaml + root_folder_id: "FOLDER_ID" + ``` + +3. **Authentication Options** + - **Option 1: Service Account (Recommended)** + - Create a **service account** in Google Cloud IAM. + - Download the JSON key file and save it as: + ``` + secrets/service_account.json + ``` + - **Share your Drive folder** with the service account’s `client_email` (found in the JSON file). + + - **Option 2: OAuth Token (User Authentication)** + - Create OAuth **Desktop App credentials** in Google Cloud. + - Save the credentials as: + ``` + secrets/oauth_credentials.json + ``` + - Generate an OAuth token by running: + ```sh + python scripts/create_update_gdrive_oauth_token.py -c secrets/oauth_credentials.json + ``` + + + Notes on the OAuth token: + Tokens are refreshed after 1 hour however keep working for 7 days (tbc) + so as long as the job doesn't last for 7 days then this method of refreshing only once per run will work + see this link for details on the token: + https://davemateer.com/2022/04/28/google-drive-with-python#tokens + + +""" +} diff --git a/src/auto_archiver/storages/gd.py b/src/auto_archiver/modules/gdrive_storage/gdrive_storage.py similarity index 61% rename from src/auto_archiver/storages/gd.py rename to src/auto_archiver/modules/gdrive_storage/gdrive_storage.py index 61c5b21..4971030 100644 --- a/src/auto_archiver/storages/gd.py +++ b/src/auto_archiver/modules/gdrive_storage/gdrive_storage.py @@ -1,79 +1,67 @@ -import shutil, os, time, json +import json +import os +import time from typing import IO -from loguru import logger -from googleapiclient.discovery import build -from googleapiclient.http import MediaFileUpload +from google.auth.transport.requests import Request from google.oauth2 import service_account from google.oauth2.credentials import Credentials -from google.auth.transport.requests import Request +from googleapiclient.discovery import build +from googleapiclient.http import MediaFileUpload +from loguru import logger + +from auto_archiver.core import Media +from auto_archiver.core import Storage + -from ..core import Media -from . import Storage class GDriveStorage(Storage): - name = "gdrive_storage" - def __init__(self, config: dict) -> None: - super().__init__(config) + def setup(self) -> None: + self.scopes = ['https://www.googleapis.com/auth/drive'] + # Initialize Google Drive service + self._setup_google_drive_service() - SCOPES = ['https://www.googleapis.com/auth/drive'] - - if self.oauth_token is not None: - """ - Tokens are refreshed after 1 hour - however keep working for 7 days (tbc) - so as long as the job doesn't last for 7 days - then this method of refreshing only once per run will work - see this link for details on the token - https://davemateer.com/2022/04/28/google-drive-with-python#tokens - """ - logger.debug(f'Using GD OAuth token {self.oauth_token}') - # workaround for missing 'refresh_token' in from_authorized_user_file - with open(self.oauth_token, 'r') as stream: - creds_json = json.load(stream) - creds_json['refresh_token'] = creds_json.get("refresh_token", "") - creds = Credentials.from_authorized_user_info(creds_json, SCOPES) - # creds = Credentials.from_authorized_user_file(self.oauth_token, SCOPES) - - if not creds or not creds.valid: - if creds and creds.expired and creds.refresh_token: - logger.debug('Requesting new GD OAuth token') - creds.refresh(Request()) - else: - raise Exception("Problem with creds - create the token again") - - # Save the credentials for the next run - with open(self.oauth_token, 'w') as token: - logger.debug('Saving new GD OAuth token') - token.write(creds.to_json()) - else: - logger.debug('GD OAuth Token valid') + def _setup_google_drive_service(self): + """Initialize Google Drive service based on provided credentials.""" + if self.oauth_token: + logger.debug(f"Using Google Drive OAuth token: {self.oauth_token}") + self.service = self._initialize_with_oauth_token() + elif self.service_account: + logger.debug(f"Using Google Drive service account: {self.service_account}") + self.service = self._initialize_with_service_account() else: - gd_service_account = self.service_account - logger.debug(f'Using GD Service Account {gd_service_account}') - creds = service_account.Credentials.from_service_account_file(gd_service_account, scopes=SCOPES) + raise ValueError("Missing credentials: either `oauth_token` or `service_account` must be provided.") - self.service = build('drive', 'v3', credentials=creds) + def _initialize_with_oauth_token(self): + """Initialize Google Drive service with OAuth token.""" + with open(self.oauth_token, 'r') as stream: + creds_json = json.load(stream) + creds_json['refresh_token'] = creds_json.get("refresh_token", "") - @staticmethod - def configs() -> dict: - return dict( - Storage.configs(), - ** { - "root_folder_id": {"default": None, "help": "root google drive folder ID to use as storage, found in URL: 'https://drive.google.com/drive/folders/FOLDER_ID'"}, - "oauth_token": {"default": None, "help": "JSON filename with Google Drive OAuth token: check auto-archiver repository scripts folder for create_update_gdrive_oauth_token.py. NOTE: storage used will count towards owner of GDrive folder, therefore it is best to use oauth_token_filename over service_account."}, - "service_account": {"default": "secrets/service_account.json", "help": "service account JSON file path, same as used for Google Sheets. NOTE: storage used will count towards the developer account."}, - }) + creds = Credentials.from_authorized_user_info(creds_json, self.scopes) + if not creds.valid and creds.expired and creds.refresh_token: + creds.refresh(Request()) + with open(self.oauth_token, 'w') as token_file: + logger.debug("Saving refreshed OAuth token.") + token_file.write(creds.to_json()) + elif not creds.valid: + raise ValueError("Invalid OAuth token. Please regenerate the token.") + + return build('drive', 'v3', credentials=creds) + + def _initialize_with_service_account(self): + """Initialize Google Drive service with service account.""" + creds = service_account.Credentials.from_service_account_file(self.service_account, scopes=self.scopes) + return build('drive', 'v3', credentials=creds) def get_cdn_url(self, media: Media) -> str: """ only support files saved in a folder for GD S3 supports folder and all stored in the root """ - # full_name = os.path.join(self.folder, media.key) parent_id, folder_id = self.root_folder_id, None path_parts = media.key.split(os.path.sep) @@ -82,13 +70,16 @@ class GDriveStorage(Storage): for folder in path_parts[0:-1]: folder_id = self._get_id_from_parent_and_name(parent_id, folder, use_mime_type=True, raise_on_missing=True) parent_id = folder_id - # get id of file inside folder (or sub folder) - file_id = self._get_id_from_parent_and_name(folder_id, filename) + file_id = self._get_id_from_parent_and_name(folder_id, filename, raise_on_missing=True) + if not file_id: + # + logger.info(f"file {filename} not found in folder {folder_id}") + return None return f"https://drive.google.com/file/d/{file_id}/view?usp=sharing" def upload(self, media: Media, **kwargs) -> bool: - logger.debug(f'[{self.__class__.name}] storing file {media.filename} with key {media.key}') + logger.debug(f'[{self.__class__.__name__}] storing file {media.filename} with key {media.key}') """ 1. for each sub-folder in the path check if exists or create 2. upload file to root_id/other_paths.../filename @@ -116,7 +107,13 @@ class GDriveStorage(Storage): # must be implemented even if unused def uploadf(self, file: IO[bytes], key: str, **kwargs: dict) -> bool: pass - def _get_id_from_parent_and_name(self, parent_id: str, name: str, retries: int = 1, sleep_seconds: int = 10, use_mime_type: bool = False, raise_on_missing: bool = True, use_cache=False): + def _get_id_from_parent_and_name(self, parent_id: str, + name: str, + retries: int = 1, + sleep_seconds: int = 10, + use_mime_type: bool = False, + raise_on_missing: bool = True, + use_cache=False): """ Retrieves the id of a folder or file from its @name and the @parent_id folder Optionally does multiple @retries and sleeps @sleep_seconds between them @@ -179,8 +176,3 @@ class GDriveStorage(Storage): gd_folder = self.service.files().create(supportsAllDrives=True, body=file_metadata, fields='id').execute() return gd_folder.get('id') - # def exists(self, key): - # try: - # self.get_cdn_url(key) - # return True - # except: return False diff --git a/src/auto_archiver/modules/generic_extractor/__init__.py b/src/auto_archiver/modules/generic_extractor/__init__.py new file mode 100644 index 0000000..5bfcd01 --- /dev/null +++ b/src/auto_archiver/modules/generic_extractor/__init__.py @@ -0,0 +1 @@ +from .generic_extractor import GenericExtractor \ No newline at end of file diff --git a/src/auto_archiver/modules/generic_extractor/__manifest__.py b/src/auto_archiver/modules/generic_extractor/__manifest__.py new file mode 100644 index 0000000..caa3ae1 --- /dev/null +++ b/src/auto_archiver/modules/generic_extractor/__manifest__.py @@ -0,0 +1,68 @@ +{ + "name": "Generic Extractor", + "version": "0.1.0", + "author": "Bellingcat", + "type": ["extractor"], + "requires_setup": False, + "dependencies": { + "python": ["yt_dlp", "requests", "loguru", "slugify"], + }, + "description": """ +This is the generic extractor used by auto-archiver, which uses `yt-dlp` under the hood. + +This module is responsible for downloading and processing media content from platforms +supported by `yt-dlp`, such as YouTube, Facebook, and others. It provides functionality +for retrieving videos, subtitles, comments, and other metadata, and it integrates with +the broader archiving framework. + +### Features +- Supports downloading videos and playlists. +- Retrieves metadata like titles, descriptions, upload dates, and durations. +- Downloads subtitles and comments when enabled. +- Configurable options for handling live streams, proxies, and more. +- Supports authentication of websites using the 'authentication' settings from your orchestration. + +### Dropins +- For websites supported by `yt-dlp` that also contain posts in addition to videos + (e.g. Facebook, Twitter, Bluesky), dropins can be created to extract post data and create + metadata objects. Some dropins are included in this generic_archiver by default, but +custom dropins can be created to handle additional websites and passed to the archiver +via the command line using the `--dropins` option (TODO!). +""", + "configs": { + "subtitles": {"default": True, "help": "download subtitles if available", "type": "bool"}, + "comments": { + "default": False, + "help": "download all comments if available, may lead to large metadata", + "type": "bool", + }, + "livestreams": { + "default": False, + "help": "if set, will download live streams, otherwise will skip them; see --max-filesize for more control", + "type": "bool", + }, + "live_from_start": { + "default": False, + "help": "if set, will download live streams from their earliest available moment, otherwise starts now.", + "type": "bool", + }, + "proxy": { + "default": "", + "help": "http/socks (https seems to not work atm) proxy to use for the webdriver, eg https://proxy-user:password@proxy-ip:port", + }, + "end_means_success": { + "default": True, + "help": "if True, any archived content will mean a 'success', if False this archiver will not return a 'success' stage; this is useful for cases when the yt-dlp will archive a video but ignore other types of content like images or text only pages that the subsequent archivers can retrieve.", + "type": "bool", + }, + "allow_playlist": { + "default": False, + "help": "If True will also download playlists, set to False if the expectation is to download a single video.", + "type": "bool", + }, + "max_downloads": { + "default": "inf", + "help": "Use to limit the number of videos to download when a channel or long page is being extracted. 'inf' means no limit.", + }, + }, +} diff --git a/src/auto_archiver/archivers/generic_archiver/bluesky.py b/src/auto_archiver/modules/generic_extractor/bluesky.py similarity index 78% rename from src/auto_archiver/archivers/generic_archiver/bluesky.py rename to src/auto_archiver/modules/generic_extractor/bluesky.py index 7aa9c39..5eef520 100644 --- a/src/auto_archiver/archivers/generic_archiver/bluesky.py +++ b/src/auto_archiver/modules/generic_extractor/bluesky.py @@ -1,17 +1,12 @@ -import os -import mimetypes - -import requests from loguru import logger -from auto_archiver.core.context import ArchivingContext -from auto_archiver.archivers.archiver import Archiver +from auto_archiver.core.extractor import Extractor from auto_archiver.core.metadata import Metadata, Media from .dropin import GenericDropin, InfoExtractor class Bluesky(GenericDropin): - def create_metadata(self, post: dict, ie_instance: InfoExtractor, archiver: Archiver, url: str) -> Metadata: + def create_metadata(self, post: dict, ie_instance: InfoExtractor, archiver: Extractor, url: str) -> Metadata: result = Metadata() result.set_url(url) result.set_title(post["record"]["text"]) @@ -28,21 +23,10 @@ class Bluesky(GenericDropin): def extract_post(self, url: str, ie_instance: InfoExtractor) -> dict: # TODO: If/when this PR (https://github.com/yt-dlp/yt-dlp/pull/12098) is merged on ytdlp, remove the comments and delete the code below - # handle, video_id = ie_instance._match_valid_url(url).group('handle', 'id') - # return ie_instance._extract_post(handle=handle, post_id=video_id) - handle, video_id = ie_instance._match_valid_url(url).group('handle', 'id') - return ie_instance._download_json( - 'https://public.api.bsky.app/xrpc/app.bsky.feed.getPostThread', - video_id, query={ - 'uri': f'at://{handle}/app.bsky.feed.post/{video_id}', - 'depth': 0, - 'parentHeight': 0, - })['thread']['post'] + return ie_instance._extract_post(handle=handle, post_id=video_id) - - - def _download_bsky_embeds(self, post: dict, archiver: Archiver) -> list[Media]: + def _download_bsky_embeds(self, post: dict, archiver: Extractor) -> list[Media]: """ Iterates over image(s) or video in a Bluesky post and downloads them """ @@ -55,11 +39,11 @@ class Bluesky(GenericDropin): for image_media in image_medias: url = media_url.format(image_media['image']['ref']['$link'], post['author']['did']) image_media = archiver.download_from_url(url) - media.append(image_media) + media.append(Media(image_media)) for video_media in video_medias: url = media_url.format(video_media['ref']['$link'], post['author']['did']) video_media = archiver.download_from_url(url) - media.append(video_media) + media.append(Media(video_media)) return media diff --git a/src/auto_archiver/archivers/generic_archiver/dropin.py b/src/auto_archiver/modules/generic_extractor/dropin.py similarity index 95% rename from src/auto_archiver/archivers/generic_archiver/dropin.py rename to src/auto_archiver/modules/generic_extractor/dropin.py index 37f3faf..c5749ff 100644 --- a/src/auto_archiver/archivers/generic_archiver/dropin.py +++ b/src/auto_archiver/modules/generic_extractor/dropin.py @@ -1,6 +1,6 @@ from yt_dlp.extractor.common import InfoExtractor from auto_archiver.core.metadata import Metadata -from auto_archiver.archivers.archiver import Archiver +from auto_archiver.core.extractor import Extractor class GenericDropin: """Base class for dropins for the generic extractor. @@ -30,7 +30,7 @@ class GenericDropin: raise NotImplementedError("This method should be implemented in the subclass") - def create_metadata(self, post: dict, ie_instance: InfoExtractor, archiver: Archiver, url: str) -> Metadata: + def create_metadata(self, post: dict, ie_instance: InfoExtractor, archiver: Extractor, url: str) -> Metadata: """ This method should create a Metadata object from the post data. """ diff --git a/src/auto_archiver/modules/generic_extractor/facebook.py b/src/auto_archiver/modules/generic_extractor/facebook.py new file mode 100644 index 0000000..fed8e09 --- /dev/null +++ b/src/auto_archiver/modules/generic_extractor/facebook.py @@ -0,0 +1,18 @@ +from .dropin import GenericDropin + + +class Facebook(GenericDropin): + def extract_post(self, url: str, ie_instance): + video_id = ie_instance._match_valid_url(url).group('id') + ie_instance._download_webpage( + url.replace('://m.facebook.com/', '://www.facebook.com/'), video_id) + webpage = ie_instance._download_webpage(url, ie_instance._match_valid_url(url).group('id')) + + # TODO: fix once https://github.com/yt-dlp/yt-dlp/pull/12275 is merged + post_data = ie_instance._extract_metadata(webpage) + return post_data + + def create_metadata(self, post: dict, ie_instance, archiver, url): + metadata = archiver.create_metadata(url) + metadata.set_title(post.get('title')).set_content(post.get('description')).set_post_data(post) + return metadata \ No newline at end of file diff --git a/src/auto_archiver/archivers/generic_archiver/generic_archiver.py b/src/auto_archiver/modules/generic_extractor/generic_extractor.py similarity index 81% rename from src/auto_archiver/archivers/generic_archiver/generic_archiver.py rename to src/auto_archiver/modules/generic_extractor/generic_extractor.py index 1574e65..6bcb249 100644 --- a/src/auto_archiver/archivers/generic_archiver/generic_archiver.py +++ b/src/auto_archiver/modules/generic_extractor/generic_extractor.py @@ -1,53 +1,17 @@ -""" -This is the generic archiver used by auto-archiver, which uses `yt-dlp` under the hood. - -This module is responsible for downloading and processing media content from platforms -supported by `yt-dlp`, such as YouTube, Facebook, and others. It provides functionality -for retrieving videos, subtitles, comments, and other metadata, and it integrates with -the broader archiving framework. - -### Features -- Supports downloading videos and playlists. -- Retrieves metadata like titles, descriptions, upload dates, and durations. -- Downloads subtitles and comments when enabled. -- Configurable options for handling live streams, proxies, and more. - -### Dropins -- For websites supported by `yt-dlp` that also contain posts in addition to videos - (e.g. Facebook, Twitter, Bluesky), dropins can be created to extract post data and create - metadata objects. Some dropins are included in this generic_archiver by default, but -custom dropins can be created to handle additional websites and passed to the archiver -via the command line using the `--dropins` option (TODO!). - -""" - - import datetime, os, yt_dlp, pysubs2 import importlib -from typing import Type +from typing import Generator, Type from yt_dlp.extractor.common import InfoExtractor from loguru import logger -from auto_archiver.archivers.archiver import Archiver -from ...core import Metadata, Media, ArchivingContext +from auto_archiver.core.extractor import Extractor +from auto_archiver.core import Metadata, Media -class GenericArchiver(Archiver): - name = "youtubedl_archiver" #left as is for backwards compat +class GenericExtractor(Extractor): _dropins = {} - def __init__(self, config: dict) -> None: - super().__init__(config) - self.subtitles = bool(self.subtitles) - self.comments = bool(self.comments) - self.livestreams = bool(self.livestreams) - self.live_from_start = bool(self.live_from_start) - self.end_means_success = bool(self.end_means_success) - self.allow_playlist = bool(self.allow_playlist) - self.max_downloads = self.max_downloads - - - def suitable_extractors(self, url: str) -> list[str]: + def suitable_extractors(self, url: str) -> Generator[str, None, None]: """ Returns a list of valid extractors for the given URL""" for info_extractor in yt_dlp.YoutubeDL()._ies.values(): @@ -152,11 +116,12 @@ class GenericArchiver(Archiver): def get_metadata_for_post(self, info_extractor: Type[InfoExtractor], url: str, ydl: yt_dlp.YoutubeDL) -> Metadata: """ - Calls into the ytdlp InfoExtract subclass to use the prive _extract_post method to get the post metadata. + Calls into the ytdlp InfoExtract subclass to use the private _extract_post method to get the post metadata. """ ie_instance = info_extractor(downloader=ydl) dropin = self.dropin_for_name(info_extractor.ie_key()) + if not dropin: # TODO: add a proper link to 'how to create your own dropin' logger.debug(f"""Could not find valid dropin for {info_extractor.IE_NAME}. @@ -207,6 +172,7 @@ class GenericArchiver(Archiver): return self.add_metadata(data, info_extractor, url, result) def dropin_for_name(self, dropin_name: str, additional_paths = [], package=__package__) -> Type[InfoExtractor]: + dropin_name = dropin_name.lower() if dropin_name == "generic": # no need for a dropin for the generic extractor (?) @@ -300,19 +266,35 @@ class GenericArchiver(Archiver): def download(self, item: Metadata) -> Metadata: url = item.get_url() - if item.netloc in ['facebook.com', 'www.facebook.com'] and self.facebook_cookie: - logger.debug('Using Facebook cookie') - yt_dlp.utils.std_headers['cookie'] = self.facebook_cookie - - ydl_options = {'outtmpl': os.path.join(ArchivingContext.get_tmp_dir(), f'%(id)s.%(ext)s'), 'quiet': False, 'noplaylist': not self.allow_playlist , 'writesubtitles': self.subtitles, 'writeautomaticsub': self.subtitles, "live_from_start": self.live_from_start, "proxy": self.proxy, "max_downloads": self.max_downloads, "playlistend": self.max_downloads} + #TODO: this is a temporary hack until this issue is closed: https://github.com/yt-dlp/yt-dlp/issues/11025 + if url.startswith("https://ya.ru"): + url = url.replace("https://ya.ru", "https://yandex.ru") + item.set("replaced_url", url) - if item.netloc in ['youtube.com', 'www.youtube.com']: - if self.cookies_from_browser: - logger.debug(f'Extracting cookies from browser {self.cookies_from_browser} for Youtube') - ydl_options['cookiesfrombrowser'] = (self.cookies_from_browser,) - elif self.cookie_file: - logger.debug(f'Using cookies from file {self.cookie_file}') - ydl_options['cookiefile'] = self.cookie_file + + ydl_options = {'outtmpl': os.path.join(self.tmp_dir, f'%(id)s.%(ext)s'), + 'quiet': False, 'noplaylist': not self.allow_playlist , + 'writesubtitles': self.subtitles,'writeautomaticsub': self.subtitles, + "live_from_start": self.live_from_start, "proxy": self.proxy, + "max_downloads": self.max_downloads, "playlistend": self.max_downloads} + + # set up auth + auth = self.auth_for_site(url, extract_cookies=False) + # order of importance: username/pasword -> api_key -> cookie -> cookie_from_browser -> cookies_file + if auth: + if 'username' in auth and 'password' in auth: + logger.debug(f'Using provided auth username and password for {url}') + ydl_options['username'] = auth['username'] + ydl_options['password'] = auth['password'] + elif 'cookie' in auth: + logger.debug(f'Using provided auth cookie for {url}') + yt_dlp.utils.std_headers['cookie'] = auth['cookie'] + elif 'cookie_from_browser' in auth: + logger.debug(f'Using extracted cookies from browser {self.cookies_from_browser} for {url}') + ydl_options['cookiesfrombrowser'] = auth['cookies_from_browser'] + elif 'cookies_file' in auth: + logger.debug(f'Using cookies from file {self.cookie_file} for {url}') + ydl_options['cookiesfile'] = auth['cookies_file'] ydl = yt_dlp.YoutubeDL(ydl_options) # allsubtitles and subtitleslangs not working as expected, so default lang is always "en" @@ -320,6 +302,6 @@ class GenericArchiver(Archiver): result = self.download_for_extractor(info_extractor, url, ydl) if result: return result - + return False diff --git a/src/auto_archiver/archivers/generic_archiver/truth.py b/src/auto_archiver/modules/generic_extractor/truth.py similarity index 97% rename from src/auto_archiver/archivers/generic_archiver/truth.py rename to src/auto_archiver/modules/generic_extractor/truth.py index bf19dce..e65b4b1 100644 --- a/src/auto_archiver/archivers/generic_archiver/truth.py +++ b/src/auto_archiver/modules/generic_extractor/truth.py @@ -2,7 +2,7 @@ from typing import Type from auto_archiver.utils import traverse_obj from auto_archiver.core.metadata import Metadata, Media -from auto_archiver.archivers.archiver import Archiver +from auto_archiver.core.extractor import Extractor from yt_dlp.extractor.common import InfoExtractor from dateutil.parser import parse as parse_dt @@ -19,7 +19,7 @@ class Truth(GenericDropin): def skip_ytdlp_download(self, url, ie_instance: Type[InfoExtractor]) -> bool: return True - def create_metadata(self, post: dict, ie_instance: InfoExtractor, archiver: Archiver, url: str) -> Metadata: + def create_metadata(self, post: dict, ie_instance: InfoExtractor, archiver: Extractor, url: str) -> Metadata: """ Creates metadata from a truth social post diff --git a/src/auto_archiver/archivers/generic_archiver/twitter.py b/src/auto_archiver/modules/generic_extractor/twitter.py similarity index 95% rename from src/auto_archiver/archivers/generic_archiver/twitter.py rename to src/auto_archiver/modules/generic_extractor/twitter.py index ce6c28d..3faed6b 100644 --- a/src/auto_archiver/archivers/generic_archiver/twitter.py +++ b/src/auto_archiver/modules/generic_extractor/twitter.py @@ -5,8 +5,8 @@ from loguru import logger from slugify import slugify from auto_archiver.core.metadata import Metadata, Media -from auto_archiver.utils import UrlUtil -from auto_archiver.archivers.archiver import Archiver +from auto_archiver.utils import url as UrlUtil +from auto_archiver.core.extractor import Extractor from .dropin import GenericDropin, InfoExtractor @@ -32,7 +32,7 @@ class Twitter(GenericDropin): twid = ie_instance._match_valid_url(url).group('id') return ie_instance._extract_status(twid=twid) - def create_metadata(self, tweet: dict, ie_instance: InfoExtractor, archiver: Archiver, url: str) -> Metadata: + def create_metadata(self, tweet: dict, ie_instance: InfoExtractor, archiver: Extractor, url: str) -> Metadata: result = Metadata() try: if not tweet.get("user") or not tweet.get("created_at"): diff --git a/src/auto_archiver/modules/gsheet_db/__init__.py b/src/auto_archiver/modules/gsheet_db/__init__.py new file mode 100644 index 0000000..01fdee6 --- /dev/null +++ b/src/auto_archiver/modules/gsheet_db/__init__.py @@ -0,0 +1 @@ +from .gsheet_db import GsheetsDb \ No newline at end of file diff --git a/src/auto_archiver/modules/gsheet_db/__manifest__.py b/src/auto_archiver/modules/gsheet_db/__manifest__.py new file mode 100644 index 0000000..cf95245 --- /dev/null +++ b/src/auto_archiver/modules/gsheet_db/__manifest__.py @@ -0,0 +1,38 @@ +{ + "name": "Google Sheets Database", + "type": ["database"], + "entry_point": "gsheet_db::GsheetsDb", + "requires_setup": True, + "dependencies": { + "python": ["loguru", "gspread", "slugify"], + }, + "configs": { + "allow_worksheets": { + "default": set(), + "help": "(CSV) only worksheets whose name is included in allow are included (overrides worksheet_block), leave empty so all are allowed", + }, + "block_worksheets": { + "default": set(), + "help": "(CSV) explicitly block some worksheets from being processed", + }, + "use_sheet_names_in_stored_paths": { + "default": True, + "type": "bool", + "help": "if True the stored files path will include 'workbook_name/worksheet_name/...'", + } + }, + "description": """ + GsheetsDatabase: + Handles integration with Google Sheets for tracking archival tasks. + +### Features +- Updates a Google Sheet with the status of the archived URLs, including in progress, success or failure, and method used. +- Saves metadata such as title, text, timestamp, hashes, screenshots, and media URLs to designated columns. +- Formats media-specific metadata, such as thumbnails and PDQ hashes for the sheet. +- Skips redundant updates for empty or invalid data fields. + +### Notes +- Currently works only with metadata provided by GsheetFeeder. +- Requires configuration of a linked Google Sheet and appropriate API credentials. + """ +} diff --git a/src/auto_archiver/databases/gsheet_db.py b/src/auto_archiver/modules/gsheet_db/gsheet_db.py similarity index 50% rename from src/auto_archiver/databases/gsheet_db.py rename to src/auto_archiver/modules/gsheet_db/gsheet_db.py index 98e72dc..c19f2ae 100644 --- a/src/auto_archiver/databases/gsheet_db.py +++ b/src/auto_archiver/modules/gsheet_db/gsheet_db.py @@ -1,47 +1,38 @@ from typing import Union, Tuple -import datetime from urllib.parse import quote from loguru import logger -from . import Database -from ..core import Metadata, Media, ArchivingContext -from ..utils import GWorksheet +from auto_archiver.core import Database +from auto_archiver.core import Metadata, Media +from auto_archiver.modules.gsheet_feeder import GWorksheet +from auto_archiver.utils.misc import get_current_timestamp class GsheetsDb(Database): """ - NB: only works if GsheetFeeder is used. - could be updated in the future to support non-GsheetFeeder metadata + NB: only works if GsheetFeeder is used. + could be updated in the future to support non-GsheetFeeder metadata """ - name = "gsheet_db" - - def __init__(self, config: dict) -> None: - # without this STEP.__init__ is not called - super().__init__(config) - - @staticmethod - def configs() -> dict: - return {} def started(self, item: Metadata) -> None: logger.warning(f"STARTED {item}") gw, row = self._retrieve_gsheet(item) - gw.set_cell(row, 'status', 'Archive in progress') + gw.set_cell(row, "status", "Archive in progress") - def failed(self, item: Metadata, reason:str) -> None: + def failed(self, item: Metadata, reason: str) -> None: logger.error(f"FAILED {item}") - self._safe_status_update(item, f'Archive failed {reason}') + self._safe_status_update(item, f"Archive failed {reason}") def aborted(self, item: Metadata) -> None: logger.warning(f"ABORTED {item}") - self._safe_status_update(item, '') + self._safe_status_update(item, "") def fetch(self, item: Metadata) -> Union[Metadata, bool]: """check if the given item has been archived already""" return False - def done(self, item: Metadata, cached: bool=False) -> None: + def done(self, item: Metadata, cached: bool = False) -> None: """archival result ready - should be saved to DB""" logger.success(f"DONE {item.get_url()}") gw, row = self._retrieve_gsheet(item) @@ -53,23 +44,25 @@ class GsheetsDb(Database): def batch_if_valid(col, val, final_value=None): final_value = final_value or val try: - if val and gw.col_exists(col) and gw.get_cell(row_values, col) == '': + if val and gw.col_exists(col) and gw.get_cell(row_values, col) == "": cell_updates.append((row, col, final_value)) except Exception as e: logger.error(f"Unable to batch {col}={final_value} due to {e}") + status_message = item.status if cached: status_message = f"[cached] {status_message}" - cell_updates.append((row, 'status', status_message)) + cell_updates.append((row, "status", status_message)) media: Media = item.get_final_media() if hasattr(media, "urls"): - batch_if_valid('archive', "\n".join(media.urls)) - batch_if_valid('date', True, datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=datetime.timezone.utc).isoformat()) - batch_if_valid('title', item.get_title()) - batch_if_valid('text', item.get("content", "")) - batch_if_valid('timestamp', item.get_timestamp()) - if media: batch_if_valid('hash', media.get("hash", "not-calculated")) + batch_if_valid("archive", "\n".join(media.urls)) + batch_if_valid("date", True, get_current_timestamp()) + batch_if_valid("title", item.get_title()) + batch_if_valid("text", item.get("content", "")) + batch_if_valid("timestamp", item.get_timestamp()) + if media: + batch_if_valid("hash", media.get("hash", "not-calculated")) # merge all pdq hashes into a single string, if present pdq_hashes = [] @@ -78,35 +71,44 @@ class GsheetsDb(Database): if pdq := m.get("pdq_hash"): pdq_hashes.append(pdq) if len(pdq_hashes): - batch_if_valid('pdq_hash', ",".join(pdq_hashes)) + batch_if_valid("pdq_hash", ",".join(pdq_hashes)) - if (screenshot := item.get_media_by_id("screenshot")) and hasattr(screenshot, "urls"): - batch_if_valid('screenshot', "\n".join(screenshot.urls)) + if (screenshot := item.get_media_by_id("screenshot")) and hasattr( + screenshot, "urls" + ): + batch_if_valid("screenshot", "\n".join(screenshot.urls)) - if (thumbnail := item.get_first_image("thumbnail")): + if thumbnail := item.get_first_image("thumbnail"): if hasattr(thumbnail, "urls"): - batch_if_valid('thumbnail', f'=IMAGE("{thumbnail.urls[0]}")') + batch_if_valid("thumbnail", f'=IMAGE("{thumbnail.urls[0]}")') - if (browsertrix := item.get_media_by_id("browsertrix")): - batch_if_valid('wacz', "\n".join(browsertrix.urls)) - batch_if_valid('replaywebpage', "\n".join([f'https://replayweb.page/?source={quote(wacz)}#view=pages&url={quote(item.get_url())}' for wacz in browsertrix.urls])) + if browsertrix := item.get_media_by_id("browsertrix"): + batch_if_valid("wacz", "\n".join(browsertrix.urls)) + batch_if_valid( + "replaywebpage", + "\n".join( + [ + f"https://replayweb.page/?source={quote(wacz)}#view=pages&url={quote(item.get_url())}" + for wacz in browsertrix.urls + ] + ), + ) gw.batch_set_cell(cell_updates) def _safe_status_update(self, item: Metadata, new_status: str) -> None: try: gw, row = self._retrieve_gsheet(item) - gw.set_cell(row, 'status', new_status) + gw.set_cell(row, "status", new_status) except Exception as e: logger.debug(f"Unable to update sheet: {e}") def _retrieve_gsheet(self, item: Metadata) -> Tuple[GWorksheet, int]: - # TODO: to make gsheet_db less coupled with gsheet_feeder's "gsheet" parameter, this method could 1st try to fetch "gsheet" from ArchivingContext and, if missing, manage its own singleton - not needed for now - if gsheet := ArchivingContext.get("gsheet"): + + if gsheet := item.get_context("gsheet"): gw: GWorksheet = gsheet.get("worksheet") row: int = gsheet.get("row") elif self.sheet_id: - print(self.sheet_id) - + logger.error(f"Unable to retrieve Gsheet for {item.get_url()}, GsheetDB must be used alongside GsheetFeeder.") return gw, row diff --git a/src/auto_archiver/modules/gsheet_feeder/__init__.py b/src/auto_archiver/modules/gsheet_feeder/__init__.py new file mode 100644 index 0000000..bb4230a --- /dev/null +++ b/src/auto_archiver/modules/gsheet_feeder/__init__.py @@ -0,0 +1,2 @@ +from .gworksheet import GWorksheet +from .gsheet_feeder import GsheetsFeeder \ No newline at end of file diff --git a/src/auto_archiver/modules/gsheet_feeder/__manifest__.py b/src/auto_archiver/modules/gsheet_feeder/__manifest__.py new file mode 100644 index 0000000..7b74072 --- /dev/null +++ b/src/auto_archiver/modules/gsheet_feeder/__manifest__.py @@ -0,0 +1,71 @@ +{ + "name": "Google Sheets Feeder", + "type": ["feeder"], + "entry_point": "gsheet_feeder::GsheetsFeeder", + "requires_setup": True, + "dependencies": { + "python": ["loguru", "gspread", "slugify"], + }, + "configs": { + "sheet": {"default": None, "help": "name of the sheet to archive"}, + "sheet_id": { + "default": None, + "help": "(alternative to sheet name) the id of the sheet to archive", + }, + "header": {"default": 1, "help": "index of the header row (starts at 1)", "type": "int"}, + "service_account": { + "default": "secrets/service_account.json", + "help": "service account JSON file path", + }, + "columns": { + "default": { + "url": "link", + "status": "archive status", + "folder": "destination folder", + "archive": "archive location", + "date": "archive date", + "thumbnail": "thumbnail", + "timestamp": "upload timestamp", + "title": "upload title", + "text": "text content", + "screenshot": "screenshot", + "hash": "hash", + "pdq_hash": "perceptual hashes", + "wacz": "wacz", + "replaywebpage": "replaywebpage", + }, + "help": "names of columns in the google sheet (stringified JSON object)", + "type": "auto_archiver.utils.json_loader", + }, + "allow_worksheets": { + "default": set(), + "help": "(CSV) only worksheets whose name is included in allow are included (overrides worksheet_block), leave empty so all are allowed", + }, + "block_worksheets": { + "default": set(), + "help": "(CSV) explicitly block some worksheets from being processed", + }, + "use_sheet_names_in_stored_paths": { + "default": True, + "help": "if True the stored files path will include 'workbook_name/worksheet_name/...'", + "type": "bool", + }, + }, + "description": """ + GsheetsFeeder + A Google Sheets-based feeder for the Auto Archiver. + + This reads data from Google Sheets and filters rows based on user-defined rules. + The filtered rows are processed into `Metadata` objects. + + ### Features + - Validates the sheet structure and filters rows based on input configurations. + - Processes only worksheets allowed by the `allow_worksheets` and `block_worksheets` configurations. + - Ensures only rows with valid URLs and unprocessed statuses are included for archival. + - Supports organizing stored files into folder paths based on sheet and worksheet names. + + ### Notes + - Requires a Google Service Account JSON file for authentication. Suggested location is `secrets/gsheets_service_account.json`. + - Create the sheet using the template provided in the docs. + """, +} diff --git a/src/auto_archiver/modules/gsheet_feeder/gsheet_feeder.py b/src/auto_archiver/modules/gsheet_feeder/gsheet_feeder.py new file mode 100644 index 0000000..8612d02 --- /dev/null +++ b/src/auto_archiver/modules/gsheet_feeder/gsheet_feeder.py @@ -0,0 +1,96 @@ +""" +GsheetsFeeder: A Google Sheets-based feeder for the Auto Archiver. + +This reads data from Google Sheets and filters rows based on user-defined rules. +The filtered rows are processed into `Metadata` objects. + +### Key properties +- validates the sheet's structure and filters rows based on input configurations. +- Ensures only rows with valid URLs and unprocessed statuses are included. +""" +import os +import gspread + +from loguru import logger +from slugify import slugify + +from auto_archiver.core import Feeder +from auto_archiver.core import Metadata +from . import GWorksheet + + +class GsheetsFeeder(Feeder): + + def setup(self) -> None: + self.gsheets_client = gspread.service_account(filename=self.service_account) + # TODO mv to validators + assert self.sheet or self.sheet_id, ( + "You need to define either a 'sheet' name or a 'sheet_id' in your manifest." + ) + + def open_sheet(self): + if self.sheet: + return self.gsheets_client.open(self.sheet) + else: # self.sheet_id + return self.gsheets_client.open_by_key(self.sheet_id) + + def __iter__(self) -> Metadata: + sh = self.open_sheet() + for ii, worksheet in enumerate(sh.worksheets()): + if not self.should_process_sheet(worksheet.title): + logger.debug(f"SKIPPED worksheet '{worksheet.title}' due to allow/block rules") + continue + logger.info(f'Opening worksheet {ii=}: {worksheet.title=} header={self.header}') + gw = GWorksheet(worksheet, header_row=self.header, columns=self.columns) + if len(missing_cols := self.missing_required_columns(gw)): + logger.warning(f"SKIPPED worksheet '{worksheet.title}' due to missing required column(s) for {missing_cols}") + continue + + # process and yield metadata here: + yield from self._process_rows(gw) + logger.success(f'Finished worksheet {worksheet.title}') + + def _process_rows(self, gw: GWorksheet): + for row in range(1 + self.header, gw.count_rows() + 1): + url = gw.get_cell(row, 'url').strip() + if not len(url): continue + original_status = gw.get_cell(row, 'status') + status = gw.get_cell(row, 'status', fresh=original_status in ['', None]) + # TODO: custom status parser(?) aka should_retry_from_status + if status not in ['', None]: continue + + # All checks done - archival process starts here + m = Metadata().set_url(url) + self._set_context(m, gw, row) + yield m + + def _set_context(self, m: Metadata, gw: GWorksheet, row: int) -> Metadata: + # TODO: Check folder value not being recognised + m.set_context("gsheet", {"row": row, "worksheet": gw}) + + if gw.get_cell_or_default(row, 'folder', "") is None: + folder = '' + else: + folder = slugify(gw.get_cell_or_default(row, 'folder', "").strip()) + if len(folder): + if self.use_sheet_names_in_stored_paths: + m.set_context("folder", os.path.join(folder, slugify(self.sheet), slugify(gw.wks.title))) + else: + m.set_context("folder", folder) + + + def should_process_sheet(self, sheet_name: str) -> bool: + if len(self.allow_worksheets) and sheet_name not in self.allow_worksheets: + # ALLOW rules exist AND sheet name not explicitly allowed + return False + if len(self.block_worksheets) and sheet_name in self.block_worksheets: + # BLOCK rules exist AND sheet name is blocked + return False + return True + + def missing_required_columns(self, gw: GWorksheet) -> list: + missing = [] + for required_col in ['url', 'status']: + if not gw.col_exists(required_col): + missing.append(required_col) + return missing diff --git a/src/auto_archiver/utils/gworksheet.py b/src/auto_archiver/modules/gsheet_feeder/gworksheet.py similarity index 100% rename from src/auto_archiver/utils/gworksheet.py rename to src/auto_archiver/modules/gsheet_feeder/gworksheet.py diff --git a/src/auto_archiver/modules/hash_enricher/__init__.py b/src/auto_archiver/modules/hash_enricher/__init__.py new file mode 100644 index 0000000..18ec885 --- /dev/null +++ b/src/auto_archiver/modules/hash_enricher/__init__.py @@ -0,0 +1 @@ +from .hash_enricher import HashEnricher \ No newline at end of file diff --git a/src/auto_archiver/modules/hash_enricher/__manifest__.py b/src/auto_archiver/modules/hash_enricher/__manifest__.py new file mode 100644 index 0000000..c7a023e --- /dev/null +++ b/src/auto_archiver/modules/hash_enricher/__manifest__.py @@ -0,0 +1,31 @@ +{ + "name": "Hash Enricher", + "type": ["enricher"], + "requires_setup": False, + "dependencies": { + "python": ["loguru"], + }, + "configs": { + "algorithm": {"default": "SHA-256", "help": "hash algorithm to use", "choices": ["SHA-256", "SHA3-512"]}, + # TODO add non-negative requirement to match previous implementation? + "chunksize": {"default": 16000000, + "help": "number of bytes to use when reading files in chunks (if this value is too large you will run out of RAM), default is 16MB", + 'type': 'int', + }, + }, + "description": """ +Generates cryptographic hashes for media files to ensure data integrity and authenticity. + +### Features +- Calculates cryptographic hashes (SHA-256 or SHA3-512) for media files stored in `Metadata` objects. +- Ensures content authenticity, integrity validation, and duplicate identification. +- Efficiently processes large files by reading file bytes in configurable chunk sizes. +- Supports dynamic configuration of hash algorithms and chunk sizes. +- Updates media metadata with the computed hash value in the format `:`. + +### Notes +- Default hash algorithm is SHA-256, but SHA3-512 is also supported. +- Chunk size defaults to 16 MB but can be adjusted based on memory requirements. +- Useful for workflows requiring hash-based content validation or deduplication. +""", +} diff --git a/src/auto_archiver/modules/hash_enricher/hash_enricher.py b/src/auto_archiver/modules/hash_enricher/hash_enricher.py new file mode 100644 index 0000000..7a0587c --- /dev/null +++ b/src/auto_archiver/modules/hash_enricher/hash_enricher.py @@ -0,0 +1,39 @@ +""" Hash Enricher for generating cryptographic hashes of media files. + +The `HashEnricher` calculates cryptographic hashes (e.g., SHA-256, SHA3-512) +for media files stored in `Metadata` objects. These hashes are used for +validating content integrity, ensuring data authenticity, and identifying +exact duplicates. The hash is computed by reading the file's bytes in chunks, +making it suitable for handling large files efficiently. + +""" +import hashlib +from loguru import logger + +from auto_archiver.core import Enricher +from auto_archiver.core import Metadata +from auto_archiver.utils.misc import calculate_file_hash + + +class HashEnricher(Enricher): + """ + Calculates hashes for Media instances + """ + + + def enrich(self, to_enrich: Metadata) -> None: + url = to_enrich.get_url() + logger.debug(f"calculating media hashes for {url=} (using {self.algorithm})") + + for i, m in enumerate(to_enrich.media): + if len(hd := self.calculate_hash(m.filename)): + to_enrich.media[i].set("hash", f"{self.algorithm}:{hd}") + + def calculate_hash(self, filename) -> str: + hash_algo = None + if self.algorithm == "SHA-256": + hash_algo = hashlib.sha256 + elif self.algorithm == "SHA3-512": + hash_algo = hashlib.sha3_512 + else: return "" + return calculate_file_hash(filename, hash_algo, self.chunksize) diff --git a/src/auto_archiver/modules/html_formatter/__init__.py b/src/auto_archiver/modules/html_formatter/__init__.py new file mode 100644 index 0000000..432ef33 --- /dev/null +++ b/src/auto_archiver/modules/html_formatter/__init__.py @@ -0,0 +1 @@ +from .html_formatter import HtmlFormatter \ No newline at end of file diff --git a/src/auto_archiver/modules/html_formatter/__manifest__.py b/src/auto_archiver/modules/html_formatter/__manifest__.py new file mode 100644 index 0000000..ec19cf8 --- /dev/null +++ b/src/auto_archiver/modules/html_formatter/__manifest__.py @@ -0,0 +1,13 @@ +{ + "name": "HTML Formatter", + "type": ["formatter"], + "requires_setup": False, + "dependencies": { + "python": ["hash_enricher", "loguru", "jinja2"], + "bin": [""] + }, + "configs": { + "detect_thumbnails": {"default": True, "help": "if true will group by thumbnails generated by thumbnail enricher by id 'thumbnail_00'"} + }, + "description": """ """, +} diff --git a/src/auto_archiver/formatters/html_formatter.py b/src/auto_archiver/modules/html_formatter/html_formatter.py similarity index 66% rename from src/auto_archiver/formatters/html_formatter.py rename to src/auto_archiver/modules/html_formatter/html_formatter.py index 5d95474..ce4e67b 100644 --- a/src/auto_archiver/formatters/html_formatter.py +++ b/src/auto_archiver/modules/html_formatter/html_formatter.py @@ -1,5 +1,4 @@ from __future__ import annotations -from dataclasses import dataclass import mimetypes, os, pathlib from jinja2 import Environment, FileSystemLoader from urllib.parse import quote @@ -7,32 +6,29 @@ from loguru import logger import json import base64 -from ..version import __version__ -from ..core import Metadata, Media, ArchivingContext -from . import Formatter -from ..enrichers import HashEnricher -from ..utils.misc import random_str +from auto_archiver.version import __version__ +from auto_archiver.core import Metadata, Media +from auto_archiver.core import Formatter +from auto_archiver.utils.misc import random_str +from auto_archiver.core.module import get_module - -@dataclass class HtmlFormatter(Formatter): - name = "html_formatter" + environment: Environment = None + template: any = None + + def setup(self) -> None: + """Sets up the Jinja2 environment and loads the template.""" + template_dir = os.path.join(pathlib.Path(__file__).parent.resolve(), "templates/") + self.environment = Environment(loader=FileSystemLoader(template_dir), autoescape=True) - def __init__(self, config: dict) -> None: - # without this STEP.__init__ is not called - super().__init__(config) - self.environment = Environment(loader=FileSystemLoader(os.path.join(pathlib.Path(__file__).parent.resolve(), "templates/")), autoescape=True) # JinjaHelper class static methods are added as filters self.environment.filters.update({ k: v.__func__ for k, v in JinjaHelpers.__dict__.items() if isinstance(v, staticmethod) }) - self.template = self.environment.get_template("html_template.html") - @staticmethod - def configs() -> dict: - return { - "detect_thumbnails": {"default": True, "help": "if true will group by thumbnails generated by thumbnail enricher by id 'thumbnail_00'"} - } + # Load a specific template or default to "html_template.html" + template_name = self.config.get("template_name", "html_template.html") + self.template = self.environment.get_template(template_name) def format(self, item: Metadata) -> Media: url = item.get_url() @@ -48,12 +44,13 @@ class HtmlFormatter(Formatter): version=__version__ ) - html_path = os.path.join(ArchivingContext.get_tmp_dir(), f"formatted{random_str(24)}.html") + html_path = os.path.join(self.tmp_dir, f"formatted{random_str(24)}.html") with open(html_path, mode="w", encoding="utf-8") as outf: outf.write(content) final_media = Media(filename=html_path, _mimetype="text/html") - he = HashEnricher({"hash_enricher": {"algorithm": ArchivingContext.get("hash_enricher.algorithm"), "chunksize": 1.6e7}}) + # get the already instantiated hash_enricher module + he = get_module('hash_enricher', self.config) if len(hd := he.calculate_hash(final_media.filename)): final_media.set("hash", f"{he.algorithm}:{hd}") diff --git a/src/auto_archiver/formatters/templates/__init__.py b/src/auto_archiver/modules/html_formatter/templates/__init__.py similarity index 100% rename from src/auto_archiver/formatters/templates/__init__.py rename to src/auto_archiver/modules/html_formatter/templates/__init__.py diff --git a/src/auto_archiver/formatters/templates/html_template.html b/src/auto_archiver/modules/html_formatter/templates/html_template.html similarity index 100% rename from src/auto_archiver/formatters/templates/html_template.html rename to src/auto_archiver/modules/html_formatter/templates/html_template.html diff --git a/src/auto_archiver/formatters/templates/macros.html b/src/auto_archiver/modules/html_formatter/templates/macros.html similarity index 100% rename from src/auto_archiver/formatters/templates/macros.html rename to src/auto_archiver/modules/html_formatter/templates/macros.html diff --git a/src/auto_archiver/modules/instagram_api_extractor/__init__.py b/src/auto_archiver/modules/instagram_api_extractor/__init__.py new file mode 100644 index 0000000..8805c07 --- /dev/null +++ b/src/auto_archiver/modules/instagram_api_extractor/__init__.py @@ -0,0 +1 @@ +from .instagram_api_extractor import InstagramAPIExtractor diff --git a/src/auto_archiver/modules/instagram_api_extractor/__manifest__.py b/src/auto_archiver/modules/instagram_api_extractor/__manifest__.py new file mode 100644 index 0000000..2d8f1d9 --- /dev/null +++ b/src/auto_archiver/modules/instagram_api_extractor/__manifest__.py @@ -0,0 +1,53 @@ +{ + "name": "Instagram API Extractor", + "type": ["extractor"], + "entry_point": "instagram_api_extractor::InstagramAPIExtractor", + "dependencies": + {"python": ["requests", + "loguru", + "retrying", + "tqdm",], + }, + "requires_setup": True, + "configs": { + "access_token": {"default": None, + "help": "a valid instagrapi-api token"}, + "api_endpoint": {"required": True, + "help": "API endpoint to use"}, + "full_profile": { + "default": False, + "type": "bool", + "help": "if true, will download all posts, tagged posts, stories, and highlights for a profile, if false, will only download the profile pic and information.", + }, + "full_profile_max_posts": { + "default": 0, + "type": "int", + "help": "Use to limit the number of posts to download when full_profile is true. 0 means no limit. limit is applied softly since posts are fetched in batch, once to: posts, tagged posts, and highlights", + }, + "minimize_json_output": { + "default": True, + "type": "bool", + "help": "if true, will remove empty values from the json output", + }, + }, + "description": """ +Archives various types of Instagram content using the Instagrapi API. + +Requires setting up an Instagrapi API deployment and providing an access token and API endpoint. + +### Features +- Connects to an Instagrapi API deployment to fetch Instagram profiles, posts, stories, highlights, reels, and tagged content. +- Supports advanced configuration options, including: + - Full profile download (all posts, stories, highlights, and tagged content). + - Limiting the number of posts to fetch for large profiles. + - Minimising JSON output to remove empty fields and redundant data. +- Provides robust error handling and retries for API calls. +- Ensures efficient media scraping, including handling nested or carousel media items. +- Adds downloaded media and metadata to the result for further processing. + +### Notes +- Requires a valid Instagrapi API token (`access_token`) and API endpoint (`api_endpoint`). +- Full-profile downloads can be limited by setting `full_profile_max_posts`. +- Designed to fetch content in batches for large profiles, minimising API load. +""", +} diff --git a/src/auto_archiver/archivers/instagram_api_archiver.py b/src/auto_archiver/modules/instagram_api_extractor/instagram_api_extractor.py similarity index 59% rename from src/auto_archiver/archivers/instagram_api_archiver.py rename to src/auto_archiver/modules/instagram_api_extractor/instagram_api_extractor.py index d0e7e87..a75e065 100644 --- a/src/auto_archiver/archivers/instagram_api_archiver.py +++ b/src/auto_archiver/modules/instagram_api_extractor/instagram_api_extractor.py @@ -1,5 +1,5 @@ """ -The `instagram_api_archiver` module provides tools for archiving various types of Instagram content +The `instagram_api_extractor` module provides tools for archiving various types of Instagram content using the [Instagrapi API](https://github.com/subzeroid/instagrapi). Connects to an Instagrapi API deployment and allows for downloading Instagram user profiles, @@ -9,87 +9,88 @@ data, reducing JSON output size, and handling large profiles. """ import re -import requests from datetime import datetime + +import requests from loguru import logger from retrying import retry from tqdm import tqdm -from . import Archiver -from ..core import Metadata -from ..core import Media +from auto_archiver.core import Extractor +from auto_archiver.core import Media +from auto_archiver.core import Metadata -class InstagramAPIArchiver(Archiver): + +class InstagramAPIExtractor(Extractor): """ Uses an https://github.com/subzeroid/instagrapi API deployment to fetch instagram posts data - + # TODO: improvement collect aggregates of locations[0].location and mentions for all posts """ - name = "instagram_api_archiver" - global_pattern = re.compile(r"(?:(?:http|https):\/\/)?(?:www.)?(?:instagram.com)\/(stories(?:\/highlights)?|p|reel)?\/?([^\/\?]*)\/?(\d+)?") + valid_url = re.compile( + r"(?:(?:http|https):\/\/)?(?:www.)?(?:instagram.com)\/(stories(?:\/highlights)?|p|reel)?\/?([^\/\?]*)\/?(\d+)?" + ) - def __init__(self, config: dict) -> None: - super().__init__(config) - self.assert_valid_string("access_token") - self.assert_valid_string("api_endpoint") - self.full_profile_max_posts = int(self.full_profile_max_posts) - if self.api_endpoint[-1] == "/": self.api_endpoint = self.api_endpoint[:-1] + def setup(self) -> None: + if self.api_endpoint[-1] == "/": + self.api_endpoint = self.api_endpoint[:-1] - self.full_profile = bool(self.full_profile) - self.minimize_json_output = bool(self.minimize_json_output) - @staticmethod - def configs() -> dict: - return { - "access_token": {"default": None, "help": "a valid instagrapi-api token"}, - "api_endpoint": {"default": None, "help": "API endpoint to use"}, - "full_profile": {"default": False, "help": "if true, will download all posts, tagged posts, stories, and highlights for a profile, if false, will only download the profile pic and information."}, - "full_profile_max_posts": {"default": 0, "help": "Use to limit the number of posts to download when full_profile is true. 0 means no limit. limit is applied softly since posts are fetched in batch, once to: posts, tagged posts, and highlights"}, - "minimize_json_output": {"default": True, "help": "if true, will remove empty values from the json output"}, - } - def download(self, item: Metadata) -> Metadata: url = item.get_url() - url.replace("instagr.com", "instagram.com").replace("instagr.am", "instagram.com") - insta_matches = self.global_pattern.findall(url) + url.replace("instagr.com", "instagram.com").replace( + "instagr.am", "instagram.com" + ) + insta_matches = self.valid_url.findall(url) logger.info(f"{insta_matches=}") - if not len(insta_matches) or len(insta_matches[0])!=3: return - if len(insta_matches) > 1: - logger.warning(f"Multiple instagram matches found in {url=}, using the first one") + if not len(insta_matches) or len(insta_matches[0]) != 3: + return + if len(insta_matches) > 1: + logger.warning( + f"Multiple instagram matches found in {url=}, using the first one" + ) return g1, g2, g3 = insta_matches[0][0], insta_matches[0][1], insta_matches[0][2] - if g1 == "": return self.download_profile(item, g2) - elif g1 == "p": return self.download_post(item, g2, context="post") - elif g1 == "reel": return self.download_post(item, g2, context="reel") - elif g1 == "stories/highlights": return self.download_highlights(item, g2) - elif g1 == "stories": - if len(g3): return self.download_post(item, id=g3, context="story") + if g1 == "": + return self.download_profile(item, g2) + elif g1 == "p": + return self.download_post(item, g2, context="post") + elif g1 == "reel": + return self.download_post(item, g2, context="reel") + elif g1 == "stories/highlights": + return self.download_highlights(item, g2) + elif g1 == "stories": + if len(g3): + return self.download_post(item, id=g3, context="story") return self.download_stories(item, g2) - else: + else: logger.warning(f"Unknown instagram regex group match {g1=} found in {url=}") return - + @retry(wait_random_min=1000, wait_random_max=3000, stop_max_attempt_number=5) def call_api(self, path: str, params: dict) -> dict: - headers = { - "accept": "application/json", - "x-access-key": self.access_token - } + headers = {"accept": "application/json", "x-access-key": self.access_token} logger.debug(f"calling {self.api_endpoint}/{path} with {params=}") - return requests.get(f"{self.api_endpoint}/{path}", headers=headers, params=params).json() + return requests.get( + f"{self.api_endpoint}/{path}", headers=headers, params=params + ).json() def cleanup_dict(self, d: dict | list) -> dict: # repeats 3 times to remove nested empty values - if not self.minimize_json_output: return d - if type(d) == list: return [self.cleanup_dict(v) for v in d] - if type(d) != dict: return d + if not self.minimize_json_output: + return d + if type(d) == list: + return [self.cleanup_dict(v) for v in d] + if type(d) != dict: + return d return { - k: clean_v - for k, v in d.items() - if (clean_v := self.cleanup_dict(v)) not in [0.0, 0, [], {}, "", None, "null"] and - k not in ["x", "y", "width", "height"] + k: clean_v + for k, v in d.items() + if (clean_v := self.cleanup_dict(v)) + not in [0.0, 0, [], {}, "", None, "null"] + and k not in ["x", "y", "width", "height"] } def download_profile(self, result: Metadata, username: str) -> Metadata: @@ -125,7 +126,9 @@ class InstagramAPIArchiver(Archiver): try: self.download_all_tagged(result, user_id) except Exception as e: - result.append("errors", f"Error downloading tagged posts for {username}") + result.append( + "errors", f"Error downloading tagged posts for {username}" + ) logger.error(f"Error downloading tagged posts for {username}: {e}") # download all highlights @@ -135,26 +138,37 @@ class InstagramAPIArchiver(Archiver): result.append("errors", f"Error downloading highlights for {username}") logger.error(f"Error downloading highlights for {username}: {e}") - - result.set_url(url) # reset as scrape_item modifies it + result.set_url(url) # reset as scrape_item modifies it return result.success("insta profile") def download_all_highlights(self, result, username, user_id): count_highlights = 0 highlights = self.call_api(f"v1/user/highlights", {"user_id": user_id}) for h in highlights: - try: + try: h_info = self._download_highlights_reusable(result, h.get("pk")) count_highlights += len(h_info.get("items", [])) except Exception as e: - result.append("errors", f"Error downloading highlight id{h.get('pk')} for {username}") - logger.error(f"Error downloading highlight id{h.get('pk')} for {username}: {e}") - if self.full_profile_max_posts and count_highlights >= self.full_profile_max_posts: - logger.info(f"HIGHLIGHTS reached full_profile_max_posts={self.full_profile_max_posts}") + result.append( + "errors", + f"Error downloading highlight id{h.get('pk')} for {username}", + ) + logger.error( + f"Error downloading highlight id{h.get('pk')} for {username}: {e}" + ) + if ( + self.full_profile_max_posts + and count_highlights >= self.full_profile_max_posts + ): + logger.info( + f"HIGHLIGHTS reached full_profile_max_posts={self.full_profile_max_posts}" + ) break result.set("#highlights", count_highlights) - def download_post(self, result: Metadata, code: str = None, id: str = None, context: str = None) -> Metadata: + def download_post( + self, result: Metadata, code: str = None, id: str = None, context: str = None + ) -> Metadata: if id: post = self.call_api(f"v1/media/by/id", {"id": id}) else: @@ -166,7 +180,8 @@ class InstagramAPIArchiver(Archiver): post = self.scrape_item(result, post, context) - if post.get("taken_at"): result.set_timestamp(post.get("taken_at")) + if post.get("taken_at"): + result.set_timestamp(post.get("taken_at")) return result.success(f"insta {context or 'post'}") def download_highlights(self, result: Metadata, id: str) -> Metadata: @@ -175,96 +190,127 @@ class InstagramAPIArchiver(Archiver): del h_info["items"] result.set_title(h_info.get("title")).set("data", h_info).set("#reels", items) return result.success("insta highlights") - - def _download_highlights_reusable(self, result: Metadata, id: str) ->dict: + + def _download_highlights_reusable(self, result: Metadata, id: str) -> dict: full_h = self.call_api(f"v2/highlight/by/id", {"id": id}) h_info = full_h.get("response", {}).get("reels", {}).get(f"highlight:{id}") assert h_info, f"Highlight {id} not found: {full_h=}" - if cover_media := h_info.get("cover_media", {}).get("cropped_image_version", {}).get("url"): + if ( + cover_media := h_info.get("cover_media", {}) + .get("cropped_image_version", {}) + .get("url") + ): filename = self.download_from_url(cover_media) result.add_media(Media(filename=filename), id=f"cover_media highlight {id}") - items = h_info.get("items", [])[::-1] # newest to oldest + items = h_info.get("items", [])[::-1] # newest to oldest for h in tqdm(items, desc="downloading highlights", unit="highlight"): - try: self.scrape_item(result, h, "highlight") + try: + self.scrape_item(result, h, "highlight") except Exception as e: result.append("errors", f"Error downloading highlight {h.get('id')}") - logger.error(f"Error downloading highlight, skipping {h.get('id')}: {e}") - + logger.error( + f"Error downloading highlight, skipping {h.get('id')}: {e}" + ) + return h_info - + def download_stories(self, result: Metadata, username: str) -> Metadata: now = datetime.now().strftime("%Y-%m-%d_%H-%M") stories = self._download_stories_reusable(result, username) - if stories == []: return result.success("insta no story") + if stories == []: + return result.success("insta no story") result.set_title(f"stories {username} at {now}").set("#stories", len(stories)) return result.success(f"insta stories {now}") - + def _download_stories_reusable(self, result: Metadata, username: str) -> list[dict]: stories = self.call_api(f"v1/user/stories/by/username", {"username": username}) - if not stories or not len(stories): return [] - stories = stories[::-1] # newest to oldest + if not stories or not len(stories): + return [] + stories = stories[::-1] # newest to oldest for s in tqdm(stories, desc="downloading stories", unit="story"): - try: self.scrape_item(result, s, "story") + try: + self.scrape_item(result, s, "story") except Exception as e: result.append("errors", f"Error downloading story {s.get('id')}") logger.error(f"Error downloading story, skipping {s.get('id')}: {e}") return stories - + def download_all_posts(self, result: Metadata, user_id: str): end_cursor = None pbar = tqdm(desc="downloading posts") post_count = 0 while end_cursor != "": - posts = self.call_api(f"v1/user/medias/chunk", {"user_id": user_id, "end_cursor": end_cursor}) - if not len(posts) or not type(posts) == list or len(posts) != 2: break + posts = self.call_api( + f"v1/user/medias/chunk", {"user_id": user_id, "end_cursor": end_cursor} + ) + if not len(posts) or not type(posts) == list or len(posts) != 2: + break posts, end_cursor = posts[0], posts[1] logger.info(f"parsing {len(posts)} posts, next {end_cursor=}") for p in posts: - try: self.scrape_item(result, p, "post") + try: + self.scrape_item(result, p, "post") except Exception as e: result.append("errors", f"Error downloading post {p.get('id')}") logger.error(f"Error downloading post, skipping {p.get('id')}: {e}") pbar.update(1) - post_count+=1 - if self.full_profile_max_posts and post_count >= self.full_profile_max_posts: - logger.info(f"POSTS reached full_profile_max_posts={self.full_profile_max_posts}") + post_count += 1 + if ( + self.full_profile_max_posts + and post_count >= self.full_profile_max_posts + ): + logger.info( + f"POSTS reached full_profile_max_posts={self.full_profile_max_posts}" + ) break result.set("#posts", post_count) - + def download_all_tagged(self, result: Metadata, user_id: str): next_page_id = "" pbar = tqdm(desc="downloading tagged posts") tagged_count = 0 while next_page_id != None: - resp = self.call_api(f"v2/user/tag/medias", {"user_id": user_id, "page_id": next_page_id}) + resp = self.call_api( + f"v2/user/tag/medias", {"user_id": user_id, "page_id": next_page_id} + ) posts = resp.get("response", {}).get("items", []) - if not len(posts): break + if not len(posts): + break next_page_id = resp.get("next_page_id") - + logger.info(f"parsing {len(posts)} tagged posts, next {next_page_id=}") for p in posts: - try: self.scrape_item(result, p, "tagged") + try: + self.scrape_item(result, p, "tagged") except Exception as e: - result.append("errors", f"Error downloading tagged post {p.get('id')}") - logger.error(f"Error downloading tagged post, skipping {p.get('id')}: {e}") + result.append( + "errors", f"Error downloading tagged post {p.get('id')}" + ) + logger.error( + f"Error downloading tagged post, skipping {p.get('id')}: {e}" + ) pbar.update(1) - tagged_count+=1 - if self.full_profile_max_posts and tagged_count >= self.full_profile_max_posts: - logger.info(f"TAGS reached full_profile_max_posts={self.full_profile_max_posts}") + tagged_count += 1 + if ( + self.full_profile_max_posts + and tagged_count >= self.full_profile_max_posts + ): + logger.info( + f"TAGS reached full_profile_max_posts={self.full_profile_max_posts}" + ) break result.set("#tagged", tagged_count) + ### reusable parsing utils below -### reusable parsing utils below - - def scrape_item(self, result:Metadata, item:dict, context:str=None) -> dict: + def scrape_item(self, result: Metadata, item: dict, context: str = None) -> dict: """ receives a Metadata and an API dict response fetches the media and adds it to the Metadata @@ -272,23 +318,25 @@ class InstagramAPIArchiver(Archiver): context can be used to give specific id prefixes to media """ if "clips_metadata" in item: - if reusable_text := item.get("clips_metadata", {}).get("reusable_text_attribute_string"): + if reusable_text := item.get("clips_metadata", {}).get( + "reusable_text_attribute_string" + ): item["clips_metadata_text"] = reusable_text - if self.minimize_json_output: + if self.minimize_json_output: del item["clips_metadata"] - if code := item.get("code") and not result.get("url"): + if code := item.get("code") and not result.get("url"): result.set_url(f"https://www.instagram.com/p/{code}/") - + resources = item.get("resources", item.get("carousel_media", [])) item, media, media_id = self.scrape_media(item, context) # if resources are present take the main media from the first resource if not media and len(resources): _, media, media_id = self.scrape_media(resources[0], context) resources = resources[1:] - + assert media, f"Image/video not found in {item=}" - + # posts with multiple items contain a resources list resources_metadata = Metadata() for r in resources: @@ -298,40 +346,54 @@ class InstagramAPIArchiver(Archiver): result.add_media(media, id=media_id) return item - - def scrape_media(self, item: dict, context:str) -> tuple[dict, Media, str]: + + def scrape_media(self, item: dict, context: str) -> tuple[dict, Media, str]: # remove unnecessary info - if self.minimize_json_output: - for k in ["image_versions", "video_versions", "video_dash_manifest", "image_versions2", "video_versions2"]: - if k in item: del item[k] + if self.minimize_json_output: + for k in [ + "image_versions", + "video_versions", + "video_dash_manifest", + "image_versions2", + "video_versions2", + ]: + if k in item: + del item[k] item = self.cleanup_dict(item) image_media = None if image_url := item.get("thumbnail_url"): filename = self.download_from_url(image_url, verbose=False) image_media = Media(filename=filename) - + # retrieve video info - best_id = item.get('id', item.get('pk')) + best_id = item.get("id", item.get("pk")) taken_at = item.get("taken_at", item.get("taken_at_ts")) code = item.get("code") caption_text = item.get("caption_text") - if "carousel_media" in item: del item["carousel_media"] + if "carousel_media" in item: + del item["carousel_media"] if video_url := item.get("video_url"): filename = self.download_from_url(video_url, verbose=False) video_media = Media(filename=filename) - if taken_at: video_media.set("date", taken_at) - if code: video_media.set("url", f"https://www.instagram.com/p/{code}") - if caption_text: video_media.set("text", caption_text) + if taken_at: + video_media.set("date", taken_at) + if code: + video_media.set("url", f"https://www.instagram.com/p/{code}") + if caption_text: + video_media.set("text", caption_text) video_media.set("preview", [image_media]) video_media.set("data", [item]) return item, video_media, f"{context or 'video'} {best_id}" elif image_media: - if taken_at: image_media.set("date", taken_at) - if code: image_media.set("url", f"https://www.instagram.com/p/{code}") - if caption_text: image_media.set("text", caption_text) + if taken_at: + image_media.set("date", taken_at) + if code: + image_media.set("url", f"https://www.instagram.com/p/{code}") + if caption_text: + image_media.set("text", caption_text) image_media.set("data", [item]) return item, image_media, f"{context or 'image'} {best_id}" - - return item, None, None \ No newline at end of file + + return item, None, None diff --git a/src/auto_archiver/modules/instagram_extractor/__init__.py b/src/auto_archiver/modules/instagram_extractor/__init__.py new file mode 100644 index 0000000..6f39171 --- /dev/null +++ b/src/auto_archiver/modules/instagram_extractor/__init__.py @@ -0,0 +1 @@ +from .instagram_extractor import InstagramExtractor \ No newline at end of file diff --git a/src/auto_archiver/modules/instagram_extractor/__manifest__.py b/src/auto_archiver/modules/instagram_extractor/__manifest__.py new file mode 100644 index 0000000..05cae19 --- /dev/null +++ b/src/auto_archiver/modules/instagram_extractor/__manifest__.py @@ -0,0 +1,36 @@ +{ + "name": "Instagram Extractor", + "type": ["extractor"], + "dependencies": { + "python": [ + "instaloader", + "loguru", + ], + }, + "requires_setup": True, + "configs": { + "username": {"required": True, + "help": "a valid Instagram username"}, + "password": { + "required": True, + "help": "the corresponding Instagram account password", + }, + "download_folder": { + "default": "instaloader", + "help": "name of a folder to temporarily download content to", + }, + "session_file": { + "default": "secrets/instaloader.session", + "help": "path to the instagram session which saves session credentials", + }, + # TODO: fine-grain + # "download_stories": {"default": True, "help": "if the link is to a user profile: whether to get stories information"}, + }, + "description": """ + Uses the [Instaloader library](https://instaloader.github.io/as-module.html) to download content from Instagram. This class handles both individual posts + and user profiles, downloading as much information as possible, including images, videos, text, stories, + highlights, and tagged posts. + Authentication is required via username/password or a session file. + + """, +} diff --git a/src/auto_archiver/archivers/instagram_archiver.py b/src/auto_archiver/modules/instagram_extractor/instagram_extractor.py similarity index 78% rename from src/auto_archiver/archivers/instagram_archiver.py rename to src/auto_archiver/modules/instagram_extractor/instagram_extractor.py index 94a8fc0..0af2c32 100644 --- a/src/auto_archiver/archivers/instagram_archiver.py +++ b/src/auto_archiver/modules/instagram_extractor/instagram_extractor.py @@ -4,33 +4,29 @@ """ import re, os, shutil, traceback -import instaloader # https://instaloader.github.io/as-module.html +import instaloader from loguru import logger -from . import Archiver -from ..core import Metadata -from ..core import Media +from auto_archiver.core import Extractor +from auto_archiver.core import Metadata +from auto_archiver.core import Media -class InstagramArchiver(Archiver): +class InstagramExtractor(Extractor): """ Uses Instaloader to download either a post (inc images, videos, text) or as much as possible from a profile (posts, stories, highlights, ...) """ - name = "instagram_archiver" - # NB: post regex should be tested before profile + + valid_url = re.compile(r"(?:(?:http|https):\/\/)?(?:www.)?(?:instagram.com|instagr.am|instagr.com)\/") + # https://regex101.com/r/MGPquX/1 - post_pattern = re.compile(r"(?:(?:http|https):\/\/)?(?:www.)?(?:instagram.com|instagr.am|instagr.com)\/(?:p|reel)\/(\w+)") + post_pattern = re.compile(r"{valid_url}(?:p|reel)\/(\w+)".format(valid_url=valid_url)) # https://regex101.com/r/6Wbsxa/1 - profile_pattern = re.compile(r"(?:(?:http|https):\/\/)?(?:www.)?(?:instagram.com|instagr.am|instagr.com)\/(\w+)") + profile_pattern = re.compile(r"{valid_url}(\w+)".format(valid_url=valid_url)) # TODO: links to stories - def __init__(self, config: dict) -> None: - super().__init__(config) - # TODO: refactor how configuration validation is done - self.assert_valid_string("username") - self.assert_valid_string("password") - self.assert_valid_string("download_folder") - self.assert_valid_string("session_file") + def setup(self) -> None: + self.insta = instaloader.Instaloader( download_geotags=True, download_comments=True, compress_json=False, dirname_pattern=self.download_folder, filename_pattern="{date_utc}_UTC_{target}__{typename}" ) @@ -45,16 +41,7 @@ class InstagramArchiver(Archiver): except Exception as e2: logger.error(f"Unable to finish login (retrying from file): {e2}\n{traceback.format_exc()}") - @staticmethod - def configs() -> dict: - return { - "username": {"default": None, "help": "a valid Instagram username"}, - "password": {"default": None, "help": "the corresponding Instagram account password"}, - "download_folder": {"default": "instaloader", "help": "name of a folder to temporarily download content to"}, - "session_file": {"default": "secrets/instaloader.session", "help": "path to the instagram session which saves session credentials"}, - #TODO: fine-grain - # "download_stories": {"default": True, "help": "if the link is to a user profile: whether to get stories information"}, - } + def download(self, item: Metadata) -> Metadata: url = item.get_url() @@ -76,7 +63,7 @@ class InstagramArchiver(Archiver): elif len(profile_matches): result = self.download_profile(url, profile_matches[0]) except Exception as e: - logger.error(f"Failed to download with instagram archiver due to: {e}, make sure your account credentials are valid.") + logger.error(f"Failed to download with instagram extractor due to: {e}, make sure your account credentials are valid.") finally: shutil.rmtree(self.download_folder, ignore_errors=True) return result diff --git a/src/auto_archiver/modules/instagram_tbot_extractor/__init__.py b/src/auto_archiver/modules/instagram_tbot_extractor/__init__.py new file mode 100644 index 0000000..aa39e63 --- /dev/null +++ b/src/auto_archiver/modules/instagram_tbot_extractor/__init__.py @@ -0,0 +1 @@ +from .instagram_tbot_extractor import InstagramTbotExtractor diff --git a/src/auto_archiver/modules/instagram_tbot_extractor/__manifest__.py b/src/auto_archiver/modules/instagram_tbot_extractor/__manifest__.py new file mode 100644 index 0000000..a24a864 --- /dev/null +++ b/src/auto_archiver/modules/instagram_tbot_extractor/__manifest__.py @@ -0,0 +1,40 @@ +{ + "name": "Instagram Telegram Bot Extractor", + "type": ["extractor"], + "dependencies": {"python": ["loguru", "telethon",], + }, + "requires_setup": True, + "configs": { + "api_id": {"default": None, "help": "telegram API_ID value, go to https://my.telegram.org/apps"}, + "api_hash": {"default": None, "help": "telegram API_HASH value, go to https://my.telegram.org/apps"}, + "session_file": {"default": "secrets/anon-insta", "help": "optional, records the telegram login session for future usage, '.session' will be appended to the provided value."}, + "timeout": {"default": 45, + "type": "int", + "help": "timeout to fetch the instagram content in seconds."}, + }, + "description": """ +The `InstagramTbotExtractor` module uses a Telegram bot (`instagram_load_bot`) to fetch and archive Instagram content, +such as posts and stories. It leverages the Telethon library to interact with the Telegram API, sending Instagram URLs +to the bot and downloading the resulting media and metadata. The downloaded content is stored as `Media` objects and +returned as part of a `Metadata` object. + +### Features +- Supports archiving Instagram posts and stories through the Telegram bot. +- Downloads and saves media files (e.g., images, videos) in a temporary directory. +- Captures and returns metadata, including titles and descriptions, as a `Metadata` object. +- Automatically manages Telegram session files for secure access. + +### Setup + +To use the `InstagramTbotExtractor`, you need to provide the following configuration settings: +- **API ID and Hash**: Telegram API credentials obtained from [my.telegram.org/apps](https://my.telegram.org/apps). +- **Session File**: Optional path to store the Telegram session file for future use. +- The session file is created automatically and should be unique for each instance. +- You may need to enter your Telegram credentials (phone) and use the a 2FA code sent to you the first time you run the extractor.: +```2025-01-30 00:43:49.348 | INFO | auto_archiver.modules.instagram_tbot_extractor.instagram_tbot_extractor:setup:36 - SETUP instagram_tbot_extractor checking login... +Please enter your phone (or bot token): +447123456789 +Please enter the code you received: 00000 +Signed in successfully as E C; remember to not break the ToS or you will risk an account ban! +``` + """, +} diff --git a/src/auto_archiver/modules/instagram_tbot_extractor/instagram_tbot_extractor.py b/src/auto_archiver/modules/instagram_tbot_extractor/instagram_tbot_extractor.py new file mode 100644 index 0000000..d4b7a8e --- /dev/null +++ b/src/auto_archiver/modules/instagram_tbot_extractor/instagram_tbot_extractor.py @@ -0,0 +1,121 @@ +""" +InstagramTbotExtractor Module + +This module provides functionality to archive Instagram content (posts, stories, etc.) using a Telegram bot (`instagram_load_bot`). +It interacts with the Telegram API via the Telethon library to send Instagram URLs to the bot, which retrieves the +relevant media and metadata. The fetched content is saved as `Media` objects in a temporary directory and returned as a +`Metadata` object. +""" + +import os +import shutil +import time +from sqlite3 import OperationalError + +from loguru import logger +from telethon.sync import TelegramClient + +from auto_archiver.core import Extractor +from auto_archiver.core import Metadata, Media +from auto_archiver.utils import random_str + + +class InstagramTbotExtractor(Extractor): + """ + calls a telegram bot to fetch instagram posts/stories... and gets available media from it + https://github.com/adw0rd/instagrapi + https://t.me/instagram_load_bot + """ + + def setup(self) -> None: + """ + 1. makes a copy of session_file that is removed in cleanup + 2. checks if the session file is valid + """ + logger.info(f"SETUP {self.name} checking login...") + self._prepare_session_file() + self._initialize_telegram_client() + + def _prepare_session_file(self): + """ + Creates a copy of the session file for exclusive use with this archiver instance. + Ensures that a valid session file exists before proceeding. + """ + new_session_file = os.path.join("secrets/", f"instabot-{time.strftime('%Y-%m-%d')}{random_str(8)}.session") + if not os.path.exists(f"{self.session_file}.session"): + raise FileNotFoundError(f"Session file {self.session_file}.session not found.") + shutil.copy(self.session_file + ".session", new_session_file) + self.session_file = new_session_file.replace(".session", "") + + def _initialize_telegram_client(self): + """Initializes the Telegram client.""" + try: + self.client = TelegramClient(self.session_file, self.api_id, self.api_hash) + except OperationalError as e: + logger.error( + f"Unable to access the {self.session_file} session. " + "Ensure that you don't use the same session file here and in telethon_extractor. " + "If you do, disable at least one of the archivers for the first-time setup of the telethon session: {e}" + ) + with self.client.start(): + logger.success(f"SETUP {self.name} login works.") + + def cleanup(self) -> None: + logger.info(f"CLEANUP {self.name}.") + session_file_name = self.session_file + ".session" + if os.path.exists(session_file_name): + os.remove(session_file_name) + + def download(self, item: Metadata) -> Metadata: + url = item.get_url() + if not "instagram.com" in url: return False + + result = Metadata() + tmp_dir = self.tmp_dir + with self.client.start(): + + chat, since_id = self._send_url_to_bot(url) + message = self._process_messages(chat, since_id, tmp_dir, result) + + if "You must enter a URL to a post" in message: + logger.debug(f"invalid link {url=} for {self.name}: {message}") + return False + # # TODO: It currently returns this as a success - is that intentional? + # if "Media not found or unavailable" in message: + # logger.debug(f"invalid link {url=} for {self.name}: {message}") + # return False + + if message: + result.set_content(message).set_title(message[:128]) + return result.success("insta-via-bot") + + def _send_url_to_bot(self, url: str): + """ + Sends the URL to the 'instagram_load_bot' and returns (chat, since_id). + """ + chat = self.client.get_entity("instagram_load_bot") + since_message = self.client.send_message(entity=chat, message=url) + return chat, since_message.id + + def _process_messages(self, chat, since_id, tmp_dir, result): + attempts = 0 + seen_media = [] + message = "" + time.sleep(3) + # media is added before text by the bot so it can be used as a stop-logic mechanism + while attempts < (self.timeout - 3) and (not message or not len(seen_media)): + attempts += 1 + time.sleep(1) + for post in self.client.iter_messages(chat, min_id=since_id): + since_id = max(since_id, post.id) + # Skip known filler message: + if post.message == 'The bot receives information through https://hikerapi.com/p/hJqpppqi': + continue + if post.media and post.id not in seen_media: + filename_dest = os.path.join(tmp_dir, f'{chat.id}_{post.id}') + media = self.client.download_media(post.media, filename_dest) + if media: + result.add_media(Media(media)) + seen_media.append(post.id) + if post.message: message += post.message + return message.strip() \ No newline at end of file diff --git a/src/auto_archiver/modules/local_storage/__init__.py b/src/auto_archiver/modules/local_storage/__init__.py new file mode 100644 index 0000000..d23147d --- /dev/null +++ b/src/auto_archiver/modules/local_storage/__init__.py @@ -0,0 +1 @@ +from .local_storage import LocalStorage \ No newline at end of file diff --git a/src/auto_archiver/modules/local_storage/__manifest__.py b/src/auto_archiver/modules/local_storage/__manifest__.py new file mode 100644 index 0000000..6d9cf53 --- /dev/null +++ b/src/auto_archiver/modules/local_storage/__manifest__.py @@ -0,0 +1,35 @@ +{ + "name": "Local Storage", + "type": ["storage"], + "requires_setup": False, + "dependencies": { + "python": ["loguru"], + }, + "configs": { + "path_generator": { + "default": "flat", + "help": "how to store the file in terms of directory structure: 'flat' sets to root; 'url' creates a directory based on the provided URL; 'random' creates a random directory.", + "choices": ["flat", "url", "random"], + }, + "filename_generator": { + "default": "static", + "help": "how to name stored files: 'random' creates a random string; 'static' uses a replicable strategy such as a hash.", + "choices": ["random", "static"], + }, + "save_to": {"default": "./local_archive", "help": "folder where to save archived content"}, + "save_absolute": {"default": False, "help": "whether the path to the stored file is absolute or relative in the output result inc. formatters (WARN: leaks the file structure)"}, + }, + "description": """ + LocalStorage: A storage module for saving archived content locally on the filesystem. + + ### Features + - Saves archived media files to a specified folder on the local filesystem. + - Maintains file metadata during storage using `shutil.copy2`. + - Supports both absolute and relative paths for stored files, configurable via `save_absolute`. + - Automatically creates directories as needed for storing files. + + ### Notes + - Default storage folder is `./archived`, but this can be changed via the `save_to` configuration. + - The `save_absolute` option can reveal the file structure in output formats; use with caution. + """ +} diff --git a/src/auto_archiver/storages/local.py b/src/auto_archiver/modules/local_storage/local_storage.py similarity index 52% rename from src/auto_archiver/storages/local.py rename to src/auto_archiver/modules/local_storage/local_storage.py index aa08e49..b995577 100644 --- a/src/auto_archiver/storages/local.py +++ b/src/auto_archiver/modules/local_storage/local_storage.py @@ -4,25 +4,11 @@ from typing import IO import os from loguru import logger -from ..core import Media -from ..storages import Storage +from auto_archiver.core import Media +from auto_archiver.core import Storage class LocalStorage(Storage): - name = "local_storage" - - def __init__(self, config: dict) -> None: - super().__init__(config) - os.makedirs(self.save_to, exist_ok=True) - - @staticmethod - def configs() -> dict: - return dict( - Storage.configs(), - ** { - "save_to": {"default": "./archived", "help": "folder where to save archived content"}, - "save_absolute": {"default": False, "help": "whether the path to the stored file is absolute or relative in the output result inc. formatters (WARN: leaks the file structure)"}, - }) def get_cdn_url(self, media: Media) -> str: # TODO: is this viable with Storage.configs on path/filename? @@ -35,7 +21,7 @@ class LocalStorage(Storage): # override parent so that we can use shutil.copy2 and keep metadata dest = os.path.join(self.save_to, media.key) os.makedirs(os.path.dirname(dest), exist_ok=True) - logger.debug(f'[{self.__class__.name}] storing file {media.filename} with key {media.key} to {dest}') + logger.debug(f'[{self.__class__.__name__}] storing file {media.filename} with key {media.key} to {dest}') res = shutil.copy2(media.filename, dest) logger.info(res) return True diff --git a/src/auto_archiver/modules/meta_enricher/__init__.py b/src/auto_archiver/modules/meta_enricher/__init__.py new file mode 100644 index 0000000..4e1d330 --- /dev/null +++ b/src/auto_archiver/modules/meta_enricher/__init__.py @@ -0,0 +1 @@ +from .meta_enricher import MetaEnricher diff --git a/src/auto_archiver/modules/meta_enricher/__manifest__.py b/src/auto_archiver/modules/meta_enricher/__manifest__.py new file mode 100644 index 0000000..37c9201 --- /dev/null +++ b/src/auto_archiver/modules/meta_enricher/__manifest__.py @@ -0,0 +1,22 @@ +{ + "name": "Archive Metadata Enricher", + "type": ["enricher"], + "requires_setup": False, + "dependencies": { + "python": ["loguru"], + }, + "description": """ + Adds metadata information about the archive operations, Adds metadata about archive operations, including file sizes and archive duration./ + To be included at the end of all enrichments. + + ### Features +- Calculates the total size of all archived media files, storing the result in human-readable and byte formats. +- Computes the duration of the archival process, storing the elapsed time in seconds. +- Ensures all enrichments are performed only if the `Metadata` object contains valid data. +- Adds detailed metadata to provide insights into file sizes and archival performance. + +### Notes +- Skips enrichment if no media or metadata is available in the `Metadata` object. +- File sizes are calculated using the `os.stat` module, ensuring accurate byte-level reporting. +""", +} diff --git a/src/auto_archiver/enrichers/meta_enricher.py b/src/auto_archiver/modules/meta_enricher/meta_enricher.py similarity index 85% rename from src/auto_archiver/enrichers/meta_enricher.py rename to src/auto_archiver/modules/meta_enricher/meta_enricher.py index b721bb5..03fb01e 100644 --- a/src/auto_archiver/enrichers/meta_enricher.py +++ b/src/auto_archiver/modules/meta_enricher/meta_enricher.py @@ -2,24 +2,14 @@ import datetime import os from loguru import logger -from . import Enricher -from ..core import Metadata +from auto_archiver.core import Enricher +from auto_archiver.core import Metadata class MetaEnricher(Enricher): """ Adds metadata information about the archive operations, to be included at the end of all enrichments """ - name = "meta_enricher" - - - def __init__(self, config: dict) -> None: - # without this STEP.__init__ is not called - super().__init__(config) - - @staticmethod - def configs() -> dict: - return {} def enrich(self, to_enrich: Metadata) -> None: url = to_enrich.get_url() @@ -28,7 +18,7 @@ class MetaEnricher(Enricher): return logger.debug(f"calculating archive metadata information for {url=}") - + self.enrich_file_sizes(to_enrich) self.enrich_archive_duration(to_enrich) @@ -40,10 +30,10 @@ class MetaEnricher(Enricher): media.set("bytes", file_stats.st_size) media.set("size", self.human_readable_bytes(file_stats.st_size)) total_size += file_stats.st_size - + to_enrich.set("total_bytes", total_size) to_enrich.set("total_size", self.human_readable_bytes(total_size)) - + def human_readable_bytes(self, size: int) -> str: # receives number of bytes and returns human readble size diff --git a/src/auto_archiver/modules/metadata_enricher/__init__.py b/src/auto_archiver/modules/metadata_enricher/__init__.py new file mode 100644 index 0000000..020bd4a --- /dev/null +++ b/src/auto_archiver/modules/metadata_enricher/__init__.py @@ -0,0 +1 @@ +from .metadata_enricher import MetadataEnricher \ No newline at end of file diff --git a/src/auto_archiver/modules/metadata_enricher/__manifest__.py b/src/auto_archiver/modules/metadata_enricher/__manifest__.py new file mode 100644 index 0000000..f8ccdc6 --- /dev/null +++ b/src/auto_archiver/modules/metadata_enricher/__manifest__.py @@ -0,0 +1,21 @@ +{ + "name": "Media Metadata Enricher", + "type": ["enricher"], + "requires_setup": True, + "dependencies": { + "python": ["loguru"], + "bin": ["exiftool"] + }, + "description": """ + Extracts metadata information from files using ExifTool. + + ### Features + - Uses ExifTool to extract detailed metadata from media files. + - Processes file-specific data like camera settings, geolocation, timestamps, and other embedded metadata. + - Adds extracted metadata to the corresponding `Media` object within the `Metadata`. + + ### Notes + - Requires ExifTool to be installed and accessible via the system's PATH. + - Skips enrichment for files where metadata extraction fails. + """ +} diff --git a/src/auto_archiver/enrichers/metadata_enricher.py b/src/auto_archiver/modules/metadata_enricher/metadata_enricher.py similarity index 82% rename from src/auto_archiver/enrichers/metadata_enricher.py rename to src/auto_archiver/modules/metadata_enricher/metadata_enricher.py index 9fe257e..c052d0a 100644 --- a/src/auto_archiver/enrichers/metadata_enricher.py +++ b/src/auto_archiver/modules/metadata_enricher/metadata_enricher.py @@ -2,23 +2,15 @@ import subprocess import traceback from loguru import logger -from . import Enricher -from ..core import Metadata +from auto_archiver.core import Enricher +from auto_archiver.core import Metadata class MetadataEnricher(Enricher): """ Extracts metadata information from files using exiftool. """ - name = "metadata_enricher" - def __init__(self, config: dict) -> None: - # without this STEP.__init__ is not called - super().__init__(config) - - @staticmethod - def configs() -> dict: - return {} def enrich(self, to_enrich: Metadata) -> None: url = to_enrich.get_url() diff --git a/src/auto_archiver/modules/mute_formatter/__init__.py b/src/auto_archiver/modules/mute_formatter/__init__.py new file mode 100644 index 0000000..b92fce9 --- /dev/null +++ b/src/auto_archiver/modules/mute_formatter/__init__.py @@ -0,0 +1 @@ +from .mute_formatter import MuteFormatter diff --git a/src/auto_archiver/modules/mute_formatter/__manifest__.py b/src/auto_archiver/modules/mute_formatter/__manifest__.py new file mode 100644 index 0000000..e81dc4c --- /dev/null +++ b/src/auto_archiver/modules/mute_formatter/__manifest__.py @@ -0,0 +1,9 @@ +{ + "name": "Mute Formatter", + "type": ["formatter"], + "requires_setup": True, + "dependencies": { + }, + "description": """ Default formatter. + """, +} diff --git a/src/auto_archiver/modules/mute_formatter/mute_formatter.py b/src/auto_archiver/modules/mute_formatter/mute_formatter.py new file mode 100644 index 0000000..129ddcb --- /dev/null +++ b/src/auto_archiver/modules/mute_formatter/mute_formatter.py @@ -0,0 +1,9 @@ +from __future__ import annotations + +from auto_archiver.core import Metadata, Media +from auto_archiver.core import Formatter + + +class MuteFormatter(Formatter): + + def format(self, item: Metadata) -> Media: return None diff --git a/src/auto_archiver/modules/pdq_hash_enricher/__init__.py b/src/auto_archiver/modules/pdq_hash_enricher/__init__.py new file mode 100644 index 0000000..b444197 --- /dev/null +++ b/src/auto_archiver/modules/pdq_hash_enricher/__init__.py @@ -0,0 +1 @@ +from .pdq_hash_enricher import PdqHashEnricher \ No newline at end of file diff --git a/src/auto_archiver/modules/pdq_hash_enricher/__manifest__.py b/src/auto_archiver/modules/pdq_hash_enricher/__manifest__.py new file mode 100644 index 0000000..133fef7 --- /dev/null +++ b/src/auto_archiver/modules/pdq_hash_enricher/__manifest__.py @@ -0,0 +1,21 @@ +{ + "name": "PDQ Hash Enricher", + "type": ["enricher"], + "requires_setup": False, + "dependencies": { + "python": ["loguru", "pdqhash", "numpy", "PIL"], + }, + "description": """ + PDQ Hash Enricher for generating perceptual hashes of media files. + + ### Features + - Calculates perceptual hashes for image files using the PDQ hashing algorithm. + - Enables detection of duplicate or near-duplicate visual content. + - Processes images stored in `Metadata` objects, adding computed hashes to the corresponding `Media` entries. + - Skips non-image media or files unsuitable for hashing (e.g., corrupted or unsupported formats). + + ### Notes + - Best used after enrichers like `thumbnail_enricher` or `screenshot_enricher` to ensure images are available. + - Uses the `pdqhash` library to compute 256-bit perceptual hashes, which are stored as hexadecimal strings. + """ +} diff --git a/src/auto_archiver/enrichers/pdq_hash_enricher.py b/src/auto_archiver/modules/pdq_hash_enricher/pdq_hash_enricher.py similarity index 88% rename from src/auto_archiver/enrichers/pdq_hash_enricher.py rename to src/auto_archiver/modules/pdq_hash_enricher/pdq_hash_enricher.py index 36f793d..e812e8b 100644 --- a/src/auto_archiver/enrichers/pdq_hash_enricher.py +++ b/src/auto_archiver/modules/pdq_hash_enricher/pdq_hash_enricher.py @@ -16,8 +16,8 @@ import numpy as np from PIL import Image, UnidentifiedImageError from loguru import logger -from . import Enricher -from ..core import Metadata +from auto_archiver.core import Enricher +from auto_archiver.core import Metadata class PdqHashEnricher(Enricher): @@ -25,15 +25,6 @@ class PdqHashEnricher(Enricher): Calculates perceptual hashes for Media instances using PDQ, allowing for (near-)duplicate detection. Ideally this enrichment is orchestrated to run after the thumbnail_enricher. """ - name = "pdq_hash_enricher" - - def __init__(self, config: dict) -> None: - # Without this STEP.__init__ is not called - super().__init__(config) - - @staticmethod - def configs() -> dict: - return {} def enrich(self, to_enrich: Metadata) -> None: url = to_enrich.get_url() diff --git a/src/auto_archiver/modules/s3_storage/__init__.py b/src/auto_archiver/modules/s3_storage/__init__.py new file mode 100644 index 0000000..cbf3237 --- /dev/null +++ b/src/auto_archiver/modules/s3_storage/__init__.py @@ -0,0 +1 @@ +from .s3_storage import S3Storage \ No newline at end of file diff --git a/src/auto_archiver/modules/s3_storage/__manifest__.py b/src/auto_archiver/modules/s3_storage/__manifest__.py new file mode 100644 index 0000000..bf032e7 --- /dev/null +++ b/src/auto_archiver/modules/s3_storage/__manifest__.py @@ -0,0 +1,54 @@ +{ + "name": "S3 Storage", + "type": ["storage"], + "requires_setup": True, + "dependencies": { + "python": ["hash_enricher", "boto3", "loguru"], + }, + "configs": { + "path_generator": { + "default": "flat", + "help": "how to store the file in terms of directory structure: 'flat' sets to root; 'url' creates a directory based on the provided URL; 'random' creates a random directory.", + "choices": ["flat", "url", "random"], + }, + "filename_generator": { + "default": "static", + "help": "how to name stored files: 'random' creates a random string; 'static' uses a replicable strategy such as a hash.", + "choices": ["random", "static"], + }, + "bucket": {"default": None, "help": "S3 bucket name"}, + "region": {"default": None, "help": "S3 region name"}, + "key": {"default": None, "help": "S3 API key"}, + "secret": {"default": None, "help": "S3 API secret"}, + "random_no_duplicate": {"default": False, + "type": "bool", + "help": "if set, it will override `path_generator`, `filename_generator` and `folder`. It will check if the file already exists and if so it will not upload it again. Creates a new root folder path `no-dups/`"}, + "endpoint_url": { + "default": 'https://{region}.digitaloceanspaces.com', + "help": "S3 bucket endpoint, {region} are inserted at runtime" + }, + "cdn_url": { + "default": 'https://{bucket}.{region}.cdn.digitaloceanspaces.com/{key}', + "help": "S3 CDN url, {bucket}, {region} and {key} are inserted at runtime" + }, + "private": {"default": False, + "type": "bool", + "help": "if true S3 files will not be readable online"}, + }, + "description": """ + S3Storage: A storage module for saving media files to an S3-compatible object storage. + + ### Features + - Uploads media files to an S3 bucket with customizable configurations. + - Supports `random_no_duplicate` mode to avoid duplicate uploads by checking existing files based on SHA-256 hashes. + - Automatically generates unique paths for files when duplicates are found. + - Configurable endpoint and CDN URL for different S3-compatible providers. + - Supports both private and public file storage, with public files being readable online. + + ### Notes + - Requires S3 credentials (API key and secret) and a bucket name to function. + - The `random_no_duplicate` option ensures no duplicate uploads by leveraging hash-based folder structures. + - Uses `boto3` for interaction with the S3 API. + - Depends on the `HashEnricher` module for hash calculation. + """ +} diff --git a/src/auto_archiver/storages/s3.py b/src/auto_archiver/modules/s3_storage/s3_storage.py similarity index 58% rename from src/auto_archiver/storages/s3.py rename to src/auto_archiver/modules/s3_storage/s3_storage.py index 5139068..6590ac9 100644 --- a/src/auto_archiver/storages/s3.py +++ b/src/auto_archiver/modules/s3_storage/s3_storage.py @@ -1,19 +1,19 @@ from typing import IO -import boto3, os -from ..utils.misc import random_str -from ..core import Media -from ..storages import Storage -from ..enrichers import HashEnricher +import boto3 +import os from loguru import logger -NO_DUPLICATES_FOLDER = "no-dups/" -class S3Storage(Storage): - name = "s3_storage" +from auto_archiver.core import Media +from auto_archiver.core import Storage +from auto_archiver.utils.misc import calculate_file_hash, random_str - def __init__(self, config: dict) -> None: - super().__init__(config) +NO_DUPLICATES_FOLDER = "no-dups/" + +class S3Storage(Storage): + + def setup(self) -> None: self.s3 = boto3.client( 's3', region_name=self.region, @@ -21,31 +21,9 @@ class S3Storage(Storage): aws_access_key_id=self.key, aws_secret_access_key=self.secret ) - self.random_no_duplicate = bool(self.random_no_duplicate) if self.random_no_duplicate: logger.warning("random_no_duplicate is set to True, this will override `path_generator`, `filename_generator` and `folder`.") - @staticmethod - def configs() -> dict: - return dict( - Storage.configs(), - ** { - "bucket": {"default": None, "help": "S3 bucket name"}, - "region": {"default": None, "help": "S3 region name"}, - "key": {"default": None, "help": "S3 API key"}, - "secret": {"default": None, "help": "S3 API secret"}, - "random_no_duplicate": {"default": False, "help": f"if set, it will override `path_generator`, `filename_generator` and `folder`. It will check if the file already exists and if so it will not upload it again. Creates a new root folder path `{NO_DUPLICATES_FOLDER}`"}, - "endpoint_url": { - "default": 'https://{region}.digitaloceanspaces.com', - "help": "S3 bucket endpoint, {region} are inserted at runtime" - }, - "cdn_url": { - "default": 'https://{bucket}.{region}.cdn.digitaloceanspaces.com/{key}', - "help": "S3 CDN url, {bucket}, {region} and {key} are inserted at runtime" - }, - "private": {"default": False, "help": "if true S3 files will not be readable online"}, - }) - def get_cdn_url(self, media: Media) -> str: return self.cdn_url.format(bucket=self.bucket, region=self.region, key=media.key) @@ -62,15 +40,13 @@ class S3Storage(Storage): extra_args['ContentType'] = media.mimetype except Exception as e: logger.warning(f"Unable to get mimetype for {media.key=}, error: {e}") - self.s3.upload_fileobj(file, Bucket=self.bucket, Key=media.key, ExtraArgs=extra_args) return True def is_upload_needed(self, media: Media) -> bool: if self.random_no_duplicate: # checks if a folder with the hash already exists, if so it skips the upload - he = HashEnricher({"hash_enricher": {"algorithm": "SHA-256", "chunksize": 1.6e7}}) - hd = he.calculate_hash(media.filename) + hd = calculate_file_hash(media.filename) path = os.path.join(NO_DUPLICATES_FOLDER, hd[:24]) if existing_key:=self.file_in_folder(path): @@ -82,8 +58,7 @@ class S3Storage(Storage): _, ext = os.path.splitext(media.key) media.key = os.path.join(path, f"{random_str(24)}{ext}") return True - - + def file_in_folder(self, path:str) -> str: # checks if path exists and is not an empty folder if not path.endswith('/'): diff --git a/src/auto_archiver/modules/screenshot_enricher/__init__.py b/src/auto_archiver/modules/screenshot_enricher/__init__.py new file mode 100644 index 0000000..393f726 --- /dev/null +++ b/src/auto_archiver/modules/screenshot_enricher/__init__.py @@ -0,0 +1 @@ +from .screenshot_enricher import ScreenshotEnricher diff --git a/src/auto_archiver/modules/screenshot_enricher/__manifest__.py b/src/auto_archiver/modules/screenshot_enricher/__manifest__.py new file mode 100644 index 0000000..831959e --- /dev/null +++ b/src/auto_archiver/modules/screenshot_enricher/__manifest__.py @@ -0,0 +1,30 @@ +{ + "name": "Screenshot Enricher", + "type": ["enricher"], + "requires_setup": True, + "dependencies": { + "python": ["loguru", "selenium"], + "bin": ["geckodriver"] + }, + "configs": { + "width": {"default": 1280, "help": "width of the screenshots"}, + "height": {"default": 720, "help": "height of the screenshots"}, + "timeout": {"default": 60, "help": "timeout for taking the screenshot"}, + "sleep_before_screenshot": {"default": 4, "help": "seconds to wait for the pages to load before taking screenshot"}, + "http_proxy": {"default": "", "help": "http proxy to use for the webdriver, eg http://proxy-user:password@proxy-ip:port"}, + "save_to_pdf": {"default": False, "help": "save the page as pdf along with the screenshot. PDF saving options can be adjusted with the 'print_options' parameter"}, + "print_options": {"default": {}, "help": "options to pass to the pdf printer"} + }, + "description": """ + Captures screenshots and optionally saves web pages as PDFs using a WebDriver. + + ### Features + - Takes screenshots of web pages, with configurable width, height, and timeout settings. + - Optionally saves pages as PDFs, with additional configuration for PDF printing options. + - Bypasses URLs detected as authentication walls. + - Integrates seamlessly with the metadata enrichment pipeline, adding screenshots and PDFs as media. + + ### Notes + - Requires a WebDriver (e.g., ChromeDriver) installed and accessible via the system's PATH. + """ +} diff --git a/src/auto_archiver/modules/screenshot_enricher/screenshot_enricher.py b/src/auto_archiver/modules/screenshot_enricher/screenshot_enricher.py new file mode 100644 index 0000000..e1da99d --- /dev/null +++ b/src/auto_archiver/modules/screenshot_enricher/screenshot_enricher.py @@ -0,0 +1,40 @@ +from loguru import logger +import time, os +import base64 + +from selenium.common.exceptions import TimeoutException + + +from auto_archiver.core import Enricher +from auto_archiver.utils import Webdriver, url as UrlUtil, random_str +from auto_archiver.core import Media, Metadata + +class ScreenshotEnricher(Enricher): + + def enrich(self, to_enrich: Metadata) -> None: + url = to_enrich.get_url() + + if UrlUtil.is_auth_wall(url): + logger.debug(f"[SKIP] SCREENSHOT since url is behind AUTH WALL: {url=}") + return + + logger.debug(f"Enriching screenshot for {url=}") + auth = self.auth_for_site(url) + with Webdriver(self.width, self.height, self.timeout, facebook_accept_cookies='facebook.com' in url, + http_proxy=self.http_proxy, print_options=self.print_options, auth=auth) as driver: + try: + driver.get(url) + time.sleep(int(self.sleep_before_screenshot)) + screenshot_file = os.path.join(self.tmp_dir, f"screenshot_{random_str(8)}.png") + driver.save_screenshot(screenshot_file) + to_enrich.add_media(Media(filename=screenshot_file), id="screenshot") + if self.save_to_pdf: + pdf_file = os.path.join(self.tmp_dir, f"pdf_{random_str(8)}.pdf") + pdf = driver.print_page(driver.print_options) + with open(pdf_file, "wb") as f: + f.write(base64.b64decode(pdf)) + to_enrich.add_media(Media(filename=pdf_file), id="pdf") + except TimeoutException: + logger.info("TimeoutException loading page for screenshot") + except Exception as e: + logger.error(f"Got error while loading webdriver for screenshot enricher: {e}") diff --git a/src/auto_archiver/modules/ssl_enricher/__init__.py b/src/auto_archiver/modules/ssl_enricher/__init__.py new file mode 100644 index 0000000..23d2bee --- /dev/null +++ b/src/auto_archiver/modules/ssl_enricher/__init__.py @@ -0,0 +1 @@ +from .ssl_enricher import SSLEnricher \ No newline at end of file diff --git a/src/auto_archiver/modules/ssl_enricher/__manifest__.py b/src/auto_archiver/modules/ssl_enricher/__manifest__.py new file mode 100644 index 0000000..9028f14 --- /dev/null +++ b/src/auto_archiver/modules/ssl_enricher/__manifest__.py @@ -0,0 +1,23 @@ +{ + "name": "SSL Certificate Enricher", + "type": ["enricher"], + "requires_setup": False, + "dependencies": { + "python": ["loguru", "slugify"], + }, + 'entry_point': 'ssl_enricher::SSLEnricher', + "configs": { + "skip_when_nothing_archived": {"default": True, "help": "if true, will skip enriching when no media is archived"}, + }, + "description": """ + Retrieves SSL certificate information for a domain and stores it as a file. + + ### Features + - Fetches SSL certificates for domains using the HTTPS protocol. + - Stores certificates in PEM format and adds them as media to the metadata. + - Skips enrichment if no media has been archived, based on the `skip_when_nothing_archived` configuration. + + ### Notes + - Requires the target URL to use the HTTPS scheme; other schemes are not supported. + """ +} diff --git a/src/auto_archiver/enrichers/ssl_enricher.py b/src/auto_archiver/modules/ssl_enricher/ssl_enricher.py similarity index 58% rename from src/auto_archiver/enrichers/ssl_enricher.py rename to src/auto_archiver/modules/ssl_enricher/ssl_enricher.py index 396df2e..b429163 100644 --- a/src/auto_archiver/enrichers/ssl_enricher.py +++ b/src/auto_archiver/modules/ssl_enricher/ssl_enricher.py @@ -3,25 +3,14 @@ from slugify import slugify from urllib.parse import urlparse from loguru import logger -from . import Enricher -from ..core import Metadata, ArchivingContext, Media +from auto_archiver.core import Enricher +from auto_archiver.core import Metadata, Media class SSLEnricher(Enricher): """ Retrieves SSL certificate information for a domain, as a file """ - name = "ssl_enricher" - - def __init__(self, config: dict) -> None: - super().__init__(config) - self. skip_when_nothing_archived = bool(self.skip_when_nothing_archived) - - @staticmethod - def configs() -> dict: - return { - "skip_when_nothing_archived": {"default": True, "help": "if true, will skip enriching when no media is archived"}, - } def enrich(self, to_enrich: Metadata) -> None: if not to_enrich.media and self.skip_when_nothing_archived: return @@ -34,6 +23,6 @@ class SSLEnricher(Enricher): logger.debug(f"fetching SSL certificate for {domain=} in {url=}") cert = ssl.get_server_certificate((domain, 443)) - cert_fn = os.path.join(ArchivingContext.get_tmp_dir(), f"{slugify(domain)}.pem") + cert_fn = os.path.join(self.tmp_dir, f"{slugify(domain)}.pem") with open(cert_fn, "w") as f: f.write(cert) to_enrich.add_media(Media(filename=cert_fn), id="ssl_certificate") diff --git a/src/auto_archiver/modules/telegram_extractor/__init__.py b/src/auto_archiver/modules/telegram_extractor/__init__.py new file mode 100644 index 0000000..1fd80c2 --- /dev/null +++ b/src/auto_archiver/modules/telegram_extractor/__init__.py @@ -0,0 +1 @@ +from .telegram_extractor import TelegramExtractor \ No newline at end of file diff --git a/src/auto_archiver/modules/telegram_extractor/__manifest__.py b/src/auto_archiver/modules/telegram_extractor/__manifest__.py new file mode 100644 index 0000000..cb0ee1e --- /dev/null +++ b/src/auto_archiver/modules/telegram_extractor/__manifest__.py @@ -0,0 +1,24 @@ +{ + "name": "Telegram Extractor", + "type": ["extractor"], + "requires_setup": False, + "dependencies": { + "python": [ + "requests", + "bs4", + "loguru", + ], + }, + "description": """ + The `TelegramExtractor` retrieves publicly available media content from Telegram message links without requiring login credentials. + It processes URLs to fetch images and videos embedded in Telegram messages, ensuring a structured output using `Metadata` + and `Media` objects. Recommended for scenarios where login-based archiving is not viable, although `telethon_archiver` + is advised for more comprehensive functionality, and higher quality media extraction. + + ### Features +- Extracts images and videos from public Telegram message links (`t.me`). +- Processes HTML content of messages to retrieve embedded media. +- Sets structured metadata, including timestamps, content, and media details. +- Does not require user authentication for Telegram. + """, +} diff --git a/src/auto_archiver/archivers/telegram_archiver.py b/src/auto_archiver/modules/telegram_extractor/telegram_extractor.py similarity index 82% rename from src/auto_archiver/archivers/telegram_archiver.py rename to src/auto_archiver/modules/telegram_extractor/telegram_extractor.py index ed57927..d612e24 100644 --- a/src/auto_archiver/archivers/telegram_archiver.py +++ b/src/auto_archiver/modules/telegram_extractor/telegram_extractor.py @@ -2,22 +2,15 @@ import requests, re, html from bs4 import BeautifulSoup from loguru import logger -from . import Archiver -from ..core import Metadata, Media +from auto_archiver.core import Extractor +from auto_archiver.core import Metadata, Media -class TelegramArchiver(Archiver): +class TelegramExtractor(Extractor): """ - Archiver for telegram that does not require login, but the telethon_archiver is much more advised, will only return if at least one image or one video is found + Extractor for telegram that does not require login, but the telethon_extractor is much more advised, + will only return if at least one image or one video is found """ - name = "telegram_archiver" - - def __init__(self, config: dict) -> None: - super().__init__(config) - - @staticmethod - def configs() -> dict: - return {} def download(self, item: Metadata) -> Metadata: url = item.get_url() diff --git a/src/auto_archiver/modules/telethon_extractor/__init__.py b/src/auto_archiver/modules/telethon_extractor/__init__.py new file mode 100644 index 0000000..2eaa57c --- /dev/null +++ b/src/auto_archiver/modules/telethon_extractor/__init__.py @@ -0,0 +1 @@ +from .telethon_extractor import TelethonExtractor \ No newline at end of file diff --git a/src/auto_archiver/modules/telethon_extractor/__manifest__.py b/src/auto_archiver/modules/telethon_extractor/__manifest__.py new file mode 100644 index 0000000..6b37654 --- /dev/null +++ b/src/auto_archiver/modules/telethon_extractor/__manifest__.py @@ -0,0 +1,44 @@ +{ + "name": "telethon_extractor", + "type": ["extractor"], + "requires_setup": True, + "dependencies": { + "python": ["telethon", + "loguru", + "tqdm", + ], + "bin": [""] + }, + "configs": { + "api_id": {"default": None, "help": "telegram API_ID value, go to https://my.telegram.org/apps"}, + "api_hash": {"default": None, "help": "telegram API_HASH value, go to https://my.telegram.org/apps"}, + "bot_token": {"default": None, "help": "optional, but allows access to more content such as large videos, talk to @botfather"}, + "session_file": {"default": "secrets/anon", "help": "optional, records the telegram login session for future usage, '.session' will be appended to the provided value."}, + "join_channels": {"default": True, "help": "disables the initial setup with channel_invites config, useful if you have a lot and get stuck"}, + "channel_invites": { + "default": {}, + "help": "(JSON string) private channel invite links (format: t.me/joinchat/HASH OR t.me/+HASH) and (optional but important to avoid hanging for minutes on startup) channel id (format: CHANNEL_ID taken from a post url like https://t.me/c/CHANNEL_ID/1), the telegram account will join any new channels on setup", + "type": "auto_archiver.utils.json_loader", + } + }, + "description": """ +The `TelethonExtractor` uses the Telethon library to archive posts and media from Telegram channels and groups. +It supports private and public channels, downloading grouped posts with media, and can join channels using invite links +if provided in the configuration. + +### Features +- Fetches posts and metadata from Telegram channels and groups, including private channels. +- Downloads media attachments (e.g., images, videos, audio) from individual posts or grouped posts. +- Handles channel invites to join channels dynamically during setup. +- Utilizes Telethon's capabilities for reliable Telegram interactions. +- Outputs structured metadata and media using `Metadata` and `Media` objects. + +### Setup +To use the `TelethonExtractor`, you must configure the following: +- **API ID and API Hash**: Obtain these from [my.telegram.org](https://my.telegram.org/apps). +- **Session File**: Optional, but records login sessions for future use (default: `secrets/anon.session`). +- **Bot Token**: Optional, allows access to additional content (e.g., large videos) but limits private channel archiving. +- **Channel Invites**: Optional, specify a JSON string of invite links to join channels during setup. + +""" +} \ No newline at end of file diff --git a/src/auto_archiver/archivers/telethon_archiver.py b/src/auto_archiver/modules/telethon_extractor/telethon_extractor.py similarity index 82% rename from src/auto_archiver/archivers/telethon_archiver.py rename to src/auto_archiver/modules/telethon_extractor/telethon_extractor.py index 2e2305d..65ea8cd 100644 --- a/src/auto_archiver/archivers/telethon_archiver.py +++ b/src/auto_archiver/modules/telethon_extractor/telethon_extractor.py @@ -6,39 +6,20 @@ from telethon.tl.functions.messages import ImportChatInviteRequest from telethon.errors.rpcerrorlist import UserAlreadyParticipantError, FloodWaitError, InviteRequestSentError, InviteHashExpiredError from loguru import logger from tqdm import tqdm -import re, time, json, os +import re, time, os -from . import Archiver -from ..core import Metadata, Media, ArchivingContext -from ..utils import random_str +from auto_archiver.core import Extractor +from auto_archiver.core import Metadata, Media +from auto_archiver.utils import random_str -class TelethonArchiver(Archiver): - name = "telethon_archiver" - link_pattern = re.compile(r"https:\/\/t\.me(\/c){0,1}\/(.+)\/(\d+)") +class TelethonExtractor(Extractor): + valid_url = re.compile(r"https:\/\/t\.me(\/c){0,1}\/(.+)\/(\d+)") invite_pattern = re.compile(r"t.me(\/joinchat){0,1}\/\+?(.+)") - def __init__(self, config: dict) -> None: - super().__init__(config) - self.assert_valid_string("api_id") - self.assert_valid_string("api_hash") - - @staticmethod - def configs() -> dict: - return { - "api_id": {"default": None, "help": "telegram API_ID value, go to https://my.telegram.org/apps"}, - "api_hash": {"default": None, "help": "telegram API_HASH value, go to https://my.telegram.org/apps"}, - "bot_token": {"default": None, "help": "optional, but allows access to more content such as large videos, talk to @botfather"}, - "session_file": {"default": "secrets/anon", "help": "optional, records the telegram login session for future usage, '.session' will be appended to the provided value."}, - "join_channels": {"default": True, "help": "disables the initial setup with channel_invites config, useful if you have a lot and get stuck"}, - "channel_invites": { - "default": {}, - "help": "(JSON string) private channel invite links (format: t.me/joinchat/HASH OR t.me/+HASH) and (optional but important to avoid hanging for minutes on startup) channel id (format: CHANNEL_ID taken from a post url like https://t.me/c/CHANNEL_ID/1), the telegram account will join any new channels on setup", - "cli_set": lambda cli_val, cur_val: dict(cur_val, **json.loads(cli_val)) - } - } def setup(self) -> None: + """ 1. makes a copy of session_file that is removed in cleanup 2. trigger login process for telegram or proceed if already saved in a session file @@ -112,7 +93,7 @@ class TelethonArchiver(Archiver): """ url = item.get_url() # detect URLs that we definitely cannot handle - match = self.link_pattern.search(url) + match = self.valid_url.search(url) logger.debug(f"TELETHON: {match=}") if not match: return False @@ -140,7 +121,7 @@ class TelethonArchiver(Archiver): media_posts = self._get_media_posts_in_group(chat, post) logger.debug(f'got {len(media_posts)=} for {url=}') - tmp_dir = ArchivingContext.get_tmp_dir() + tmp_dir = self.tmp_dir group_id = post.grouped_id if post.grouped_id is not None else post.id title = post.message diff --git a/src/auto_archiver/modules/thumbnail_enricher/__init__.py b/src/auto_archiver/modules/thumbnail_enricher/__init__.py new file mode 100644 index 0000000..fe20719 --- /dev/null +++ b/src/auto_archiver/modules/thumbnail_enricher/__init__.py @@ -0,0 +1 @@ +from .thumbnail_enricher import ThumbnailEnricher diff --git a/src/auto_archiver/modules/thumbnail_enricher/__manifest__.py b/src/auto_archiver/modules/thumbnail_enricher/__manifest__.py new file mode 100644 index 0000000..e47397f --- /dev/null +++ b/src/auto_archiver/modules/thumbnail_enricher/__manifest__.py @@ -0,0 +1,27 @@ +{ + "name": "Thumbnail Enricher", + "type": ["enricher"], + "requires_setup": False, + "dependencies": { + "python": ["loguru", "ffmpeg"], + "bin": ["ffmpeg"] + }, + "configs": { + "thumbnails_per_minute": {"default": 60, "help": "how many thumbnails to generate per minute of video, can be limited by max_thumbnails"}, + "max_thumbnails": {"default": 16, "help": "limit the number of thumbnails to generate per video, 0 means no limit"}, + }, + "description": """ + Generates thumbnails for video files to provide visual previews. + + ### Features + - Processes video files and generates evenly distributed thumbnails. + - Calculates the number of thumbnails based on video duration, `thumbnails_per_minute`, and `max_thumbnails`. + - Distributes thumbnails equally across the video's duration and stores them as media objects. + - Adds metadata for each thumbnail, including timestamps and IDs. + + ### Notes + - Requires `ffmpeg` to be installed and accessible via the system's PATH. + - Handles videos without pre-existing duration metadata by probing with `ffmpeg`. + - Skips enrichment for non-video media files. + """ +} diff --git a/src/auto_archiver/enrichers/thumbnail_enricher.py b/src/auto_archiver/modules/thumbnail_enricher/thumbnail_enricher.py similarity index 75% rename from src/auto_archiver/enrichers/thumbnail_enricher.py rename to src/auto_archiver/modules/thumbnail_enricher/thumbnail_enricher.py index 5d8bee2..e0ac937 100644 --- a/src/auto_archiver/enrichers/thumbnail_enricher.py +++ b/src/auto_archiver/modules/thumbnail_enricher/thumbnail_enricher.py @@ -9,29 +9,15 @@ and identify important moments without watching the entire video. import ffmpeg, os from loguru import logger -from . import Enricher -from ..core import Media, Metadata, ArchivingContext -from ..utils.misc import random_str +from auto_archiver.core import Enricher +from auto_archiver.core import Media, Metadata +from auto_archiver.utils.misc import random_str class ThumbnailEnricher(Enricher): """ Generates thumbnails for all the media """ - name = "thumbnail_enricher" - - def __init__(self, config: dict) -> None: - # without this STEP.__init__ is not called - super().__init__(config) - self.thumbnails_per_second = int(self.thumbnails_per_minute) / 60 - self.max_thumbnails = int(self.max_thumbnails) - - @staticmethod - def configs() -> dict: - return { - "thumbnails_per_minute": {"default": 60, "help": "how many thumbnails to generate per minute of video, can be limited by max_thumbnails"}, - "max_thumbnails": {"default": 16, "help": "limit the number of thumbnails to generate per video, 0 means no limit"}, - } def enrich(self, to_enrich: Metadata) -> None: """ @@ -42,7 +28,7 @@ class ThumbnailEnricher(Enricher): logger.debug(f"generating thumbnails for {to_enrich.get_url()}") for m_id, m in enumerate(to_enrich.media[::]): if m.is_video(): - folder = os.path.join(ArchivingContext.get_tmp_dir(), random_str(24)) + folder = os.path.join(self.tmp_dir, random_str(24)) os.makedirs(folder, exist_ok=True) logger.debug(f"generating thumbnails for {m.filename}") duration = m.get("duration") @@ -56,7 +42,7 @@ class ThumbnailEnricher(Enricher): logger.error(f"error getting duration of video {m.filename}: {e}") return - num_thumbs = int(min(max(1, duration * self.thumbnails_per_second), self.max_thumbnails)) + num_thumbs = int(min(max(1, duration * self.thumbnails_per_minute), self.max_thumbnails)) timestamps = [duration / (num_thumbs + 1) * i for i in range(1, num_thumbs + 1)] thumbnails_media = [] diff --git a/src/auto_archiver/modules/timestamping_enricher/__init__.py b/src/auto_archiver/modules/timestamping_enricher/__init__.py new file mode 100644 index 0000000..62d358a --- /dev/null +++ b/src/auto_archiver/modules/timestamping_enricher/__init__.py @@ -0,0 +1 @@ +from .timestamping_enricher import TimestampingEnricher diff --git a/src/auto_archiver/modules/timestamping_enricher/__manifest__.py b/src/auto_archiver/modules/timestamping_enricher/__manifest__.py new file mode 100644 index 0000000..6ad9c57 --- /dev/null +++ b/src/auto_archiver/modules/timestamping_enricher/__manifest__.py @@ -0,0 +1,54 @@ +{ + "name": "Timestamping Enricher", + "type": ["enricher"], + "requires_setup": True, + "dependencies": { + "python": [ + "loguru", + "slugify", + "tsp_client", + "asn1crypto", + "certvalidator", + "certifi" + ], + }, + "configs": { + "tsa_urls": { + "default": [ + # [Adobe Approved Trust List] and [Windows Cert Store] + "http://timestamp.digicert.com", + "http://timestamp.identrust.com", + # "https://timestamp.entrust.net/TSS/RFC3161sha2TS", # not valid for timestamping + # "https://timestamp.sectigo.com", # wait 15 seconds between each request. + + # [Adobe: European Union Trusted Lists]. + # "https://timestamp.sectigo.com/qualified", # wait 15 seconds between each request. + + # [Windows Cert Store] + "http://timestamp.globalsign.com/tsa/r6advanced1", + # [Adobe: European Union Trusted Lists] and [Windows Cert Store] + # "http://ts.quovadisglobal.com/eu", # not valid for timestamping + # "http://tsa.belgium.be/connect", # self-signed certificate in certificate chain + # "https://timestamp.aped.gov.gr/qtss", # self-signed certificate in certificate chain + # "http://tsa.sep.bg", # self-signed certificate in certificate chain + # "http://tsa.izenpe.com", #unable to get local issuer certificate + # "http://kstamp.keynectis.com/KSign", # unable to get local issuer certificate + "http://tss.accv.es:8318/tsa", + ], + "help": "List of RFC3161 Time Stamp Authorities to use, separate with commas if passed via the command line.", + } + }, + "description": """ + Generates RFC3161-compliant timestamp tokens using Time Stamp Authorities (TSA) for archived files. + + ### Features + - Creates timestamp tokens to prove the existence of files at a specific time, useful for legal and authenticity purposes. + - Aggregates file hashes into a text file and timestamps the concatenated data. + - Uses multiple Time Stamp Authorities (TSAs) to ensure reliability and redundancy. + - Validates timestamping certificates against trusted Certificate Authorities (CAs) using the `certifi` trust store. + + ### Notes + - Should be run after the `hash_enricher` to ensure file hashes are available. + - Requires internet access to interact with the configured TSAs. + """ +} diff --git a/src/auto_archiver/enrichers/timestamping_enricher.py b/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py similarity index 69% rename from src/auto_archiver/enrichers/timestamping_enricher.py rename to src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py index dffa1a3..078c1ba 100644 --- a/src/auto_archiver/enrichers/timestamping_enricher.py +++ b/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py @@ -8,10 +8,8 @@ from certvalidator import CertificateValidator, ValidationContext from asn1crypto import pem import certifi -from . import Enricher -from ..core import Metadata, ArchivingContext, Media -from ..archivers import Archiver - +from auto_archiver.core import Enricher +from auto_archiver.core import Metadata, Media class TimestampingEnricher(Enricher): """ @@ -21,41 +19,6 @@ class TimestampingEnricher(Enricher): See https://gist.github.com/Manouchehri/fd754e402d98430243455713efada710 for list of timestamp authorities. """ - name = "timestamping_enricher" - - def __init__(self, config: dict) -> None: - super().__init__(config) - - @staticmethod - def configs() -> dict: - return { - "tsa_urls": { - "default": [ - # [Adobe Approved Trust List] and [Windows Cert Store] - "http://timestamp.digicert.com", - "http://timestamp.identrust.com", - # "https://timestamp.entrust.net/TSS/RFC3161sha2TS", # not valid for timestamping - # "https://timestamp.sectigo.com", # wait 15 seconds between each request. - - # [Adobe: European Union Trusted Lists]. - # "https://timestamp.sectigo.com/qualified", # wait 15 seconds between each request. - - # [Windows Cert Store] - "http://timestamp.globalsign.com/tsa/r6advanced1", - - # [Adobe: European Union Trusted Lists] and [Windows Cert Store] - # "http://ts.quovadisglobal.com/eu", # not valid for timestamping - # "http://tsa.belgium.be/connect", # self-signed certificate in certificate chain - # "https://timestamp.aped.gov.gr/qtss", # self-signed certificate in certificate chain - # "http://tsa.sep.bg", # self-signed certificate in certificate chain - # "http://tsa.izenpe.com", #unable to get local issuer certificate - # "http://kstamp.keynectis.com/KSign", # unable to get local issuer certificate - "http://tss.accv.es:8318/tsa", - ], - "help": "List of RFC3161 Time Stamp Authorities to use, separate with commas if passed via the command line.", - "cli_set": lambda cli_val, cur_val: set(cli_val.split(",")) - } - } def enrich(self, to_enrich: Metadata) -> None: url = to_enrich.get_url() @@ -68,7 +31,7 @@ class TimestampingEnricher(Enricher): logger.warning(f"No hashes found in {url=}") return - tmp_dir = ArchivingContext.get_tmp_dir() + tmp_dir = self.tmp_dir hashes_fn = os.path.join(tmp_dir, "hashes.txt") data_to_sign = "\n".join(hashes) @@ -128,7 +91,7 @@ class TimestampingEnricher(Enricher): cert_chain = [] for cert in path: - cert_fn = os.path.join(ArchivingContext.get_tmp_dir(), f"{str(cert.serial_number)[:20]}.crt") + cert_fn = os.path.join(self.tmp_dir, f"{str(cert.serial_number)[:20]}.crt") with open(cert_fn, "wb") as f: f.write(cert.dump()) cert_chain.append(Media(filename=cert_fn).set("subject", cert.subject.native["common_name"])) diff --git a/src/auto_archiver/modules/twitter_api_extractor/__init__.py b/src/auto_archiver/modules/twitter_api_extractor/__init__.py new file mode 100644 index 0000000..7005965 --- /dev/null +++ b/src/auto_archiver/modules/twitter_api_extractor/__init__.py @@ -0,0 +1 @@ +from .twitter_api_extractor import TwitterApiExtractor \ No newline at end of file diff --git a/src/auto_archiver/modules/twitter_api_extractor/__manifest__.py b/src/auto_archiver/modules/twitter_api_extractor/__manifest__.py new file mode 100644 index 0000000..05d1ac0 --- /dev/null +++ b/src/auto_archiver/modules/twitter_api_extractor/__manifest__.py @@ -0,0 +1,44 @@ +{ + "name": "Twitter API Extractor", + "type": ["extractor"], + "requires_setup": True, + "dependencies": { + "python": ["requests", + "loguru", + "pytwitter", + "slugify",], + "bin": [""] + }, + "configs": { + "bearer_token": {"default": None, "help": "[deprecated: see bearer_tokens] twitter API bearer_token which is enough for archiving, if not provided you will need consumer_key, consumer_secret, access_token, access_secret"}, + "bearer_tokens": {"default": [], "help": " a list of twitter API bearer_token which is enough for archiving, if not provided you will need consumer_key, consumer_secret, access_token, access_secret, if provided you can still add those for better rate limits. CSV of bearer tokens if provided via the command line", + }, + "consumer_key": {"default": None, "help": "twitter API consumer_key"}, + "consumer_secret": {"default": None, "help": "twitter API consumer_secret"}, + "access_token": {"default": None, "help": "twitter API access_token"}, + "access_secret": {"default": None, "help": "twitter API access_secret"}, + }, + "description": """ + The `TwitterApiExtractor` fetches tweets and associated media using the Twitter API. + It supports multiple API configurations for extended rate limits and reliable access. + Features include URL expansion, media downloads (e.g., images, videos), and structured output + via `Metadata` and `Media` objects. Requires Twitter API credentials such as bearer tokens + or consumer key/secret and access token/secret. + + ### Features + - Fetches tweets and their metadata, including text, creation timestamp, and author information. + - Downloads media attachments (e.g., images, videos) in high quality. + - Supports multiple API configurations for improved rate limiting. + - Expands shortened URLs (e.g., `t.co` links). + - Outputs structured metadata and media using `Metadata` and `Media` objects. + + ### Setup + To use the `TwitterApiExtractor`, you must provide valid Twitter API credentials via configuration: + - **Bearer Token(s)**: A single token or a list for rate-limited API access. + - **Consumer Key and Secret**: Required for user-authenticated API access. + - **Access Token and Secret**: Complements the consumer key for enhanced API capabilities. + + Credentials can be obtained by creating a Twitter developer account at [Twitter Developer Platform](https://developer.twitter.com/en). + """ +, +} diff --git a/src/auto_archiver/archivers/twitter_api_archiver.py b/src/auto_archiver/modules/twitter_api_extractor/twitter_api_extractor.py similarity index 73% rename from src/auto_archiver/archivers/twitter_api_archiver.py rename to src/auto_archiver/modules/twitter_api_extractor/twitter_api_extractor.py index d1e4dee..72fd2f2 100644 --- a/src/auto_archiver/archivers/twitter_api_archiver.py +++ b/src/auto_archiver/modules/twitter_api_extractor/twitter_api_extractor.py @@ -8,43 +8,25 @@ from loguru import logger from pytwitter import Api from slugify import slugify -from . import Archiver -from ..core import Metadata,Media +from auto_archiver.core import Extractor +from auto_archiver.core import Metadata, Media -class TwitterApiArchiver(Archiver): - name = "twitter_api_archiver" - link_pattern = re.compile(r"(?:twitter|x).com\/(?:\#!\/)?(\w+)\/status(?:es)?\/(\d+)") +class TwitterApiExtractor(Extractor): - def __init__(self, config: dict) -> None: - super().__init__(config) + valid_url: re.Pattern = re.compile(r"(?:twitter|x).com\/(?:\#!\/)?(\w+)\/status(?:es)?\/(\d+)") + def setup(self) -> None: self.api_index = 0 self.apis = [] if len(self.bearer_tokens): self.apis.extend([Api(bearer_token=bearer_token) for bearer_token in self.bearer_tokens]) if self.bearer_token: - self.assert_valid_string("bearer_token") self.apis.append(Api(bearer_token=self.bearer_token)) if self.consumer_key and self.consumer_secret and self.access_token and self.access_secret: - self.assert_valid_string("consumer_key") - self.assert_valid_string("consumer_secret") - self.assert_valid_string("access_token") - self.assert_valid_string("access_secret") self.apis.append(Api(consumer_key=self.consumer_key, consumer_secret=self.consumer_secret, access_token=self.access_token, access_secret=self.access_secret)) assert self.api_client is not None, "Missing Twitter API configurations, please provide either AND/OR (consumer_key, consumer_secret, access_token, access_secret) to use this archiver, you can provide both for better rate-limit results." - @staticmethod - def configs() -> dict: - return { - "bearer_token": {"default": None, "help": "[deprecated: see bearer_tokens] twitter API bearer_token which is enough for archiving, if not provided you will need consumer_key, consumer_secret, access_token, access_secret"}, - "bearer_tokens": {"default": [], "help": " a list of twitter API bearer_token which is enough for archiving, if not provided you will need consumer_key, consumer_secret, access_token, access_secret, if provided you can still add those for better rate limits. CSV of bearer tokens if provided via the command line", "cli_set": lambda cli_val, cur_val: list(set(cli_val.split(",")))}, - "consumer_key": {"default": None, "help": "twitter API consumer_key"}, - "consumer_secret": {"default": None, "help": "twitter API consumer_secret"}, - "access_token": {"default": None, "help": "twitter API access_token"}, - "access_secret": {"default": None, "help": "twitter API access_secret"}, - } - @property # getter .mimetype def api_client(self) -> str: return self.apis[self.api_index] @@ -71,7 +53,7 @@ class TwitterApiArchiver(Archiver): def get_username_tweet_id(self, url): # detect URLs that we definitely cannot handle - matches = self.link_pattern.findall(url) + matches = self.valid_url.findall(url) if not len(matches): return False, False username, tweet_id = matches[0] # only one URL supported diff --git a/src/auto_archiver/modules/vk_extractor/__init__.py b/src/auto_archiver/modules/vk_extractor/__init__.py new file mode 100644 index 0000000..0f9bcad --- /dev/null +++ b/src/auto_archiver/modules/vk_extractor/__init__.py @@ -0,0 +1 @@ +from .vk_extractor import VkExtractor diff --git a/src/auto_archiver/modules/vk_extractor/__manifest__.py b/src/auto_archiver/modules/vk_extractor/__manifest__.py new file mode 100644 index 0000000..61e454e --- /dev/null +++ b/src/auto_archiver/modules/vk_extractor/__manifest__.py @@ -0,0 +1,39 @@ +{ + "name": "VKontakte Extractor", + "type": ["extractor"], + "requires_setup": True, + "depends": ["core", "utils"], + "dependencies": { + "python": ["loguru", "vk_url_scraper"], + }, + "configs": { + "username": {"required": True, + "help": "valid VKontakte username"}, + "password": {"required": True, + "help": "valid VKontakte password"}, + "session_file": { + "default": "secrets/vk_config.v2.json", + "help": "valid VKontakte password", + }, + }, + "description": """ +The `VkExtractor` fetches posts, text, and images from VK (VKontakte) social media pages. +This archiver is specialized for `/wall` posts and uses the `VkScraper` library to extract +and download content. Note that VK videos are handled separately by the `YTDownloader`. + +### Features +- Extracts text, timestamps, and metadata from VK `/wall` posts. +- Downloads associated images and attaches them to the resulting `Metadata` object. +- Processes multiple segments of VK URLs that contain mixed content (e.g., wall, photo). +- Outputs structured metadata and media using `Metadata` and `Media` objects. + +### Setup +To use the `VkArchiver`, you must provide valid VKontakte login credentials and session information: +- **Username**: A valid VKontakte account username. +- **Password**: The corresponding password for the VKontakte account. +- **Session File**: Optional. Path to a session configuration file (`.json`) for persistent VK login. + +Credentials can be set in the configuration file or directly via environment variables. Ensure you +have access to the VKontakte API by creating an account at [VKontakte](https://vk.com/). +""", +} diff --git a/src/auto_archiver/archivers/vk_archiver.py b/src/auto_archiver/modules/vk_extractor/vk_extractor.py similarity index 59% rename from src/auto_archiver/archivers/vk_archiver.py rename to src/auto_archiver/modules/vk_extractor/vk_extractor.py index f8bb60a..99527c4 100644 --- a/src/auto_archiver/archivers/vk_archiver.py +++ b/src/auto_archiver/modules/vk_extractor/vk_extractor.py @@ -1,32 +1,20 @@ from loguru import logger from vk_url_scraper import VkScraper -from ..utils.misc import dump_payload -from . import Archiver -from ..core import Metadata, Media, ArchivingContext +from auto_archiver.utils.misc import dump_payload +from auto_archiver.core import Extractor +from auto_archiver.core import Metadata, Media -class VkArchiver(Archiver): +class VkExtractor(Extractor): """" VK videos are handled by YTDownloader, this archiver gets posts text and images. Currently only works for /wall posts """ - name = "vk_archiver" - def __init__(self, config: dict) -> None: - super().__init__(config) - self.assert_valid_string("username") - self.assert_valid_string("password") + def setup(self) -> None: self.vks = VkScraper(self.username, self.password, session_file=self.session_file) - @staticmethod - def configs() -> dict: - return { - "username": {"default": None, "help": "valid VKontakte username"}, - "password": {"default": None, "help": "valid VKontakte password"}, - "session_file": {"default": "secrets/vk_config.v2.json", "help": "valid VKontakte password"}, - } - def download(self, item: Metadata) -> Metadata: url = item.get_url() @@ -46,7 +34,7 @@ class VkArchiver(Archiver): result.set_content(dump_payload(vk_scrapes)) - filenames = self.vks.download_media(vk_scrapes, ArchivingContext.get_tmp_dir()) + filenames = self.vks.download_media(vk_scrapes, self.tmp_dir) for filename in filenames: result.add_media(Media(filename)) diff --git a/src/auto_archiver/modules/wacz_enricher/__init__.py b/src/auto_archiver/modules/wacz_enricher/__init__.py new file mode 100644 index 0000000..686b8d8 --- /dev/null +++ b/src/auto_archiver/modules/wacz_enricher/__init__.py @@ -0,0 +1 @@ +from .wacz_enricher import WaczExtractorEnricher diff --git a/src/auto_archiver/modules/wacz_enricher/__manifest__.py b/src/auto_archiver/modules/wacz_enricher/__manifest__.py new file mode 100644 index 0000000..46ce05e --- /dev/null +++ b/src/auto_archiver/modules/wacz_enricher/__manifest__.py @@ -0,0 +1,41 @@ +{ + "name": "WACZ Enricher", + "type": ["enricher", "archiver"], + "entry_point": "wacz_enricher::WaczExtractorEnricher", + "requires_setup": True, + "dependencies": { + "python": [ + "loguru", + "jsonlines", + "warcio" + ], + # TODO? + "bin": [ + "docker" + ] + }, + "configs": { + "profile": {"default": None, "help": "browsertrix-profile (for profile generation see https://github.com/webrecorder/browsertrix-crawler#creating-and-using-browser-profiles)."}, + "docker_commands": {"default": None, "help":"if a custom docker invocation is needed"}, + "timeout": {"default": 120, "help": "timeout for WACZ generation in seconds"}, + "extract_media": {"default": False, "help": "If enabled all the images/videos/audio present in the WACZ archive will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched."}, + "extract_screenshot": {"default": True, "help": "If enabled the screenshot captured by browsertrix will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched."}, + "socks_proxy_host": {"default": None, "help": "SOCKS proxy host for browsertrix-crawler, use in combination with socks_proxy_port. eg: user:password@host"}, + "socks_proxy_port": {"default": None, "help": "SOCKS proxy port for browsertrix-crawler, use in combination with socks_proxy_host. eg 1234"}, + "proxy_server": {"default": None, "help": "SOCKS server proxy URL, in development"}, + }, + "description": """ + Creates .WACZ archives of web pages using the `browsertrix-crawler` tool, with options for media extraction and screenshot saving. + [Browsertrix-crawler](https://crawler.docs.browsertrix.com/user-guide/) is a headless browser-based crawler that archives web pages in WACZ format. + + ### Features + - Archives web pages into .WACZ format using Docker or direct invocation of `browsertrix-crawler`. + - Supports custom profiles for archiving private or dynamic content. + - Extracts media (images, videos, audio) and screenshots from the archive, optionally adding them to the enrichment pipeline. + - Generates metadata from the archived page's content and structure (e.g., titles, text). + + ### Notes + - Requires Docker for running `browsertrix-crawler` . + - Configurable via parameters for timeout, media extraction, screenshots, and proxy settings. + """ +} diff --git a/src/auto_archiver/enrichers/wacz_enricher.py b/src/auto_archiver/modules/wacz_enricher/wacz_enricher.py similarity index 84% rename from src/auto_archiver/enrichers/wacz_enricher.py rename to src/auto_archiver/modules/wacz_enricher/wacz_enricher.py index 3c39056..ff7314a 100644 --- a/src/auto_archiver/enrichers/wacz_enricher.py +++ b/src/auto_archiver/modules/wacz_enricher/wacz_enricher.py @@ -5,39 +5,21 @@ from zipfile import ZipFile from loguru import logger from warcio.archiveiterator import ArchiveIterator -from ..core import Media, Metadata, ArchivingContext -from . import Enricher -from ..archivers import Archiver -from ..utils import UrlUtil, random_str +from auto_archiver.core import Media, Metadata +from auto_archiver.core import Extractor, Enricher +from auto_archiver.utils import url as UrlUtil, random_str -class WaczArchiverEnricher(Enricher, Archiver): +class WaczExtractorEnricher(Enricher, Extractor): """ Uses https://github.com/webrecorder/browsertrix-crawler to generate a .WACZ archive of the URL If used with [profiles](https://github.com/webrecorder/browsertrix-crawler#creating-and-using-browser-profiles) it can become quite powerful for archiving private content. When used as an archiver it will extract the media from the .WACZ archive so it can be enriched. """ - name = "wacz_archiver_enricher" - def __init__(self, config: dict) -> None: - # without this STEP.__init__ is not called - super().__init__(config) - - @staticmethod - def configs() -> dict: - return { - "profile": {"default": None, "help": "browsertrix-profile (for profile generation see https://github.com/webrecorder/browsertrix-crawler#creating-and-using-browser-profiles)."}, - "docker_commands": {"default": None, "help":"if a custom docker invocation is needed"}, - "timeout": {"default": 120, "help": "timeout for WACZ generation in seconds"}, - "extract_media": {"default": False, "help": "If enabled all the images/videos/audio present in the WACZ archive will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched."}, - "extract_screenshot": {"default": True, "help": "If enabled the screenshot captured by browsertrix will be extracted into separate Media and appear in the html report. The .wacz file will be kept untouched."}, - "socks_proxy_host": {"default": None, "help": "SOCKS proxy host for browsertrix-crawler, use in combination with socks_proxy_port. eg: user:password@host"}, - "socks_proxy_port": {"default": None, "help": "SOCKS proxy port for browsertrix-crawler, use in combination with socks_proxy_host. eg 1234"}, - "proxy_server": {"default": None, "help": "SOCKS server proxy URL, in development"}, - } - def setup(self) -> None: + self.use_docker = os.environ.get('WACZ_ENABLE_DOCKER') or not os.environ.get('RUNNING_IN_DOCKER') self.docker_in_docker = os.environ.get('WACZ_ENABLE_DOCKER') and os.environ.get('RUNNING_IN_DOCKER') @@ -68,7 +50,7 @@ class WaczArchiverEnricher(Enricher, Archiver): url = to_enrich.get_url() collection = random_str(8) - browsertrix_home_host = self.browsertrix_home_host or os.path.abspath(ArchivingContext.get_tmp_dir()) + browsertrix_home_host = self.browsertrix_home_host or os.path.abspath(self.tmp_dir) browsertrix_home_container = self.browsertrix_home_container or browsertrix_home_host cmd = [ @@ -171,7 +153,7 @@ class WaczArchiverEnricher(Enricher, Archiver): logger.info(f"WACZ extract_media or extract_screenshot flag is set, extracting media from {wacz_filename=}") # unzipping the .wacz - tmp_dir = ArchivingContext.get_tmp_dir() + tmp_dir = self.tmp_dir unzipped_dir = os.path.join(tmp_dir, "unzipped") with ZipFile(wacz_filename, 'r') as z_obj: z_obj.extractall(path=unzipped_dir) @@ -239,4 +221,4 @@ class WaczArchiverEnricher(Enricher, Archiver): to_enrich.add_media(m, warc_fn) counter += 1 seen_urls.add(record_url) - logger.info(f"WACZ extract_media/extract_screenshot finished, found {counter} relevant media file(s)") + logger.info(f"WACZ extract_media/extract_screenshot finished, found {counter} relevant media file(s)") \ No newline at end of file diff --git a/src/auto_archiver/modules/wayback_extractor_enricher/__init__.py b/src/auto_archiver/modules/wayback_extractor_enricher/__init__.py new file mode 100644 index 0000000..b69332d --- /dev/null +++ b/src/auto_archiver/modules/wayback_extractor_enricher/__init__.py @@ -0,0 +1 @@ +from .wayback_extractor_enricher import WaybackExtractorEnricher \ No newline at end of file diff --git a/src/auto_archiver/modules/wayback_extractor_enricher/__manifest__.py b/src/auto_archiver/modules/wayback_extractor_enricher/__manifest__.py new file mode 100644 index 0000000..baecc14 --- /dev/null +++ b/src/auto_archiver/modules/wayback_extractor_enricher/__manifest__.py @@ -0,0 +1,56 @@ +{ + "name": "Wayback Machine Enricher", + "type": ["enricher", "archiver"], + "entry_point": "wayback_extractor_enricher::WaybackExtractorEnricher", + "requires_setup": True, + "dependencies": { + "python": ["loguru", "requests"], + }, + "configs": { + "timeout": { + "default": 15, + "help": "seconds to wait for successful archive confirmation from wayback, if more than this passes the result contains the job_id so the status can later be checked manually.", + }, + "if_not_archived_within": { + "default": None, + "help": "only tell wayback to archive if no archive is available before the number of seconds specified, use None to ignore this option. For more information: https://docs.google.com/document/d/1Nsv52MvSjbLb2PCpHlat0gkzw0EvtSgpKHu4mk0MnrA", + }, + "key": { + "required": True, + "help": "wayback API key. to get credentials visit https://archive.org/account/s3.php", + }, + "secret": { + "required": True, + "help": "wayback API secret. to get credentials visit https://archive.org/account/s3.php", + }, + "proxy_http": { + "default": None, + "help": "http proxy to use for wayback requests, eg http://proxy-user:password@proxy-ip:port", + }, + "proxy_https": { + "default": None, + "help": "https proxy to use for wayback requests, eg https://proxy-user:password@proxy-ip:port", + }, + }, + "description": """ + Submits the current URL to the Wayback Machine for archiving and returns either a job ID or the completed archive URL. + + ### Features + - Archives URLs using the Internet Archive's Wayback Machine API. + - Supports conditional archiving based on the existence of prior archives within a specified time range. + - Provides proxies for HTTP and HTTPS requests. + - Fetches and confirms the archive URL or provides a job ID for later status checks. + + ### Notes + - Requires a valid Wayback Machine API key and secret. + - Handles rate-limiting by Wayback Machine and retries status checks with exponential backoff. + + ### Steps to Get an Wayback API Key: + - Sign up for an account at [Internet Archive](https://archive.org/account/signup). + - Log in to your account. + - Navigte to your [account settings](https://archive.org/account). + - or: https://archive.org/developers/tutorial-get-ia-credentials.html + - Under Wayback Machine API Keys, generate a new key. + - Note down your API key and secret, as they will be required for authentication. + """, +} diff --git a/src/auto_archiver/enrichers/wayback_enricher.py b/src/auto_archiver/modules/wayback_extractor_enricher/wayback_extractor_enricher.py similarity index 69% rename from src/auto_archiver/enrichers/wayback_enricher.py rename to src/auto_archiver/modules/wayback_extractor_enricher/wayback_extractor_enricher.py index 305bfcf..1763b12 100644 --- a/src/auto_archiver/enrichers/wayback_enricher.py +++ b/src/auto_archiver/modules/wayback_extractor_enricher/wayback_extractor_enricher.py @@ -2,35 +2,16 @@ import json from loguru import logger import time, requests -from . import Enricher -from ..archivers import Archiver -from ..utils import UrlUtil -from ..core import Metadata +from auto_archiver.core import Extractor, Enricher +from auto_archiver.utils import url as UrlUtil +from auto_archiver.core import Metadata -class WaybackArchiverEnricher(Enricher, Archiver): +class WaybackExtractorEnricher(Enricher, Extractor): """ Submits the current URL to the webarchive and returns a job_id or completed archive. The Wayback machine will rate-limit IP heavy usage. """ - name = "wayback_archiver_enricher" - - def __init__(self, config: dict) -> None: - # without this STEP.__init__ is not called - super().__init__(config) - assert type(self.secret) == str and len(self.secret) > 0, "please provide a value for the wayback_enricher API key" - assert type(self.secret) == str and len(self.secret) > 0, "please provide a value for the wayback_enricher API secret" - - @staticmethod - def configs() -> dict: - return { - "timeout": {"default": 15, "help": "seconds to wait for successful archive confirmation from wayback, if more than this passes the result contains the job_id so the status can later be checked manually."}, - "if_not_archived_within": {"default": None, "help": "only tell wayback to archive if no archive is available before the number of seconds specified, use None to ignore this option. For more information: https://docs.google.com/document/d/1Nsv52MvSjbLb2PCpHlat0gkzw0EvtSgpKHu4mk0MnrA"}, - "key": {"default": None, "help": "wayback API key. to get credentials visit https://archive.org/account/s3.php"}, - "secret": {"default": None, "help": "wayback API secret. to get credentials visit https://archive.org/account/s3.php"}, - "proxy_http": {"default": None, "help": "http proxy to use for wayback requests, eg http://proxy-user:password@proxy-ip:port"}, - "proxy_https": {"default": None, "help": "https proxy to use for wayback requests, eg https://proxy-user:password@proxy-ip:port"}, - } def download(self, item: Metadata) -> Metadata: # this new Metadata object is required to avoid duplication diff --git a/src/auto_archiver/modules/whisper_enricher/__init__.py b/src/auto_archiver/modules/whisper_enricher/__init__.py new file mode 100644 index 0000000..d3d3526 --- /dev/null +++ b/src/auto_archiver/modules/whisper_enricher/__init__.py @@ -0,0 +1 @@ +from .whisper_enricher import WhisperEnricher \ No newline at end of file diff --git a/src/auto_archiver/modules/whisper_enricher/__manifest__.py b/src/auto_archiver/modules/whisper_enricher/__manifest__.py new file mode 100644 index 0000000..98e743e --- /dev/null +++ b/src/auto_archiver/modules/whisper_enricher/__manifest__.py @@ -0,0 +1,35 @@ +{ + "name": "Whisper Enricher", + "type": ["enricher"], + "requires_setup": True, + "dependencies": { + "python": ["s3_storage", "loguru", "requests"], + }, + "configs": { + "api_endpoint": {"required": True, + "help": "WhisperApi api endpoint, eg: https://whisperbox-api.com/api/v1, a deployment of https://github.com/bellingcat/whisperbox-transcribe."}, + "api_key": {"required": True, + "help": "WhisperApi api key for authentication"}, + "include_srt": {"default": False, "help": "Whether to include a subtitle SRT (SubRip Subtitle file) for the video (can be used in video players)."}, + "timeout": {"default": 90, "help": "How many seconds to wait at most for a successful job completion."}, + "action": {"default": "translate", + "help": "which Whisper operation to execute", + "choices": ["transcribe", "translate", "language_detection"]}, + }, + "description": """ + Integrates with a Whisper API service to transcribe, translate, or detect the language of audio and video files. + + ### Features + - Submits audio or video files to a Whisper API deployment for processing. + - Supports operations such as transcription, translation, and language detection. + - Optionally generates SRT subtitle files for video content. + - Integrates with S3-compatible storage systems to make files publicly accessible for processing. + - Handles job submission, status checking, artifact retrieval, and cleanup. + + ### Notes + - Requires a Whisper API endpoint and API key for authentication. + - Only compatible with S3-compatible storage systems for media file accessibility. + - ** This stores the media files in S3 prior to enriching them as Whisper requires public URLs to access the media files. + - Handles multiple jobs and retries for failed or incomplete processing. + """ +} diff --git a/src/auto_archiver/enrichers/whisper_enricher.py b/src/auto_archiver/modules/whisper_enricher/whisper_enricher.py similarity index 74% rename from src/auto_archiver/enrichers/whisper_enricher.py rename to src/auto_archiver/modules/whisper_enricher/whisper_enricher.py index c0089a4..89579f9 100644 --- a/src/auto_archiver/enrichers/whisper_enricher.py +++ b/src/auto_archiver/modules/whisper_enricher/whisper_enricher.py @@ -2,10 +2,9 @@ import traceback import requests, time from loguru import logger -from . import Enricher -from ..core import Metadata, Media, ArchivingContext -from ..storages import S3Storage - +from auto_archiver.core import Enricher +from auto_archiver.core import Metadata, Media +from auto_archiver.core.module import get_module class WhisperEnricher(Enricher): """ @@ -13,38 +12,26 @@ class WhisperEnricher(Enricher): whisper API repository: https://github.com/bellingcat/whisperbox-transcribe/ Only works if an S3 compatible storage is used """ - name = "whisper_enricher" - def __init__(self, config: dict) -> None: - # without this STEP.__init__ is not called - super().__init__(config) - assert type(self.api_endpoint) == str and len(self.api_endpoint) > 0, "please provide a value for the whisper_enricher api_endpoint" - assert type(self.api_key) == str and len(self.api_key) > 0, "please provide a value for the whisper_enricher api_key" - self.timeout = int(self.timeout) - - @staticmethod - def configs() -> dict: - return { - "api_endpoint": {"default": None, "help": "WhisperApi api endpoint, eg: https://whisperbox-api.com/api/v1, a deployment of https://github.com/bellingcat/whisperbox-transcribe."}, - "api_key": {"default": None, "help": "WhisperApi api key for authentication"}, - "include_srt": {"default": False, "help": "Whether to include a subtitle SRT (SubRip Subtitle file) for the video (can be used in video players)."}, - "timeout": {"default": 90, "help": "How many seconds to wait at most for a successful job completion."}, - "action": {"default": "translate", "help": "which Whisper operation to execute", "choices": ["transcribe", "translate", "language_detection"]}, - - } - - def enrich(self, to_enrich: Metadata) -> None: - if not self._get_s3_storage(): + def setup(self) -> None: + self.stores = self.config['steps']['storages'] + self.s3 = get_module("s3_storage", self.config) + if not "s3_storage" in self.stores: logger.error("WhisperEnricher: To use the WhisperEnricher you need to use S3Storage so files are accessible publicly to the whisper service being called.") return + + def enrich(self, to_enrich: Metadata) -> None: + url = to_enrich.get_url() logger.debug(f"WHISPER[{self.action}]: iterating media items for {url=}.") job_results = {} for i, m in enumerate(to_enrich.media): if m.is_video() or m.is_audio(): - m.store(url=url, metadata=to_enrich) + # TODO: this used to pass all storage items to store now + # Now only passing S3, the rest will get added later in the usual order (?) + m.store(url=url, metadata=to_enrich, storages=[self.s3]) try: job_id = self.submit_job(m) job_results[job_id] = False @@ -72,8 +59,8 @@ class WhisperEnricher(Enricher): to_enrich.set_content(f"\n[automatic video transcript]: {v}") def submit_job(self, media: Media): - s3 = self._get_s3_storage() - s3_url = s3.get_cdn_url(media) + + s3_url = self.s3.get_cdn_url(media) assert s3_url in media.urls, f"Could not find S3 url ({s3_url}) in list of stored media urls " payload = { "url": s3_url, @@ -126,10 +113,3 @@ class WhisperEnricher(Enricher): logger.debug(f"DELETE whisper {job_id=} result: {r_del.status_code}") return result return False - - def _get_s3_storage(self) -> S3Storage: - try: - return next(s for s in ArchivingContext.get("storages") if s.__class__ == S3Storage) - except: - logger.warning("No S3Storage instance found in storages") - return diff --git a/src/auto_archiver/storages/__init__.py b/src/auto_archiver/storages/__init__.py deleted file mode 100644 index bff83e6..0000000 --- a/src/auto_archiver/storages/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -""" This module contains the storage classes for the auto-archiver. - -""" -from .storage import Storage -from .s3 import S3Storage -from .local import LocalStorage -from .gd import GDriveStorage -from .atlos import AtlosStorage \ No newline at end of file diff --git a/src/auto_archiver/storages/storage.py b/src/auto_archiver/storages/storage.py deleted file mode 100644 index c9b55e0..0000000 --- a/src/auto_archiver/storages/storage.py +++ /dev/null @@ -1,86 +0,0 @@ -from __future__ import annotations -from abc import abstractmethod -from dataclasses import dataclass -from typing import IO, Optional -import os - -from ..utils.misc import random_str - -from ..core import Media, Step, ArchivingContext, Metadata -from ..enrichers import HashEnricher -from loguru import logger -from slugify import slugify - - -@dataclass -class Storage(Step): - name = "storage" - PATH_GENERATOR_OPTIONS = ["flat", "url", "random"] - FILENAME_GENERATOR_CHOICES = ["random", "static"] - - def __init__(self, config: dict) -> None: - # without this STEP.__init__ is not called - super().__init__(config) - assert self.path_generator in Storage.PATH_GENERATOR_OPTIONS, f"path_generator must be one of {Storage.PATH_GENERATOR_OPTIONS}" - assert self.filename_generator in Storage.FILENAME_GENERATOR_CHOICES, f"filename_generator must be one of {Storage.FILENAME_GENERATOR_CHOICES}" - - @staticmethod - def configs() -> dict: - return { - "path_generator": { - "default": "url", - "help": "how to store the file in terms of directory structure: 'flat' sets to root; 'url' creates a directory based on the provided URL; 'random' creates a random directory.", - "choices": Storage.PATH_GENERATOR_OPTIONS - }, - "filename_generator": { - "default": "random", - "help": "how to name stored files: 'random' creates a random string; 'static' uses a replicable strategy such as a hash.", - "choices": Storage.FILENAME_GENERATOR_CHOICES - } - } - - def init(name: str, config: dict) -> Storage: - # only for typing... - return Step.init(name, config, Storage) - - def store(self, media: Media, url: str, metadata: Optional[Metadata]=None) -> None: - if media.is_stored(): - logger.debug(f"{media.key} already stored, skipping") - return - self.set_key(media, url) - self.upload(media, metadata=metadata) - media.add_url(self.get_cdn_url(media)) - - @abstractmethod - def get_cdn_url(self, media: Media) -> str: pass - - @abstractmethod - def uploadf(self, file: IO[bytes], key: str, **kwargs: dict) -> bool: pass - - def upload(self, media: Media, **kwargs) -> bool: - logger.debug(f'[{self.__class__.name}] storing file {media.filename} with key {media.key}') - with open(media.filename, 'rb') as f: - return self.uploadf(f, media, **kwargs) - - def set_key(self, media: Media, url) -> None: - """takes the media and optionally item info and generates a key""" - if media.key is not None and len(media.key) > 0: return - folder = ArchivingContext.get("folder", "") - filename, ext = os.path.splitext(media.filename) - - # path_generator logic - if self.path_generator == "flat": - path = "" - filename = slugify(filename) # in case it comes with os.sep - elif self.path_generator == "url": path = slugify(url) - elif self.path_generator == "random": - path = ArchivingContext.get("random_path", random_str(24), True) - - # filename_generator logic - if self.filename_generator == "random": filename = random_str(24) - elif self.filename_generator == "static": - he = HashEnricher({"hash_enricher": {"algorithm": ArchivingContext.get("hash_enricher.algorithm"), "chunksize": 1.6e7}}) - hd = he.calculate_hash(media.filename) - filename = hd[:24] - - media.key = os.path.join(folder, path, f"{filename}{ext}") diff --git a/src/auto_archiver/utils/__init__.py b/src/auto_archiver/utils/__init__.py index 788f159..ed2d3bb 100644 --- a/src/auto_archiver/utils/__init__.py +++ b/src/auto_archiver/utils/__init__.py @@ -1,10 +1,7 @@ """ Auto Archiver Utilities. """ # we need to explicitly expose the available imports here -from .gworksheet import GWorksheet from .misc import * from .webdriver import Webdriver -from .gsheet import Gsheets -from .url import UrlUtil from .atlos import get_atlos_config_options # handy utils from ytdlp diff --git a/src/auto_archiver/utils/gsheet.py b/src/auto_archiver/utils/gsheet.py index f84aab2..e69de29 100644 --- a/src/auto_archiver/utils/gsheet.py +++ b/src/auto_archiver/utils/gsheet.py @@ -1,52 +0,0 @@ -import json, gspread - -from ..core import Step - - -class Gsheets(Step): - name = "gsheets" - - def __init__(self, config: dict) -> None: - # without this STEP.__init__ is not called - super().__init__(config) - self.gsheets_client = gspread.service_account(filename=self.service_account) - # TODO: config should be responsible for conversions - try: self.header = int(self.header) - except: pass - assert type(self.header) == int, f"header ({self.header}) value must be an integer not {type(self.header)}" - assert self.sheet is not None or self.sheet_id is not None, "You need to define either a 'sheet' name or a 'sheet_id' in your orchestration file when using gsheets." - - @staticmethod - def configs() -> dict: - return { - "sheet": {"default": None, "help": "name of the sheet to archive"}, - "sheet_id": {"default": None, "help": "(alternative to sheet name) the id of the sheet to archive"}, - "header": {"default": 1, "help": "index of the header row (starts at 1)"}, - "service_account": {"default": "secrets/service_account.json", "help": "service account JSON file path"}, - "columns": { - "default": { - 'url': 'link', - 'status': 'archive status', - 'folder': 'destination folder', - 'archive': 'archive location', - 'date': 'archive date', - 'thumbnail': 'thumbnail', - 'timestamp': 'upload timestamp', - 'title': 'upload title', - 'text': 'text content', - 'screenshot': 'screenshot', - 'hash': 'hash', - 'pdq_hash': 'perceptual hashes', - 'wacz': 'wacz', - 'replaywebpage': 'replaywebpage', - }, - "help": "names of columns in the google sheet (stringified JSON object)", - "cli_set": lambda cli_val, cur_val: dict(cur_val, **json.loads(cli_val)) - }, - } - - def open_sheet(self): - if self.sheet: - return self.gsheets_client.open(self.sheet) - else: # self.sheet_id - return self.gsheets_client.open_by_key(self.sheet_id) diff --git a/src/auto_archiver/utils/misc.py b/src/auto_archiver/utils/misc.py index e312fc6..cd03b49 100644 --- a/src/auto_archiver/utils/misc.py +++ b/src/auto_archiver/utils/misc.py @@ -1,7 +1,9 @@ - -import os, json, requests +import os +import json import uuid -from datetime import datetime +from datetime import datetime, timezone +import requests +import hashlib from loguru import logger @@ -51,6 +53,52 @@ def update_nested_dict(dictionary, update_dict): else: dictionary[key] = value + def random_str(length: int = 32) -> str: assert length <= 32, "length must be less than 32 as UUID4 is used" - return str(uuid.uuid4()).replace("-", "")[:length] \ No newline at end of file + return str(uuid.uuid4()).replace("-", "")[:length] + + +def json_loader(cli_val): + return json.loads(cli_val) + + +def calculate_file_hash(filename: str, hash_algo = hashlib.sha256, chunksize: int = 16000000) -> str: + hash = hash_algo() + with open(filename, "rb") as f: + while True: + buf = f.read(chunksize) + if not buf: break + hash.update(buf) + return hash.hexdigest() + +def get_current_datetime_iso() -> str: + return datetime.now(timezone.utc).replace(tzinfo=timezone.utc).isoformat() + + +def get_datetime_from_str(dt_str: str, fmt: str | None = None) -> datetime | None: + # parse a datetime string with option of passing a specific format + try: + return datetime.strptime(dt_str, fmt) if fmt else datetime.fromisoformat(dt_str) + except ValueError as e: + logger.error(f"Unable to parse datestring {dt_str}: {e}") + return None + + +def get_timestamp(ts, utc=True, iso=True) -> str | datetime | None: + # Consistent parsing of timestamps + # If utc=True, the timezone is set to UTC, + # if iso=True, the output is an iso string + if not ts: return + try: + if isinstance(ts, str): ts = datetime.fromisoformat(ts) + if isinstance(ts, (int, float)): ts = datetime.fromtimestamp(ts) + if utc: ts = ts.replace(tzinfo=timezone.utc) + if iso: return ts.isoformat() + return ts + except Exception as e: + logger.error(f"Unable to parse timestamp {ts}: {e}") + return None + +def get_current_timestamp() -> str: + return get_timestamp(datetime.now()) \ No newline at end of file diff --git a/src/auto_archiver/utils/url.py b/src/auto_archiver/utils/url.py index 7586cca..40884da 100644 --- a/src/auto_archiver/utils/url.py +++ b/src/auto_archiver/utils/url.py @@ -1,79 +1,84 @@ import re from urllib.parse import urlparse, urlunparse -class UrlUtil: - telegram_private = re.compile(r"https:\/\/t\.me(\/c)\/(.+)\/(\d+)") - is_istagram = re.compile(r"https:\/\/www\.instagram\.com") - @staticmethod - def clean(url: str) -> str: return url +AUTHWALL_URLS = [ + re.compile(r"https:\/\/t\.me(\/c)\/(.+)\/(\d+)"), # telegram private channels + re.compile(r"https:\/\/www\.instagram\.com"), # instagram +] - @staticmethod - def is_auth_wall(url: str) -> bool: - """ - checks if URL is behind an authentication wall meaning steps like wayback, wacz, ... may not work - """ - if UrlUtil.telegram_private.match(url): return True - if UrlUtil.is_istagram.match(url): return True +def domain_for_url(url: str) -> str: + """ + SECURITY: parse the domain using urllib to avoid any potential security issues + """ + return urlparse(url).netloc - return False +def clean(url: str) -> str: + return url - @staticmethod - def remove_get_parameters(url: str) -> str: - # http://example.com/file.mp4?t=1 -> http://example.com/file.mp4 - # useful for mimetypes to work - parsed_url = urlparse(url) - new_url = urlunparse(parsed_url._replace(query='')) - return new_url +def is_auth_wall(url: str) -> bool: + """ + checks if URL is behind an authentication wall meaning steps like wayback, wacz, ... may not work + """ + for regex in AUTHWALL_URLS: + if regex.match(url): + return True - @staticmethod - def is_relevant_url(url: str) -> bool: - """ - Detect if a detected media URL is recurring and therefore irrelevant to a specific archive. Useful, for example, for the enumeration of the media files in WARC files which include profile pictures, favicons, etc. - """ - clean_url = UrlUtil.remove_get_parameters(url) + return False - # favicons - if "favicon" in url: return False - # ifnore icons - if clean_url.endswith(".ico"): return False - # ignore SVGs - if UrlUtil.remove_get_parameters(url).endswith(".svg"): return False +def remove_get_parameters(url: str) -> str: + # http://example.com/file.mp4?t=1 -> http://example.com/file.mp4 + # useful for mimetypes to work + parsed_url = urlparse(url) + new_url = urlunparse(parsed_url._replace(query='')) + return new_url - # twitter profile pictures - if "twimg.com/profile_images" in url: return False - if "twimg.com" in url and "/default_profile_images" in url: return False +def is_relevant_url(url: str) -> bool: + """ + Detect if a detected media URL is recurring and therefore irrelevant to a specific archive. Useful, for example, for the enumeration of the media files in WARC files which include profile pictures, favicons, etc. + """ + clean_url = remove_get_parameters(url) - # instagram profile pictures - if "https://scontent.cdninstagram.com/" in url and "150x150" in url: return False - # instagram recurring images - if "https://static.cdninstagram.com/rsrc.php/" in url: return False + # favicons + if "favicon" in url: return False + # ifnore icons + if clean_url.endswith(".ico"): return False + # ignore SVGs + if remove_get_parameters(url).endswith(".svg"): return False - # telegram - if "https://telegram.org/img/emoji/" in url: return False + # twitter profile pictures + if "twimg.com/profile_images" in url: return False + if "twimg.com" in url and "/default_profile_images" in url: return False - # youtube - if "https://www.youtube.com/s/gaming/emoji/" in url: return False - if "https://yt3.ggpht.com" in url and "default-user=" in url: return False - if "https://www.youtube.com/s/search/audio/" in url: return False + # instagram profile pictures + if "https://scontent.cdninstagram.com/" in url and "150x150" in url: return False + # instagram recurring images + if "https://static.cdninstagram.com/rsrc.php/" in url: return False - # ok - if " https://ok.ru/res/i/" in url: return False + # telegram + if "https://telegram.org/img/emoji/" in url: return False - # vk - if "https://vk.com/emoji/" in url: return False - if "vk.com/images/" in url: return False - if "vk.com/images/reaction/" in url: return False + # youtube + if "https://www.youtube.com/s/gaming/emoji/" in url: return False + if "https://yt3.ggpht.com" in url and "default-user=" in url: return False + if "https://www.youtube.com/s/search/audio/" in url: return False - # wikipedia - if "wikipedia.org/static" in url: return False + # ok + if " https://ok.ru/res/i/" in url: return False - return True + # vk + if "https://vk.com/emoji/" in url: return False + if "vk.com/images/" in url: return False + if "vk.com/images/reaction/" in url: return False - @staticmethod - def twitter_best_quality_url(url: str) -> str: - """ - some twitter image URLs point to a less-than best quality - this returns the URL pointing to the highest (original) quality - """ - return re.sub(r"name=(\w+)", "name=orig", url, 1) + # wikipedia + if "wikipedia.org/static" in url: return False + + return True + +def twitter_best_quality_url(url: str) -> str: + """ + some twitter image URLs point to a less-than best quality + this returns the URL pointing to the highest (original) quality + """ + return re.sub(r"name=(\w+)", "name=orig", url, 1) diff --git a/src/auto_archiver/utils/webdriver.py b/src/auto_archiver/utils/webdriver.py index cf84c35..db26d04 100644 --- a/src/auto_archiver/utils/webdriver.py +++ b/src/auto_archiver/utils/webdriver.py @@ -9,12 +9,79 @@ from loguru import logger from selenium.webdriver.common.by import By import time +#import domain_for_url +from urllib.parse import urlparse, urlunparse +from http.cookiejar import MozillaCookieJar +class CookieSettingDriver(webdriver.Firefox): + + facebook_accept_cookies: bool + cookies: str + cookiejar: MozillaCookieJar + + def __init__(self, cookies, cookiejar, facebook_accept_cookies, *args, **kwargs): + super(CookieSettingDriver, self).__init__(*args, **kwargs) + self.cookies = cookies + self.cookiejar = cookiejar + self.facebook_accept_cookies = facebook_accept_cookies + + def get(self, url: str): + if self.cookies or self.cookiejar: + # set up the driver to make it not 'cookie averse' (needs a context/URL) + # get the 'robots.txt' file which should be quick and easy + robots_url = urlunparse(urlparse(url)._replace(path='/robots.txt', query='', fragment='')) + super(CookieSettingDriver, self).get(robots_url) + + if self.cookies: + # an explicit cookie is set for this site, use that first + for cookie in self.cookies.split(";"): + for name, value in cookie.split("="): + self.driver.add_cookie({'name': name, 'value': value}) + elif self.cookiejar: + domain = urlparse(url).netloc.lstrip("www.") + for cookie in self.cookiejar: + if domain in cookie.domain: + try: + self.add_cookie({ + 'name': cookie.name, + 'value': cookie.value, + 'path': cookie.path, + 'domain': cookie.domain, + 'secure': bool(cookie.secure), + 'expiry': cookie.expires + }) + except Exception as e: + logger.warning(f"Failed to add cookie to webdriver: {e}") + + if self.facebook_accept_cookies: + try: + logger.debug(f'Trying fb click accept cookie popup.') + super(CookieSettingDriver, self).get("http://www.facebook.com") + essential_only = self.find_element(By.XPATH, "//span[contains(text(), 'Decline optional cookies')]") + essential_only.click() + logger.debug(f'fb click worked') + # linux server needs a sleep otherwise facebook cookie won't have worked and we'll get a popup on next page + time.sleep(2) + except Exception as e: + logger.warning(f'Failed on fb accept cookies.', e) + # now get the actual URL + super(CookieSettingDriver, self).get(url) + if self.facebook_accept_cookies: + # try and click the 'close' button on the 'login' window to close it + close_button = self.find_element(By.XPATH, "//div[@role='dialog']//div[@aria-label='Close']") + if close_button: + close_button.click() + + + class Webdriver: - def __init__(self, width: int, height: int, timeout_seconds: int, facebook_accept_cookies: bool = False, http_proxy: str = "", print_options: dict = {}) -> webdriver: + def __init__(self, width: int, height: int, timeout_seconds: int, + facebook_accept_cookies: bool = False, http_proxy: str = "", + print_options: dict = {}, auth: dict = {}) -> webdriver: self.width = width self.height = height self.timeout_seconds = timeout_seconds + self.auth = auth self.facebook_accept_cookies = facebook_accept_cookies self.http_proxy = http_proxy # create and set print options @@ -23,32 +90,26 @@ class Webdriver: setattr(self.print_options, k, v) def __enter__(self) -> webdriver: + options = webdriver.FirefoxOptions() options.add_argument("--headless") options.add_argument(f'--proxy-server={self.http_proxy}') options.set_preference('network.protocol-handler.external.tg', False) + # if facebook cookie popup is present, force the browser to English since then it's easier to click the 'Decline optional cookies' option + if self.facebook_accept_cookies: + options.add_argument('--lang=en') + try: - self.driver = webdriver.Firefox(options=options) + self.driver = CookieSettingDriver(cookies=self.auth.get('cookies'), cookiejar=self.auth.get('cookies_jar'), + facebook_accept_cookies=self.facebook_accept_cookies, options=options) self.driver.set_window_size(self.width, self.height) self.driver.set_page_load_timeout(self.timeout_seconds) self.driver.print_options = self.print_options except TimeoutException as e: logger.error(f"failed to get new webdriver, possibly due to insufficient system resources or timeout settings: {e}") - if self.facebook_accept_cookies: - try: - logger.debug(f'Trying fb click accept cookie popup.') - self.driver.get("http://www.facebook.com") - foo = self.driver.find_element(By.XPATH, "//button[@data-cookiebanner='accept_only_essential_button']") - foo.click() - logger.debug(f'fb click worked') - # linux server needs a sleep otherwise facebook cookie won't have worked and we'll get a popup on next page - time.sleep(2) - except: - logger.warning(f'Failed on fb accept cookies.') - return self.driver - + def __exit__(self, exc_type, exc_val, exc_tb): self.driver.close() self.driver.quit() diff --git a/tests/__init__.py b/tests/__init__.py index 3d66aff..e69de29 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,6 +0,0 @@ -import tempfile - -from auto_archiver.core.context import ArchivingContext - -ArchivingContext.reset(full_reset=True) -ArchivingContext.set_tmp_dir(tempfile.gettempdir()) \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index 553b573..8675fbc 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,11 +1,14 @@ """ pytest conftest file, for shared fixtures and configuration """ - +import os +import pickle +from tempfile import TemporaryDirectory from typing import Dict, Tuple - +import hashlib import pytest from auto_archiver.core.metadata import Metadata +from auto_archiver.core.module import get_module, _LAZY_LOADED_MODULES # Test names inserted into this list will be run last. This is useful for expensive/costly tests # that you only want to run if everything else succeeds (e.g. API calls). The order here is important @@ -13,6 +16,38 @@ from auto_archiver.core.metadata import Metadata # format is the name of the module (python file) without the .py extension TESTS_TO_RUN_LAST = ['test_twitter_api_archiver'] +@pytest.fixture +def setup_module(request): + def _setup_module(module_name, config={}): + + if isinstance(module_name, type): + # get the module name: + # if the class does not have a .name, use the name of the parent folder + module_name = module_name.__module__.rsplit(".",2)[-2] + + m = get_module(module_name, {module_name: config}) + + # add the tmp_dir to the module + tmp_dir = TemporaryDirectory() + m.tmp_dir = tmp_dir.name + + def cleanup(): + _LAZY_LOADED_MODULES.pop(module_name) + tmp_dir.cleanup() + request.addfinalizer(cleanup) + + return m + + return _setup_module + +@pytest.fixture +def check_hash(): + def _check_hash(filename: str, hash: str): + with open(filename, "rb") as f: + buf = f.read() + assert hash == hashlib.sha256(buf).hexdigest() + + return _check_hash @pytest.fixture def make_item(): @@ -79,4 +114,18 @@ def pytest_runtest_setup(item): test_name = _test_failed_incremental[cls_name].get((), None) # if name found, test has failed for the combination of class name & test name if test_name is not None: - pytest.xfail(f"previous test failed ({test_name})") \ No newline at end of file + pytest.xfail(f"previous test failed ({test_name})") + + + +@pytest.fixture() +def unpickle(): + """ + Returns a helper function that unpickles a file + ** gets the file from the test_files directory: tests/data/test_files ** + """ + def _unpickle(path): + test_data_dir = os.path.join(os.path.dirname(__file__), "data", "test_files") + with open(os.path.join(test_data_dir, path), "rb") as f: + return pickle.load(f) + return _unpickle \ No newline at end of file diff --git a/tests/data/csv_no_headers.csv b/tests/data/csv_no_headers.csv new file mode 100644 index 0000000..cd66b33 --- /dev/null +++ b/tests/data/csv_no_headers.csv @@ -0,0 +1,2 @@ +https://example.com/1/,data 1 +https://example.com/2/,data 2 \ No newline at end of file diff --git a/tests/data/csv_with_headers.csv b/tests/data/csv_with_headers.csv new file mode 100644 index 0000000..c3e296d --- /dev/null +++ b/tests/data/csv_with_headers.csv @@ -0,0 +1,3 @@ +webpages,other data +https://example.com/1/,data 1 +https://example.com/2/,data 2 \ No newline at end of file diff --git a/tests/data/test_modules/example_module/__init__.py b/tests/data/test_modules/example_module/__init__.py new file mode 100644 index 0000000..560a9b9 --- /dev/null +++ b/tests/data/test_modules/example_module/__init__.py @@ -0,0 +1 @@ +from .example_module import ExampleModule \ No newline at end of file diff --git a/tests/data/test_modules/example_module/__manifest__.py b/tests/data/test_modules/example_module/__manifest__.py new file mode 100644 index 0000000..f2ebdbf --- /dev/null +++ b/tests/data/test_modules/example_module/__manifest__.py @@ -0,0 +1,11 @@ +{ + "name": "Example Module", + "type": ["extractor", "feeder", "formatter", "storage", "enricher", "database"], + "requires_setup": False, + "dependencies": {"python": ["loguru"] + }, + "configs": { + "csv_file": {"default": "db.csv", "help": "CSV file name"}, + "required_field": {"required": True, "help": "required field in the CSV file"}, + }, +} \ No newline at end of file diff --git a/tests/data/test_modules/example_module/example_module.py b/tests/data/test_modules/example_module/example_module.py new file mode 100644 index 0000000..7def054 --- /dev/null +++ b/tests/data/test_modules/example_module/example_module.py @@ -0,0 +1,28 @@ +from auto_archiver.core import Extractor, Enricher, Feeder, Database, Storage, Formatter, Metadata + +class ExampleModule(Extractor, Enricher, Feeder, Database, Storage, Formatter): + def download(self, item): + print("download") + + def __iter__(self): + yield Metadata().set_url("https://example.com") + + + def done(self, result): + print("done") + + def enrich(self, to_enrich): + print("enrich") + + def get_cdn_url(self, media): + return "nice_url" + + def save(self, item): + print("save") + + def uploadf(self, file, key, **kwargs): + print("uploadf") + + + def format(self, item): + print("format") diff --git a/tests/data/test_orchestration.yaml b/tests/data/test_orchestration.yaml new file mode 100644 index 0000000..ec6af35 --- /dev/null +++ b/tests/data/test_orchestration.yaml @@ -0,0 +1,16 @@ +steps: + feeders: + - example_module + extractors: + - example_module + formatters: + - example_module + storages: + - example_module + databases: + - example_module + enrichers: + - example_module + + +# Global configuration \ No newline at end of file diff --git a/tests/databases/test_csv_db.py b/tests/databases/test_csv_db.py index 4395ef0..afca0d8 100644 --- a/tests/databases/test_csv_db.py +++ b/tests/databases/test_csv_db.py @@ -1,15 +1,13 @@ -from auto_archiver.databases.csv_db import CSVDb +from auto_archiver.modules.csv_db import CSVDb from auto_archiver.core import Metadata -def test_store_item(tmp_path): +def test_store_item(tmp_path, setup_module): """Tests storing an item in the CSV database""" temp_db = tmp_path / "temp_db.csv" - db = CSVDb({ - "csv_db": {"csv_file": temp_db.as_posix()} - }) + db = setup_module(CSVDb, {"csv_file": temp_db.as_posix()}) item = Metadata().set_url("http://example.com").set_title("Example").set_content("Example content").success("my-archiver") diff --git a/tests/databases/test_gsheet_db.py b/tests/databases/test_gsheet_db.py new file mode 100644 index 0000000..18a22f1 --- /dev/null +++ b/tests/databases/test_gsheet_db.py @@ -0,0 +1,142 @@ +from datetime import datetime, timezone +from unittest.mock import MagicMock, patch + +import pytest + +from auto_archiver.core import Metadata, Media +from auto_archiver.modules.gsheet_db import GsheetsDb +from auto_archiver.modules.gsheet_feeder import GWorksheet + + +@pytest.fixture +def mock_gworksheet(): + mock_gworksheet = MagicMock(spec=GWorksheet) + mock_gworksheet.col_exists.return_value = True + mock_gworksheet.get_cell.return_value = "" + mock_gworksheet.get_row.return_value = {} + return mock_gworksheet + + +@pytest.fixture +def mock_metadata(): + metadata: Metadata = MagicMock(spec=Metadata) + metadata.get_url.return_value = "http://example.com" + metadata.status = "done" + metadata.get_title.return_value = "Example Title" + metadata.get.return_value = "Example Content" + metadata.get_timestamp.return_value = "2025-01-01T00:00:00" + metadata.get_final_media.return_value = MagicMock(spec=Media) + metadata.get_all_media.return_value = [] + metadata.get_media_by_id.return_value = None + metadata.get_first_image.return_value = None + return metadata + +@pytest.fixture +def metadata(): + metadata = Metadata() + metadata.add_media(Media(filename="screenshot", urls=["http://example.com/screenshot.png"])) + metadata.add_media(Media(filename="browsertrix", urls=["http://example.com/browsertrix.wacz"])) + metadata.add_media(Media(filename="thumbnail", urls=["http://example.com/thumbnail.png"])) + metadata.set_url("http://example.com") + metadata.set_title("Example Title") + metadata.set_content("Example Content") + metadata.success("my-archiver") + metadata.set("timestamp", "2025-01-01T00:00:00") + metadata.set("date", "2025-02-04T18:22:24.909112+00:00") + return metadata + + +@pytest.fixture +def mock_media(): + """Fixture for a mock Media object.""" + mock_media = MagicMock(spec=Media) + mock_media.urls = ["http://example.com/media"] + mock_media.get.return_value = "not-calculated" + return mock_media + +@pytest.fixture +def gsheets_db(mock_gworksheet, setup_module): + db = setup_module("gsheet_db", { + "allow_worksheets": "set()", + "block_worksheets": "set()", + "use_sheet_names_in_stored_paths": "True", + }) + db._retrieve_gsheet = MagicMock(return_value=(mock_gworksheet, 1)) + return db + + +@pytest.fixture +def fixed_timestamp(): + """Fixture for a fixed timestamp.""" + return datetime(2025, 1, 1, tzinfo=timezone.utc) + + +@pytest.fixture +def expected_calls(mock_media, fixed_timestamp): + """Fixture for the expected cell updates.""" + return [ + (1, 'status', 'my-archiver: success'), + (1, 'archive', 'http://example.com/screenshot.png'), + (1, 'date', '2025-02-01T00:00:00+00:00'), + (1, 'title', 'Example Title'), + (1, 'text', 'Example Content'), + (1, 'timestamp', '2025-01-01T00:00:00+00:00'), + (1, 'hash', 'not-calculated'), + # (1, 'screenshot', 'http://example.com/screenshot.png'), + # (1, 'thumbnail', '=IMAGE("http://example.com/thumbnail.png")'), + # (1, 'wacz', 'http://example.com/browsertrix.wacz'), + # (1, 'replaywebpage', 'https://replayweb.page/?source=http%3A%2F%2Fexample.com%2Fbrowsertrix.wacz#view=pages&url=') + ] + +def test_retrieve_gsheet(gsheets_db, metadata, mock_gworksheet): + gw, row = gsheets_db._retrieve_gsheet(metadata) + assert gw == mock_gworksheet + assert row == 1 + + +def test_started(gsheets_db, mock_metadata, mock_gworksheet): + gsheets_db.started(mock_metadata) + mock_gworksheet.set_cell.assert_called_once_with(1, 'status', 'Archive in progress') + +def test_failed(gsheets_db, mock_metadata, mock_gworksheet): + reason = "Test failure" + gsheets_db.failed(mock_metadata, reason) + mock_gworksheet.set_cell.assert_called_once_with(1, 'status', f'Archive failed {reason}') + + +def test_aborted(gsheets_db, mock_metadata, mock_gworksheet): + gsheets_db.aborted(mock_metadata) + mock_gworksheet.set_cell.assert_called_once_with(1, 'status', '') + + +def test_done(gsheets_db, metadata, mock_gworksheet, expected_calls): + with patch("auto_archiver.modules.gsheet_db.gsheet_db.get_current_timestamp", return_value='2025-02-01T00:00:00+00:00'): + gsheets_db.done(metadata) + mock_gworksheet.batch_set_cell.assert_called_once_with(expected_calls) + + +def test_done_cached(gsheets_db, metadata, mock_gworksheet): + with patch("auto_archiver.modules.gsheet_db.gsheet_db.get_current_timestamp", return_value='2025-02-01T00:00:00+00:00'): + gsheets_db.done(metadata, cached=True) + + # Verify the status message includes "[cached]" + call_args = mock_gworksheet.batch_set_cell.call_args[0][0] + assert any(call[2].startswith("[cached]") for call in call_args) + + +def test_done_missing_media(gsheets_db, metadata, mock_gworksheet): + # clear media from metadata + metadata.media = [] + with patch("auto_archiver.modules.gsheet_db.gsheet_db.get_current_timestamp", + return_value='2025-02-01T00:00:00+00:00'): + gsheets_db.done(metadata) + # Verify nothing media-related gets updated + call_args = mock_gworksheet.batch_set_cell.call_args[0][0] + media_fields = {'archive', 'screenshot', 'thumbnail', 'wacz', 'replaywebpage'} + assert all(call[1] not in media_fields for call in call_args) + +def test_safe_status_update(gsheets_db, metadata, mock_gworksheet): + gsheets_db._safe_status_update(metadata, "Test status") + mock_gworksheet.set_cell.assert_called_once_with(1, 'status', 'Test status') + + diff --git a/tests/enrichers/test_hash_enricher.py b/tests/enrichers/test_hash_enricher.py index 99f8117..4b61fc2 100644 --- a/tests/enrichers/test_hash_enricher.py +++ b/tests/enrichers/test_hash_enricher.py @@ -1,7 +1,8 @@ import pytest -from auto_archiver.enrichers.hash_enricher import HashEnricher +from auto_archiver.modules.hash_enricher import HashEnricher from auto_archiver.core import Metadata, Media +from auto_archiver.core.module import get_module_lazy @pytest.mark.parametrize("algorithm, filename, expected_hash", [ ("SHA-256", "tests/data/testfile_1.txt", "1b4f0e9851971998e732078544c96b36c3d01cedf7caa332359d6f1d83567014"), @@ -9,36 +10,29 @@ from auto_archiver.core import Metadata, Media ("SHA3-512", "tests/data/testfile_1.txt", "d2d8cc4f369b340130bd2b29b8b54e918b7c260c3279176da9ccaa37c96eb71735fc97568e892dc6220bf4ae0d748edb46bd75622751556393be3f482e6f794e"), ("SHA3-512", "tests/data/testfile_2.txt", "e35970edaa1e0d8af7d948491b2da0450a49fd9cc1e83c5db4c6f175f9550cf341f642f6be8cfb0bfa476e4258e5088c5ad549087bf02811132ac2fa22b734c6") ]) -def test_calculate_hash(algorithm, filename, expected_hash): +def test_calculate_hash(algorithm, filename, expected_hash, setup_module): # test SHA-256 - he = HashEnricher({"algorithm": algorithm, "chunksize": 1}) + he = setup_module(HashEnricher, {"algorithm": algorithm, "chunksize": 100}) assert he.calculate_hash(filename) == expected_hash -def test_default_config_values(): - he = HashEnricher(config={}) +def test_default_config_values(setup_module): + he = setup_module(HashEnricher) assert he.algorithm == "SHA-256" assert he.chunksize == 16000000 -def test_invalid_chunksize(): - with pytest.raises(AssertionError): - he = HashEnricher({"chunksize": "-100"}) - -def test_invalid_algorithm(): - with pytest.raises(AssertionError): - HashEnricher({"algorithm": "SHA-123"}) - def test_config(): # test default config - c = HashEnricher.configs() + c = get_module_lazy('hash_enricher').configs assert c["algorithm"]["default"] == "SHA-256" assert c["chunksize"]["default"] == 16000000 assert c["algorithm"]["choices"] == ["SHA-256", "SHA3-512"] assert c["algorithm"]["help"] == "hash algorithm to use" assert c["chunksize"]["help"] == "number of bytes to use when reading files in chunks (if this value is too large you will run out of RAM), default is 16MB" -def test_hash_media(): - he = HashEnricher({"algorithm": "SHA-256", "chunksize": 1}) +def test_hash_media(setup_module): + + he = setup_module(HashEnricher, {"algorithm": "SHA-256", "chunksize": 1}) # generate metadata with two test files m = Metadata().set_url("https://example.com") diff --git a/tests/enrichers/test_meta_enricher.py b/tests/enrichers/test_meta_enricher.py new file mode 100644 index 0000000..a09aaa9 --- /dev/null +++ b/tests/enrichers/test_meta_enricher.py @@ -0,0 +1,103 @@ +import datetime +from datetime import datetime, timedelta, timezone +from unittest.mock import MagicMock, patch + +import pytest + +from auto_archiver.core import Metadata, Media +from auto_archiver.modules.meta_enricher import MetaEnricher + + +@pytest.fixture +def mock_metadata(): + """Creates a mock Metadata object.""" + mock: Metadata = MagicMock(spec=Metadata) + mock.get_url.return_value = "https://example.com" + mock.is_empty.return_value = False # Default to not empty + mock.get_all_media.return_value = [] + return mock + +@pytest.fixture +def mock_media(): + """Creates a mock Media object.""" + mock: Media = MagicMock(spec=Media) + mock.filename = "mock_file.txt" + return mock + +@pytest.fixture +def metadata(): + m = Metadata() + m.set_url("https://example.com") + m.set_title("Test Title") + m.set_content("Test Content") + return m + + +@pytest.fixture(autouse=True) +def meta_enricher(setup_module): + return setup_module(MetaEnricher, {}) + + +def test_enrich_skips_empty_metadata(meta_enricher, mock_metadata): + """Test that enrich() does nothing when Metadata is empty.""" + mock_metadata.is_empty.return_value = True + meta_enricher.enrich(mock_metadata) + mock_metadata.get_url.assert_called_once() + + +def test_enrich_file_sizes(meta_enricher, metadata, tmp_path): + """Test that enrich_file_sizes() calculates and sets file sizes correctly.""" + file1 = tmp_path / "testfile_1.txt" + file2 = tmp_path / "testfile_2.txt" + file1.write_text("A" * 1000) + file2.write_text("B" * 2000) + metadata.add_media(Media(str(file1))) + metadata.add_media(Media(str(file2))) + + meta_enricher.enrich_file_sizes(metadata) + + # Verify individual media file sizes + media1 = metadata.get_all_media()[0] + media2 = metadata.get_all_media()[1] + + assert media1.get("bytes") == 1000 + assert media1.get("size") == "1000.0 bytes" + assert media2.get("bytes") == 2000 + assert media2.get("size") == "2.0 KB" + + assert metadata.get("total_bytes") == 3000 + assert metadata.get("total_size") == "2.9 KB" + +@pytest.mark.parametrize( + "size, expected", + [ + (500, "500.0 bytes"), + (1024, "1.0 KB"), + (2048, "2.0 KB"), + (1048576, "1.0 MB"), + (1073741824, "1.0 GB"), + ], +) +def test_human_readable_bytes(size, expected): + """Test that human_readable_bytes() converts sizes correctly.""" + enricher = MetaEnricher() + assert enricher.human_readable_bytes(size) == expected + +def test_enrich_file_sizes_no_media(meta_enricher, metadata): + """Test that enrich_file_sizes() handles empty media list gracefully.""" + meta_enricher.enrich_file_sizes(metadata) + assert metadata.get("total_bytes") == 0 + assert metadata.get("total_size") == "0.0 bytes" + + +def test_enrich_archive_duration(meta_enricher, metadata): + # Set fixed "processed at" time in the past + processed_at = datetime.now(timezone.utc) - timedelta(minutes=10, seconds=30) + metadata.set("_processed_at", processed_at) + # patch datetime + with patch("datetime.datetime") as mock_datetime: + mock_now = datetime.now(timezone.utc) + mock_datetime.now.return_value = mock_now + meta_enricher.enrich_archive_duration(metadata) + + assert metadata.get("archive_duration_seconds") == 630 \ No newline at end of file diff --git a/tests/archivers/__init__.py b/tests/extractors/__init__.py similarity index 100% rename from tests/archivers/__init__.py rename to tests/extractors/__init__.py diff --git a/tests/archivers/test_archiver_base.py b/tests/extractors/test_extractor_base.py similarity index 59% rename from tests/archivers/test_archiver_base.py rename to tests/extractors/test_extractor_base.py index d793706..6e77ec3 100644 --- a/tests/archivers/test_archiver_base.py +++ b/tests/extractors/test_extractor_base.py @@ -1,19 +1,22 @@ +from typing import Type + import pytest -from auto_archiver.core import Metadata -from auto_archiver.core import Step from auto_archiver.core.metadata import Metadata -from auto_archiver.archivers.archiver import Archiver -class TestArchiverBase(object): +from auto_archiver.core.extractor import Extractor - archiver_class: str = None + +class TestExtractorBase(object): + + extractor_module: str = None config: dict = None @pytest.fixture(autouse=True) - def setup_archiver(self): - assert self.archiver_class is not None, "self.archiver_class must be set on the subclass" + def setup_extractor(self, setup_module): + assert self.extractor_module is not None, "self.extractor_module must be set on the subclass" assert self.config is not None, "self.config must be a dict set on the subclass" - self.archiver: Archiver = self.archiver_class({self.archiver_class.name: self.config}) + + self.extractor: Type[Extractor] = setup_module(self.extractor_module, self.config) def assertValidResponseMetadata(self, test_response: Metadata, title: str, timestamp: str, status: str = ""): assert test_response is not False diff --git a/tests/archivers/test_generic_archiver.py b/tests/extractors/test_generic_extractor.py similarity index 85% rename from tests/archivers/test_generic_archiver.py rename to tests/extractors/test_generic_extractor.py index 6e249e8..c70a51f 100644 --- a/tests/archivers/test_generic_archiver.py +++ b/tests/extractors/test_generic_extractor.py @@ -6,13 +6,15 @@ from os.path import dirname import pytest -from auto_archiver.archivers.generic_archiver import GenericArchiver -from .test_archiver_base import TestArchiverBase +from auto_archiver.modules.generic_extractor.generic_extractor import GenericExtractor +from .test_extractor_base import TestExtractorBase -class TestGenericArchiver(TestArchiverBase): - """Tests Base Archiver +class TestGenericExtractor(TestExtractorBase): + """Tests Generic Extractor """ - archiver_class = GenericArchiver + extractor_module = 'generic_extractor' + extractor: GenericExtractor + config = { 'subtitles': False, 'comments': False, @@ -28,12 +30,12 @@ class TestGenericArchiver(TestArchiverBase): def test_load_dropin(self): # test loading dropins that are in the generic_archiver package - package = "auto_archiver.archivers.generic_archiver" - assert self.archiver.dropin_for_name("bluesky", package=package) + package = "auto_archiver.modules.generic_extractor" + assert self.extractor.dropin_for_name("bluesky", package=package) # test loading dropings via filepath path = os.path.join(dirname(dirname(__file__)), "data/") - assert self.archiver.dropin_for_name("dropin", additional_paths=[path]) + assert self.extractor.dropin_for_name("dropin", additional_paths=[path]) @@ -51,12 +53,12 @@ class TestGenericArchiver(TestArchiverBase): This behaviour may be changed in the future (e.g. if we want the youtubedl archiver to just handle URLs it has extractors for, and then if and only if all archivers fails, does it fall back to the generic archiver) """ - assert self.archiver.suitable(url) == is_suitable + assert self.extractor.suitable(url) == is_suitable @pytest.mark.download def test_download_tiktok(self, make_item): item = make_item("https://www.tiktok.com/@funnycats0ftiktok/video/7345101300750748970") - result = self.archiver.download(item) + result = self.extractor.download(item) assert result.get_url() == "https://www.tiktok.com/@funnycats0ftiktok/video/7345101300750748970" @pytest.mark.download @@ -72,7 +74,7 @@ class TestGenericArchiver(TestArchiverBase): It should return 'False' """ item = make_item(url) - result = self.archiver.download(item) + result = self.extractor.download(item) assert not result @@ -80,7 +82,7 @@ class TestGenericArchiver(TestArchiverBase): def test_youtube_download(self, make_item): # url https://www.youtube.com/watch?v=5qap5aO4i9A item = make_item("https://www.youtube.com/watch?v=J---aiyznGQ") - result = self.archiver.download(item) + result = self.extractor.download(item) assert result.get_url() == "https://www.youtube.com/watch?v=J---aiyznGQ" assert result.get_title() == "Keyboard Cat! - THE ORIGINAL!" assert result.get('description') == "Buy NEW Keyboard Cat Merch! https://keyboardcat.creator-spring.com\n\nxo Keyboard Cat memes make your day better!\nhttp://www.keyboardcatstore.com/\nhttps://www.facebook.com/thekeyboardcat\nhttp://www.charlieschmidt.com/" @@ -91,78 +93,78 @@ class TestGenericArchiver(TestArchiverBase): @pytest.mark.download def test_bluesky_download_multiple_images(self, make_item): item = make_item("https://bsky.app/profile/bellingcat.com/post/3lffjoxcu7k2w") - result = self.archiver.download(item) + result = self.extractor.download(item) assert result is not False @pytest.mark.download def test_bluesky_download_single_image(self, make_item): item = make_item("https://bsky.app/profile/bellingcat.com/post/3lfn3hbcxgc2q") - result = self.archiver.download(item) + result = self.extractor.download(item) assert result is not False @pytest.mark.download def test_bluesky_download_no_media(self, make_item): item = make_item("https://bsky.app/profile/bellingcat.com/post/3lfphwmcs4c2z") - result = self.archiver.download(item) + result = self.extractor.download(item) assert result is not False @pytest.mark.download def test_bluesky_download_video(self, make_item): item = make_item("https://bsky.app/profile/bellingcat.com/post/3le2l4gsxlk2i") - result = self.archiver.download(item) + result = self.extractor.download(item) assert result is not False @pytest.mark.download def test_truthsocial_download_video(self, make_item): item = make_item("https://truthsocial.com/@DaynaTrueman/posts/110602446619561579") - result = self.archiver.download(item) + result = self.extractor.download(item) assert len(result.media) == 1 assert result is not False @pytest.mark.download def test_truthsocial_download_no_media(self, make_item): item = make_item("https://truthsocial.com/@bbcnewa/posts/109598702184774628") - result = self.archiver.download(item) + result = self.extractor.download(item) assert result is not False @pytest.mark.download def test_truthsocial_download_poll(self, make_item): item = make_item("https://truthsocial.com/@CNN_US/posts/113724326568555098") - result = self.archiver.download(item) + result = self.extractor.download(item) assert result is not False @pytest.mark.download def test_truthsocial_download_single_image(self, make_item): item = make_item("https://truthsocial.com/@mariabartiromo/posts/113861116433335006") - result = self.archiver.download(item) + result = self.extractor.download(item) assert len(result.media) == 1 assert result is not False @pytest.mark.download def test_truthsocial_download_multiple_images(self, make_item): item = make_item("https://truthsocial.com/@trrth/posts/113861302149349135") - result = self.archiver.download(item) + result = self.extractor.download(item) assert len(result.media) == 3 @pytest.mark.download def test_twitter_download_nonexistend_tweet(self, make_item): # this tweet does not exist url = "https://x.com/Bellingcat/status/17197025860711058" - response = self.archiver.download(make_item(url)) + response = self.extractor.download(make_item(url)) assert not response @pytest.mark.download def test_twitter_download_malformed_tweetid(self, make_item): # this tweet does not exist url = "https://x.com/Bellingcat/status/1719702a586071100058" - response = self.archiver.download(make_item(url)) + response = self.extractor.download(make_item(url)) assert not response @pytest.mark.download def test_twitter_download_tweet_no_media(self, make_item): item = make_item("https://twitter.com/MeCookieMonster/status/1617921633456640001?s=20&t=3d0g4ZQis7dCbSDg-mE7-w") - post = self.archiver.download(item) + post = self.extractor.download(item) self.assertValidResponseMetadata( post, @@ -174,7 +176,7 @@ class TestGenericArchiver(TestArchiverBase): @pytest.mark.download def test_twitter_download_video(self, make_item): url = "https://x.com/bellingcat/status/1871552600346415571" - post = self.archiver.download(make_item(url)) + post = self.extractor.download(make_item(url)) self.assertValidResponseMetadata( post, "Bellingcat - This month's Bellingchat Premium is with @KolinaKoltai. She reveals how she investigated a platform allowing users to create AI-generated child sexual abuse material and explains why it's crucial to investigate the people behind these services", @@ -193,7 +195,7 @@ class TestGenericArchiver(TestArchiverBase): """Download tweets with sensitive media""" - post = self.archiver.download(make_item(url)) + post = self.extractor.download(make_item(url)) self.assertValidResponseMetadata( post, title, diff --git a/tests/extractors/test_instagram_api_extractor.py b/tests/extractors/test_instagram_api_extractor.py new file mode 100644 index 0000000..c119e3f --- /dev/null +++ b/tests/extractors/test_instagram_api_extractor.py @@ -0,0 +1,188 @@ +from datetime import datetime +from typing import Type + +import pytest +from unittest.mock import patch, MagicMock + +from auto_archiver.core import Metadata +from auto_archiver.modules.instagram_api_extractor.instagram_api_extractor import InstagramAPIExtractor +from .test_extractor_base import TestExtractorBase + + + +@pytest.fixture +def mock_user_response(): + return { + "user": { + "pk": "123", + "username": "test_user", + "full_name": "Test User", + "profile_pic_url_hd": "http://example.com/profile.jpg", + "profile_pic_url": "http://example.com/profile_lowres.jpg" + } + } + +@pytest.fixture +def mock_post_response(): + return { + "id": "post_123", + "code": "abc123", + "caption_text": "Test Caption", + "taken_at": datetime.now().timestamp(), + "video_url": "http://example.com/video.mp4", + "thumbnail_url": "http://example.com/thumbnail.jpg" + } + +@pytest.fixture +def mock_story_response(): + return [{ + "id": "story_123", + "taken_at": datetime.now().timestamp(), + "video_url": "http://example.com/story.mp4" + }] + +@pytest.fixture +def mock_highlight_response(): + return { + "response": { + "reels": { + "highlight:123": { + "id": "123", + "title": "Test Highlight", + "items": [{ + "id": "item_123", + "taken_at": datetime.now().timestamp(), + "video_url": "http://example.com/highlight.mp4" + }] + } + } + } + } + + +# @pytest.mark.incremental +class TestInstagramAPIExtractor(TestExtractorBase): + """ + Test suite for InstagramAPIExtractor. + """ + + extractor_module = "instagram_api_extractor" + extractor: InstagramAPIExtractor + + config = { + "access_token": "test_access_token", + "api_endpoint": "https://api.instagram.com/v1", + "full_profile": False, + # "full_profile_max_posts": 0, + # "minimize_json_output": True, + } + + @pytest.fixture + def metadata(self): + m = Metadata() + m.set_url("https://instagram.com/test_user") + m.set("netloc", "instagram.com") + return m + + @pytest.mark.parametrize("url,expected", [ + ("https://instagram.com/user", [("", "user", "")]), + ("https://instagr.am/p/post_id", []), + ("https://youtube.com", []), + ("https://www.instagram.com/reel/reel_id", [("reel", "reel_id", "")]), + ("https://instagram.com/stories/highlights/123", [("stories/highlights", "123", "")]), + ("https://instagram.com/stories/user/123", [("stories", "user", "123")]), + ]) + def test_url_parsing(self, url, expected): + assert self.extractor.valid_url.findall(url) == expected + + def test_initialize(self): + assert self.extractor.api_endpoint[-1] != "/" + + @pytest.mark.parametrize("input_dict,expected", [ + ({"x": 0, "valid": "data"}, {"valid": "data"}), + ({"nested": {"y": None, "valid": [{}]}}, {"nested": {"valid": [{}]}}), + ]) + def test_cleanup_dict(self, input_dict, expected): + assert self.extractor.cleanup_dict(input_dict) == expected + + def test_download(self): + pass + + def test_download_post(self, metadata, mock_user_response): + # test with context=reel + # test with context=post + # test with multiple images + # test gets text (metadata title) + pass + + def test_download_profile_basic(self, metadata, mock_user_response): + """Test basic profile download without full_profile""" + with patch.object(self.extractor, 'call_api') as mock_call, \ + patch.object(self.extractor, 'download_from_url') as mock_download: + # Mock API responses + mock_call.return_value = mock_user_response + mock_download.return_value = "profile.jpg" + + result = self.extractor.download_profile(metadata, "test_user") + assert result.status == "insta profile: success" + assert result.get_title() == "Test User" + assert result.get("data") == self.extractor.cleanup_dict(mock_user_response["user"]) + # Verify profile picture download + mock_call.assert_called_once_with("v2/user/by/username", {"username": "test_user"}) + mock_download.assert_called_once_with("http://example.com/profile.jpg") + assert len(result.media) == 1 + assert result.media[0].filename == "profile.jpg" + + def test_download_profile_full(self, metadata, mock_user_response, mock_story_response): + """Test full profile download with stories/posts""" + with patch.object(self.extractor, 'call_api') as mock_call, \ + patch.object(self.extractor, 'download_all_posts') as mock_posts, \ + patch.object(self.extractor, 'download_all_highlights') as mock_highlights, \ + patch.object(self.extractor, 'download_all_tagged') as mock_tagged, \ + patch.object(self.extractor, '_download_stories_reusable') as mock_stories: + + self.extractor.full_profile = True + mock_call.side_effect = [ + mock_user_response, + mock_story_response + ] + mock_highlights.return_value = None + mock_stories.return_value = mock_story_response + mock_posts.return_value = None + mock_tagged.return_value = None + + result = self.extractor.download_profile(metadata, "test_user") + assert result.get("#stories") == len(mock_story_response) + mock_posts.assert_called_once_with(result, "123") + assert "errors" not in result.metadata + + def test_download_profile_not_found(self, metadata): + """Test profile not found error""" + with patch.object(self.extractor, 'call_api') as mock_call: + mock_call.return_value = {"user": None} + with pytest.raises(AssertionError) as exc_info: + self.extractor.download_profile(metadata, "invalid_user") + assert "User invalid_user not found" in str(exc_info.value) + + def test_download_profile_error_handling(self, metadata, mock_user_response): + """Test error handling in full profile mode""" + with (patch.object(self.extractor, 'call_api') as mock_call, \ + patch.object(self.extractor, 'download_all_highlights') as mock_highlights, \ + patch.object(self.extractor, 'download_all_tagged') as mock_tagged, \ + patch.object(self.extractor, '_download_stories_reusable') as stories_tagged, \ + patch.object(self.extractor, 'download_all_posts') as mock_posts + ): + self.extractor.full_profile = True + mock_call.side_effect = [ + mock_user_response, + Exception("Stories API failed"), + Exception("Posts API failed") + ] + mock_highlights.return_value = None + mock_tagged.return_value = None + stories_tagged.return_value = None + mock_posts.return_value = None + result = self.extractor.download_profile(metadata, "test_user") + + assert result.is_success() + assert "Error downloading stories for test_user" in result.metadata["errors"] \ No newline at end of file diff --git a/tests/extractors/test_instagram_extractor.py b/tests/extractors/test_instagram_extractor.py new file mode 100644 index 0000000..7efe1b1 --- /dev/null +++ b/tests/extractors/test_instagram_extractor.py @@ -0,0 +1,21 @@ +import pytest + +from auto_archiver.modules.instagram_extractor import InstagramExtractor +from .test_extractor_base import TestExtractorBase + +class TestInstagramExtractor(TestExtractorBase): + + extractor_module: str = 'instagram_extractor' + config: dict = {} + + @pytest.mark.parametrize("url", [ + "https://www.instagram.com/p/", + "https://www.instagram.com/p/1234567890/", + "https://www.instagram.com/reel/1234567890/", + "https://www.instagram.com/username/", + "https://www.instagram.com/username/stories/", + "https://www.instagram.com/username/highlights/", + ]) + def test_regex_matches(self, url): + # post + assert InstagramExtractor.valid_url.match(url) diff --git a/tests/extractors/test_instagram_tbot_extractor.py b/tests/extractors/test_instagram_tbot_extractor.py new file mode 100644 index 0000000..d7a1e53 --- /dev/null +++ b/tests/extractors/test_instagram_tbot_extractor.py @@ -0,0 +1,94 @@ +import os +from typing import Type +from unittest.mock import patch, MagicMock + +import pytest + +from auto_archiver.core import Metadata +from auto_archiver.core.extractor import Extractor +from auto_archiver.modules.instagram_tbot_extractor import InstagramTbotExtractor +from tests.extractors.test_extractor_base import TestExtractorBase + +TESTFILES = os.path.join(os.path.dirname(__file__), "testfiles") + + +@pytest.fixture +def session_file(tmpdir): + """Fixture to create a test session file.""" + session_file = os.path.join(tmpdir, "test_session.session") + with open(session_file, "w") as f: + f.write("mock_session_data") + return session_file.replace(".session", "") + + +@pytest.fixture(autouse=True) +def patch_extractor_methods(request, setup_module): + with patch.object(InstagramTbotExtractor, '_prepare_session_file', return_value=None), \ + patch.object(InstagramTbotExtractor, '_initialize_telegram_client', return_value=None): + if hasattr(request, 'cls') and hasattr(request.cls, 'config'): + request.cls.extractor = setup_module("instagram_tbot_extractor", request.cls.config) + + yield + +@pytest.fixture +def metadata_sample(): + m = Metadata() + m.set_title("Test Title") + m.set_timestamp("2021-01-01T00:00:00Z") + m.set_url("https://www.instagram.com/p/1234567890") + return m + + +class TestInstagramTbotExtractor: + + extractor_module = "instagram_tbot_extractor" + extractor: InstagramTbotExtractor + config = { + "api_id": 12345, + "api_hash": "test_api_hash", + "session_file": "test_session", + } + + @pytest.fixture + def mock_telegram_client(self): + """Fixture to mock TelegramClient interactions.""" + with patch("auto_archiver.modules.instagram_tbot_extractor._initialize_telegram_client") as mock_client: + instance = MagicMock() + mock_client.return_value = instance + yield instance + + def test_extractor_is_initialized(self): + assert self.extractor is not None + + + @patch("time.sleep") + @pytest.mark.parametrize("url, expected_status, bot_responses", [ + ("https://www.instagram.com/p/C4QgLbrIKXG", "insta-via-bot: success", [MagicMock(id=101, media=None, message="Are you new to Bellingcat? - The way we share our investigations is different. 💭\nWe want you to read our story but also learn ou")]), + ("https://www.instagram.com/reel/DEVLK8qoIbg/", "insta-via-bot: success", [MagicMock(id=101, media=None, message="Our volunteer community is at the centre of many incredible Bellingcat investigations and tools. Stephanie Ladel is one such vol")]), + # todo tbot not working for stories :( + ("https://www.instagram.com/stories/bellingcatofficial/3556336382743057476/", False, [MagicMock(id=101, media=None, message="Media not found or unavailable")]), + ("https://www.youtube.com/watch?v=ymCMy8OffHM", False, []), + ("https://www.instagram.com/p/INVALID", False, [MagicMock(id=101, media=None, message="You must enter a URL to a post")]), + ]) + def test_download(self, mock_sleep, url, expected_status, bot_responses, metadata_sample): + """Test the `download()` method with various Instagram URLs.""" + metadata_sample.set_url(url) + self.extractor.client = MagicMock() + result = self.extractor.download(metadata_sample) + pass + # TODO fully mock or use as authenticated test + # if expected_status: + # assert result.is_success() + # assert result.status == expected_status + # assert result.metadata.get("title") in [msg.message[:128] for msg in bot_responses if msg.message] + # else: + # assert result is False + + + + + # Test story +# Test expired story +# Test requires login/ access (?) +# Test post +# Test multiple images? \ No newline at end of file diff --git a/tests/archivers/test_twitter_api_archiver.py b/tests/extractors/test_twitter_api_extractor.py similarity index 84% rename from tests/archivers/test_twitter_api_archiver.py rename to tests/extractors/test_twitter_api_extractor.py index a95f2c7..d9a8eb0 100644 --- a/tests/archivers/test_twitter_api_archiver.py +++ b/tests/extractors/test_twitter_api_extractor.py @@ -1,17 +1,18 @@ import os import datetime - +import hashlib import pytest from pytwitter.models.media import MediaVariant -from .test_archiver_base import TestArchiverBase -from auto_archiver.archivers import TwitterApiArchiver +from .test_extractor_base import TestExtractorBase +from auto_archiver.modules.twitter_api_extractor import TwitterApiExtractor @pytest.mark.incremental -class TestTwitterApiArchiver(TestArchiverBase): +class TestTwitterApiExtractor(TestExtractorBase): + + extractor_module = 'twitter_api_extractor' - archiver_class = TwitterApiArchiver config = { "bearer_tokens": [], "bearer_token": os.environ.get("TWITTER_BEARER_TOKEN", "TEST_KEY"), @@ -30,7 +31,7 @@ class TestTwitterApiArchiver(TestArchiverBase): ("https://www.bellingcat.com/category/resources/?s=20&t=3d0g4ZQis7dCbSDg-mE7-w", "https://www.bellingcat.com/category/resources/?s=20&t=3d0g4ZQis7dCbSDg-mE7-w"), # shouldn't strip params from non-twitter/x URLs ]) def test_sanitize_url(self, url, expected): - assert expected == self.archiver.sanitize_url(url) + assert expected == self.extractor.sanitize_url(url) @pytest.mark.parametrize("url, exptected_username, exptected_tweetid", [ ("https://twitter.com/bellingcat/status/1874097816571961839", "bellingcat", "1874097816571961839"), @@ -39,7 +40,7 @@ class TestTwitterApiArchiver(TestArchiverBase): ]) def test_get_username_tweet_id_from_url(self, url, exptected_username, exptected_tweetid): - username, tweet_id = self.archiver.get_username_tweet_id(url) + username, tweet_id = self.extractor.get_username_tweet_id(url) assert exptected_username == username assert exptected_tweetid == tweet_id @@ -50,7 +51,7 @@ class TestTwitterApiArchiver(TestArchiverBase): MediaVariant(bit_rate=832000, content_type='video/mp4', url='https://video.twimg.com/ext_tw_video/1871551993677852672/pu/vid/avc1/640x360/uiDZDSmZ8MZn9hsi.mp4?tag=12'), MediaVariant(bit_rate=2176000, content_type='video/mp4', url='https://video.twimg.com/ext_tw_video/1871551993677852672/pu/vid/avc1/1280x720/6Y340Esh568WZnRZ.mp4?tag=12') ] - chosen_variant = self.archiver.choose_variant(variant_list) + chosen_variant = self.extractor.choose_variant(variant_list) assert chosen_variant == variant_list[3] @pytest.mark.skipif(not os.environ.get("TWITTER_BEARER_TOKEN"), reason="No Twitter bearer token provided") @@ -58,7 +59,7 @@ class TestTwitterApiArchiver(TestArchiverBase): def test_download_nonexistent_tweet(self, make_item): # this tweet does not exist url = "https://x.com/Bellingcat/status/17197025860711058" - response = self.archiver.download(make_item(url)) + response = self.extractor.download(make_item(url)) assert not response @pytest.mark.skipif(not os.environ.get("TWITTER_BEARER_TOKEN"), reason="No Twitter bearer token provided") @@ -66,7 +67,7 @@ class TestTwitterApiArchiver(TestArchiverBase): def test_download_malformed_tweetid(self, make_item): # this tweet does not exist url = "https://x.com/Bellingcat/status/1719702586071100058" - response = self.archiver.download(make_item(url)) + response = self.extractor.download(make_item(url)) assert not response @pytest.mark.skipif(not os.environ.get("TWITTER_BEARER_TOKEN"), reason="No Twitter bearer token provided") @@ -74,7 +75,7 @@ class TestTwitterApiArchiver(TestArchiverBase): def test_download_tweet_no_media(self, make_item): item = make_item("https://twitter.com/MeCookieMonster/status/1617921633456640001?s=20&t=3d0g4ZQis7dCbSDg-mE7-w") - post = self.archiver.download(item) + post = self.extractor.download(item) self.assertValidResponseMetadata( post, @@ -87,7 +88,7 @@ class TestTwitterApiArchiver(TestArchiverBase): @pytest.mark.download def test_download_video(self, make_item): url = "https://x.com/bellingcat/status/1871552600346415571" - post = self.archiver.download(make_item(url)) + post = self.extractor.download(make_item(url)) self.assertValidResponseMetadata( post, "This month's Bellingchat Premium is with @KolinaKoltai. She reveals how she investigated a platform allowing users to create AI-generated child sexual abuse material and explains why it's crucial to investigate the people behind these services https://t.co/SfBUq0hSD0 https://t.co/rIHx0WlKp8", @@ -95,22 +96,23 @@ class TestTwitterApiArchiver(TestArchiverBase): ) @pytest.mark.skipif(not os.environ.get("TWITTER_BEARER_TOKEN"), reason="No Twitter bearer token provided") - @pytest.mark.parametrize("url, title, timestamp, image_src", [ - ("https://x.com/SozinhoRamalho/status/1876710769913450647", "ignore tweet, testing sensitivity warning nudity https://t.co/t3u0hQsSB1", datetime.datetime(2024, 12, 31, 14, 18, 33, tzinfo=datetime.timezone.utc), "https://pbs.twimg.com/media/GgtqkomWkAAHUUl.jpg"), - ("https://x.com/SozinhoRamalho/status/1876710875475681357", "ignore tweet, testing sensitivity warning violence https://t.co/syYDSkpjZD", datetime.datetime(2024, 12, 31, 14, 18, 33, tzinfo=datetime.timezone.utc), "https://pbs.twimg.com/media/GgtqkomWkAAHUUl.jpg"), - ("https://x.com/SozinhoRamalho/status/1876711053813227618", "ignore tweet, testing sensitivity warning sensitive https://t.co/XE7cRdjzYq", datetime.datetime(2024, 12, 31, 14, 18, 33, tzinfo=datetime.timezone.utc), "https://pbs.twimg.com/media/GgtqkomWkAAHUUl.jpg"), - ("https://x.com/SozinhoRamalho/status/1876711141314801937", "ignore tweet, testing sensitivity warning nudity, violence, sensitivity https://t.co/YxCFbbhYE3", datetime.datetime(2024, 12, 31, 14, 18, 33, tzinfo=datetime.timezone.utc), "https://pbs.twimg.com/media/GgtqkomWkAAHUUl.jpg"), + @pytest.mark.parametrize("url, title, timestamp", [ + ("https://x.com/SozinhoRamalho/status/1876710769913450647", "ignore tweet, testing sensitivity warning nudity https://t.co/t3u0hQsSB1", datetime.datetime(2024, 12, 31, 14, 18, 33, tzinfo=datetime.timezone.utc)), + ("https://x.com/SozinhoRamalho/status/1876710875475681357", "ignore tweet, testing sensitivity warning violence https://t.co/syYDSkpjZD", datetime.datetime(2024, 12, 31, 14, 18, 33, tzinfo=datetime.timezone.utc)), + ("https://x.com/SozinhoRamalho/status/1876711053813227618", "ignore tweet, testing sensitivity warning sensitive https://t.co/XE7cRdjzYq", datetime.datetime(2024, 12, 31, 14, 18, 33, tzinfo=datetime.timezone.utc)), + ("https://x.com/SozinhoRamalho/status/1876711141314801937", "ignore tweet, testing sensitivity warning nudity, violence, sensitivity https://t.co/YxCFbbhYE3", datetime.datetime(2024, 12, 31, 14, 18, 33, tzinfo=datetime.timezone.utc)), ]) @pytest.mark.download - def test_download_sensitive_media(self, url, title, timestamp, image_src, make_item): + def test_download_sensitive_media(self, url, title, timestamp, check_hash, make_item): """Download tweets with sensitive media""" - post = self.archiver.download(make_item(url)) + post = self.extractor.download(make_item(url)) self.assertValidResponseMetadata( post, title, timestamp ) assert len(post.media) == 1 - assert post.media[0].get('src') == image_src \ No newline at end of file + # check the SHA1 hash (quick) of the media, to make sure it's valid + check_hash(post.media[0].filename, "3eea9c03b2dcedd1eb9a169d8bfd1cf877996fab4961de019a96eb9d32d2d733") \ No newline at end of file diff --git a/tests/feeders/test_csv_feeder.py b/tests/feeders/test_csv_feeder.py new file mode 100644 index 0000000..546c3a7 --- /dev/null +++ b/tests/feeders/test_csv_feeder.py @@ -0,0 +1,57 @@ +import pytest + +@pytest.fixture +def headerless_csv_file(): + return "tests/data/csv_no_headers.csv" + +@pytest.fixture +def header_csv_file(): + return "tests/data/csv_with_headers.csv" + +@pytest.fixture +def header_csv_file_non_default_column(): + return "tests/data/csv_with_headers_non_default_column.csv" + + +def test_csv_feeder_no_headers(headerless_csv_file, setup_module): + from auto_archiver.modules.csv_feeder.csv_feeder import CSVFeeder + + feeder = setup_module(CSVFeeder, {"files": [headerless_csv_file]}) + + urls = list(feeder) + assert len(urls) == 2 + assert urls[0].get_url() == "https://example.com/1/" + assert urls[1].get_url() == "https://example.com/2/" + +def test_csv_feeder_with_headers(header_csv_file, setup_module): + from auto_archiver.modules.csv_feeder.csv_feeder import CSVFeeder + + feeder = setup_module(CSVFeeder, {"files": [header_csv_file]}) + + urls = list(feeder) + assert len(urls) == 2 + assert urls[0].get_url() == "https://example.com/1/" + assert urls[1].get_url() == "https://example.com/2/" + +def test_csv_feeder_wrong_column(header_csv_file, setup_module, caplog): + from auto_archiver.modules.csv_feeder.csv_feeder import CSVFeeder + + + with caplog.at_level("WARNING"): + feeder = setup_module(CSVFeeder, {"files": [header_csv_file], "column": 1}) + urls = list(feeder) + + assert len(urls) == 0 + assert "Not a valid URL in row" in caplog.text + assert len(caplog.records) == 2 + + +def test_csv_feeder_column_by_name(header_csv_file, setup_module): + from auto_archiver.modules.csv_feeder.csv_feeder import CSVFeeder + + feeder = setup_module(CSVFeeder, {"files": [header_csv_file], "column": "webpages"}) + + urls = list(feeder) + assert len(urls) == 2 + assert urls[0].get_url() == "https://example.com/1/" + assert urls[1].get_url() == "https://example.com/2/" \ No newline at end of file diff --git a/tests/feeders/test_gsheet_feeder.py b/tests/feeders/test_gsheet_feeder.py new file mode 100644 index 0000000..b86e329 --- /dev/null +++ b/tests/feeders/test_gsheet_feeder.py @@ -0,0 +1,273 @@ +from typing import Type + +import gspread +import pytest +from unittest.mock import patch, MagicMock +from auto_archiver.modules.gsheet_feeder import GsheetsFeeder +from auto_archiver.core import Metadata, Feeder + + +def test_setup_without_sheet_and_sheet_id(setup_module): + # Ensure setup() raises AssertionError if neither sheet nor sheet_id is set. + with patch("gspread.service_account"): + with pytest.raises(AssertionError): + setup_module( + "gsheet_feeder", + {"service_account": "dummy.json", "sheet": None, "sheet_id": None}, + ) + + +@pytest.fixture +def gsheet_feeder(setup_module) -> GsheetsFeeder: + with patch("gspread.service_account"): + feeder = setup_module( + "gsheet_feeder", + { + "service_account": "dummy.json", + "sheet": "test-auto-archiver", + "sheet_id": None, + "header": 1, + "columns": { + "url": "link", + "status": "archive status", + "folder": "destination folder", + "archive": "archive location", + "date": "archive date", + "thumbnail": "thumbnail", + "timestamp": "upload timestamp", + "title": "upload title", + "text": "text content", + "screenshot": "screenshot", + "hash": "hash", + "pdq_hash": "perceptual hashes", + "wacz": "wacz", + "replaywebpage": "replaywebpage", + }, + "allow_worksheets": set(), + "block_worksheets": set(), + "use_sheet_names_in_stored_paths": True, + }, + ) + feeder.gsheets_client = MagicMock() + return feeder + + +class MockWorksheet: + """ + mimics the bits we need from gworksheet + """ + + class SheetSheet: + title = "TestSheet" + + rows = [ + {"row": 2, "url": "http://example.com", "status": "", "folder": ""}, + {"row": 3, "url": "http://example.com", "status": "", "folder": ""}, + {"row": 4, "url": "", "status": "", "folder": ""}, + {"row": 5, "url": "https://another.com", "status": None, "folder": ""}, + { + "row": 6, + "url": "https://another.com", + "status": "success", + "folder": "some_folder", + }, + ] + + def __init__(self): + self.wks = self.SheetSheet() + + def count_rows(self): + if not self.rows: + return 0 + return max(r["row"] for r in self.rows) + + def get_cell(self, row, col_name, fresh=False): + matching = next((r for r in self.rows if r["row"] == row), {}) + return matching.get(col_name, "") + + def get_cell_or_default(self, row, col_name, default): + matching = next((r for r in self.rows if r["row"] == row), {}) + return matching.get(col_name, default) + + +def test__process_rows(gsheet_feeder: GsheetsFeeder): + testworksheet = MockWorksheet() + metadata_items = list(gsheet_feeder._process_rows(testworksheet)) + assert len(metadata_items) == 3 + assert isinstance(metadata_items[0], Metadata) + assert metadata_items[0].get("url") == "http://example.com" + + +def test__set_metadata(gsheet_feeder: GsheetsFeeder): + worksheet = MockWorksheet() + metadata = Metadata() + gsheet_feeder._set_context(metadata, worksheet, 1) + assert metadata.get_context("gsheet") == {"row": 1, "worksheet": worksheet} + + +@pytest.mark.skip(reason="Not recognising folder column") +def test__set_metadata_with_folder_pickled(gsheet_feeder: GsheetsFeeder, worksheet): + gsheet_feeder._set_context(worksheet, 7) + assert Metadata.get_context("gsheet") == {"row": 1, "worksheet": worksheet} + + +def test__set_metadata_with_folder(gsheet_feeder: GsheetsFeeder): + testworksheet = MockWorksheet() + metadata = Metadata() + testworksheet.wks.title = "TestSheet" + gsheet_feeder._set_context(metadata, testworksheet, 6) + assert metadata.get_context("gsheet") == {"row": 6, "worksheet": testworksheet} + assert metadata.get_context("folder") == "some-folder/test-auto-archiver/testsheet" + + +@pytest.mark.usefixtures("setup_module") +@pytest.mark.parametrize( + "sheet, sheet_id, expected_method, expected_arg, description", + [ + ("TestSheet", None, "open", "TestSheet", "opening by sheet name"), + (None, "ABC123", "open_by_key", "ABC123", "opening by sheet ID"), + ], +) +def test_open_sheet_with_name_or_id( + setup_module, sheet, sheet_id, expected_method, expected_arg, description +): + """Ensure open_sheet() correctly opens by name or ID based on configuration.""" + with patch("gspread.service_account") as mock_service_account: + mock_client = MagicMock() + mock_service_account.return_value = mock_client + mock_client.open.return_value = "MockSheet" + mock_client.open_by_key.return_value = "MockSheet" + + # Setup module with parameterized values + feeder = setup_module( + "gsheet_feeder", + {"service_account": "dummy.json", "sheet": sheet, "sheet_id": sheet_id}, + ) + sheet_result = feeder.open_sheet() + # Validate the correct method was called + getattr(mock_client, expected_method).assert_called_once_with( + expected_arg + ), f"Failed: {description}" + assert sheet_result == "MockSheet", f"Failed: {description}" + + +@pytest.mark.usefixtures("setup_module") +def test_open_sheet_with_sheet_id(setup_module): + """Ensure open_sheet() correctly opens a sheet by ID.""" + with patch("gspread.service_account") as mock_service_account: + mock_client = MagicMock() + mock_service_account.return_value = mock_client + mock_client.open_by_key.return_value = "MockSheet" + feeder = setup_module( + "gsheet_feeder", + {"service_account": "dummy.json", "sheet": None, "sheet_id": "ABC123"}, + ) + sheet = feeder.open_sheet() + mock_client.open_by_key.assert_called_once_with("ABC123") + assert sheet == "MockSheet" + + +def test_should_process_sheet(setup_module): + with patch("gspread.service_account"): + gdb = setup_module( + "gsheet_feeder", + { + "service_account": "dummy.json", + "sheet": "TestSheet", + "sheet_id": None, + "allow_worksheets": {"TestSheet", "Sheet2"}, + "block_worksheets": {"Sheet3"}, + }, + ) + assert gdb.should_process_sheet("TestSheet") == True + assert gdb.should_process_sheet("Sheet3") == False + # False if allow_worksheets is set + assert gdb.should_process_sheet("AnotherSheet") == False + + +@pytest.mark.skip(reason="Requires a real connection") +class TestGSheetsFeederReal: + """Testing GSheetsFeeder class""" + + module_name: str = "gsheet_feeder" + feeder: GsheetsFeeder + # You must follow the setup process explain in the docs for this to work + config: dict = { + "service_account": "secrets/service_account.json", + "sheet": "test-auto-archiver", + "sheet_id": None, + "header": 1, + "columns": { + "url": "link", + "status": "archive status", + "folder": "destination folder", + "archive": "archive location", + "date": "archive date", + "thumbnail": "thumbnail", + "timestamp": "upload timestamp", + "title": "upload title", + "text": "text content", + "screenshot": "screenshot", + "hash": "hash", + "pdq_hash": "perceptual hashes", + "wacz": "wacz", + "replaywebpage": "replaywebpage", + }, + "allow_worksheets": set(), + "block_worksheets": set(), + "use_sheet_names_in_stored_paths": True, + } + + @pytest.fixture(autouse=True) + def setup_feeder(self, setup_module): + assert ( + self.module_name is not None + ), "self.module_name must be set on the subclass" + assert self.config is not None, "self.config must be a dict set on the subclass" + self.feeder: Type[Feeder] = setup_module(self.module_name, self.config) + + def reset_test_sheet(self): + """Clears test sheet and re-adds headers to ensure consistent test results.""" + client = gspread.service_account(self.config["service_account"]) + sheet = client.open(self.config["sheet"]) + worksheet = sheet.get_worksheet(0) + worksheet.clear() + worksheet.append_row(["Link", "Archive Status"]) + + def test_setup(self): + assert hasattr(self.feeder, "gsheets_client") + + def test_open_sheet_real_connection(self): + """Ensure open_sheet() connects to a real Google Sheets instance.""" + sheet = self.feeder.open_sheet() + assert sheet is not None, "open_sheet() should return a valid sheet instance" + assert hasattr( + sheet, "worksheets" + ), "Returned object should have worksheets method" + + def test_iter_yields_metadata_real_data(self): + """Ensure __iter__() yields Metadata objects for real test sheet data.""" + self.reset_test_sheet() + client = gspread.service_account(self.config["service_account"]) + sheet = client.open(self.config["sheet"]) + worksheet = sheet.get_worksheet(0) + # Insert test rows as a temp method + # Next we will refactor the feeder for better testing + test_rows = [ + ["https://example.com", ""], + ["", ""], + ["https://example.com", "done"], + ] + worksheet.append_rows(test_rows) + metadata_list = list(self.feeder) + + # Validate that only the first row is processed + assert len(metadata_list) == 1 + assert metadata_list[0].metadata.get("url") == "https://example.com" + + +# TODO + +# Test two sheets +# test two sheets with different columns +# test folder implementation diff --git a/tests/feeders/test_gworksheet.py b/tests/feeders/test_gworksheet.py new file mode 100644 index 0000000..e6f5cc6 --- /dev/null +++ b/tests/feeders/test_gworksheet.py @@ -0,0 +1,144 @@ +import pytest +from unittest.mock import MagicMock + +from auto_archiver.modules.gsheet_feeder import GWorksheet + + +class TestGWorksheet: + @pytest.fixture + def mock_worksheet(self): + mock_ws = MagicMock() + mock_ws.get_values.return_value = [ + ["Link", "Archive Status", "Archive Location", "Archive Date"], + ["url1", "archived", "filepath1", "2023-01-01"], + ["url2", "pending", "filepath2", "2023-01-02"], + ] + return mock_ws + + @pytest.fixture + def gworksheet(self, mock_worksheet): + return GWorksheet(mock_worksheet) + + # Test initialization and basic properties + def test_initialization_sets_headers(self, gworksheet): + assert gworksheet.headers == ["link", "archive status", "archive location", "archive date"] + + def test_count_rows_returns_correct_value(self, gworksheet): + # inc header row + assert gworksheet.count_rows() == 3 + + # Test column validation and lookup + @pytest.mark.parametrize( + "col,expected_index", + [ + ("url", 0), + ("status", 1), + ("archive", 2), + ("date", 3), + ], + ) + def test_col_index_returns_correct_index(self, gworksheet, col, expected_index): + assert gworksheet._col_index(col) == expected_index + + def test_check_col_exists_raises_for_invalid_column(self, gworksheet): + with pytest.raises(Exception, match="Column invalid_col"): + gworksheet._check_col_exists("invalid_col") + + # Test data retrieval + @pytest.mark.parametrize( + "row,expected", + [ + (1, ["Link", "Archive Status", "Archive Location", "Archive Date"]), + (2, ["url1", "archived", "filepath1", "2023-01-01"]), + (3, ["url2", "pending", "filepath2", "2023-01-02"]), + ], + ) + def test_get_row_returns_correct_data(self, gworksheet, row, expected): + assert gworksheet.get_row(row) == expected + + @pytest.mark.parametrize( + "row,col,expected", + [ + (2, "url", "url1"), + (2, "status", "archived"), + (3, "date", "2023-01-02"), + ], + ) + def test_get_cell_returns_correct_value(self, gworksheet, row, col, expected): + assert gworksheet.get_cell(row, col) == expected + + def test_get_cell_handles_fresh_data(self, mock_worksheet, gworksheet): + mock_worksheet.cell.return_value.value = "fresh_value" + result = gworksheet.get_cell(2, "url", fresh=True) + assert result == "fresh_value" + mock_worksheet.cell.assert_called_once_with(2, 1) + + # Test edge cases and error handling + @pytest.mark.parametrize( + "when_empty,expected", + [ + (True, "default"), + (False, ""), + ], + ) + def test_get_cell_or_default_handles_empty_values( + self, mock_worksheet, when_empty, expected + ): + mock_worksheet.get_values.return_value[1][0] = "" # Empty URL cell + g = GWorksheet(mock_worksheet) + assert ( + g.get_cell_or_default( + 2, "url", default="default", when_empty_use_default=when_empty + ) + == expected + ) + + def test_get_cell_or_default_handles_missing_columns(self, gworksheet): + assert ( + gworksheet.get_cell_or_default(1, "invalid_col", default="safe") == "safe" + ) + + # Test write operations + def test_set_cell_updates_correct_position(self, mock_worksheet, gworksheet): + gworksheet.set_cell(2, "url", "new_url") + mock_worksheet.update_cell.assert_called_once_with(2, 1, "new_url") + + def test_batch_set_cell_formats_requests_correctly( + self, mock_worksheet, gworksheet + ): + updates = [(2, "url", "new_url"), (3, "status", "processed")] + gworksheet.batch_set_cell(updates) + expected_batch = [ + {"range": "A2", "values": [["new_url"]]}, + {"range": "B3", "values": [["processed"]]}, + ] + mock_worksheet.batch_update.assert_called_once_with( + expected_batch, value_input_option="USER_ENTERED" + ) + + def test_batch_set_cell_truncates_long_values(self, mock_worksheet, gworksheet): + long_value = "x" * 50000 + gworksheet.batch_set_cell([(1, "url", long_value)]) + submitted_value = mock_worksheet.batch_update.call_args[0][0][0]["values"][0][0] + assert len(submitted_value) == 49999 + + # Test coordinate conversion + @pytest.mark.parametrize( + "row,col,expected", + [ + (1, "url", "A1"), + (2, "status", "B2"), + (3, "archive", "C3"), + (4, "date", "D4"), + ], + ) + def test_to_a1_conversion(self, gworksheet, row, col, expected): + assert gworksheet.to_a1(row, col) == expected + + # Test empty worksheet + def test_empty_worksheet_initialization(self): + mock_ws = MagicMock() + mock_ws.get_values.return_value = [] + g = GWorksheet(mock_ws) + assert g.headers == [] + assert g.count_rows() == 0 diff --git a/tests/formatters/test_html_formatter.py b/tests/formatters/test_html_formatter.py index 3540062..60abaa7 100644 --- a/tests/formatters/test_html_formatter.py +++ b/tests/formatters/test_html_formatter.py @@ -1,10 +1,10 @@ -from auto_archiver.core.context import ArchivingContext -from auto_archiver.formatters.html_formatter import HtmlFormatter +from auto_archiver.modules.html_formatter import HtmlFormatter from auto_archiver.core import Metadata, Media -def test_format(): - formatter = HtmlFormatter({}) +def test_format(setup_module): + formatter = setup_module(HtmlFormatter) + metadata = Metadata().set("content", "Hello, world!").set_url('https://example.com') final_media = formatter.format(metadata) diff --git a/tests/storages/test_S3_storage.py b/tests/storages/test_S3_storage.py new file mode 100644 index 0000000..2a5d026 --- /dev/null +++ b/tests/storages/test_S3_storage.py @@ -0,0 +1,124 @@ +from typing import Type +import pytest +from unittest.mock import MagicMock, patch +from auto_archiver.core import Media +from auto_archiver.modules.s3_storage import S3Storage + + +class TestS3Storage: + """ + Test suite for S3Storage. + """ + module_name: str = "s3_storage" + storage: Type[S3Storage] + s3: MagicMock + config: dict = { + "path_generator": "flat", + "filename_generator": "static", + "bucket": "test-bucket", + "region": "test-region", + "key": "test-key", + "secret": "test-secret", + "random_no_duplicate": False, + "endpoint_url": "https://{region}.example.com", + "cdn_url": "https://cdn.example.com/{key}", + "private": False, + } + + @patch('boto3.client') + @pytest.fixture(autouse=True) + def setup_storage(self, setup_module): + self.storage = setup_module(self.module_name, self.config) + + def test_client_initialization(self): + """Test that S3 client is initialized with correct parameters""" + assert self.storage.s3 is not None + assert self.storage.s3.meta.region_name == 'test-region' + + def test_get_cdn_url_generation(self): + """Test CDN URL formatting """ + media = Media("test.txt") + media.key = "path/to/file.txt" + url = self.storage.get_cdn_url(media) + assert url == "https://cdn.example.com/path/to/file.txt" + media.key = "another/path.jpg" + assert self.storage.get_cdn_url(media) == "https://cdn.example.com/another/path.jpg" + + def test_uploadf_sets_acl_public(self): + media = Media("test.txt") + mock_file = MagicMock() + with patch.object(self.storage.s3, 'upload_fileobj') as mock_s3_upload, \ + patch.object(self.storage, 'is_upload_needed', return_value=True): + self.storage.uploadf(mock_file, media) + mock_s3_upload.assert_called_once_with( + mock_file, + Bucket='test-bucket', + Key=media.key, + ExtraArgs={'ACL': 'public-read', 'ContentType': 'text/plain'} + ) + + def test_upload_decision_logic(self): + """Test is_upload_needed under different conditions""" + media = Media("test.txt") + # Test default state (random_no_duplicate=False) + assert self.storage.is_upload_needed(media) is True + # Set duplicate checking config to true: + + self.storage.random_no_duplicate = True + with patch('auto_archiver.modules.s3_storage.s3_storage.calculate_file_hash') as mock_calc_hash, \ + patch.object(self.storage, 'file_in_folder') as mock_file_in_folder: + mock_calc_hash.return_value = 'beepboop123beepboop123beepboop123' + mock_file_in_folder.return_value = 'existing_key.txt' + # Test duplicate result + assert self.storage.is_upload_needed(media) is False + assert media.key == 'existing_key.txt' + mock_file_in_folder.assert_called_with( + # (first 24 chars of hash) + 'no-dups/beepboop123beepboop123be' + ) + + + @patch.object(S3Storage, 'file_in_folder') + def test_skips_upload_when_duplicate_exists(self, mock_file_in_folder): + """Test that upload skips when file_in_folder finds existing object""" + self.storage.random_no_duplicate = True + mock_file_in_folder.return_value = "existing_folder/existing_file.txt" + # Create test media with calculated hash + media = Media("test.txt") + media.key = "original_path.txt" + with patch('auto_archiver.modules.s3_storage.s3_storage.calculate_file_hash') as mock_calculate_hash: + mock_calculate_hash.return_value = "beepboop123beepboop123beepboop123" + # Verify upload + assert self.storage.is_upload_needed(media) is False + assert media.key == "existing_folder/existing_file.txt" + assert media.get("previously archived") is True + with patch.object(self.storage.s3, 'upload_fileobj') as mock_upload: + result = self.storage.uploadf(None, media) + mock_upload.assert_not_called() + assert result is True + + @patch.object(S3Storage, 'is_upload_needed') + def test_uploads_with_correct_parameters(self, mock_upload_needed): + media = Media("test.txt") + media.key = "original_key.txt" + mock_upload_needed.return_value = True + media.mimetype = 'image/png' + mock_file = MagicMock() + + with patch.object(self.storage.s3, 'upload_fileobj') as mock_upload: + self.storage.uploadf(mock_file, media) + # verify call occured with these params + mock_upload.assert_called_once_with( + mock_file, + Bucket='test-bucket', + Key='original_key.txt', + ExtraArgs={ + 'ACL': 'public-read', + 'ContentType': 'image/png' + } + ) + + def test_file_in_folder_exists(self): + with patch.object(self.storage.s3, 'list_objects') as mock_list_objects: + mock_list_objects.return_value = {'Contents': [{'Key': 'path/to/file.txt'}]} + assert self.storage.file_in_folder('path/to/') == 'path/to/file.txt' \ No newline at end of file diff --git a/tests/storages/test_gdrive_storage.py b/tests/storages/test_gdrive_storage.py new file mode 100644 index 0000000..aba0a25 --- /dev/null +++ b/tests/storages/test_gdrive_storage.py @@ -0,0 +1,68 @@ +from typing import Type +import pytest +from unittest.mock import MagicMock, patch +from auto_archiver.core import Media +from auto_archiver.modules.gdrive_storage import GDriveStorage +from auto_archiver.core.metadata import Metadata +from tests.storages.test_storage_base import TestStorageBase + + +class TestGDriveStorage: + """ + Test suite for GDriveStorage. + """ + + module_name: str = "gdrive_storage" + storage: Type[GDriveStorage] + config: dict = {'path_generator': 'url', + 'filename_generator': 'static', + 'root_folder_id': "fake_root_folder_id", + 'oauth_token': None, + 'service_account': 'fake_service_account.json' + } + + @pytest.fixture(autouse=True) + def gdrive(self, setup_module): + with patch('google.oauth2.service_account.Credentials.from_service_account_file') as mock_creds: + self.storage = setup_module(self.module_name, self.config) + + def test_initialize_fails_with_non_existent_creds(self): + """ + Test that the Google Drive service raises a FileNotFoundError when the service account file does not exist. + """ + # Act and Assert + with pytest.raises(FileNotFoundError) as exc_info: + self.storage.setup() + assert "No such file or directory" in str(exc_info.value) + + + def test_path_parts(self): + media = Media(filename="test.jpg") + media.key = "folder1/folder2/test.jpg" + + +@pytest.mark.skip(reason="Requires real credentials") +@pytest.mark.download +class TestGDriveStorageConnected(TestStorageBase): + """ + 'Real' tests for GDriveStorage. + """ + + module_name: str = "gdrive_storage" + storage: Type[GDriveStorage] + config: dict = {'path_generator': 'url', + 'filename_generator': 'static', + # TODO: replace with real root folder id + 'root_folder_id': "1TVY_oJt95_dmRSEdP9m5zFy7l50TeCSk", + 'oauth_token': None, + 'service_account': 'secrets/service_account.json' + } + + + def test_initialize_with_real_credentials(self): + """ + Test that the Google Drive service can be initialized with real credentials. + """ + assert self.storage.service is not None + + diff --git a/tests/storages/test_storage_base.py b/tests/storages/test_storage_base.py new file mode 100644 index 0000000..7578acd --- /dev/null +++ b/tests/storages/test_storage_base.py @@ -0,0 +1,22 @@ +from typing import Type + +import pytest + +from auto_archiver.core.metadata import Metadata +from auto_archiver.core.storage import Storage + + +class TestStorageBase(object): + + module_name: str = None + config: dict = None + + @pytest.fixture(autouse=True) + def setup_storage(self, setup_module): + assert ( + self.module_name is not None + ), "self.module_name must be set on the subclass" + assert self.config is not None, "self.config must be a dict set on the subclass" + self.storage: Type[Storage] = setup_module( + self.module_name, self.config + ) diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..75fe515 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,103 @@ +import pytest +from auto_archiver.core import config +from ruamel.yaml.scanner import ScannerError +from ruamel.yaml.comments import CommentedMap + +def test_return_default_config_for_nonexistent_file(): + assert config.read_yaml("nonexistent_file.yaml") == config.EMPTY_CONFIG + +def test_return_default_config_for_empty_file(tmp_path): + empty_file = tmp_path / "empty_file.yaml" + empty_file.write_text("") + assert config.read_yaml(empty_file) == config.EMPTY_CONFIG + +def test_raise_error_on_invalid_yaml(tmp_path): + invalid_yaml = tmp_path / "invalid_yaml.yaml" + invalid_yaml.write_text("key: \"value_without_end_quote") + # make sure it raises ScannerError + with pytest.raises(ScannerError): + config.read_yaml(invalid_yaml) + +def test_write_yaml(tmp_path): + yaml_file = tmp_path / "write_yaml.yaml" + config.store_yaml(config.EMPTY_CONFIG, yaml_file.as_posix()) + assert "steps:\n" in yaml_file.read_text() + +def test_round_trip_comments(tmp_path): + yaml_file = tmp_path / "round_trip_comments.yaml" + + with open(yaml_file, "w") as f: + f.write("generic_extractor:\n facebook_cookie: abc # end of line comment\n subtitles: true\n # comments: false\n # livestreams: false\n list_type:\n - value1\n - value2") + + loaded = config.read_yaml(yaml_file) + # check the comments are preserved + assert loaded['generic_extractor']['facebook_cookie'] == "abc" + assert loaded['generic_extractor'].ca.items['facebook_cookie'][2].value == "# end of line comment\n" + + # add some more items to my_settings + loaded['generic_extractor']['list_type'].append("bellingcat") + config.store_yaml(loaded, yaml_file.as_posix()) + + assert "# comments: false" in yaml_file.read_text() + assert "facebook_cookie: abc # end of line comment" in yaml_file.read_text() + assert "abc # end of line comment" in yaml_file.read_text() + assert "- value2\n - bellingcat" in yaml_file.read_text() + +def test_merge_dicts(): + yaml_dict = config.EMPTY_CONFIG + yaml_dict['settings'] = CommentedMap(**{ + "key1": ["a"], + "key2": "old_value", + "key3": ["a", "b", "c"], + "key5": "value5", + }) + + dotdict = { + "settings.key1": ["b", "c"], + "settings.key2": "new_value", + "settings.key3": ["b", "c", "d"], + "settings.key4": "value4", + } + merged = config.merge_dicts(dotdict, yaml_dict) + assert merged["settings"]["key1"] == ["a", "b", "c"] + assert merged["settings"]["key2"] == "new_value" + assert merged["settings"]["key3"] == ["a", "b", "c", "d"] + assert merged["settings"]["key4"] == "value4" + assert merged["settings"]["key5"] == "value5" + + +def test_check_types(): + assert config.is_list_type([]) == True + assert config.is_list_type(()) == True + assert config.is_list_type(set()) == True + assert config.is_list_type({}) == False + assert config.is_list_type("") == False + assert config.is_dict_type({}) == True + assert config.is_dict_type(CommentedMap()) == True + assert config.is_dict_type([]) == False + assert config.is_dict_type("") == False + +def test_from_dot_notation(): + dotdict = { + "settings.key1": ["a", "b", "c"], + "settings.key2": "new_value", + "settings.key3.key4": "value", + } + normal_dict = config.from_dot_notation(dotdict) + assert normal_dict["settings"]["key1"] == ["a", "b", "c"] + assert normal_dict["settings"]["key2"] == "new_value" + assert normal_dict["settings"]["key3"]["key4"] == "value" + +def test_to_dot_notation(): + yaml_dict = config.EMPTY_CONFIG + yaml_dict['settings'] = { + "key1": ["a", "b", "c"], + "key2": "new_value", + "key3": { + "key4": "value", + } + } + dotdict = config.to_dot_notation(yaml_dict) + assert dotdict["settings.key1"] == ["a", "b", "c"] + assert dotdict["settings.key2"] == "new_value" + assert dotdict["settings.key3.key4"] == "value" \ No newline at end of file diff --git a/tests/test_implementation.py b/tests/test_implementation.py new file mode 100644 index 0000000..7e33651 --- /dev/null +++ b/tests/test_implementation.py @@ -0,0 +1,62 @@ +import sys +import pytest + +from auto_archiver.__main__ import main + + +@pytest.fixture +def orchestration_file_path(tmp_path): + return (tmp_path / "example_orch.yaml").as_posix() + +@pytest.fixture +def orchestration_file(orchestration_file_path): + def _orchestration_file(content=''): + with open(orchestration_file_path, "w") as f: + f.write(content) + return orchestration_file_path + + return _orchestration_file + +@pytest.fixture +def autoarchiver(tmp_path, monkeypatch, request): + def _autoarchiver(args=[]): + + def cleanup(): + from loguru import logger + if not logger._core.handlers.get(0): + logger._core.handlers_count = 0 + logger.add(sys.stderr) + + request.addfinalizer(cleanup) + + # change dir to tmp_path + monkeypatch.chdir(tmp_path) + with monkeypatch.context() as m: + m.setattr(sys, "argv", ["auto-archiver"] + args) + return main() + + return _autoarchiver + + +def test_run_auto_archiver_no_args(caplog, autoarchiver): + with pytest.raises(SystemExit): + autoarchiver() + + assert "provide at least one URL via the command line, or set up an alternative feeder" in caplog.text + +def test_run_auto_archiver_invalid_file(caplog, autoarchiver): + # exec 'auto-archiver' on the command lin + with pytest.raises(SystemExit): + autoarchiver(["--config", "nonexistent_file.yaml"]) + + assert "Make sure the file exists and try again, or run without th" in caplog.text + +def test_run_auto_archiver_empty_file(caplog, autoarchiver, orchestration_file): + # create a valid (empty) orchestration file + path = orchestration_file(content="") + # exec 'auto-archiver' on the command lin + with pytest.raises(SystemExit): + autoarchiver(["--config", path]) + + # should treat an empty file as if there is no file at all + assert " No URLs provided. Please provide at least one URL via the com" in caplog.text diff --git a/tests/test_metadata.py b/tests/test_metadata.py new file mode 100644 index 0000000..b07e107 --- /dev/null +++ b/tests/test_metadata.py @@ -0,0 +1,165 @@ +import pytest +from datetime import datetime, timezone +from dataclasses import dataclass +from typing import Any +from auto_archiver.core.metadata import Metadata + + +@pytest.fixture +def basic_metadata(): + m = Metadata() + m.set_url("https://example.com") + m.set("title", "Test Page") + return m + + +@dataclass +class MockMedia: + filename: str = "" + mimetype: str = "" + data: dict = None + + def get(self, key: str, default: Any = None) -> Any: + return self.data.get(key, default) if self.data else default + + def set(self, key: str, value: Any) -> None: + if not self.data: + self.data = {} + self.data[key] = value + + +@pytest.fixture +def media_file(): + def _create(filename="test.txt", mimetype="text/plain", hash_value=None): + m = MockMedia(filename=filename, mimetype=mimetype) + if hash_value: + m.set("hash", hash_value) + return m + + return _create + + +def test_initial_state(): + m = Metadata() + assert m.status == "no archiver" + assert m.metadata == {"_processed_at": m.get("_processed_at")} + assert m.media == [] + assert isinstance(m.get("_processed_at"), datetime) + + +def test_url_properties(basic_metadata): + assert basic_metadata.get_url() == "https://example.com" + assert basic_metadata.netloc == "example.com" + + +def test_simple_merge(basic_metadata): + right = Metadata(status="success") + right.set("title", "Test Title") + + basic_metadata.merge(right) + assert basic_metadata.status == "success" + assert basic_metadata.get("title") == "Test Title" + + +def test_left_merge(): + left = ( + Metadata() + .set("tags", ["a"]) + .set("stats", {"views": 10}) + .set("status", "success") + ) + right = ( + Metadata() + .set("tags", ["b"]) + .set("stats", {"likes": 5}) + .set("status", "no archiver") + ) + + left.merge(right, overwrite_left=True) + assert left.get("status") == "no archiver" + assert left.get("tags") == ["a", "b"] + assert left.get("stats") == {"views": 10, "likes": 5} + + +def test_media_management(basic_metadata, media_file): + media1 = media_file(hash_value="abc") + media2 = media_file(hash_value="abc") # Duplicate + media3 = media_file(hash_value="def") + + basic_metadata.add_media(media1, "m1") + basic_metadata.add_media(media2, "m2") + basic_metadata.add_media(media3) + + assert len(basic_metadata.media) == 3 + basic_metadata.remove_duplicate_media_by_hash() + assert len(basic_metadata.media) == 2 + assert basic_metadata.get_media_by_id("m1") == media1 + + +def test_success(): + m = Metadata() + assert not m.is_success() + m.success("context") + assert m.is_success() + assert m.status == "context: success" + + +def test_is_empty(): + m = Metadata() + assert m.is_empty() + # meaningless ids + ( + m.set("url", "example.com") + .set("total_bytes", 100) + .set("archive_duration_seconds", 10) + .set("_processed_at", datetime.now(timezone.utc)) + ) + assert m.is_empty() + + +def test_store(): + pass + +# Test Media operations + + +# Test custom getter/setters + + +def test_get_set_url(): + m = Metadata() + m.set_url("http://example.com") + assert m.get_url() == "http://example.com" + with pytest.raises(AssertionError): + m.set_url("") + assert m.get("url") == "http://example.com" + + +def test_set_content(): + m = Metadata() + m.set_content("Some content") + assert m.get("content") == "Some content" + # Test appending + m.set_content("New content") + # Do we want to add a line break to the method? + assert m.get("content") == "Some contentNew content" + + +def test_choose_most_complex(): + pass + + +def test_get_context(): + m = Metadata() + m.set_context("somekey", "somevalue") + assert m.get_context("somekey") == "somevalue" + assert m.get_context("nonexistent") is None + m.set_context("anotherkey", "anothervalue") + # check the previous is retained + assert m.get_context("somekey") == "somevalue" + assert m.get_context("anotherkey") == "anothervalue" + assert len(m._context) == 2 + + +def test_choose_most_complete(): + pass \ No newline at end of file diff --git a/tests/test_modules.py b/tests/test_modules.py new file mode 100644 index 0000000..854edb5 --- /dev/null +++ b/tests/test_modules.py @@ -0,0 +1,104 @@ +import sys +import pytest +from auto_archiver.core.module import get_module_lazy, BaseModule, LazyBaseModule, _LAZY_LOADED_MODULES + +@pytest.fixture +def example_module(): + import auto_archiver + + previous_path = auto_archiver.modules.__path__ + auto_archiver.modules.__path__.append("tests/data/test_modules/") + + module = get_module_lazy("example_module") + yield module + # cleanup + try: + del module._manifest + except AttributeError: + pass + del _LAZY_LOADED_MODULES["example_module"] + sys.modules.pop("auto_archiver.modules.example_module.example_module", None) + auto_archiver.modules.__path__ = previous_path + +def test_get_module_lazy(example_module): + assert example_module.name == "example_module" + assert example_module.display_name == "Example Module" + + assert example_module.manifest is not None + +def test_python_dependency_check(example_module): + # example_module requires loguru, which is not installed + # monkey patch the manifest to include a nonexistnet dependency + example_module.manifest["dependencies"]["python"] = ["does_not_exist"] + + with pytest.raises(SystemExit) as load_error: + example_module.load({}) + + assert load_error.value.code == 1 + +def test_binary_dependency_check(example_module): + # example_module requires ffmpeg, which is not installed + # monkey patch the manifest to include a nonexistnet dependency + example_module.manifest["dependencies"]["binary"] = ["does_not_exist"] + +def test_module_dependency_check_loads_module(example_module): + # example_module requires cli_feeder, which is not installed + # monkey patch the manifest to include a nonexistnet dependency + example_module.manifest["dependencies"]["python"] = ["hash_enricher"] + + loaded_module = example_module.load({}) + assert loaded_module is not None + + # check the dependency is loaded + assert _LAZY_LOADED_MODULES["hash_enricher"] is not None + assert _LAZY_LOADED_MODULES["hash_enricher"]._instance is not None + +def test_load_module(example_module): + + # setup the module, and check that config is set to the default values + loaded_module = example_module.load({}) + assert loaded_module is not None + assert isinstance(loaded_module, BaseModule) + assert loaded_module.name == "example_module" + assert loaded_module.display_name == "Example Module" + assert loaded_module.config["example_module"] == {"csv_file" : "db.csv"} + + # check that the vlaue is set on the module itself + assert loaded_module.csv_file == "db.csv" + +@pytest.mark.parametrize("module_name", ["local_storage", "generic_extractor", "html_formatter", "csv_db"]) +def test_load_modules(module_name): + # test that specific modules can be loaded + module = get_module_lazy(module_name) + assert module is not None + assert isinstance(module, LazyBaseModule) + assert module.name == module_name + + loaded_module = module.load({}) + assert isinstance(loaded_module, BaseModule) + assert loaded_module.name == module_name + assert loaded_module.display_name == module.display_name + + # check that default settings are applied + default_config = module.configs + assert loaded_module.name in loaded_module.config.keys() + + +@pytest.mark.parametrize("module_name", ["local_storage", "generic_extractor", "html_formatter", "csv_db"]) +def test_lazy_base_module(module_name): + lazy_module = get_module_lazy(module_name) + + assert lazy_module is not None + assert isinstance(lazy_module, LazyBaseModule) + assert lazy_module.name == module_name + assert len(lazy_module.display_name) > 0 + assert module_name in lazy_module.path + assert isinstance(lazy_module.manifest, dict) + + assert lazy_module.requires_setup == lazy_module.manifest.get("requires_setup", True) + assert len(lazy_module.entry_point) > 0 + assert len(lazy_module.configs) > 0 + assert len(lazy_module.description) > 0 + assert len(lazy_module.version) > 0 + + diff --git a/tests/test_orchestrator.py b/tests/test_orchestrator.py new file mode 100644 index 0000000..5ba57d0 --- /dev/null +++ b/tests/test_orchestrator.py @@ -0,0 +1,134 @@ +import pytest +import sys +from argparse import ArgumentParser, ArgumentTypeError +from auto_archiver.core.orchestrator import ArchivingOrchestrator +from auto_archiver.version import __version__ +from auto_archiver.core.config import read_yaml, store_yaml +from auto_archiver.core.module import _LAZY_LOADED_MODULES + +TEST_ORCHESTRATION = "tests/data/test_orchestration.yaml" +TEST_MODULES = "tests/data/test_modules/" + +@pytest.fixture +def test_args(): + return ["--config", TEST_ORCHESTRATION, + "--module_paths", TEST_MODULES, + "--example_module.required_field", "some_value"] # just set this for normal testing, we will remove it later + +@pytest.fixture +def orchestrator(): + yield ArchivingOrchestrator() + # hack - the loguru logger starts with one logger, but if orchestrator has run before + # it'll remove the default logger, add it back in: + + from loguru import logger + + if not logger._core.handlers.get(0): + logger._core.handlers_count = 0 + logger.add(sys.stderr) + # and remove the custom logger + if logger._core.handlers.get(1): + logger.remove(1) + + # delete out any loaded modules + _LAZY_LOADED_MODULES.clear() + + +@pytest.fixture +def basic_parser(orchestrator) -> ArgumentParser: + return orchestrator.setup_basic_parser() + +def test_setup_orchestrator(orchestrator): + assert orchestrator is not None + +def test_parse_config(): + pass + +def test_parse_basic(basic_parser): + args = basic_parser.parse_args(["--config", TEST_ORCHESTRATION]) + assert args.config_file == TEST_ORCHESTRATION + +@pytest.mark.parametrize("mode", ["simple", "full"]) +def test_mode(basic_parser, mode): + args = basic_parser.parse_args(["--mode", mode]) + assert args.mode == mode + +def test_mode_invalid(basic_parser, capsys): + with pytest.raises(SystemExit) as exit_error: + basic_parser.parse_args(["--mode", "invalid"]) + assert exit_error.value.code == 2 + assert "invalid choice" in capsys.readouterr().err + +def test_version(basic_parser, capsys): + with pytest.raises(SystemExit) as exit_error: + basic_parser.parse_args(["--version"]) + assert exit_error.value.code == 0 + assert capsys.readouterr().out == f"{__version__}\n" + +def test_help(orchestrator, basic_parser, capsys): + + args = basic_parser.parse_args(["--help"]) + assert args.help == True + + # test the show_help() on orchestrator + with pytest.raises(SystemExit) as exit_error: + orchestrator.show_help(args) + + assert exit_error.value.code == 0 + assert "Usage: auto-archiver [--help] [--version] [--config CONFIG_FILE]" in capsys.readouterr().out + + +def test_add_custom_modules_path(orchestrator, test_args): + orchestrator.run(test_args) + + import auto_archiver + assert "tests/data/test_modules/" in auto_archiver.modules.__path__ + +def test_add_custom_modules_path_invalid(orchestrator, caplog, test_args): + + orchestrator.run(test_args + # we still need to load the real path to get the example_module + ["--module_paths", "tests/data/invalid_test_modules/"]) + + assert caplog.records[0].message == "Path 'tests/data/invalid_test_modules/' does not exist. Skipping..." + + +def test_check_required_values(orchestrator, caplog, test_args): + # drop the example_module.required_field from the test_args + test_args = test_args[:-2] + + with pytest.raises(SystemExit) as exit_error: + orchestrator.run(test_args) + + assert caplog.records[1].message == "the following arguments are required: --example_module.required_field" + +def test_get_required_values_from_config(orchestrator, test_args, tmp_path): + + # load the default example yaml, add a required field, then run the orchestrator + test_yaml = read_yaml(TEST_ORCHESTRATION) + test_yaml['example_module'] = {'required_field': 'some_value'} + # write it to a temp file + tmp_file = (tmp_path / "temp_config.yaml").as_posix() + store_yaml(test_yaml, tmp_file) + + # run the orchestrator + orchestrator.run(["--config", tmp_file, "--module_paths", TEST_MODULES]) + assert orchestrator.config is not None + +def test_load_authentication_string(orchestrator, test_args): + + orchestrator.run(test_args + ["--authentication", '{"facebook.com": {"username": "my_username", "password": "my_password"}}']) + assert orchestrator.config['authentication'] == {"facebook.com": {"username": "my_username", "password": "my_password"}} + +def test_load_authentication_string_concat_site(orchestrator, test_args): + + orchestrator.run(test_args + ["--authentication", '{"x.com,twitter.com": {"api_key": "my_key"}}']) + assert orchestrator.config['authentication'] == {"x.com": {"api_key": "my_key"}, + "twitter.com": {"api_key": "my_key"}} + +def test_load_invalid_authentication_string(orchestrator, test_args): + with pytest.raises(ArgumentTypeError): + orchestrator.run(test_args + ["--authentication", "{\''invalid_json"]) + +def test_load_authentication_invalid_dict(orchestrator, test_args): + with pytest.raises(ArgumentTypeError): + orchestrator.run(test_args + ["--authentication", "[true, false]"]) \ No newline at end of file