Compare commits

...

30 Commits

Author SHA1 Message Date
Logan Williams
cc66ee3fd4 bump to patch 23 2023-06-06 12:24:43 -06:00
Logan Williams
b3b727b005 Fix ValueError 2023-06-06 12:13:08 -06:00
msramalho
ee37b20e6c fix: on missing col 2023-05-24 20:25:30 +01:00
msramalho
a184bf7b97 Bump version to v0.5.20 for release 2023-05-24 20:24:35 +01:00
msramalho
e535f44a88 optional folder 2023-05-24 20:24:15 +01:00
msramalho
0f28bf0e35 Bump version to v0.5.19 for release 2023-05-24 19:57:51 +01:00
msramalho
18a8636552 feat: new DB for auto-archiver-api 2023-05-24 19:24:53 +01:00
msramalho
81be65c828 Bump version to v0.5.18 for release 2023-05-24 11:19:02 +01:00
msramalho
0a91863212 typing fixes 2023-05-24 11:18:39 +01:00
msramalho
3ad8349e3f Bump version to v0.5.17 for release 2023-05-23 19:05:53 +01:00
msramalho
2768225cd1 fix: generator not called 2023-05-23 19:05:47 +01:00
msramalho
3e44b9b577 Bump version to v0.5.16 for release 2023-05-23 18:12:56 +01:00
msramalho
1a5797d0f8 feat: orchestrator fed returns archive result 2023-05-23 18:12:04 +01:00
msramalho
768b8fce9f Bump version to v0.5.15 for release 2023-05-19 12:35:26 +01:00
msramalho
613b1f1e50 properly overwrite configs 2023-05-19 12:35:19 +01:00
msramalho
919c37bfb6 Bump version to v0.5.14 for release 2023-05-19 12:18:02 +01:00
msramalho
a655b3c987 gsheet accepts ID too 2023-05-19 12:17:34 +01:00
msramalho
d645b840ee disable duplicate GH actions 2023-05-19 12:17:03 +01:00
msramalho
3da9c9cf8f Bump version to v0.5.13 for release 2023-05-19 11:49:38 +01:00
msramalho
987bbcaad0 removes conflicting unused dep 2023-05-19 11:49:29 +01:00
msramalho
68e9d2a2ce allows yaml config to be overwritten 2023-05-19 11:49:02 +01:00
Logan Williams
76be271c18 Update workflows to work with main branch 2023-05-15 10:14:53 +02:00
Logan Williams
074f132ad9 Merge branch 'dockerize' 2023-05-11 15:09:02 +02:00
Logan Williams
c47da0a46f Fix issue with profiles in browsertrix 2023-05-11 15:08:27 +02:00
Miguel Sozinho Ramalho
eb82936a04 Merge pull request #76 from bellingcat/dockerize 2023-05-11 13:57:37 +01:00
Miguel Sozinho Ramalho
cc03ad7c49 Update README.md 2023-05-11 13:55:28 +01:00
Logan Williams
6d2aa3dd7a Add invocation example 2023-05-11 14:32:23 +02:00
Logan Williams
f2e580de4e Update README images 2023-05-11 14:30:27 +02:00
Logan Williams
3f48d75d8f Merge branch 'dockerize' of github.com:bellingcat/auto-archiver into dockerize 2023-05-11 11:33:47 +02:00
Logan Williams
80ea912d0e Update README 2023-05-11 11:32:46 +02:00
23 changed files with 294 additions and 563 deletions

View File

@@ -9,7 +9,7 @@ on:
release: release:
types: [published] types: [published]
push: push:
branches: [ "dockerize" ] # branches: [ "main" ]
tags: [ "v*.*.*" ] tags: [ "v*.*.*" ]
env: env:

View File

@@ -12,7 +12,7 @@ on:
release: release:
types: [published] types: [published]
push: push:
branches: [ "dockerize" ] # branches: [ "main" ]
tags: [ "v*.*.*" ] tags: [ "v*.*.*" ]
permissions: permissions:

View File

@@ -33,7 +33,6 @@ vk-url-scraper = "*"
uwsgi = "*" uwsgi = "*"
requests = {extras = ["socks"], version = "*"} requests = {extras = ["socks"], version = "*"}
# wacz = "==0.4.8" # wacz = "==0.4.8"
pywb = ">=2.7.3"
[requires] [requires]
python_version = "3.10" python_version = "3.10"

660
Pipfile.lock generated
View File

@@ -1,7 +1,7 @@
{ {
"_meta": { "_meta": {
"hash": { "hash": {
"sha256": "1d7402ecf980bb765b8b289b23ed4d8699dbe16edfcde7c8191a4abb225a02c0" "sha256": "d8e565338c61f172d75ca6dbe9f7c74093035936ad04e315a6af87001a79f1f4"
}, },
"pipfile-spec": 6, "pipfile-spec": 6,
"requires": { "requires": {
@@ -172,21 +172,29 @@
"index": "pypi", "index": "pypi",
"version": "==4.12.2" "version": "==4.12.2"
}, },
"blinker": {
"hashes": [
"sha256:4afd3de66ef3a9f8067559fb7a1cbe555c17dcbe15971b05d1b625c3e7abe213",
"sha256:c3d739772abb7bc2860abf5f2ec284223d9ad5c76da018234f6f50d6f31ab1f0"
],
"markers": "python_version >= '3.7'",
"version": "==1.6.2"
},
"boto3": { "boto3": {
"hashes": [ "hashes": [
"sha256:061d3270472b9be09901bb08a45e9871ac8f86a9b1c9c615535ca0223acd7582", "sha256:110263d6be92383b9d5de61f4b36f732f0c98080dcfaf0aeb2f53f93b45b030a",
"sha256:5b2b13d9f3430e3d5e768bf32097d5d6d16f47a4719f2656de67da49dd3e4de1" "sha256:82d92428599c107d8cf7c716e4d805193f07585968d261d9751a0e0777d8b178"
], ],
"index": "pypi", "index": "pypi",
"version": "==1.26.131" "version": "==1.26.136"
}, },
"botocore": { "botocore": {
"hashes": [ "hashes": [
"sha256:d0dea23bccdfd7c2f6d0cd3216cfbd7065bc3e9e7b1ef6fee0952b04f5d2cffd", "sha256:134c9a84d3ce112fd7ed00b626d6a48f9d232742dffd9ee77f190b53c44bbcc8",
"sha256:ffbd85915b2624c545438a33c2624a809593720a10648f6e757fe50be4893188" "sha256:1f36cfa1586c9ceeaf2835a886c5272dd81c9467cf1d515f2dc6a82a2a658da0"
], ],
"markers": "python_version >= '3.7'", "markers": "python_version >= '3.7'",
"version": "==1.29.131" "version": "==1.29.136"
}, },
"brotli": { "brotli": {
"hashes": [ "hashes": [
@@ -276,57 +284,6 @@
"markers": "platform_python_implementation >= 'CPython'", "markers": "platform_python_implementation >= 'CPython'",
"version": "==1.0.9" "version": "==1.0.9"
}, },
"brotlipy": {
"hashes": [
"sha256:07194f4768eb62a4f4ea76b6d0df6ade185e24ebd85877c351daa0a069f1111a",
"sha256:08a16ebe2ffc52f645c076f96b138f185e74e5d59b4a65e84af17d5997d82890",
"sha256:091b299bf36dd6ef7a06570dbc98c0f80a504a56c5b797f31934d2ad01ae7d17",
"sha256:09ec3e125d16749b31c74f021aba809541b3564e5359f8c265cbae442810b41a",
"sha256:0be698678a114addcf87a4b9496c552c68a2c99bf93cf8e08f5738b392e82057",
"sha256:0fa6088a9a87645d43d7e21e32b4a6bf8f7c3939015a50158c10972aa7f425b7",
"sha256:1379347337dc3d20b2d61456d44ccce13e0625db2611c368023b4194d5e2477f",
"sha256:1ea4e578241504b58f2456a6c69952c88866c794648bdc74baee74839da61d44",
"sha256:22a53ccebcce2425e19f99682c12be510bf27bd75c9b77a1720db63047a77554",
"sha256:2699945a0a992c04fc7dc7fa2f1d0575a2c8b4b769f2874a08e8eae46bef36ae",
"sha256:2a80319ae13ea8dd60ecdc4f5ccf6da3ae64787765923256b62c598c5bba4121",
"sha256:2e5c64522364a9ebcdf47c5744a5ddeb3f934742d31e61ebfbbc095460b47162",
"sha256:36def0b859beaf21910157b4c33eb3b06d8ce459c942102f16988cca6ea164df",
"sha256:382971a641125323e90486244d6266ffb0e1f4dd920fbdcf508d2a19acc7c3b3",
"sha256:3a3e56ced8b15fbbd363380344f70f3b438e0fd1fcf27b7526b6172ea950e867",
"sha256:3c1d5e2cf945a46975bdb11a19257fa057b67591eb232f393d260e7246d9e571",
"sha256:4864ac52c116ea3e3a844248a9c9fbebb8797891cbca55484ecb6eed3ebeba24",
"sha256:4bac11c1ffba9eaa2894ec958a44e7f17778b3303c2ee9f99c39fcc511c26668",
"sha256:4e4638b49835d567d447a2cfacec109f9a777f219f071312268b351b6839436d",
"sha256:50ca336374131cfad20612f26cc43c637ac0bfd2be3361495e99270883b52962",
"sha256:5664fe14f3a613431db622172bad923096a303d3adce55536f4409c8e2eafba4",
"sha256:5de6f7d010b7558f72f4b061a07395c5c3fd57f0285c5af7f126a677b976a868",
"sha256:637847560d671657f993313ecc6c6c6666a936b7a925779fd044065c7bc035b9",
"sha256:653faef61241bf8bf99d73ca7ec4baa63401ba7b2a2aa88958394869379d67c7",
"sha256:786afc8c9bd67de8d31f46e408a3386331e126829114e4db034f91eacb05396d",
"sha256:79aaf217072840f3e9a3b641cccc51f7fc23037496bd71e26211856b93f4b4cb",
"sha256:79ab3bca8dd12c17e092273484f2ac48b906de2b4828dcdf6a7d520f99646ab3",
"sha256:7b21341eab7c939214e457e24b265594067a6ad268305289148ebaf2dacef325",
"sha256:7e31f7adcc5851ca06134705fcf3478210da45d35ad75ec181e1ce9ce345bb38",
"sha256:7ff18e42f51ebc9d9d77a0db33f99ad95f01dd431e4491f0eca519b90e9415a9",
"sha256:82f61506d001e626ec3a1ac8a69df11eb3555a4878599befcb672c8178befac8",
"sha256:890b973039ba26c3ad2e86e8908ab527ed64f9b1357f81a676604da8088e4bf9",
"sha256:8b39abc3256c978f575df5cd7893153277216474f303e26f0e43ba3d3969ef96",
"sha256:8ef230ca9e168ce2b7dc173a48a0cc3d78bcdf0bd0ea7743472a317041a4768e",
"sha256:9448227b0df082e574c45c983fa5cd4bda7bfb11ea6b59def0940c1647be0c3c",
"sha256:96bc59ff9b5b5552843dc67999486a220e07a0522dddd3935da05dc194fa485c",
"sha256:a07647886e24e2fb2d68ca8bf3ada398eb56fd8eac46c733d4d95c64d17f743b",
"sha256:ac1d66c9774ee62e762750e399a0c95e93b180e96179b645f28b162b55ae8adc",
"sha256:af65d2699cb9f13b26ec3ba09e75e80d31ff422c03675fcb36ee4dabe588fdc2",
"sha256:b4c98b0d2c9c7020a524ca5bbff42027db1004c6571f8bc7b747f2b843128e7a",
"sha256:b7cf5bb69e767a59acc3da0d199d4b5d0c9fed7bef3ffa3efa80c6f39095686b",
"sha256:c6cc0036b1304dd0073eec416cb2f6b9e37ac8296afd9e481cac3b1f07f9db25",
"sha256:d2c1c724c4ac375feb2110f1af98ecdc0e5a8ea79d068efb5891f621a5b235cb",
"sha256:dc6c5ee0df9732a44d08edab32f8a616b769cc5a4155a12d2d010d248eb3fb07",
"sha256:e5c549ae5928dda952463196180445c24d6fad2d73cb13bd118293aced31b771",
"sha256:fd1d1c64214af5d90014d82cee5d8141b13d44c92ada7a0c0ec0679c6f15a471"
],
"version": "==0.7.0"
},
"bs4": { "bs4": {
"hashes": [ "hashes": [
"sha256:36ecea1fd7cc5c0c6e4a1ff075df26d50da647b75376626cc186e2212886dd3a" "sha256:36ecea1fd7cc5c0c6e4a1ff075df26d50da647b75376626cc186e2212886dd3a"
@@ -342,13 +299,6 @@
"markers": "python_version ~= '3.7'", "markers": "python_version ~= '3.7'",
"version": "==5.3.0" "version": "==5.3.0"
}, },
"certauth": {
"hashes": [
"sha256:7862d5deff0b33d2fb28d36861ba63d91c82d700bfdfc4bd848a8711ca72b8fb",
"sha256:f84b8c7075d0e445614d5ec4662056511453f19228cf4fcf8278cccae17b316b"
],
"version": "==1.3.0"
},
"certifi": { "certifi": {
"hashes": [ "hashes": [
"sha256:0f0d56dc5a6ad56fd4ba36484d6cc34451e1c6548c61daad8c320169f91eddc7", "sha256:0f0d56dc5a6ad56fd4ba36484d6cc34451e1c6548c61daad8c320169f91eddc7",
@@ -504,16 +454,16 @@
"sha256:f8303414c7b03f794347ad062c0516cee0e15f7a612abd0ce1e25caf6ceb47df", "sha256:f8303414c7b03f794347ad062c0516cee0e15f7a612abd0ce1e25caf6ceb47df",
"sha256:fca62a8301b605b954ad2e9c3666f9d97f63872aa4efcae5492baca2056b74ab" "sha256:fca62a8301b605b954ad2e9c3666f9d97f63872aa4efcae5492baca2056b74ab"
], ],
"markers": "python_version >= '3.6'", "markers": "python_full_version >= '3.7.0'",
"version": "==3.1.0" "version": "==3.1.0"
}, },
"click": { "click": {
"hashes": [ "hashes": [
"sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a", "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e",
"sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc" "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"
], ],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "markers": "python_version >= '3.7'",
"version": "==7.1.2" "version": "==8.1.3"
}, },
"cloudscraper": { "cloudscraper": {
"hashes": [ "hashes": [
@@ -570,14 +520,6 @@
"index": "pypi", "index": "pypi",
"version": "==1.1.8" "version": "==1.1.8"
}, },
"defusedxml": {
"hashes": [
"sha256:1bb3032db185915b62d7c6209c5a8792be6a32ab2fedacc84e01b52c51aa3e69",
"sha256:a352e7e428770286cc899e2542b6cdaedb2b4953ff269a210103ec58f6198a61"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
"version": "==0.7.1"
},
"exceptiongroup": { "exceptiongroup": {
"hashes": [ "hashes": [
"sha256:232c37c63e4f682982c8b6459f33a8981039e5fb8756b2074364e5055c498c9e", "sha256:232c37c63e4f682982c8b6459f33a8981039e5fb8756b2074364e5055c498c9e",
@@ -586,13 +528,6 @@
"markers": "python_version < '3.11'", "markers": "python_version < '3.11'",
"version": "==1.1.1" "version": "==1.1.1"
}, },
"fakeredis": {
"hashes": [
"sha256:0b03dfdced20169f945bde3cfd505ad5f7b8cf18ad0a124548563cbc587878da",
"sha256:ba8a820203ba5a7a7ef62e3619318d513b87f60328c5583634df398ae4b7af00"
],
"version": "==0.16.0"
},
"ffmpeg-python": { "ffmpeg-python": {
"hashes": [ "hashes": [
"sha256:65225db34627c578ef0e11c8b1eb528bb35e024752f6f10b78c011f6f64c4127", "sha256:65225db34627c578ef0e11c8b1eb528bb35e024752f6f10b78c011f6f64c4127",
@@ -611,11 +546,11 @@
}, },
"flask": { "flask": {
"hashes": [ "hashes": [
"sha256:0fbeb6180d383a9186d0d6ed954e0042ad9f18e0e8de088b2b419d526927d196", "sha256:77fd4e1249d8c9923de34907236b747ced06e5467ecac1a7bb7115ae0e9670b0",
"sha256:c34f04500f2cbbea882b1acb02002ad6fe6b7ffa64a6164577995657f50aed22" "sha256:8c2f9abd47a9e8df7f0c3f091ce9497d011dc3b31effcf4c85a6e2b50f4114ef"
], ],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "markers": "python_version >= '3.8'",
"version": "==1.1.4" "version": "==2.3.2"
}, },
"frozenlist": { "frozenlist": {
"hashes": [ "hashes": [
@@ -704,45 +639,6 @@
"markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
"version": "==0.18.3" "version": "==0.18.3"
}, },
"gevent": {
"hashes": [
"sha256:0082d8a5d23c35812ce0e716a91ede597f6dd2c5ff508a02a998f73598c59397",
"sha256:01928770972181ad8866ee37ea3504f1824587b188fcab782ef1619ce7538766",
"sha256:05c5e8a50cd6868dd36536c92fb4468d18090e801bd63611593c0717bab63692",
"sha256:08b4c17064e28f4eb85604486abc89f442c7407d2aed249cf54544ce5c9baee6",
"sha256:177f93a3a90f46a5009e0841fef561601e5c637ba4332ab8572edd96af650101",
"sha256:22ce1f38fdfe2149ffe8ec2131ca45281791c1e464db34b3b4321ae9d8d2efbb",
"sha256:24d3550fbaeef5fddd794819c2853bca45a86c3d64a056a2c268d981518220d1",
"sha256:2afa3f3ad528155433f6ac8bd64fa5cc303855b97004416ec719a6b1ca179481",
"sha256:2bcec9f80196c751fdcf389ca9f7141e7b0db960d8465ed79be5e685bfcad682",
"sha256:2cfff82f05f14b7f5d9ed53ccb7a609ae8604df522bb05c971bca78ec9d8b2b9",
"sha256:3baeeccc4791ba3f8db27179dff11855a8f9210ddd754f6c9b48e0d2561c2aea",
"sha256:3c012c73e6c61f13c75e3a4869dbe6a2ffa025f103421a6de9c85e627e7477b1",
"sha256:3dad62f55fad839d498c801e139481348991cee6e1c7706041b5fe096cb6a279",
"sha256:542ae891e2aa217d2cf6d8446538fcd2f3263a40eec123b970b899bac391c47a",
"sha256:6a02a88723ed3f0fd92cbf1df3c4cd2fbd87d82b0a4bac3e36a8875923115214",
"sha256:74fc1ef16b86616cfddcc74f7292642b0f72dde4dd95aebf4c45bb236744be54",
"sha256:7909780f0cf18a1fc32aafd8c8e130cdd93c6e285b11263f7f2d1a0f3678bc50",
"sha256:7ccffcf708094564e442ac6fde46f0ae9e40015cb69d995f4b39cc29a7643881",
"sha256:8c21cb5c9f4e14d75b3fe0b143ec875d7dbd1495fad6d49704b00e57e781ee0f",
"sha256:973749bacb7bc4f4181a8fb2a7e0e2ff44038de56d08e856dd54a5ac1d7331b4",
"sha256:9d86438ede1cbe0fde6ef4cc3f72bf2f1ecc9630d8b633ff344a3aeeca272cdd",
"sha256:9f9652d1e4062d4b5b5a0a49ff679fa890430b5f76969d35dccb2df114c55e0f",
"sha256:a5ad4ed8afa0a71e1927623589f06a9b5e8b5e77810be3125cb4d93050d3fd1f",
"sha256:b7709c64afa8bb3000c28bb91ec42c79594a7cb0f322e20427d57f9762366a5b",
"sha256:bb5cb8db753469c7a9a0b8a972d2660fe851aa06eee699a1ca42988afb0aaa02",
"sha256:c43f081cbca41d27fd8fef9c6a32cf83cb979345b20abc07bf68df165cdadb24",
"sha256:cc2fef0f98ee180704cf95ec84f2bc2d86c6c3711bb6b6740d74e0afe708b62c",
"sha256:da8d2d51a49b2a5beb02ad619ca9ddbef806ef4870ba04e5ac7b8b41a5b61db3",
"sha256:e1899b921219fc8959ff9afb94dae36be82e0769ed13d330a393594d478a0b3a",
"sha256:eae3c46f9484eaacd67ffcdf4eaf6ca830f587edd543613b0f5c4eb3c11d052d",
"sha256:ec21f9eaaa6a7b1e62da786132d6788675b314f25f98d9541f1bf00584ed4749",
"sha256:f289fae643a3f1c3b909d6b033e6921b05234a4907e9c9c8c3f1fe403e6ac452",
"sha256:f48b64578c367b91fa793bf8eaaaf4995cb93c8bc45860e473bf868070ad094e"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
"version": "==21.12.0"
},
"google-api-core": { "google-api-core": {
"hashes": [ "hashes": [
"sha256:4b9bb5d5a380a0befa0573b302651b8a9a89262c1730e37bf423cec511804c22", "sha256:4b9bb5d5a380a0befa0573b302651b8a9a89262c1730e37bf423cec511804c22",
@@ -761,11 +657,11 @@
}, },
"google-auth": { "google-auth": {
"hashes": [ "hashes": [
"sha256:ce311e2bc58b130fddf316df57c9b3943c2a7b4f6ec31de9663a9333e4064efc", "sha256:55a395cdfd3f3dd3f649131d41f97c17b4ed8a2aac1be3502090c716314e8a37",
"sha256:f586b274d3eb7bd932ea424b1c702a30e0393a2e2bc4ca3eae8263ffd8be229f" "sha256:d7a3249027e7f464fbbfd7ee8319a08ad09d2eea51578575c4bd360ffa049ccb"
], ],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
"version": "==2.17.3" "version": "==2.18.1"
}, },
"google-auth-httplib2": { "google-auth-httplib2": {
"hashes": [ "hashes": [
@@ -791,85 +687,13 @@
"markers": "python_version >= '3.7'", "markers": "python_version >= '3.7'",
"version": "==1.59.0" "version": "==1.59.0"
}, },
"greenlet": {
"hashes": [
"sha256:0120a879aa2b1ac5118bce959ea2492ba18783f65ea15821680a256dfad04754",
"sha256:025b8de2273d2809f027d347aa2541651d2e15d593bbce0d5f502ca438c54136",
"sha256:05ae7383f968bba4211b1fbfc90158f8e3da86804878442b4fb6c16ccbcaa519",
"sha256:0914f02fcaa8f84f13b2df4a81645d9e82de21ed95633765dd5cc4d3af9d7403",
"sha256:0971d37ae0eaf42344e8610d340aa0ad3d06cd2eee381891a10fe771879791f9",
"sha256:0a954002064ee919b444b19c1185e8cce307a1f20600f47d6f4b6d336972c809",
"sha256:0aa1845944e62f358d63fcc911ad3b415f585612946b8edc824825929b40e59e",
"sha256:104f29dd822be678ef6b16bf0035dcd43206a8a48668a6cae4d2fe9c7a7abdeb",
"sha256:11fc7692d95cc7a6a8447bb160d98671ab291e0a8ea90572d582d57361360f05",
"sha256:17a69967561269b691747e7f436d75a4def47e5efcbc3c573180fc828e176d80",
"sha256:2794eef1b04b5ba8948c72cc606aab62ac4b0c538b14806d9c0d88afd0576d6b",
"sha256:2c6e942ca9835c0b97814d14f78da453241837419e0d26f7403058e8db3e38f8",
"sha256:2ccdc818cc106cc238ff7eba0d71b9c77be868fdca31d6c3b1347a54c9b187b2",
"sha256:325f272eb997916b4a3fc1fea7313a8adb760934c2140ce13a2117e1b0a8095d",
"sha256:39464518a2abe9c505a727af7c0b4efff2cf242aa168be5f0daa47649f4d7ca8",
"sha256:3a24f3213579dc8459e485e333330a921f579543a5214dbc935bc0763474ece3",
"sha256:3aeac044c324c1a4027dca0cde550bd83a0c0fbff7ef2c98df9e718a5086c194",
"sha256:3c22998bfef3fcc1b15694818fc9b1b87c6cc8398198b96b6d355a7bcb8c934e",
"sha256:467b73ce5dcd89e381292fb4314aede9b12906c18fab903f995b86034d96d5c8",
"sha256:4a8b58232f5b72973350c2b917ea3df0bebd07c3c82a0a0e34775fc2c1f857e9",
"sha256:4f74aa0092602da2069df0bc6553919a15169d77bcdab52a21f8c5242898f519",
"sha256:5662492df0588a51d5690f6578f3bbbd803e7f8d99a99f3bf6128a401be9c269",
"sha256:5c2d21c2b768d8c86ad935e404cc78c30d53dea009609c3ef3a9d49970c864b5",
"sha256:5edf75e7fcfa9725064ae0d8407c849456553a181ebefedb7606bac19aa1478b",
"sha256:60839ab4ea7de6139a3be35b77e22e0398c270020050458b3d25db4c7c394df5",
"sha256:62723e7eb85fa52e536e516ee2ac91433c7bb60d51099293671815ff49ed1c21",
"sha256:64e10f303ea354500c927da5b59c3802196a07468332d292aef9ddaca08d03dd",
"sha256:66aa4e9a726b70bcbfcc446b7ba89c8cec40f405e51422c39f42dfa206a96a05",
"sha256:695d0d8b5ae42c800f1763c9fce9d7b94ae3b878919379150ee5ba458a460d57",
"sha256:70048d7b2c07c5eadf8393e6398595591df5f59a2f26abc2f81abca09610492f",
"sha256:7afa706510ab079fd6d039cc6e369d4535a48e202d042c32e2097f030a16450f",
"sha256:7cf37343e43404699d58808e51f347f57efd3010cc7cee134cdb9141bd1ad9ea",
"sha256:8149a6865b14c33be7ae760bcdb73548bb01e8e47ae15e013bf7ef9290ca309a",
"sha256:814f26b864ed2230d3a7efe0336f5766ad012f94aad6ba43a7c54ca88dd77cba",
"sha256:82a38d7d2077128a017094aff334e67e26194f46bd709f9dcdacbf3835d47ef5",
"sha256:83a7a6560df073ec9de2b7cb685b199dfd12519bc0020c62db9d1bb522f989fa",
"sha256:8415239c68b2ec9de10a5adf1130ee9cb0ebd3e19573c55ba160ff0ca809e012",
"sha256:88720794390002b0c8fa29e9602b395093a9a766b229a847e8d88349e418b28a",
"sha256:890f633dc8cb307761ec566bc0b4e350a93ddd77dc172839be122be12bae3e10",
"sha256:8926a78192b8b73c936f3e87929931455a6a6c6c385448a07b9f7d1072c19ff3",
"sha256:8c0581077cf2734569f3e500fab09c0ff6a2ab99b1afcacbad09b3c2843ae743",
"sha256:8fda1139d87ce5f7bd80e80e54f9f2c6fe2f47983f1a6f128c47bf310197deb6",
"sha256:91a84faf718e6f8b888ca63d0b2d6d185c8e2a198d2a7322d75c303e7097c8b7",
"sha256:924df1e7e5db27d19b1359dc7d052a917529c95ba5b8b62f4af611176da7c8ad",
"sha256:949c9061b8c6d3e6e439466a9be1e787208dec6246f4ec5fffe9677b4c19fcc3",
"sha256:9649891ab4153f217f319914455ccf0b86986b55fc0573ce803eb998ad7d6854",
"sha256:96656c5f7c95fc02c36d4f6ef32f4e94bb0b6b36e6a002c21c39785a4eec5f5d",
"sha256:a812df7282a8fc717eafd487fccc5ba40ea83bb5b13eb3c90c446d88dbdfd2be",
"sha256:a8d24eb5cb67996fb84633fdc96dbc04f2d8b12bfcb20ab3222d6be271616b67",
"sha256:bef49c07fcb411c942da6ee7d7ea37430f830c482bf6e4b72d92fd506dd3a427",
"sha256:bffba15cff4802ff493d6edcf20d7f94ab1c2aee7cfc1e1c7627c05f1102eee8",
"sha256:c0643250dd0756f4960633f5359884f609a234d4066686754e834073d84e9b51",
"sha256:c6f90234e4438062d6d09f7d667f79edcc7c5e354ba3a145ff98176f974b8132",
"sha256:c8c9301e3274276d3d20ab6335aa7c5d9e5da2009cccb01127bddb5c951f8870",
"sha256:c8ece5d1a99a2adcb38f69af2f07d96fb615415d32820108cd340361f590d128",
"sha256:cb863057bed786f6622982fb8b2c122c68e6e9eddccaa9fa98fd937e45ee6c4f",
"sha256:ccbe7129a282ec5797df0451ca1802f11578be018a32979131065565da89b392",
"sha256:d25cdedd72aa2271b984af54294e9527306966ec18963fd032cc851a725ddc1b",
"sha256:d75afcbb214d429dacdf75e03a1d6d6c5bd1fa9c35e360df8ea5b6270fb2211c",
"sha256:d7815e1519a8361c5ea2a7a5864945906f8e386fa1bc26797b4d443ab11a4589",
"sha256:eb6ac495dccb1520667cfea50d89e26f9ffb49fa28496dea2b95720d8b45eb54",
"sha256:ec615d2912b9ad807afd3be80bf32711c0ff9c2b00aa004a45fd5d5dde7853d9",
"sha256:f5e09dc5c6e1796969fd4b775ea1417d70e49a5df29aaa8e5d10675d9e11872c",
"sha256:f6661b58412879a2aa099abb26d3c93e91dedaba55a6394d1fb1512a77e85de9",
"sha256:f7d20c3267385236b4ce54575cc8e9f43e7673fc761b069c820097092e318e3b",
"sha256:fe7c51f8a2ab616cb34bc33d810c887e89117771028e1e3d3b77ca25ddeace04"
],
"markers": "platform_python_implementation == 'CPython'",
"version": "==1.1.3.post0"
},
"gspread": { "gspread": {
"hashes": [ "hashes": [
"sha256:b6125120049fd92a3e1e7544ac395f336cec3ff8be04550cf737d90af007685d", "sha256:34b97834bbefaccf72497702bae26d12f96d0685e49a418afe6a92a9bbcb9c9c",
"sha256:f9739e2b839d7fa1f8a5f0cf0d4ee68c776e2fbf4bfe3167ad2e910bd588fb44" "sha256:85f296f3ee95d1742e44be368d5a7c0ca3957b8ad2cb6833f7094055f3390e32"
], ],
"index": "pypi", "index": "pypi",
"version": "==5.8.0" "version": "==5.9.0"
}, },
"h11": { "h11": {
"hashes": [ "hashes": [
@@ -881,11 +705,11 @@
}, },
"httpcore": { "httpcore": {
"hashes": [ "hashes": [
"sha256:0fdfea45e94f0c9fd96eab9286077f9ff788dd186635ae61b312693e4d943599", "sha256:628e768aaeec1f7effdc6408ba1c3cdbd7487c1fc570f7d66844ec4f003e1ca4",
"sha256:cc045a3241afbf60ce056202301b4d8b6af08845e3294055eb26b09913ef903c" "sha256:caf508597c525f9b8bfff187e270666309f63115af30f7d68b16143a403c8356"
], ],
"markers": "python_version >= '3.7'", "markers": "python_version >= '3.7'",
"version": "==0.17.0" "version": "==0.17.1"
}, },
"httplib2": { "httplib2": {
"hashes": [ "hashes": [
@@ -897,11 +721,11 @@
}, },
"httpx": { "httpx": {
"hashes": [ "hashes": [
"sha256:447556b50c1921c351ea54b4fe79d91b724ed2b027462ab9a329465d147d5a4e", "sha256:06781eb9ac53cde990577af654bd990a4949de37a28bdb4a230d434f3a30b9bd",
"sha256:507d676fc3e26110d41df7d35ebd8b3b8585052450f4097401c9be59d928c63e" "sha256:5853a43053df830c20f8110c5e69fe44d035d850b2dfe795e196f00fdb774bdd"
], ],
"markers": "python_version >= '3.7'", "markers": "python_version >= '3.7'",
"version": "==0.24.0" "version": "==0.24.1"
}, },
"idna": { "idna": {
"hashes": [ "hashes": [
@@ -920,19 +744,19 @@
}, },
"itsdangerous": { "itsdangerous": {
"hashes": [ "hashes": [
"sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19", "sha256:2c2349112351b88699d8d4b6b075022c0808887cb7ad10069318a8b0bc88db44",
"sha256:b12271b2047cb23eeb98c8b5622e2e5c5e9abd9784a153e9d8ef9cb4dd09d749" "sha256:5dbbc68b317e5e42f327f9021763545dc3fc3bfe22e6deb96aaf1fc38874156a"
], ],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "markers": "python_version >= '3.7'",
"version": "==1.1.0" "version": "==2.1.2"
}, },
"jinja2": { "jinja2": {
"hashes": [ "hashes": [
"sha256:03e47ad063331dd6a3f04a43eddca8a966a26ba0c5b7207a9a9e4e08f1b29419", "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852",
"sha256:a6d58433de0ae800347cab1fa3043cebbabe8baa9d29e668f1c768cb87a333c6" "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"
], ],
"index": "pypi", "index": "pypi",
"version": "==2.11.3" "version": "==3.1.2"
}, },
"jmespath": { "jmespath": {
"hashes": [ "hashes": [
@@ -1043,78 +867,59 @@
}, },
"markupsafe": { "markupsafe": {
"hashes": [ "hashes": [
"sha256:01a9b8ea66f1658938f65b93a85ebe8bc016e6769611be228d797c9d998dd298", "sha256:0576fe974b40a400449768941d5d0858cc624e3249dfd1e0c33674e5c7ca7aed",
"sha256:023cb26ec21ece8dc3907c0e8320058b2e0cb3c55cf9564da612bc325bed5e64", "sha256:085fd3201e7b12809f9e6e9bc1e5c96a368c8523fad5afb02afe3c051ae4afcc",
"sha256:0446679737af14f45767963a1a9ef7620189912317d095f2d9ffa183a4d25d2b", "sha256:090376d812fb6ac5f171e5938e82e7f2d7adc2b629101cec0db8b267815c85e2",
"sha256:04635854b943835a6ea959e948d19dcd311762c5c0c6e1f0e16ee57022669194", "sha256:0b462104ba25f1ac006fdab8b6a01ebbfbce9ed37fd37fd4acd70c67c973e460",
"sha256:0717a7390a68be14b8c793ba258e075c6f4ca819f15edfc2a3a027c823718567", "sha256:137678c63c977754abe9086a3ec011e8fd985ab90631145dfb9294ad09c102a7",
"sha256:0955295dd5eec6cb6cc2fe1698f4c6d84af2e92de33fbcac4111913cd100a6ff", "sha256:1bea30e9bf331f3fef67e0a3877b2288593c98a21ccb2cf29b74c581a4eb3af0",
"sha256:0d4b31cc67ab36e3392bbf3862cfbadac3db12bdd8b02a2731f509ed5b829724", "sha256:22152d00bf4a9c7c83960521fc558f55a1adbc0631fbb00a9471e097b19d72e1",
"sha256:10f82115e21dc0dfec9ab5c0223652f7197feb168c940f3ef61563fc2d6beb74", "sha256:22731d79ed2eb25059ae3df1dfc9cb1546691cc41f4e3130fe6bfbc3ecbbecfa",
"sha256:168cd0a3642de83558a5153c8bd34f175a9a6e7f6dc6384b9655d2697312a646", "sha256:2298c859cfc5463f1b64bd55cb3e602528db6fa0f3cfd568d3605c50678f8f03",
"sha256:1d609f577dc6e1aa17d746f8bd3c31aa4d258f4070d61b2aa5c4166c1539de35", "sha256:28057e985dace2f478e042eaa15606c7efccb700797660629da387eb289b9323",
"sha256:1f2ade76b9903f39aa442b4aadd2177decb66525062db244b35d71d0ee8599b6", "sha256:2e7821bffe00aa6bd07a23913b7f4e01328c3d5cc0b40b36c0bd81d362faeb65",
"sha256:20dca64a3ef2d6e4d5d615a3fd418ad3bde77a47ec8a23d984a12b5b4c74491a", "sha256:2ec4f2d48ae59bbb9d1f9d7efb9236ab81429a764dedca114f5fdabbc3788013",
"sha256:2a7d351cbd8cfeb19ca00de495e224dea7e7d919659c2841bbb7f420ad03e2d6", "sha256:340bea174e9761308703ae988e982005aedf427de816d1afe98147668cc03036",
"sha256:2d7d807855b419fc2ed3e631034685db6079889a1f01d5d9dac950f764da3dad", "sha256:40627dcf047dadb22cd25ea7ecfe9cbf3bbbad0482ee5920b582f3809c97654f",
"sha256:2ef54abee730b502252bcdf31b10dacb0a416229b72c18b19e24a4509f273d26", "sha256:40dfd3fefbef579ee058f139733ac336312663c6706d1163b82b3003fb1925c4",
"sha256:36bc903cbb393720fad60fc28c10de6acf10dc6cc883f3e24ee4012371399a38", "sha256:4cf06cdc1dda95223e9d2d3c58d3b178aa5dacb35ee7e3bbac10e4e1faacb419",
"sha256:37205cac2a79194e3750b0af2a5720d95f786a55ce7df90c3af697bfa100eaac", "sha256:50c42830a633fa0cf9e7d27664637532791bfc31c731a87b202d2d8ac40c3ea2",
"sha256:3c112550557578c26af18a1ccc9e090bfe03832ae994343cfdacd287db6a6ae7", "sha256:55f44b440d491028addb3b88f72207d71eeebfb7b5dbf0643f7c023ae1fba619",
"sha256:3dd007d54ee88b46be476e293f48c85048603f5f516008bee124ddd891398ed6", "sha256:608e7073dfa9e38a85d38474c082d4281f4ce276ac0010224eaba11e929dd53a",
"sha256:4296f2b1ce8c86a6aea78613c34bb1a672ea0e3de9c6ba08a960efe0b0a09047", "sha256:63ba06c9941e46fa389d389644e2d8225e0e3e5ebcc4ff1ea8506dce646f8c8a",
"sha256:47ab1e7b91c098ab893b828deafa1203de86d0bc6ab587b160f78fe6c4011f75", "sha256:65608c35bfb8a76763f37036547f7adfd09270fbdbf96608be2bead319728fcd",
"sha256:49e3ceeabbfb9d66c3aef5af3a60cc43b85c33df25ce03d0031a608b0a8b2e3f", "sha256:665a36ae6f8f20a4676b53224e33d456a6f5a72657d9c83c2aa00765072f31f7",
"sha256:4dc8f9fb58f7364b63fd9f85013b780ef83c11857ae79f2feda41e270468dd9b", "sha256:6d6607f98fcf17e534162f0709aaad3ab7a96032723d8ac8750ffe17ae5a0666",
"sha256:4efca8f86c54b22348a5467704e3fec767b2db12fc39c6d963168ab1d3fc9135", "sha256:7313ce6a199651c4ed9d7e4cfb4aa56fe923b1adf9af3b420ee14e6d9a73df65",
"sha256:53edb4da6925ad13c07b6d26c2a852bd81e364f95301c66e930ab2aef5b5ddd8", "sha256:7668b52e102d0ed87cb082380a7e2e1e78737ddecdde129acadb0eccc5423859",
"sha256:5855f8438a7d1d458206a2466bf82b0f104a3724bf96a1c781ab731e4201731a", "sha256:7df70907e00c970c60b9ef2938d894a9381f38e6b9db73c5be35e59d92e06625",
"sha256:594c67807fb16238b30c44bdf74f36c02cdf22d1c8cda91ef8a0ed8dabf5620a", "sha256:7e007132af78ea9df29495dbf7b5824cb71648d7133cf7848a2a5dd00d36f9ff",
"sha256:5b6d930f030f8ed98e3e6c98ffa0652bdb82601e7a016ec2ab5d7ff23baa78d1", "sha256:835fb5e38fd89328e9c81067fd642b3593c33e1e17e2fdbf77f5676abb14a156",
"sha256:5bb28c636d87e840583ee3adeb78172efc47c8b26127267f54a9c0ec251d41a9", "sha256:8bca7e26c1dd751236cfb0c6c72d4ad61d986e9a41bbf76cb445f69488b2a2bd",
"sha256:60bf42e36abfaf9aff1f50f52644b336d4f0a3fd6d8a60ca0d054ac9f713a864", "sha256:8db032bf0ce9022a8e41a22598eefc802314e81b879ae093f36ce9ddf39ab1ba",
"sha256:611d1ad9a4288cf3e3c16014564df047fe08410e628f89805e475368bd304914", "sha256:99625a92da8229df6d44335e6fcc558a5037dd0a760e11d84be2260e6f37002f",
"sha256:6300b8454aa6930a24b9618fbb54b5a68135092bc666f7b06901f897fa5c2fee", "sha256:9cad97ab29dfc3f0249b483412c85c8ef4766d96cdf9dcf5a1e3caa3f3661cf1",
"sha256:63f3268ba69ace99cab4e3e3b5840b03340efed0948ab8f78d2fd87ee5442a4f", "sha256:a4abaec6ca3ad8660690236d11bfe28dfd707778e2442b45addd2f086d6ef094",
"sha256:6557b31b5e2c9ddf0de32a691f2312a32f77cd7681d8af66c2692efdbef84c18", "sha256:a6e40afa7f45939ca356f348c8e23048e02cb109ced1eb8420961b2f40fb373a",
"sha256:693ce3f9e70a6cf7d2fb9e6c9d8b204b6b39897a2c4a1aa65728d5ac97dcc1d8", "sha256:a6f2fcca746e8d5910e18782f976489939d54a91f9411c32051b4aab2bd7c513",
"sha256:6a7fae0dd14cf60ad5ff42baa2e95727c3d81ded453457771d02b7d2b3f9c0c2", "sha256:a806db027852538d2ad7555b203300173dd1b77ba116de92da9afbc3a3be3eed",
"sha256:6c4ca60fa24e85fe25b912b01e62cb969d69a23a5d5867682dd3e80b5b02581d", "sha256:abcabc8c2b26036d62d4c746381a6f7cf60aafcc653198ad678306986b09450d",
"sha256:6fcf051089389abe060c9cd7caa212c707e58153afa2c649f00346ce6d260f1b", "sha256:b8526c6d437855442cdd3d87eede9c425c4445ea011ca38d937db299382e6fa3",
"sha256:7d91275b0245b1da4d4cfa07e0faedd5b0812efc15b702576d103293e252af1b", "sha256:bb06feb762bade6bf3c8b844462274db0c76acc95c52abe8dbed28ae3d44a147",
"sha256:89c687013cb1cd489a0f0ac24febe8c7a666e6e221b783e53ac50ebf68e45d86", "sha256:c0a33bc9f02c2b17c3ea382f91b4db0e6cde90b63b296422a939886a7a80de1c",
"sha256:8d206346619592c6200148b01a2142798c989edcb9c896f9ac9722a99d4e77e6", "sha256:c4a549890a45f57f1ebf99c067a4ad0cb423a05544accaf2b065246827ed9603",
"sha256:905fec760bd2fa1388bb5b489ee8ee5f7291d692638ea5f67982d968366bef9f", "sha256:ca244fa73f50a800cf8c3ebf7fd93149ec37f5cb9596aa8873ae2c1d23498601",
"sha256:97383d78eb34da7e1fa37dd273c20ad4320929af65d156e35a5e2d89566d9dfb", "sha256:cf877ab4ed6e302ec1d04952ca358b381a882fbd9d1b07cccbfd61783561f98a",
"sha256:984d76483eb32f1bcb536dc27e4ad56bba4baa70be32fa87152832cdd9db0833", "sha256:d9d971ec1e79906046aa3ca266de79eac42f1dbf3612a05dc9368125952bd1a1",
"sha256:99df47edb6bda1249d3e80fdabb1dab8c08ef3975f69aed437cb69d0a5de1e28", "sha256:da25303d91526aac3672ee6d49a2f3db2d9502a4a60b55519feb1a4c7714e07d",
"sha256:9f02365d4e99430a12647f09b6cc8bab61a6564363f313126f775eb4f6ef798e", "sha256:e55e40ff0cc8cc5c07996915ad367fa47da6b3fc091fdadca7f5403239c5fec3",
"sha256:a30e67a65b53ea0a5e62fe23682cfe22712e01f453b95233b25502f7c61cb415", "sha256:f03a532d7dee1bed20bc4884194a16160a2de9ffc6354b3878ec9682bb623c54",
"sha256:ab3ef638ace319fa26553db0624c4699e31a28bb2a835c5faca8f8acf6a5a902", "sha256:f1cd098434e83e656abf198f103a8207a8187c0fc110306691a2e94a78d0abb2",
"sha256:aca6377c0cb8a8253e493c6b451565ac77e98c2951c45f913e0b52facdcff83f", "sha256:f2bfb563d0211ce16b63c7cb9395d2c682a23187f54c3d79bfec33e6705473c6",
"sha256:add36cb2dbb8b736611303cd3bfcee00afd96471b09cda130da3581cbdc56a6d", "sha256:f8ffb705ffcf5ddd0e80b65ddf7bed7ee4f5a441ea7d3419e861a12eaf41af58"
"sha256:b2f4bf27480f5e5e8ce285a8c8fd176c0b03e93dcc6646477d4630e83440c6a9",
"sha256:b7f2d075102dc8c794cbde1947378051c4e5180d52d276987b8d28a3bd58c17d",
"sha256:baa1a4e8f868845af802979fcdbf0bb11f94f1cb7ced4c4b8a351bb60d108145",
"sha256:be98f628055368795d818ebf93da628541e10b75b41c559fdf36d104c5787066",
"sha256:bf5d821ffabf0ef3533c39c518f3357b171a1651c1ff6827325e4489b0e46c3c",
"sha256:c47adbc92fc1bb2b3274c4b3a43ae0e4573d9fbff4f54cd484555edbf030baf1",
"sha256:cdfba22ea2f0029c9261a4bd07e830a8da012291fbe44dc794e488b6c9bb353a",
"sha256:d6c7ebd4e944c85e2c3421e612a7057a2f48d478d79e61800d81468a8d842207",
"sha256:d7f9850398e85aba693bb640262d3611788b1f29a79f0c93c565694658f4071f",
"sha256:d8446c54dc28c01e5a2dbac5a25f071f6653e6e40f3a8818e8b45d790fe6ef53",
"sha256:deb993cacb280823246a026e3b2d81c493c53de6acfd5e6bfe31ab3402bb37dd",
"sha256:e0f138900af21926a02425cf736db95be9f4af72ba1bb21453432a07f6082134",
"sha256:e9936f0b261d4df76ad22f8fee3ae83b60d7c3e871292cd42f40b81b70afae85",
"sha256:f0567c4dc99f264f49fe27da5f735f414c4e7e7dd850cfd8e69f0862d7c74ea9",
"sha256:f5653a225f31e113b152e56f154ccbe59eeb1c7487b39b9d9f9cdb58e6c79dc5",
"sha256:f826e31d18b516f653fe296d967d700fddad5901ae07c622bb3705955e1faa94",
"sha256:f8ba0e8349a38d3001fae7eadded3f6606f0da5d748ee53cc1dab1d6527b9509",
"sha256:f9081981fe268bd86831e5c75f7de206ef275defcb82bc70740ae6dc507aee51",
"sha256:fa130dd50c57d53368c9d59395cb5526eda596d3ffe36666cd81a44d56e48872"
], ],
"markers": "python_version >= '3.6'", "markers": "python_version >= '3.7'",
"version": "==2.0.1" "version": "==2.1.2"
}, },
"marshmallow": { "marshmallow": {
"hashes": [ "hashes": [
@@ -1267,38 +1072,24 @@
"markers": "python_version >= '3.7'", "markers": "python_version >= '3.7'",
"version": "==23.1" "version": "==23.1"
}, },
"portalocker": {
"hashes": [
"sha256:032e81d534a88ec1736d03f780ba073f047a06c478b06e2937486f334e955c51",
"sha256:a07c5b4f3985c3cf4798369631fb7011adb498e2a46d8440efc75a8f29a0f983"
],
"markers": "python_version >= '3.5'",
"version": "==2.7.0"
},
"protobuf": { "protobuf": {
"hashes": [ "hashes": [
"sha256:03eee35b60317112a72d19c54d0bff7bc58ff12fea4cd7b018232bd99758ffdf", "sha256:2036a3a1e7fc27f973fa0a7888dce712393af644f4695385f117886abc792e39",
"sha256:2b94bd6df92d71bd1234a2ffe7ce96ddf6d10cf637a18d6b55ad0a89fbb7fc21", "sha256:32e78beda26d7a101fecf15d7a4a792278a0d26a31bc327ff05564a9d68ab8ee",
"sha256:36f5370a930cb77c8ad2f4135590c672d0d2c72d4a707c7d0058dce4b4b4a598", "sha256:346990f634272caac1f09efbcfbbacb23098b1f606d172534c6fa2d9758bb436",
"sha256:5f1eba1da2a2f3f7df469fccddef3cc060b8a16cfe3cc65961ad36b4dbcf59c5", "sha256:3b8905eafe4439076e1f58e9d1fa327025fd2777cf90f14083092ae47f77b0aa",
"sha256:6c16657d6717a0c62d5d740cb354fbad1b0d8cb811669e06fc1caa0ff4799ddd", "sha256:3ce113b3f3362493bddc9069c2163a38f240a9ed685ff83e7bcb756b05e1deb0",
"sha256:6fe180b56e1169d72ecc4acbd39186339aed20af5384531b8e8979b02bbee159", "sha256:410bcc0a5b279f634d3e16082ce221dfef7c3392fac723500e2e64d1806dd2be",
"sha256:7cb5b9a05ce52c6a782bb97de52679bd3438ff2b7460eff5da348db65650f227", "sha256:5b9cd6097e6acae48a68cb29b56bc79339be84eca65b486910bb1e7a30e2b7c1",
"sha256:9744e934ea5855d12191040ea198eaf704ac78665d365a89d9572e3b627c2688", "sha256:65f0ac96ef67d7dd09b19a46aad81a851b6f85f89725577f16de38f2d68ad477",
"sha256:9f5a0fbfcdcc364f3986f9ed9f8bb1328fb84114fd790423ff3d7fdb0f85c2d1", "sha256:91fac0753c3c4951fbb98a93271c43cc7cf3b93cf67747b3e600bb1e5cc14d61",
"sha256:baca40d067dddd62141a129f244703160d278648b569e90bb0e3753067644711", "sha256:95789b569418a3e32a53f43d7763be3d490a831e9c08042539462b6d972c2d7e",
"sha256:d5a35ff54e3f62e8fc7be02bb0d2fbc212bba1a5a9cc2748090690093996f07b", "sha256:ac50be82491369a9ec3710565777e4da87c6d2e20404e0abb1f3a8f10ffd20f0",
"sha256:e62fb869762b4ba18666370e2f8a18f17f8ab92dd4467295c6d38be6f8fef60b", "sha256:decf119d54e820f298ee6d89c72d6b289ea240c32c521f00433f9dc420595f38",
"sha256:ebde3a023b8e11bfa6c890ef34cd6a8b47d586f26135e86c21344fe433daf2e2" "sha256:f9510cac91e764e86acd74e2b7f7bc5e6127a7f3fb646d7c8033cfb84fd1176a"
], ],
"markers": "python_version >= '3.7'", "markers": "python_version >= '3.7'",
"version": "==4.23.0" "version": "==4.23.1"
},
"py3amf": {
"hashes": [
"sha256:dd81e7f98640a70a2ecefc2ea39b11bb75887239dbb1a82cf4e753a294aafb68"
],
"version": "==0.8.11"
}, },
"pyaes": { "pyaes": {
"hashes": [ "hashes": [
@@ -1331,42 +1122,41 @@
}, },
"pycryptodomex": { "pycryptodomex": {
"hashes": [ "hashes": [
"sha256:0af93aad8d62e810247beedef0261c148790c52f3cd33643791cc6396dd217c1", "sha256:160a39a708c36fa0b168ab79386dede588e62aec06eb505add870739329aecc6",
"sha256:12056c38e49d972f9c553a3d598425f8a1c1d35b2e4330f89d5ff1ffb70de041", "sha256:192306cf881fe3467dda0e174a4f47bb3a8bb24b90c9cdfbdc248eec5fc0578c",
"sha256:23d83b610bd97704f0cd3acc48d99b76a15c8c1540d8665c94d514a49905bad7", "sha256:1949e09ea49b09c36d11a951b16ff2a05a0ffe969dda1846e4686ee342fe8646",
"sha256:2d4d395f109faba34067a08de36304e846c791808524614c731431ee048fe70a", "sha256:215be2980a6b70704c10796dd7003eb4390e7be138ac6fb8344bf47e71a8d470",
"sha256:32e764322e902bbfac49ca1446604d2839381bbbdd5a57920c9daaf2e0b778df", "sha256:27072a494ce621cc7a9096bbf60ed66826bb94db24b49b7359509e7951033e74",
"sha256:3c2516b42437ae6c7a29ef3ddc73c8d4714e7b6df995b76be4695bbe4b3b5cd2", "sha256:2dc4eab20f4f04a2d00220fdc9258717b82d31913552e766d5f00282c031b70a",
"sha256:40e8a11f578bd0851b02719c862d55d3ee18d906c8b68a9c09f8c564d6bb5b92", "sha256:302a8f37c224e7b5d72017d462a2be058e28f7be627bdd854066e16722d0fc0c",
"sha256:4b51e826f0a04d832eda0790bbd0665d9bfe73e5a4d8ea93b6a9b38beeebe935", "sha256:3d9314ac785a5b75d5aaf924c5f21d6ca7e8df442e5cf4f0fefad4f6e284d422",
"sha256:4c4674f4b040321055c596aac926d12f7f6859dfe98cd12f4d9453b43ab6adc8", "sha256:3e3ecb5fe979e7c1bb0027e518340acf7ee60415d79295e5251d13c68dde576e",
"sha256:55eed98b4150a744920597c81b3965b632038781bab8a08a12ea1d004213c600", "sha256:4d9379c684efea80fdab02a3eb0169372bca7db13f9332cb67483b8dc8b67c37",
"sha256:599bb4ae4bbd614ca05f49bd4e672b7a250b80b13ae1238f05fd0f09d87ed80a", "sha256:50308fcdbf8345e5ec224a5502b4215178bdb5e95456ead8ab1a69ffd94779cb",
"sha256:5c23482860302d0d9883404eaaa54b0615eefa5274f70529703e2c43cc571827", "sha256:5594a125dae30d60e94f37797fc67ce3c744522de7992c7c360d02fdb34918f8",
"sha256:64b876d57cb894b31056ad8dd6a6ae1099b117ae07a3d39707221133490e5715", "sha256:58fc0aceb9c961b9897facec9da24c6a94c5db04597ec832060f53d4d6a07196",
"sha256:67a3648025e4ddb72d43addab764336ba2e670c8377dba5dd752e42285440d31", "sha256:6421d23d6a648e83ba2670a352bcd978542dad86829209f59d17a3f087f4afef",
"sha256:6feedf4b0e36b395329b4186a805f60f900129cdf0170e120ecabbfcb763995d", "sha256:6875eb8666f68ddbd39097867325bd22771f595b4e2b0149739b5623c8bf899b",
"sha256:78f0ddd4adc64baa39b416f3637aaf99f45acb0bcdc16706f0cc7ebfc6f10109", "sha256:6ed3606832987018615f68e8ed716a7065c09a0fe94afd7c9ca1b6777f0ac6eb",
"sha256:7a6651a07f67c28b6e978d63aa3a3fccea0feefed9a8453af3f7421a758461b7", "sha256:71687eed47df7e965f6e0bf3cadef98f368d5221f0fb89d2132effe1a3e6a194",
"sha256:7a8dc3ee7a99aae202a4db52de5a08aa4d01831eb403c4d21da04ec2f79810db", "sha256:73d64b32d84cf48d9ec62106aa277dbe99ab5fbfd38c5100bc7bddd3beb569f7",
"sha256:7cc28dd33f1f3662d6da28ead4f9891035f63f49d30267d3b41194c8778997c8", "sha256:75672205148bdea34669173366df005dbd52be05115e919551ee97171083423d",
"sha256:7fa0b52df90343fafe319257b31d909be1d2e8852277fb0376ba89d26d2921db", "sha256:76f0a46bee539dae4b3dfe37216f678769349576b0080fdbe431d19a02da42ff",
"sha256:88b0d5bb87eaf2a31e8a759302b89cf30c97f2f8ca7d83b8c9208abe8acb447a", "sha256:8ff129a5a0eb5ff16e45ca4fa70a6051da7f3de303c33b259063c19be0c43d35",
"sha256:a4fa037078e92c7cc49f6789a8bac3de06856740bb2038d05f2d9a2e4b165d59", "sha256:ac614363a86cc53d8ba44b6c469831d1555947e69ab3276ae8d6edc219f570f7",
"sha256:a57e3257bacd719769110f1f70dd901c5b6955e9596ad403af11a3e6e7e3311c", "sha256:ba95abd563b0d1b88401658665a260852a8e6c647026ee6a0a65589287681df8",
"sha256:ab33c2d9f275e05e235dbca1063753b5346af4a5cac34a51fa0da0d4edfb21d7", "sha256:bbdcce0a226d9205560a5936b05208c709b01d493ed8307792075dedfaaffa5f",
"sha256:c84689c73358dfc23f9fdcff2cb9e7856e65e2ce3b5ed8ff630d4c9bdeb1867b", "sha256:bec6c80994d4e7a38312072f89458903b65ec99bed2d65aa4de96d997a53ea7a",
"sha256:c92537b596bd5bffb82f8964cabb9fef1bca8a28a9e0a69ffd3ec92a4a7ad41b", "sha256:c2953afebf282a444c51bf4effe751706b4d0d63d7ca2cc51db21f902aa5b84e",
"sha256:caa937ff29d07a665dfcfd7a84f0d4207b2ebf483362fa9054041d67fdfacc20", "sha256:d35a8ffdc8b05e4b353ba281217c8437f02c57d7233363824e9d794cf753c419",
"sha256:d38ab9e53b1c09608ba2d9b8b888f1e75d6f66e2787e437adb1fecbffec6b112", "sha256:d56c9ec41258fd3734db9f5e4d2faeabe48644ba9ca23b18e1839b3bdf093222",
"sha256:d4cf0128da167562c49b0e034f09e9cedd733997354f2314837c2fa461c87bb1", "sha256:d84e105787f5e5d36ec6a581ff37a1048d12e638688074b2a00bcf402f9aa1c2",
"sha256:db23d7341e21b273d2440ec6faf6c8b1ca95c8894da612e165be0b89a8688340", "sha256:e00a4bacb83a2627e8210cb353a2e31f04befc1155db2976e5e239dd66482278",
"sha256:ee8bf4fdcad7d66beb744957db8717afc12d176e3fd9c5d106835133881a049b", "sha256:f237278836dda412a325e9340ba2e6a84cb0f56b9244781e5b61f10b3905de88",
"sha256:f854c8476512cebe6a8681cc4789e4fcff6019c17baa0fd72b459155dc605ab4", "sha256:f9ab5ef0718f6a8716695dea16d83b671b22c45e9c0c78fd807c32c0192e54b5"
"sha256:fd29d35ac80755e5c0a99d96b44fb9abbd7e871849581ea6a4cb826d24267537"
], ],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
"version": "==3.17" "version": "==3.18.0"
}, },
"pygments": { "pygments": {
"hashes": [ "hashes": [
@@ -1376,14 +1166,6 @@
"markers": "python_version >= '3.7'", "markers": "python_version >= '3.7'",
"version": "==2.15.1" "version": "==2.15.1"
}, },
"pyopenssl": {
"hashes": [
"sha256:841498b9bec61623b1b6c47ebbc02367c07d60e0e195f19790817f10cc8db0b7",
"sha256:9e0c526404a210df9d2b18cd33364beadb0dc858a739b885677bc65e105d4a4c"
],
"markers": "python_version >= '3.6'",
"version": "==23.1.1"
},
"pyparsing": { "pyparsing": {
"hashes": [ "hashes": [
"sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb", "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb",
@@ -1431,22 +1213,6 @@
], ],
"version": "==2023.3" "version": "==2023.3"
}, },
"pytz-deprecation-shim": {
"hashes": [
"sha256:8314c9692a636c8eb3bda879b9f119e350e93223ae83e70e80c31675a0fdc1a6",
"sha256:af097bae1b616dde5c5744441e2ddc69e74dfdcb0c263129610d85b87445a59d"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
"version": "==0.1.0.post0"
},
"pywb": {
"hashes": [
"sha256:52463f09b89f9df7267af90c799b7d88c27906fad214d1361cab227923258d25",
"sha256:b94d85f3aff169aa22477281bda520716e307f9a2f8e2cd48a2105bdfb322753"
],
"index": "pypi",
"version": "==2.7.3"
},
"pyyaml": { "pyyaml": {
"hashes": [ "hashes": [
"sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf", "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf",
@@ -1493,13 +1259,6 @@
"index": "pypi", "index": "pypi",
"version": "==6.0" "version": "==6.0"
}, },
"redis": {
"hashes": [
"sha256:8a1900a9f2a0a44ecf6e8b5eb3e967a9909dfed219ad66df094f27f7d6f330fb",
"sha256:a22ca993cea2962dbb588f9f30d0015ac4afcc45bee27d3978c0dbe9e97c6c0f"
],
"version": "==2.10.6"
},
"regex": { "regex": {
"hashes": [ "hashes": [
"sha256:02f4541550459c08fdd6f97aa4e24c6f1932eec780d58a2faa2068253df7d6ff", "sha256:02f4541550459c08fdd6f97aa4e24c6f1932eec780d58a2faa2068253df7d6ff",
@@ -1605,13 +1364,6 @@
"index": "pypi", "index": "pypi",
"version": "==2.30.0" "version": "==2.30.0"
}, },
"requests-file": {
"hashes": [
"sha256:07d74208d3389d01c38ab89ef403af0cfec63957d53a0081d8eca738d0247d8e",
"sha256:dfe5dae75c12481f68ba353183c53a65e6044c923e64c24b2209f6c7570ca953"
],
"version": "==1.5.1"
},
"requests-oauthlib": { "requests-oauthlib": {
"hashes": [ "hashes": [
"sha256:2577c501a2fb8d05a304c09d090d6e47c306fef15809d102b327cf8364bddab5", "sha256:2577c501a2fb8d05a304c09d090d6e47c306fef15809d102b327cf8364bddab5",
@@ -1633,7 +1385,7 @@
"sha256:2d11b9b8dd03868f09b4fffadc84a6a8cda574e40dc90821bd845720ebb8e89c", "sha256:2d11b9b8dd03868f09b4fffadc84a6a8cda574e40dc90821bd845720ebb8e89c",
"sha256:69cdf53799e63f38b95b9bf9c875f8c90e78dd62b2f00c13a911c7a3b9fa4704" "sha256:69cdf53799e63f38b95b9bf9c875f8c90e78dd62b2f00c13a911c7a3b9fa4704"
], ],
"markers": "python_version >= '3.7'", "markers": "python_full_version >= '3.7.0'",
"version": "==13.3.5" "version": "==13.3.5"
}, },
"rsa": { "rsa": {
@@ -1641,7 +1393,7 @@
"sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7", "sha256:90260d9058e514786967344d0ef75fa8727eed8a7d2e43ce9f4bcf1b536174f7",
"sha256:e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21" "sha256:e38464a49c6c85d7f1351b0126661487a7e0a14a50f1675ec50eb34d4f20ef21"
], ],
"markers": "python_version >= '3.6' and python_version < '4.0'", "markers": "python_version >= '3.6' and python_version < '4'",
"version": "==4.9" "version": "==4.9"
}, },
"s3transfer": { "s3transfer": {
@@ -1660,14 +1412,6 @@
"index": "pypi", "index": "pypi",
"version": "==4.9.1" "version": "==4.9.1"
}, },
"setuptools": {
"hashes": [
"sha256:23aaf86b85ca52ceb801d32703f12d77517b2556af839621c641fca11287952b",
"sha256:f104fa03692a2602fa0fec6c6a9e63b6c8a968de13e17c026957dd1f53d80990"
],
"markers": "python_version >= '3.7'",
"version": "==67.7.2"
},
"six": { "six": {
"hashes": [ "hashes": [
"sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
@@ -1707,12 +1451,6 @@
"markers": "python_version >= '3.7'", "markers": "python_version >= '3.7'",
"version": "==2.4.1" "version": "==2.4.1"
}, },
"surt": {
"hashes": [
"sha256:24167eb6c01f24f757eef9bca6bf0ec089ec05ad5b6213c3b727a5e58c0c4720"
],
"version": "==0.3.1"
},
"telethon": { "telethon": {
"hashes": [ "hashes": [
"sha256:b3990ec22351a3f3e1af376729c985025bbdd3bdabdde8c156112c3d3dfe1941", "sha256:b3990ec22351a3f3e1af376729c985025bbdd3bdabdde8c156112c3d3dfe1941",
@@ -1735,14 +1473,6 @@
"index": "pypi", "index": "pypi",
"version": "==0.3.5" "version": "==0.3.5"
}, },
"tldextract": {
"hashes": [
"sha256:26f646987b01ae2946e7491cce4aaf54129f3489a196a274e6c843ec72968313",
"sha256:fa9e50c4a03bede2a1d95dca620d661678484626858ccf388cf9671a0dd497a4"
],
"markers": "python_version >= '3.7'",
"version": "==3.4.1"
},
"tqdm": { "tqdm": {
"hashes": [ "hashes": [
"sha256:1871fb68a86b8fb3b59ca4cdd3dcccbc7e6d613eeed31f4c332531977b89beb5", "sha256:1871fb68a86b8fb3b59ca4cdd3dcccbc7e6d613eeed31f4c332531977b89beb5",
@@ -1782,28 +1512,13 @@
], ],
"version": "==0.8.0" "version": "==0.8.0"
}, },
"tzdata": {
"hashes": [
"sha256:11ef1e08e54acb0d4f95bdb1be05da659673de4acbd21bf9c69e94cc5e907a3a",
"sha256:7e65763eef3120314099b6939b5546db7adce1e7d6f2e179e3df563c70511eda"
],
"markers": "python_version >= '3.6'",
"version": "==2023.3"
},
"tzlocal": { "tzlocal": {
"hashes": [ "hashes": [
"sha256:3f21d09e1b2aa9f2dacca12da240ca37de3ba5237a93addfd6d593afe9073355", "sha256:46eb99ad4bdb71f3f72b7d24f4267753e240944ecfc16f25d2719ba89827a803",
"sha256:b44c4388f3d34f25862cfbb387578a4d70fec417649da694a132f628a23367e2" "sha256:f3596e180296aaf2dbd97d124fe76ae3a0e3d32b258447de7b939b3fd4be992f"
], ],
"markers": "python_version >= '3.7'", "markers": "python_version >= '3.7'",
"version": "==4.3" "version": "==5.0.1"
},
"ua-parser": {
"hashes": [
"sha256:ed3efc695f475ffe56248c9789b3016247e9c20e3556cfa4d5aadc78ab4b26c6",
"sha256:f97126300df8ac0f8f2c9d8559669532d626a1af529265fd253cba56e73ab36e"
],
"version": "==0.16.1"
}, },
"uritemplate": { "uritemplate": {
"hashes": [ "hashes": [
@@ -1843,27 +1558,6 @@
"index": "pypi", "index": "pypi",
"version": "==0.3.24" "version": "==0.3.24"
}, },
"warcio": {
"hashes": [
"sha256:ced1a162d76434d56abd81b37ac152821d1a11e1db835ead5d649f58068c2203",
"sha256:e1889dad9ecac654de5b0973247f335a55827b1b14a8203772d18c749143ea51"
],
"version": "==1.7.4"
},
"webassets": {
"hashes": [
"sha256:167132337677c8cedc9705090f6d48da3fb262c8e0b2773b29f3352f050181cd",
"sha256:a31a55147752ba1b3dc07dee0ad8c8efff274464e08bbdb88c1fd59ffd552724"
],
"version": "==2.0"
},
"webencodings": {
"hashes": [
"sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78",
"sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923"
],
"version": "==0.5.1"
},
"websockets": { "websockets": {
"hashes": [ "hashes": [
"sha256:01f5567d9cf6f502d655151645d4e8b72b453413d3819d2b6f1185abc23e82dd", "sha256:01f5567d9cf6f502d655151645d4e8b72b453413d3819d2b6f1185abc23e82dd",
@@ -1942,25 +1636,18 @@
}, },
"werkzeug": { "werkzeug": {
"hashes": [ "hashes": [
"sha256:2de2a5db0baeae7b2d2664949077c2ac63fbd16d98da0ff71837f7d1dea3fd43", "sha256:1d5a58e0377d1fe39d061a5de4469e414e78ccb1e1e59c0f5ad6fa1c36c52b76",
"sha256:6c80b1e5ad3665290ea39320b91e1be1e0d5f60652b964a3070216de83d2e47c" "sha256:48e5e61472fee0ddee27ebad085614ebedb7af41e88f687aaf881afb723a162f"
], ],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "markers": "python_version >= '3.8'",
"version": "==1.0.1" "version": "==2.3.4"
},
"wsgiprox": {
"hashes": [
"sha256:3a97f6f8d51122aa5e3a1c6c8c73a84a15a3f9b69f312a3aae8bef9b52a8da85",
"sha256:8dab64cef38ff39d525d246bc1b34b5a378c1476be7127ef79c85c29ab92765c"
],
"version": "==1.5.2"
}, },
"wsproto": { "wsproto": {
"hashes": [ "hashes": [
"sha256:ad565f26ecb92588a3e43bc3d96164de84cd9902482b130d0ddbaa9664a85065", "sha256:ad565f26ecb92588a3e43bc3d96164de84cd9902482b130d0ddbaa9664a85065",
"sha256:b9acddd652b585d75b20477888c56642fdade28bdfd3579aa24a4d2c037dd736" "sha256:b9acddd652b585d75b20477888c56642fdade28bdfd3579aa24a4d2c037dd736"
], ],
"markers": "python_version >= '3.7'", "markers": "python_full_version >= '3.7.0'",
"version": "==1.2.0" "version": "==1.2.0"
}, },
"yarl": { "yarl": {
@@ -2050,49 +1737,6 @@
], ],
"index": "pypi", "index": "pypi",
"version": "==2023.3.4" "version": "==2023.3.4"
},
"zope.event": {
"hashes": [
"sha256:73d9e3ef750cca14816a9c322c7250b0d7c9dbc337df5d1b807ff8d3d0b9e97c",
"sha256:81d98813046fc86cc4136e3698fee628a3282f9c320db18658c21749235fce80"
],
"version": "==4.6"
},
"zope.interface": {
"hashes": [
"sha256:042f2381118b093714081fd82c98e3b189b68db38ee7d35b63c327c470ef8373",
"sha256:0ec9653825f837fbddc4e4b603d90269b501486c11800d7c761eee7ce46d1bbb",
"sha256:12175ca6b4db7621aedd7c30aa7cfa0a2d65ea3a0105393e05482d7a2d367446",
"sha256:1592f68ae11e557b9ff2bc96ac8fc30b187e77c45a3c9cd876e3368c53dc5ba8",
"sha256:23ac41d52fd15dd8be77e3257bc51bbb82469cf7f5e9a30b75e903e21439d16c",
"sha256:424d23b97fa1542d7be882eae0c0fc3d6827784105264a8169a26ce16db260d8",
"sha256:4407b1435572e3e1610797c9203ad2753666c62883b921318c5403fb7139dec2",
"sha256:48f4d38cf4b462e75fac78b6f11ad47b06b1c568eb59896db5b6ec1094eb467f",
"sha256:4c3d7dfd897a588ec27e391edbe3dd320a03684457470415870254e714126b1f",
"sha256:5171eb073474a5038321409a630904fd61f12dd1856dd7e9d19cd6fe092cbbc5",
"sha256:5a158846d0fca0a908c1afb281ddba88744d403f2550dc34405c3691769cdd85",
"sha256:6ee934f023f875ec2cfd2b05a937bd817efcc6c4c3f55c5778cbf78e58362ddc",
"sha256:790c1d9d8f9c92819c31ea660cd43c3d5451df1df61e2e814a6f99cebb292788",
"sha256:809fe3bf1a91393abc7e92d607976bbb8586512913a79f2bf7d7ec15bd8ea518",
"sha256:87b690bbee9876163210fd3f500ee59f5803e4a6607d1b1238833b8885ebd410",
"sha256:89086c9d3490a0f265a3c4b794037a84541ff5ffa28bb9c24cc9f66566968464",
"sha256:99856d6c98a326abbcc2363827e16bd6044f70f2ef42f453c0bd5440c4ce24e5",
"sha256:aab584725afd10c710b8f1e6e208dbee2d0ad009f57d674cb9d1b3964037275d",
"sha256:af169ba897692e9cd984a81cb0f02e46dacdc07d6cf9fd5c91e81f8efaf93d52",
"sha256:b39b8711578dcfd45fc0140993403b8a81e879ec25d53189f3faa1f006087dca",
"sha256:b3f543ae9d3408549a9900720f18c0194ac0fe810cecda2a584fd4dca2eb3bb8",
"sha256:d0583b75f2e70ec93f100931660328965bb9ff65ae54695fb3fa0a1255daa6f2",
"sha256:dfbbbf0809a3606046a41f8561c3eada9db811be94138f42d9135a5c47e75f6f",
"sha256:e538f2d4a6ffb6edfb303ce70ae7e88629ac6e5581870e66c306d9ad7b564a58",
"sha256:eba51599370c87088d8882ab74f637de0c4f04a6d08a312dce49368ba9ed5c2a",
"sha256:ee4b43f35f5dc15e1fec55ccb53c130adb1d11e8ad8263d68b1284b66a04190d",
"sha256:f2363e5fd81afb650085c6686f2ee3706975c54f331b426800b53531191fdf28",
"sha256:f299c020c6679cb389814a3b81200fe55d428012c5e76da7e722491f5d205990",
"sha256:f72f23bab1848edb7472309e9898603141644faec9fd57a823ea6b4d1c4c8995",
"sha256:fa90bac61c9dc3e1a563e5babb3fd2c0c1c80567e815442ddbe561eadc803b30"
],
"markers": "python_version >= '3.7'",
"version": "==6.0"
} }
}, },
"develop": { "develop": {

View File

@@ -1,7 +1,7 @@
<h1 align="center">Auto Archiver</h1> <h1 align="center">Auto Archiver</h1>
[![PyPI version](https://badge.fury.io/py/auto-archiver.svg)](https://badge.fury.io/py/auto-archiver) [![PyPI version](https://badge.fury.io/py/auto-archiver.svg)](https://badge.fury.io/py/auto-archiver)
[![Docker Image Version (latest by date)](https://img.shields.io/docker/v/bellingcat/auto-archiver?label=version&logo=docker)](https://pypi.org/project/auto-archiver/) [![Docker Image Version (latest by date)](https://img.shields.io/docker/v/bellingcat/auto-archiver?label=version&logo=docker)](https://hub.docker.com/r/bellingcat/auto-archiver)
<!-- ![Docker Pulls](https://img.shields.io/docker/pulls/bellingcat/auto-archiver) --> <!-- ![Docker Pulls](https://img.shields.io/docker/pulls/bellingcat/auto-archiver) -->
<!-- [![PyPI download month](https://img.shields.io/pypi/dm/auto-archiver.svg)](https://pypi.python.org/pypi/auto-archiver/) --> <!-- [![PyPI download month](https://img.shields.io/pypi/dm/auto-archiver.svg)](https://pypi.python.org/pypi/auto-archiver/) -->
<!-- [![Documentation Status](https://readthedocs.org/projects/vk-url-scraper/badge/?version=latest)](https://vk-url-scraper.readthedocs.io/en/latest/?badge=latest) --> <!-- [![Documentation Status](https://readthedocs.org/projects/vk-url-scraper/badge/?version=latest)](https://vk-url-scraper.readthedocs.io/en/latest/?badge=latest) -->
@@ -20,12 +20,10 @@ There are 3 ways to use the auto-archiver:
But **you always need a configuration/orchestration file**, which is where you'll configure where/what/how to archive. Make sure you read [orchestration](#orchestration). But **you always need a configuration/orchestration file**, which is where you'll configure where/what/how to archive. Make sure you read [orchestration](#orchestration).
## How to run the auto-archiver ## How to install and run the auto-archiver
### Option 1 - docker ### Option 1 - docker
<details><summary><code>Docker instructions</code></summary>
[![dockeri.co](https://dockerico.blankenship.io/image/bellingcat/auto-archiver)](https://hub.docker.com/r/bellingcat/auto-archiver) [![dockeri.co](https://dockerico.blankenship.io/image/bellingcat/auto-archiver)](https://hub.docker.com/r/bellingcat/auto-archiver)
Docker works like a virtual machine running inside your computer, it isolates everything and makes installation simple. Since it is an isolated environment when you need to pass it your orchestration file or get downloaded media out of docker you will need to connect folders on your machine with folders inside docker with the `-v` volume flag. Docker works like a virtual machine running inside your computer, it isolates everything and makes installation simple. Since it is an isolated environment when you need to pass it your orchestration file or get downloaded media out of docker you will need to connect folders on your machine with folders inside docker with the `-v` volume flag.
@@ -45,8 +43,6 @@ Docker works like a virtual machine running inside your computer, it isolates ev
2. `$PWD/local_archive` is a folder `local_archive/` in case you want to archive locally and have the files accessible outside docker 2. `$PWD/local_archive` is a folder `local_archive/` in case you want to archive locally and have the files accessible outside docker
3. `/app/local_archive` is a folder inside docker that you can reference in your orchestration.yml file 3. `/app/local_archive` is a folder inside docker that you can reference in your orchestration.yml file
</details>
### Option 2 - python package ### Option 2 - python package
<details><summary><code>Python package instructions</code></summary> <details><summary><code>Python package instructions</code></summary>
@@ -54,8 +50,9 @@ Docker works like a virtual machine running inside your computer, it isolates ev
1. make sure you have python 3.8 or higher installed 1. make sure you have python 3.8 or higher installed
2. install the package `pip/pipenv/conda install auto-archiver` 2. install the package `pip/pipenv/conda install auto-archiver`
3. test it's installed with `auto-archiver --help` 3. test it's installed with `auto-archiver --help`
4. run it with your orchestration file and pass any flags you want in the command line `auto-archiver --config secrets/orchestration.yaml` 4. run it with your orchestration file and pass any flags you want in the command line `auto-archiver --config secrets/orchestration.yaml` if your orchestration file is inside a `secrets/`, which we advise
1. if your orchestration file is inside a `secrets/` which we advise
You will also need [ffmpeg](https://www.ffmpeg.org/), [firefox](https://www.mozilla.org/en-US/firefox/new/) and [geckodriver](https://github.com/mozilla/geckodriver/releases), and optionally [fonts-noto](https://fonts.google.com/noto). Similar to the local installation.
</details> </details>
@@ -69,7 +66,7 @@ This can also be used for development.
Install the following locally: Install the following locally:
1. [ffmpeg](https://www.ffmpeg.org/) must also be installed locally for this tool to work. 1. [ffmpeg](https://www.ffmpeg.org/) must also be installed locally for this tool to work.
2. [firefox](https://www.mozilla.org/en-US/firefox/new/) and [geckodriver](https://github.com/mozilla/geckodriver/releases) on a path folder like `/usr/local/bin`. 2. [firefox](https://www.mozilla.org/en-US/firefox/new/) and [geckodriver](https://github.com/mozilla/geckodriver/releases) on a path folder like `/usr/local/bin`.
3. [fonts-noto](https://fonts.google.com/noto) to deal with multiple unicode characters during selenium/geckodriver's screenshots: `sudo apt install fonts-noto -y`. 3. (optional) [fonts-noto](https://fonts.google.com/noto) to deal with multiple unicode characters during selenium/geckodriver's screenshots: `sudo apt install fonts-noto -y`.
Clone and run: Clone and run:
1. `git clone https://github.com/bellingcat/auto-archiver` 1. `git clone https://github.com/bellingcat/auto-archiver`
@@ -87,7 +84,7 @@ The archiver work is orchestrated by the following workflow (we call each a **st
4. **Formatter** creates a report from all the archived content (HTML, PDF, ...) 4. **Formatter** creates a report from all the archived content (HTML, PDF, ...)
5. **Database** knows what's been archived and also stores the archive result (spreadsheet, CSV, or just the console) 5. **Database** knows what's been archived and also stores the archive result (spreadsheet, CSV, or just the console)
To setup an auto-archiver instance, instance, create an `orchestration.yaml` which contains the workflow you would like. We advise you put this file into a `secrets/` folder and do not share it with others because it will contain passwords and other secrets. To setup an auto-archiver instance create an `orchestration.yaml` which contains the workflow you would like. We advise you put this file into a `secrets/` folder and do not share it with others because it will contain passwords and other secrets.
The structure of orchestration file is split into 2 parts: `steps` (what **steps** to use) and `configurations` (how those steps should behave), here's a simplification: The structure of orchestration file is split into 2 parts: `steps` (what **steps** to use) and `configurations` (how those steps should behave), here's a simplification:
```yaml ```yaml
@@ -147,19 +144,30 @@ Use this to make sure you help making sure you did all the required steps:
* [ ] (optional for browsertrix) `profile.tar.gz` file * [ ] (optional for browsertrix) `profile.tar.gz` file
#### Example invocations #### Example invocations
These assume you've installed with pipenv, see docker section above for how to run through docker The recommended way to run the auto-archiver is through Docker. The invocations below will run the auto-archiver Docker image using a configuration file that you have specified
```bash
# all the configurations come from ./secrets/orchestration.yaml
docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver --config secrets/orchestration.yaml
# uses the same configurations but for another google docs sheet
# with a header on row 2 and with some different column names
# notice that columns is a dictionary so you need to pass it as JSON and it will override only the values provided
docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver --config secrets/orchestration.yaml --gsheet_feeder.sheet="use it on another sheets doc" --gsheet_feeder.header=2 --gsheet_feeder.columns='{"url": "link"}'
# all the configurations come from orchestration.yaml and specifies that s3 files should be private
docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver --config secrets/orchestration.yaml --s3_storage.private=1
```
The auto-archiver can also be run locally, if pre-requisites are correctly configured. Equivalent invocations are below.
```bash ```bash
# all the configurations come from ./orchestration.yaml
auto-archiver
# all the configurations come from ./secrets/orchestration.yaml # all the configurations come from ./secrets/orchestration.yaml
auto-archiver --config secrets/orchestration.yaml auto-archiver --config secrets/orchestration.yaml
# uses the same configurations but for another google docs sheet # uses the same configurations but for another google docs sheet
# with a header on row 2 and with some different column names # with a header on row 2 and with some different column names
# notice that columns is a dictionary so you need to pass it as JSON and it will override only the values provided # notice that columns is a dictionary so you need to pass it as JSON and it will override only the values provided
auto-archiver --config orchestration.yaml --gsheet_feeder.sheet="use it on another sheets doc" --gsheet_feeder.header=2 --gsheet_feeder.columns='{"url": "link"}' auto-archiver --config secrets/orchestration.yaml --gsheet_feeder.sheet="use it on another sheets doc" --gsheet_feeder.header=2 --gsheet_feeder.columns='{"url": "link"}'
# all the configurations come from orchestration.yaml and specifies that s3 files should be private # all the configurations come from orchestration.yaml and specifies that s3 files should be private
auto-archiver --s3_storage.private=1 auto-archiver --config secrets/orchestration.yaml --s3_storage.private=1
``` ```
### Extra notes on configuration ### Extra notes on configuration
@@ -173,18 +181,45 @@ The first time you run, you will be prompted to do a authentication with the pho
## Running on Google Sheets Feeder (gsheet_feeder) ## Running on Google Sheets Feeder (gsheet_feeder)
The `--gseets_feeder.sheet` property is the name of the Google Sheet to check for URLs. The `--gseets_feeder.sheet` property is the name of the Google Sheet to check for URLs.
This sheet must have been shared with the Google Service account used by `gspread`. This sheet must have been shared with the Google Service account used by `gspread`.
This sheet must also have specific columns (case-insensitive) in the `header` row - see [Gsheet.configs](src/auto_archiver/utils/gsheet.py) for all their names. This sheet must also have specific columns (case-insensitive) in the `header` as specified in [Gsheet.configs](src/auto_archiver/utils/gsheet.py). The default names of these columns and their purpose is:
For example, for use with this spreadsheet: Inputs:
![A screenshot of a Google Spreadsheet with column headers defined as above, and several Youtube and Twitter URLs in the "Media URL" column](docs/demo-before.png) * **Link** *(required)*: the URL of the post to archive
* **Destination folder**: custom folder for archived file (regardless of storage)
Outputs:
* **Archive status** *(required)*: Status of archive operation
* **Archive location**: URL of archived post
* **Archive date**: Date archived
* **Thumbnail**: Embeds a thumbnail for the post in the spreadsheet
* **Timestamp**: Timestamp of original post
* **Title**: Post title
* **Text**: Post text
* **Screenshot**: Link to screenshot of post
* **Hash**: Hash of archived HTML file (which contains hashes of post media)
* **WACZ**: Link to a WACZ web archive of post
* **ReplayWebpage**: Link to a ReplayWebpage viewer of the WACZ archive
For example, this is a spreadsheet configured with all of the columns for the auto archiver and a few URLs to archive. (Note that the column names are not case sensitive.)
![A screenshot of a Google Spreadsheet with column headers defined as above, and several Youtube and Twitter URLs in the "Link" column](docs/demo-before.png)
Now the auto archiver can be invoked, with this command in this example: `docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver:dockerize --config secrets/orchestration-global.yaml --gsheet_feeder.sheet "Auto archive test 2023-2"`. Note that the sheet name has been overridden/specified in the command line invocation.
When the auto archiver starts running, it updates the "Archive status" column. When the auto archiver starts running, it updates the "Archive status" column.
![A screenshot of a Google Spreadsheet with column headers defined as above, and several Youtube and Twitter URLs in the "Media URL" column. The auto archiver has added "archive in progress" to one of the status columns.](docs/demo-progress.png)
![A screenshot of a Google Spreadsheet with column headers defined as above, and several Youtube and Twitter URLs in the "Link" column. The auto archiver has added "archive in progress" to one of the status columns.](docs/demo-progress.png)
The links are downloaded and archived, and the spreadsheet is updated to the following: The links are downloaded and archived, and the spreadsheet is updated to the following:
![A screenshot of a Google Spreadsheet with videos archived and metadata added per the description of the columns above.](docs/demo-after.png) ![A screenshot of a Google Spreadsheet with videos archived and metadata added per the description of the columns above.](docs/demo-after.png)
Note that the first row is skipped, as it is assumed to be a header row (`--gsheet_feeder.header=1` and you can change it if you use more rows above). Rows with an empty URL column, or a non-empty archive column are also skipped. All sheets in the document will be checked. Note that the first row is skipped, as it is assumed to be a header row (`--gsheet_feeder.header=1` and you can change it if you use more rows above). Rows with an empty URL column, or a non-empty archive column are also skipped. All sheets in the document will be checked.
The "archive location" link contains the path of the archived file, in local storage, S3, or in Google Drive.
![The archive result for a link in the demo sheet.](docs/demo-archive.png)
--- ---
## Development ## Development
@@ -193,7 +228,7 @@ Use `python -m src.auto_archiver --config secrets/orchestration.yaml` to run fro
#### Docker development #### Docker development
working with docker locally: working with docker locally:
* `docker build . -t auto-archiver` to build a local image * `docker build . -t auto-archiver` to build a local image
* `docker run --rm -v $PWD/secrets:/app/secrets aa pipenv run python3 -m auto_archiver --config secrets/orchestration.yaml` * `docker run --rm -v $PWD/secrets:/app/secrets auto-archiver pipenv run python3 -m auto_archiver --config secrets/orchestration.yaml`
* to use local archive, also create a volume `-v` for it by adding `-v $PWD/local_archive:/app/local_archive` * to use local archive, also create a volume `-v` for it by adding `-v $PWD/local_archive:/app/local_archive`

Binary file not shown.

Before

Width:  |  Height:  |  Size: 183 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 486 KiB

After

Width:  |  Height:  |  Size: 1.5 MiB

BIN
docs/demo-archive.png Normal file

Binary file not shown.

After

Width:  |  Height:  |  Size: 819 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 223 KiB

After

Width:  |  Height:  |  Size: 664 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 241 KiB

After

Width:  |  Height:  |  Size: 698 KiB

View File

@@ -5,7 +5,7 @@ def main():
config = Config() config = Config()
config.parse() config.parse()
orchestrator = ArchivingOrchestrator(config) orchestrator = ArchivingOrchestrator(config)
orchestrator.feed() for r in orchestrator.feed(): pass
if __name__ == "__main__": if __name__ == "__main__":

View File

@@ -13,6 +13,7 @@ from ..formatters import Formatter
from ..storages import Storage from ..storages import Storage
from ..enrichers import Enricher from ..enrichers import Enricher
from . import Step from . import Step
from ..utils import update_nested_dict
@dataclass @dataclass
@@ -38,10 +39,11 @@ class Config:
self.cli_ops = {} self.cli_ops = {}
self.config = {} self.config = {}
def parse(self, use_cli=True, yaml_config_filename: str = None): def parse(self, use_cli=True, yaml_config_filename: str = None, overwrite_configs: str = {}):
""" """
if yaml_config_filename is provided, the --config argument is ignored, if yaml_config_filename is provided, the --config argument is ignored,
useful for library usage when the config values are preloaded useful for library usage when the config values are preloaded
overwrite_configs is a dict that overwrites the yaml file contents
""" """
# 1. parse CLI values # 1. parse CLI values
if use_cli: if use_cli:
@@ -80,6 +82,7 @@ class Config:
# 2. read YAML config file (or use provided value) # 2. read YAML config file (or use provided value)
self.yaml_config = self.read_yaml(yaml_config_filename) self.yaml_config = self.read_yaml(yaml_config_filename)
update_nested_dict(self.yaml_config, overwrite_configs)
# 3. CONFIGS: decide value with priority: CLI >> config.yaml >> default # 3. CONFIGS: decide value with priority: CLI >> config.yaml >> default
self.config = defaultdict(dict) self.config = defaultdict(dict)

View File

@@ -1,7 +1,6 @@
from __future__ import annotations from __future__ import annotations
from ast import List from typing import Any, List
from typing import Any
from dataclasses import dataclass, field from dataclasses import dataclass, field
from dataclasses_json import dataclass_json, config from dataclasses_json import dataclass_json, config
import mimetypes import mimetypes

View File

@@ -1,7 +1,6 @@
from __future__ import annotations from __future__ import annotations
from ast import List, Set from typing import Any, List, Union, Dict
from typing import Any, Union, Dict
from dataclasses import dataclass, field from dataclasses import dataclass, field
from dataclasses_json import dataclass_json, config from dataclasses_json import dataclass_json, config
import datetime import datetime

View File

@@ -1,6 +1,5 @@
from __future__ import annotations from __future__ import annotations
from ast import List from typing import Generator, Union, List
from typing import Union
from .context import ArchivingContext from .context import ArchivingContext
@@ -10,7 +9,6 @@ from ..formatters import Formatter
from ..storages import Storage from ..storages import Storage
from ..enrichers import Enricher from ..enrichers import Enricher
from ..databases import Database from ..databases import Database
from .media import Media
from .metadata import Metadata from .metadata import Metadata
import tempfile, traceback import tempfile, traceback
@@ -29,9 +27,9 @@ class ArchivingOrchestrator:
for a in self.archivers: a.setup() for a in self.archivers: a.setup()
def feed(self) -> None: def feed(self) -> Generator[Metadata]:
for item in self.feeder: for item in self.feeder:
self.feed_item(item) yield self.feed_item(item)
def feed_item(self, item: Metadata) -> Metadata: def feed_item(self, item: Metadata) -> Metadata:
try: try:

View File

@@ -21,7 +21,7 @@ class Step(ABC):
def init(name: str, config: dict, child: Type[Step]) -> Step: def init(name: str, config: dict, child: Type[Step]) -> Step:
""" """
looks into direct subclasses of child for name and returns such ab object looks into direct subclasses of child for name and returns such an object
TODO: cannot find subclasses of child.subclasses TODO: cannot find subclasses of child.subclasses
""" """
for sub in child.__subclasses__(): for sub in child.__subclasses__():

View File

@@ -2,3 +2,4 @@ from .database import Database
from .gsheet_db import GsheetsDb from .gsheet_db import GsheetsDb
from .console_db import ConsoleDb from .console_db import ConsoleDb
from .csv_db import CSVDb from .csv_db import CSVDb
from .api_db import AAApiDb

View File

@@ -0,0 +1,41 @@
import requests, os
from loguru import logger
from . import Database
from ..core import Metadata
class AAApiDb(Database):
"""
Connects to auto-archiver-api instance
"""
name = "auto_archiver_api_db"
def __init__(self, config: dict) -> None:
# without this STEP.__init__ is not called
super().__init__(config)
self.assert_valid_string("api_endpoint")
self.assert_valid_string("api_secret")
@staticmethod
def configs() -> dict:
return {
"api_endpoint": {"default": None, "help": "API endpoint where calls are made to"},
"api_secret": {"default": None, "help": "API authentication secret"},
"public": {"default": False, "help": "whether the URL should be publicly available via the API"},
"author_id": {"default": None, "help": "which email to assign as author"},
"group_id": {"default": None, "help": "which group of users have access to the archive in case public=false as author"},
"tags": {"default": [], "help": "what tags to add to the archived URL", "cli_set": lambda cli_val, cur_val: set(cli_val.split(","))},
}
def done(self, item: Metadata) -> None:
"""archival result ready - should be saved to DB"""
logger.info(f"saving archive of {item.get_url()} to the AA API.")
payload = {'result': item.to_json(), 'public': self.public, 'author_id': self.author_id, 'group_id': self.group_id, 'tags': list(self.tags)}
response = requests.post(os.path.join(self.api_endpoint, "submit-archive"), json=payload, auth=("abc", self.api_secret))
if response.status_code == 200:
logger.success(f"AA API: {response.json()}")
else:
logger.error(f"AA API FAIL ({response.status_code}): {response.json()}")

View File

@@ -25,14 +25,8 @@ class WaczEnricher(Enricher):
} }
def enrich(self, to_enrich: Metadata) -> bool: def enrich(self, to_enrich: Metadata) -> bool:
# TODO: figure out support for browsertrix in docker
url = to_enrich.get_url() url = to_enrich.get_url()
if UrlUtil.is_auth_wall(url):
logger.debug(f"[SKIP] SCREENSHOT since url is behind AUTH WALL: {url=}")
return
collection = str(uuid.uuid4())[0:8] collection = str(uuid.uuid4())[0:8]
browsertrix_home = os.path.abspath(ArchivingContext.get_tmp_dir()) browsertrix_home = os.path.abspath(ArchivingContext.get_tmp_dir())
@@ -50,9 +44,10 @@ class WaczEnricher(Enricher):
"--saveState", "never", "--saveState", "never",
"--behaviors", "autoscroll,autoplay,autofetch,siteSpecific", "--behaviors", "autoscroll,autoplay,autofetch,siteSpecific",
"--behaviorTimeout", str(self.timeout), "--behaviorTimeout", str(self.timeout),
"--timeout", str(self.timeout), "--timeout", str(self.timeout)]
"--profile", str(self.profile)
] if self.profile:
cmd.extend(["--profile", os.path.join("/app", str(self.profile))])
else: else:
logger.debug(f"generating WACZ in Docker for {url=}") logger.debug(f"generating WACZ in Docker for {url=}")
@@ -75,9 +70,7 @@ class WaczEnricher(Enricher):
if self.profile: if self.profile:
profile_fn = os.path.join(browsertrix_home, "profile.tar.gz") profile_fn = os.path.join(browsertrix_home, "profile.tar.gz")
shutil.copyfile(self.profile, profile_fn) shutil.copyfile(self.profile, profile_fn)
# TODO: test which is right cmd.extend(["--profile", os.path.join("/crawls", "profile.tar.gz")])
cmd.extend(["--profile", profile_fn])
# cmd.extend(["--profile", "/crawls/profile.tar.gz"])
try: try:
logger.info(f"Running browsertrix-crawler: {' '.join(cmd)}") logger.info(f"Running browsertrix-crawler: {' '.join(cmd)}")

View File

@@ -39,7 +39,7 @@ class GsheetsFeeder(Gsheets, Feeder):
}) })
def __iter__(self) -> Metadata: def __iter__(self) -> Metadata:
sh = self.gsheets_client.open(self.sheet) sh = self.open_sheet()
for ii, wks in enumerate(sh.worksheets()): for ii, wks in enumerate(sh.worksheets()):
if not self.should_process_sheet(wks.title): if not self.should_process_sheet(wks.title):
logger.debug(f"SKIPPED worksheet '{wks.title}' due to allow/block rules") logger.debug(f"SKIPPED worksheet '{wks.title}' due to allow/block rules")
@@ -64,7 +64,10 @@ class GsheetsFeeder(Gsheets, Feeder):
# All checks done - archival process starts here # All checks done - archival process starts here
m = Metadata().set_url(url) m = Metadata().set_url(url)
ArchivingContext.set("gsheet", {"row": row, "worksheet": gw}, keep_on_reset=True) ArchivingContext.set("gsheet", {"row": row, "worksheet": gw}, keep_on_reset=True)
folder = slugify(gw.get_cell(row, 'folder').strip()) if gw.get_cell_or_default(row, 'folder', "") is None:
folder = ''
else:
folder = slugify(gw.get_cell_or_default(row, 'folder', "").strip())
if len(folder): if len(folder):
if self.use_sheet_names_in_stored_paths: if self.use_sheet_names_in_stored_paths:
ArchivingContext.set("folder", os.path.join(folder, slugify(self.sheet), slugify(wks.title)), True) ArchivingContext.set("folder", os.path.join(folder, slugify(self.sheet), slugify(wks.title)), True)

View File

@@ -10,16 +10,17 @@ class Gsheets(Step):
# without this STEP.__init__ is not called # without this STEP.__init__ is not called
super().__init__(config) super().__init__(config)
self.gsheets_client = gspread.service_account(filename=self.service_account) self.gsheets_client = gspread.service_account(filename=self.service_account)
#TODO: config should be responsible for conversions # TODO: config should be responsible for conversions
try: self.header = int(self.header) try: self.header = int(self.header)
except: pass except: pass
assert type(self.header) == int, f"header ({self.header}) value must be an integer not {type(self.header)}" assert type(self.header) == int, f"header ({self.header}) value must be an integer not {type(self.header)}"
assert self.sheet is not None, "You need to define a sheet name in your orchestration file when using gsheets." assert self.sheet is not None or self.sheet_id is not None, "You need to define either a 'sheet' name or a 'sheet_id' in your orchestration file when using gsheets."
@staticmethod @staticmethod
def configs() -> dict: def configs() -> dict:
return { return {
"sheet": {"default": None, "help": "name of the sheet to archive"}, "sheet": {"default": None, "help": "name of the sheet to archive"},
"sheet_id": {"default": None, "help": "(alternative to sheet name) the id of the sheet to archive"},
"header": {"default": 1, "help": "index of the header row (starts at 1)"}, "header": {"default": 1, "help": "index of the header row (starts at 1)"},
"service_account": {"default": "secrets/service_account.json", "help": "service account JSON file path"}, "service_account": {"default": "secrets/service_account.json", "help": "service account JSON file path"},
"columns": { "columns": {
@@ -42,3 +43,9 @@ class Gsheets(Step):
"cli_set": lambda cli_val, cur_val: dict(cur_val, **json.loads(cli_val)) "cli_set": lambda cli_val, cur_val: dict(cur_val, **json.loads(cli_val))
}, },
} }
def open_sheet(self):
if self.sheet:
return self.gsheets_client.open(self.sheet)
else: # self.sheet_id
return self.gsheets_client.open_by_key(self.sheet_id)

View File

@@ -40,3 +40,12 @@ class DateTimeEncoder(json.JSONEncoder):
def dump_payload(p): def dump_payload(p):
return json.dumps(p, ensure_ascii=False, indent=4, cls=DateTimeEncoder) return json.dumps(p, ensure_ascii=False, indent=4, cls=DateTimeEncoder)
def update_nested_dict(dictionary, update_dict):
# takes 2 dicts and overwrites the first with the second only on the changed balues
for key, value in update_dict.items():
if key in dictionary and isinstance(value, dict) and isinstance(dictionary[key], dict):
update_nested_dict(dictionary[key], value)
else:
dictionary[key] = value

View File

@@ -3,7 +3,7 @@ _MAJOR = "0"
_MINOR = "5" _MINOR = "5"
# On main and in a nightly release the patch should be one ahead of the last # On main and in a nightly release the patch should be one ahead of the last
# released build. # released build.
_PATCH = "12" _PATCH = "23"
# This is mainly for nightly builds which have the suffix ".dev$DATE". See # This is mainly for nightly builds which have the suffix ".dev$DATE". See
# https://semver.org/#is-v123-a-semantic-version for the semantics. # https://semver.org/#is-v123-a-semantic-version for the semantics.
_SUFFIX = "" _SUFFIX = ""