diff --git a/Pipfile b/Pipfile index cb39b78..fc582be 100644 --- a/Pipfile +++ b/Pipfile @@ -29,6 +29,7 @@ ocrd-pyexiftool = "*" gabber = {git = "https://github.com/stanfordio/gabber.git"} snscrape = {git = "https://github.com/bellingcat/snscrape"} polyphemus = {git = "https://github.com/bellingcat/polyphemus"} +filelock = "*" [dev-packages] pytest = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 6875bcd..ae4c410 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "13cc50755a59b2cd8bf93049a9a695aa27d35b973b0bdc154af5d21ce48fd57f" + "sha256": "7906836fd50f71b63c52c7d2fd6518d0ccb5a1b7316279a256abf7d18f692f4e" }, "pipfile-spec": 6, "requires": { @@ -21,46 +21,53 @@ "sha256:58d5c3d29f5a36ffeb94f02f0d786cd53014cf9b3b3951d42e0080d8a9498d30", "sha256:ad9aa55b65ef2808eb405f46cf74df7fcb7044d5cbc26487f96eb2ef2e436693" ], + "markers": "python_version >= '3.6'", "version": "==4.11.1" }, "blis": { "hashes": [ - "sha256:148f59a0a47a38ce82e3afc50c709494d5e5a494bef28ce1519c7a17346c645b", - "sha256:1667db8439d9ca41c0c1f0ea954d87462be01b125436c4b264f73603c9fb4e82", - "sha256:3e024f103522e72a27019cfcfe14569522a394f5d651565560a18040fdd69a6c", - "sha256:4a48eeaa506f176bcac306378f5e8063697c93e26d2418fcbe053e8912019090", - "sha256:5d4a81f9438db7a19ac8e64ad41331f65a659ea8f3bb1889a9c2088cfd9fe104", - "sha256:64bef63b1abd5b41819ea53897bdbc03c631a59c1757a9393e6ae0828692f31c", - "sha256:680480dfa16b354f2e4d584edb8d36f0505ed8df12939beee2d161aea7bb3609", - "sha256:76d13dbcd648ca33dfc83569bb219d0696e4f6e5ad00b9f538332a3bdb28ff30", - "sha256:7865e39cac4e10506afc49213938fb7e13bf73ca980c9c20ffad2de4ef858f43", - "sha256:929a6e70b800f9df505f08ed3e863bf5fd0a209aed45fb38a0fd2b8342c04981", - "sha256:a0183760604b14e8eb671a431d06606594def03c36aaaa2a2e7b7f88382dac76", - "sha256:b1e0567cde024e6ef677fe825d934baa7362cd71450c98e5198538026a86e896", - "sha256:b5e0acc760daf5c3b45bce44653943e3a04d81c21c5b92213ed51664525dc24e", - "sha256:bead485e5d79d3eb62a8df55618743878fb3cba606aaf926153db5803270b185", - "sha256:cfb7d730fef706f3ea4389196ce5f610f24cc83f828c498a275c12f05f0cf5c4", - "sha256:d6055ced65d6581ab4f1da0d3f6ec14c60512474c5c9b3210c9f30dd7dd1447d", - "sha256:e22145110864bcffb1d52cb57050b67b8a8ecd43c7c0a1ac0bcdb2c85c8bf416", - "sha256:ee19fddb5964570d97c2096a9a1e595fa48abdde187b14f99dcea7bb546989a6", - "sha256:f4109cce38e644e81d923836b34024905d59e88c8fb48b89b420f4d7661cd89f" + "sha256:0c3245e42c7c6ba0d68d7dff4171d11bb08174e639bc8edd52a1fd831de1d903", + "sha256:0f7bfdee74ac695c35360ace00f2630c1b47406dc0b99ba9211bfa8588bfbed9", + "sha256:159a1a9b32213d99d1415789ac66ed8d23442a696d9d376c66d7b791d3eae575", + "sha256:17df5ac7d9a9dbbf0415f8f8392fbdf1790fa394f89d695bae5e2e7e361c852b", + "sha256:1e970ba1eb12ca38fb5d57f379472125bc3f5106c8214dc847fe79b027212135", + "sha256:1f5fa330ab66d0e92a845b1db361ec8bf3dc4bc7e0dc0ded94f36b8e9f731650", + "sha256:2056b0d9722f5505cfa94904c6248021197c63ecf45804dcf117f8f1c6160ab6", + "sha256:25cdb9775699c1b926df514a5d4c28016c11722a66211f1024b2f21373f50de2", + "sha256:2778fe0ba0e25c157839fdd19ed66b9a340c92d4e92e707b7fa9aa21c51cb254", + "sha256:294421b720c2de904908de841464c667e1a5c5e9f3db6931dfa29cf369d3653a", + "sha256:2db369a4f95927be37e11790dd1ccbf99fd6201eaffbcf408546db847b7b5740", + "sha256:354cadff661a1452c886f541b84018770ddb4c134844c56e7c1a30aa4bcc473d", + "sha256:4e7b7b8bc8cf5e82958bbc393e0167318a930d394cbbf04c1ba18cfabaef5818", + "sha256:63735128c9cae44dc6cbf7557327385df0c4ed2dc2c45a00dabfde1e4d00802d", + "sha256:66b8ca1a2eb8f1e0563a592aae4b8682b66189ad560e3b8221d93eab0cb76582", + "sha256:90f17543e0aa3bc379d139867467df2c365ffaf5b61988de12dbba6dbbc9fab4", + "sha256:95d22d3007cb454d11a478331690629861f7d40b4668f9fccfd13b6507ed099b", + "sha256:96ff4c0c1ceab9f94c14b3281f3cef82f593c48c3b5f6169bd51cdcd315e0a6e", + "sha256:ae5b06fe3b94645ac5d93cbc7c0129639cc3e0d50b4efb361a20a9e160277a92", + "sha256:bf60f634481c3d0faf831ac4f2d1c75343e98f714dc88e3fb3c329758577e772", + "sha256:bfa56e7ef14ae607d8444eb344d22f252a2e0b0f9bfa4bdc9b0c48a9f96b5461", + "sha256:cac120e3c0517095d3c39278e8b6b1102b1add0e1f4e161a87f313d8ee7c12e1", + "sha256:f576ad64b772b6fd7df6ef94986235f321983dc870d0f76d78c931bafc41cfa4", + "sha256:f7d541bb06323aa350163ba4a3ad00e8effb3b53d4c58ee6228224f3928b6c57" ], - "version": "==0.7.7" + "version": "==0.7.8" }, "boto3": { "hashes": [ - "sha256:1c13d555172cf88eb645af2429e4a7f42be85e365d6ffc110c952a556d3f8808", - "sha256:4af6a8bc5110b5f9d2fbd00a3c110e4c4cc36fae78d05afa354831f5789e363b" + "sha256:67d404c643091d4aa37fc485193289ad859f1f65f94d0fa544e13bdd1d4187c1", + "sha256:c9a9f893561f64f5b81de197714ac4951251a328672a8dba28ad4c4a589c3adf" ], "index": "pypi", - "version": "==1.24.6" + "version": "==1.24.22" }, "botocore": { "hashes": [ - "sha256:97c909a6ec5ad421573c18ae67fc6ea4232502cd30cffaf03bfcb584d9df652d", - "sha256:eeebe304161db6828413dc358ea80ece52f4ddbc8ecde4dd58978d5861a09293" + "sha256:7145d9b7cae87999a9f074de700d02a1b3222ee7d1863aa631ff56c5fc868035", + "sha256:f57cb33446deef92e552b0be0e430d475c73cf64bc9e46cdb4783cdfe39cb6bb" ], - "version": "==1.27.6" + "markers": "python_version >= '3.7'", + "version": "==1.27.22" }, "brotli": { "hashes": [ @@ -142,6 +149,7 @@ "sha256:6a94c6402995a99c3970cc7e4884bb60b4a8639938157eeed436098bf9831757", "sha256:f9f17d2aec496a9aa6b76f53e3b614c965223c061982d434d160f930c698a9db" ], + "markers": "python_version ~= '3.7'", "version": "==5.2.0" }, "catalogue": { @@ -149,27 +157,31 @@ "sha256:535d33ae79ebd21ca298551d85da186ae8b8e1df36b0fb0246da774163ec2d6b", "sha256:cab4feda641fe05da1e6a1a9d123b0869d5ca324dcd93d4a5c384408ab62e7fb" ], + "markers": "python_version >= '3.6'", "version": "==2.0.7" }, "certifi": { "hashes": [ - "sha256:9c5705e395cd70084351dd8ad5c41e65655e08ce46f2ec9cf6c2c08390f71eb7", - "sha256:f1d53542ee8cbedbe2118b5686372fb33c297fcd6379b050cca0ef13a597382a" + "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d", + "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412" ], - "version": "==2022.5.18.1" + "markers": "python_version >= '3.6'", + "version": "==2022.6.15" }, "charset-normalizer": { "hashes": [ - "sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597", - "sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df" + "sha256:5189b6f22b01957427f35b6a08d9a0bc45b46d3788ef5a92e978433c7a35f8a5", + "sha256:575e708016ff3a5e3681541cb9d79312c416835686d054a23accb873b254f413" ], - "version": "==2.0.12" + "markers": "python_version >= '3.6'", + "version": "==2.1.0" }, "click": { "hashes": [ "sha256:6a7a62563bbfabfda3a38f3023a1db4a35978c0abd76f6c9605ecd6554d6d9b1", "sha256:8458d7b1287c5fb128c90e23381cf99dcde74beaf6c7ff6384ce84d6fe090adb" ], + "markers": "python_version >= '3.6'", "version": "==8.0.4" }, "cryptg": { @@ -220,6 +232,7 @@ "sha256:169725b5816959d34de2545b33fee6a8021a6e08818794a426c5a4f981f17e5e", "sha256:228bd261a85d92d870ed358f263ee028ac026302304f2186827377a3895c5819", "sha256:2aa3fa467d906cd2c27fa0a2e2952dd7925f5fcc7973fab6d815ef6acb25aad8", + "sha256:38b51ac23f914d85b197dbd0fb2d3e2de9bf6112b9b30f16b45dbb6c9b4e509d", "sha256:4749f220e4c06ec44eb10de13794ff0508cdc4f8eff656cf49cab2cdb3122c0c", "sha256:492084aef23ac2ff3da3729e9d36340bc91a96c2dc8c3a82a1926e384ab52412", "sha256:4f87fe087f2ae36c3e20e2b1a29d7f76a28c035372d0a97655f26223d975235a", @@ -232,9 +245,13 @@ "sha256:af3c01e6b20f9e6c07c7d7cdb7f710e49889d3906c9a3e039546ee6636a34b9a", "sha256:b8e1c18bb00800425576710468299153caad20c64ddb6819d40a6a34e21ee21c", "sha256:c59293b232b53ebb47427f16cf648e937022f489cff36c11d1d8a1f0075b6609", + "sha256:c59ca1072769cb6c3eade59de9e080ff2cecde0122f7e0ca0dc9ef2ed9240f0e", + "sha256:cd818356b635d8ae546e152a6f2b95f00e959d128a16155c275b0c202cd6312b", + "sha256:d4dc378fb9dda3b0529361fe32cfe1a6de0fc16bb40c710aaec8d217534928d2", "sha256:d7a59cef8f2fa25d12e2c30138f8623acbd43ad2715e730a709e49c5eef8e1b0", "sha256:dd52d8a81881804625df88453611175ab7e0099b34f52204da1f6940cf2e83c9", - "sha256:ea535f74ab6024e7416f93de564e5c81fb7c0964b96280de66f60aeb05f0cf53" + "sha256:ea535f74ab6024e7416f93de564e5c81fb7c0964b96280de66f60aeb05f0cf53", + "sha256:ee73a48c5a7e0f2acf6830ddc958ffafd7a614cfb79d14017a459bc7a7145ecd" ], "version": "==2.0.6" }, @@ -254,14 +271,24 @@ "index": "pypi", "version": "==0.2.0" }, + "filelock": { + "hashes": [ + "sha256:37def7b658813cda163b56fc564cdc75e86d338246458c4c28ae84cabefa2404", + "sha256:3a0fd85166ad9dbab54c9aec96737b744106dc5f15c0b09a6744a445299fcf04" + ], + "index": "pypi", + "version": "==3.7.1" + }, "future": { "hashes": [ "sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d" ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.18.2" }, "gabber": { - "git": "https://github.com/stanfordio/gabber.git" + "git": "https://github.com/stanfordio/gabber.git", + "ref": "a032db8047fa6b762b2fc127b08ee37d6ad9e110" }, "gogettr": { "hashes": [ @@ -273,78 +300,20 @@ }, "google-auth": { "hashes": [ - "sha256:8a954960f852d5f19e6af14dd8e75c20159609e85d8db37e4013cc8c3824a7e1", - "sha256:df549a1433108801b11bdcc0e312eaf0d5f0500db42f0523e4d65c78722e8475" + "sha256:3b2f9d2f436cc7c3b363d0ac66470f42fede249c3bafcc504e9f0bcbe983cff0", + "sha256:75b3977e7e22784607e074800048f44d6a56df589fb2abe58a11d4d20c97c314" ], - "version": "==2.7.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", + "version": "==2.9.0" }, "google-auth-oauthlib": { "hashes": [ "sha256:6d6161d0ec0a62e2abf2207c6071c117ec5897b300823c4bb2d963ee86e20e4f", "sha256:d5e98a71203330699f92a26bc08847a92e8c3b1b8d82a021f1af34164db143ae" ], + "markers": "python_version >= '3.6'", "version": "==0.5.2" }, - "greenlet": { - "hashes": [ - "sha256:004aed447382d80a56ecc354a6d807f305e6c808714ce6ccbca4839c94fae81d", - "sha256:068d68fad6bd623e29a2d36e74538c9b9d6dc6464931cd27d93da6cfc6a7f242", - "sha256:06fd4075754009c9817c6b4e1dc0af4616de52757b6ca973a81c3c1aadc28257", - "sha256:1004cb542451814b12a4f38e835a47734e2b2c683acbf463d5ae76282a3974cf", - "sha256:10c358633a8b27bfc32d27114ef2ca2ddc9f1f89f1643d1157b85e1fdd695315", - "sha256:115bc25fefbdc692c4483e9ddb9011ccd0251590ed59dbfff0f4eb7050bf99c4", - "sha256:1d987a2579336792f73ae6b106c2f087e32afc8573fbf9566f123ac6d8cfb72f", - "sha256:2128d727fd1e8afba8e68feb2cdcf88c90163b69ddc9707722a3e491c5280720", - "sha256:230132c241fe284f93f2e7b3969e9b22bbd76ef98cf93e382c945d378907f5a4", - "sha256:23558f7bd08a663386c032ab8d302d613d2d02ae0c9758ad410bab6035b58d3d", - "sha256:255d520d3e4a5f16883b182e1a94219fe455ab4f50aaaf534bfd6d64ee728397", - "sha256:2a6bc19a728f6f643cfc89b876159a1a25a8f7d8700c013d48a73691f80b4550", - "sha256:379bed346ef8ba0a0e698b3c5975a44d15dd4a5bbff40bbd7fd548b445d5550b", - "sha256:3b12d0866759db93b0a893b4e50a7d7d1681519d2346c26695bb8bb2c652230e", - "sha256:40d491944f69e350e1e8b25f6ca49459824ede1678ec0cd4b5541f41edc06614", - "sha256:471484c7b9d7b7867263051aa81cdeed6e06b455e629a7f05eb91a6cb8bd0836", - "sha256:488c557080557bc01aabb3e1bda7225c68455b853733a8652857ac0d810dad1b", - "sha256:49c2e76e7aa81ba889b3c183e2341af3cc6161ee38852085110ae49d5b5d9a40", - "sha256:52d13ec90236e5935ed6da044e78faa1371d5116cc43fe6d7ca8994dd619ef96", - "sha256:57898c69a253d81f487787bdd538629fabd671fab8a9e31b041ca30965fd9556", - "sha256:5d577eef5beb5730ef01ab39983eb852a97c359b7a546809adf70c409f4b2ecc", - "sha256:6a41987c1474c9158a0c0c96611530a8f299bc547d35bee8add981b8b2534f74", - "sha256:6ae67b7df8db3626af8e042e9c6949cfa27d1a3bbbfdff29e45b72bb6673a650", - "sha256:6c42c27e9d12e8a481aff469ffe8dd4ce0484c354a418470960f760f6ae41e7c", - "sha256:6c4a90c9f6128b4d0905a89930bd325e0491574e5cb453f606bb7094a3197587", - "sha256:6e64518e5833ac2d9359b6d9bd4df2c0cf441a0f3a4eca9e735fbea99009fa70", - "sha256:6fd3a270c23c5b42d86a9c7c6b0229f23ee4a7a4cabdaaa1693ad7a0982d13cb", - "sha256:70db73351e0fcf11a76288c47a0469d9a330bcb2e7618c5eb57432b8caa82403", - "sha256:771f401692046845626cbdf1dd0f04e999413ede0ee9ad39033fe30b5fa2e845", - "sha256:7935026ec61b967cbc6b746c0ca75c1651ea118d7fee4d259cff9e6866153374", - "sha256:7b76b1cac9baac1980210e29145800954e7b42e91ef69c4d695de1cab87ce41f", - "sha256:7e3f37c11b6699b1a1e0fcc0e88829dba4f2866546381b05ab8b3f4db645a823", - "sha256:8370fa65ad421484894f559055f951843754153b72b9bca2ebdc5288efe2e3f0", - "sha256:8ae9c443d44a4e23252632e4d7775f419f992d0df3eff923e23775f5cc551d39", - "sha256:8b31d85f2781e44f1ffaaf7ea07f484e7d42317c677c355fa77b4a1a4bea7394", - "sha256:8b450336b27f3b375cadc474c6704838eaa8dd3ca312aac3bb69d92264a8e638", - "sha256:9ce84357388a76d886febff4e50e321c212ffd3248b590960b2da6e02404a5c9", - "sha256:a23e986fb0ba8e7407286add41fa0d4207be44e3dce1b04789f4757800eca1cf", - "sha256:a81610ee00d0da9cd2c8679479b7791149365b6dfb3971b01b22ee29b04787ce", - "sha256:b4e40444975e5ab0ed3004369209c39a28e084951daaeee4919f164b6b849b14", - "sha256:b66600de16702b9dfa74bea34524b55183a2183e5fd92f20fe6c2fcae550a64c", - "sha256:ba6ee18694d3673796b7a31b7d21254e87e9e43ca5be56f323fd396111255315", - "sha256:bd03837da28293baa39bdfc3cada69e2f8807f423ae06168aa28d2b32c63a6b6", - "sha256:bd2192070f88c0778ae1d68a0980fdece3473498c1db37f3794e3454f91e3ecf", - "sha256:c1f6f1a3cc013012cd1da913c40b13e6d721046a8c8a0ea0cde94069645a75db", - "sha256:ce10a8e7e067bde3c1fbf494d2b8859db510206030b0b67bc3af90b0eb1887b9", - "sha256:d31386d208303a5a6cf0819ef9f6db6680bab9e4ca8e48adb3d4b26ead89beb7", - "sha256:d83b3af53b201970973c5574b39df226746194063bb248a53fd12b470ac34319", - "sha256:df9657b212c054ac6d803290d7c4bcd7790af0b725984fce1eeb0a1e3f2d9798", - "sha256:e576e5fd3f129e6b3595dc734ac7f2b8c548f19ef07781194bc538dc9c0cdbbc", - "sha256:e7400358558094c1bcedc75f3b3c4f400c53130b44833848890a99968dee6a64", - "sha256:eb6a385f8577d30e4cb43dd555fb134ddaae1edeb84205e09dabec332bf49fd0", - "sha256:f27f0875e0873f6bf5df09a456bfcac0667824cabac4cad30b43f36e0382ffe7", - "sha256:fcd4a6d04995f1d66bc78b503e4e59ae72fd32aaec4f661657fe5ae5c1aa4ce3" - ], - "markers": "python_version >= '3' and (platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32'))))))", - "version": "==2.0.0a2" - }, "gspread": { "hashes": [ "sha256:21704b47d007c3b5fd34eddfa4c4a9dcd1ecc1dc615083b9c636127726e66c18", @@ -358,34 +327,38 @@ "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff", "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d" ], + "markers": "python_version >= '3.5'", "version": "==3.3" }, "instaloader": { "hashes": [ - "sha256:77d4a140aafd1a9f48765db1f5ede9b74136eda67f428bfc392d7440b26ae74c" + "sha256:bb4a3a89d0476d5e30a1f9dfa2263c94ed6e2cbd880e5159fcf5843310b5346f" ], "index": "pypi", - "version": "==4.9.1" + "version": "==4.9.2" }, "jinja2": { "hashes": [ "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852", "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61" ], + "markers": "python_version >= '3.7'", "version": "==3.1.2" }, "jmespath": { "hashes": [ - "sha256:a490e280edd1f57d6de88636992d05b71e97d69a26a19f058ecf7d304474bf5e", - "sha256:e8dcd576ed616f14ec02eed0005c85973b5890083313860136657e24784e4c04" + "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", + "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe" ], - "version": "==1.0.0" + "markers": "python_version >= '3.7'", + "version": "==1.0.1" }, "langcodes": { "hashes": [ "sha256:4d89fc9acb6e9c8fdef70bcdf376113a3db09b67285d9e1d534de6d8818e7e69", "sha256:794d07d5a28781231ac335a1561b8442f8648ca07cd518310aeb45d6f0807ef6" ], + "markers": "python_version >= '3.6'", "version": "==3.3.0" }, "langdetect": { @@ -398,80 +371,88 @@ }, "loguru": { "hashes": [ - "sha256:066bd06758d0a513e9836fd9c6b5a75bfb3fd36841f4b996bc60b547a309d41c", - "sha256:4e2414d534a2ab57573365b3e6d0234dfb1d84b68b7f3b948e6fb743860a77c3" + "sha256:b28e72ac7a98be3d28ad28570299a393dfcd32e5e3f6a353dec94675767b6319", + "sha256:f8087ac396b5ee5f67c963b495d615ebbceac2796379599820e324419d53667c" ], "index": "pypi", - "version": "==0.6.0" + "version": "==0.5.3" }, "lxml": { "hashes": [ - "sha256:00f3a6f88fd5f4357844dd91a1abac5f466c6799f1b7f1da2df6665253845b11", - "sha256:024684e0c5cfa121c22140d3a0898a3a9b2ea0f0fd2c229b6658af4bdf1155e5", - "sha256:03370ec37fe562238d385e2c53089076dee53aabf8325cab964fdb04a9130fa0", - "sha256:0aa4cce579512c33373ca4c5e23c21e40c1aa1a33533a75e51b654834fd0e4f2", - "sha256:1057356b808d149bc14eb8f37bb89129f237df488661c1e0fc0376ca90e1d2c3", - "sha256:11d62c97ceff9bab94b6b29c010ea5fb6831743459bb759c917f49ba75601cd0", - "sha256:1254a79f8a67a3908de725caf59eae62d86738f6387b0a34b32e02abd6ae73db", - "sha256:1bfb791a8fcdbf55d1d41b8be940393687bec0e9b12733f0796668086d1a23ff", - "sha256:28cf04a1a38e961d4a764d2940af9b941b66263ed5584392ef875ee9c1e360a3", - "sha256:2b9c2341d96926b0d0e132e5c49ef85eb53fa92ae1c3a70f9072f3db0d32bc07", - "sha256:2d10659e6e5c53298e6d718fd126e793285bff904bb71d7239a17218f6a197b7", - "sha256:3af00ee88376022589ceeb8170eb67dacf5f7cd625ea59fa0977d719777d4ae8", - "sha256:3cf816aed8125cfc9e6e5c6c31ff94278320d591bd7970c4a0233bee0d1c8790", - "sha256:4becd16750ca5c2a1b1588269322b2cebd10c07738f336c922b658dbab96a61c", - "sha256:4cd69bca464e892ea4ed544ba6a7850aaff6f8d792f8055a10638db60acbac18", - "sha256:4e97c8fc761ad63909198acc892f34c20f37f3baa2c50a62d5ec5d7f1efc68a1", - "sha256:520461c36727268a989790aef08884347cd41f2d8ae855489ccf40b50321d8d7", - "sha256:53b0410b220766321759f7f9066da67b1d0d4a7f6636a477984cbb1d98483955", - "sha256:56e19fb6e4b8bd07fb20028d03d3bc67bcc0621347fbde64f248e44839771756", - "sha256:5a49ad78543925e1a4196e20c9c54492afa4f1502c2a563f73097e2044c75190", - "sha256:5d52e1173f52020392f593f87a6af2d4055dd800574a5cb0af4ea3878801d307", - "sha256:607224ffae9a0cf0a2f6e14f5f6bce43e83a6fbdaa647891729c103bdd6a5593", - "sha256:612ef8f2795a89ba3a1d4c8c1af84d8453fd53ee611aa5ad460fdd2cab426fc2", - "sha256:615886ee84b6f42f1bdf1852a9669b5fe3b96b6ff27f1a7a330b67ad9911200a", - "sha256:63419db39df8dc5564f6f103102c4665f7e4d9cb64030e98cf7a74eae5d5760d", - "sha256:6467626fa74f96f4d80fc6ec2555799e97fff8f36e0bfc7f67769f83e59cff40", - "sha256:65b3b5f12c6fb5611e79157214f3cd533083f9b058bf2fc8a1c5cc5ee40fdc5a", - "sha256:686565ac77ff94a8965c11829af253d9e2ce3bf0d9225b1d2eb5c4d4666d0dca", - "sha256:6af7f51a6010748fc1bb71917318d953c9673e4ae3f6d285aaf93ef5b2eb11c1", - "sha256:70a198030d26f5e569367f0f04509b63256faa76a22886280eea69a4f535dd40", - "sha256:754a1dd04bff8a509a31146bd8f3a5dc8191a8694d582dd5fb71ff09f0722c22", - "sha256:75da29a0752c8f2395df0115ac1681cefbdd4418676015be8178b733704cbff2", - "sha256:81c29c8741fa07ecec8ec7417c3d8d1e2f18cf5a10a280f4e1c3f8c3590228b2", - "sha256:9093a359a86650a3dbd6532c3e4d21a6f58ba2cb60d0e72db0848115d24c10ba", - "sha256:915ecf7d486df17cc65aeefdb680d5ad4390cc8c857cf8db3fe241ed234f856a", - "sha256:94b181dd2777890139e49a5336bf3a9a3378ce66132c665fe8db4e8b7683cde2", - "sha256:94f2e45b054dd759bed137b6e14ae8625495f7d90ddd23cf62c7a68f72b62656", - "sha256:9af19eb789d674b59a9bee5005779757aab857c40bf9cc313cb01eafac55ce55", - "sha256:9cae837b988f44925d14d048fa6a8c54f197c8b1223fd9ee9c27084f84606143", - "sha256:aa7447bf7c1a15ef24e2b86a277b585dd3f055e8890ac7f97374d170187daa97", - "sha256:b1e22f3ee4d75ca261b6bffbf64f6f178cb194b1be3191065a09f8d98828daa9", - "sha256:b5031d151d6147eac53366d6ec87da84cd4d8c5e80b1d9948a667a7164116e39", - "sha256:b62d1431b4c40cda43cc986f19b8c86b1d2ae8918cfc00f4776fdf070b65c0c4", - "sha256:b71c52d69b91af7d18c13aef1b0cc3baee36b78607c711eb14a52bf3aa7c815e", - "sha256:b7679344f2270840dc5babc9ccbedbc04f7473c1f66d4676bb01680c0db85bcc", - "sha256:bb7c1b029e54e26e01b1d1d912fc21abb65650d16ea9a191d026def4ed0859ed", - "sha256:c2a57755e366e0ac7ebdb3e9207f159c3bf1afed02392ab18453ce81f5ee92ee", - "sha256:cf9ec915857d260511399ab87e1e70fa13d6b2972258f8e620a3959468edfc32", - "sha256:d0d03b9636f1326772e6854459728676354d4c7731dae9902b180e2065ba3da6", - "sha256:d1690c4d37674a5f0cdafbc5ed7e360800afcf06928c2a024c779c046891bf09", - "sha256:d76da27f5e3e9bc40eba6ed7a9e985f57547e98cf20521d91215707f2fb57e0f", - "sha256:d882c2f3345261e898b9f604be76b61c901fbfa4ac32e3f51d5dc1edc89da3cb", - "sha256:d8e5021e770b0a3084c30dda5901d5fce6d4474feaf0ced8f8e5a82702502fbb", - "sha256:dd00d28d1ab5fa7627f5abc957f29a6338a7395b724571a8cbff8fbed83aaa82", - "sha256:e35a298691b9e10e5a5631f8f0ba605b30ebe19208dc8f58b670462f53753641", - "sha256:e4d020ecf3740b7312bacab2cb966bb720fd4d3490562d373b4ad91dd1857c0d", - "sha256:e564d5a771b4015f34166a05ea2165b7e283635c41b1347696117f780084b46d", - "sha256:ea3f2e9eb41f973f73619e88bf7bd950b16b4c2ce73d15f24a11800ce1eaf276", - "sha256:eabdbe04ee0a7e760fa6cd9e799d2b020d098c580ba99107d52e1e5e538b1ecb", - "sha256:f17b9df97c5ecdfb56c5e85b3c9df9831246df698f8581c6e111ac664c7c656e", - "sha256:f386def57742aacc3d864169dfce644a8c396f95aa35b41b69df53f558d56dd0", - "sha256:f6d23a01921b741774f35e924d418a43cf03eca1444f3fdfd7978d35a5aaab8b", - "sha256:fcdf70191f0d1761d190a436db06a46f05af60e1410e1507935f0332280c9268" + "sha256:04da965dfebb5dac2619cb90fcf93efdb35b3c6994fea58a157a834f2f94b318", + "sha256:0538747a9d7827ce3e16a8fdd201a99e661c7dee3c96c885d8ecba3c35d1032c", + "sha256:0645e934e940107e2fdbe7c5b6fb8ec6232444260752598bc4d09511bd056c0b", + "sha256:079b68f197c796e42aa80b1f739f058dcee796dc725cc9a1be0cdb08fc45b000", + "sha256:0f3f0059891d3254c7b5fb935330d6db38d6519ecd238ca4fce93c234b4a0f73", + "sha256:10d2017f9150248563bb579cd0d07c61c58da85c922b780060dcc9a3aa9f432d", + "sha256:1355755b62c28950f9ce123c7a41460ed9743c699905cbe664a5bcc5c9c7c7fb", + "sha256:13c90064b224e10c14dcdf8086688d3f0e612db53766e7478d7754703295c7c8", + "sha256:1423631e3d51008871299525b541413c9b6c6423593e89f9c4cfbe8460afc0a2", + "sha256:1436cf0063bba7888e43f1ba8d58824f085410ea2025befe81150aceb123e345", + "sha256:1a7c59c6ffd6ef5db362b798f350e24ab2cfa5700d53ac6681918f314a4d3b94", + "sha256:1e1cf47774373777936c5aabad489fef7b1c087dcd1f426b621fda9dcc12994e", + "sha256:206a51077773c6c5d2ce1991327cda719063a47adc02bd703c56a662cdb6c58b", + "sha256:21fb3d24ab430fc538a96e9fbb9b150029914805d551deeac7d7822f64631dfc", + "sha256:27e590352c76156f50f538dbcebd1925317a0f70540f7dc8c97d2931c595783a", + "sha256:287605bede6bd36e930577c5925fcea17cb30453d96a7b4c63c14a257118dbb9", + "sha256:2aaf6a0a6465d39b5ca69688fce82d20088c1838534982996ec46633dc7ad6cc", + "sha256:32a73c53783becdb7eaf75a2a1525ea8e49379fb7248c3eeefb9412123536387", + "sha256:41fb58868b816c202e8881fd0f179a4644ce6e7cbbb248ef0283a34b73ec73bb", + "sha256:4780677767dd52b99f0af1f123bc2c22873d30b474aa0e2fc3fe5e02217687c7", + "sha256:4878e667ebabe9b65e785ac8da4d48886fe81193a84bbe49f12acff8f7a383a4", + "sha256:487c8e61d7acc50b8be82bda8c8d21d20e133c3cbf41bd8ad7eb1aaeb3f07c97", + "sha256:49a866923e69bc7da45a0565636243707c22752fc38f6b9d5c8428a86121022c", + "sha256:4beea0f31491bc086991b97517b9683e5cfb369205dac0148ef685ac12a20a67", + "sha256:4cfbe42c686f33944e12f45a27d25a492cc0e43e1dc1da5d6a87cbcaf2e95627", + "sha256:4d5bae0a37af799207140652a700f21a85946f107a199bcb06720b13a4f1f0b7", + "sha256:4e285b5f2bf321fc0857b491b5028c5f276ec0c873b985d58d7748ece1d770dd", + "sha256:57e4d637258703d14171b54203fd6822fda218c6c2658a7d30816b10995f29f3", + "sha256:5974895115737a74a00b321e339b9c3f45c20275d226398ae79ac008d908bff7", + "sha256:5ef87fca280fb15342726bd5f980f6faf8b84a5287fcc2d4962ea8af88b35130", + "sha256:603a464c2e67d8a546ddaa206d98e3246e5db05594b97db844c2f0a1af37cf5b", + "sha256:6653071f4f9bac46fbc30f3c7838b0e9063ee335908c5d61fb7a4a86c8fd2036", + "sha256:6ca2264f341dd81e41f3fffecec6e446aa2121e0b8d026fb5130e02de1402785", + "sha256:6d279033bf614953c3fc4a0aa9ac33a21e8044ca72d4fa8b9273fe75359d5cca", + "sha256:6d949f53ad4fc7cf02c44d6678e7ff05ec5f5552b235b9e136bd52e9bf730b91", + "sha256:6daa662aba22ef3258934105be2dd9afa5bb45748f4f702a3b39a5bf53a1f4dc", + "sha256:6eafc048ea3f1b3c136c71a86db393be36b5b3d9c87b1c25204e7d397cee9536", + "sha256:830c88747dce8a3e7525defa68afd742b4580df6aa2fdd6f0855481e3994d391", + "sha256:86e92728ef3fc842c50a5cb1d5ba2bc66db7da08a7af53fb3da79e202d1b2cd3", + "sha256:8caf4d16b31961e964c62194ea3e26a0e9561cdf72eecb1781458b67ec83423d", + "sha256:8d1a92d8e90b286d491e5626af53afef2ba04da33e82e30744795c71880eaa21", + "sha256:8f0a4d179c9a941eb80c3a63cdb495e539e064f8054230844dcf2fcb812b71d3", + "sha256:9232b09f5efee6a495a99ae6824881940d6447debe272ea400c02e3b68aad85d", + "sha256:927a9dd016d6033bc12e0bf5dee1dde140235fc8d0d51099353c76081c03dc29", + "sha256:93e414e3206779ef41e5ff2448067213febf260ba747fc65389a3ddaa3fb8715", + "sha256:98cafc618614d72b02185ac583c6f7796202062c41d2eeecdf07820bad3295ed", + "sha256:9c3a88d20e4fe4a2a4a84bf439a5ac9c9aba400b85244c63a1ab7088f85d9d25", + "sha256:9f36de4cd0c262dd9927886cc2305aa3f2210db437aa4fed3fb4940b8bf4592c", + "sha256:a60f90bba4c37962cbf210f0188ecca87daafdf60271f4c6948606e4dabf8785", + "sha256:a614e4afed58c14254e67862456d212c4dcceebab2eaa44d627c2ca04bf86837", + "sha256:ae06c1e4bc60ee076292e582a7512f304abdf6c70db59b56745cca1684f875a4", + "sha256:b122a188cd292c4d2fcd78d04f863b789ef43aa129b233d7c9004de08693728b", + "sha256:b570da8cd0012f4af9fa76a5635cd31f707473e65a5a335b186069d5c7121ff2", + "sha256:bcaa1c495ce623966d9fc8a187da80082334236a2a1c7e141763ffaf7a405067", + "sha256:bd34f6d1810d9354dc7e35158aa6cc33456be7706df4420819af6ed966e85448", + "sha256:be9eb06489bc975c38706902cbc6888f39e946b81383abc2838d186f0e8b6a9d", + "sha256:c4b2e0559b68455c085fb0f6178e9752c4be3bba104d6e881eb5573b399d1eb2", + "sha256:c62e8dd9754b7debda0c5ba59d34509c4688f853588d75b53c3791983faa96fc", + "sha256:c852b1530083a620cb0de5f3cd6826f19862bafeaf77586f1aef326e49d95f0c", + "sha256:d9fc0bf3ff86c17348dfc5d322f627d78273eba545db865c3cd14b3f19e57fa5", + "sha256:dad7b164905d3e534883281c050180afcf1e230c3d4a54e8038aa5cfcf312b84", + "sha256:e5f66bdf0976ec667fc4594d2812a00b07ed14d1b44259d19a41ae3fff99f2b8", + "sha256:e8f0c9d65da595cfe91713bc1222af9ecabd37971762cb830dea2fc3b3bb2acf", + "sha256:edffbe3c510d8f4bf8640e02ca019e48a9b72357318383ca60e3330c23aaffc7", + "sha256:eea5d6443b093e1545ad0210e6cf27f920482bfcf5c77cdc8596aec73523bb7e", + "sha256:ef72013e20dd5ba86a8ae1aed7f56f31d3374189aa8b433e7b12ad182c0d2dfb", + "sha256:f05251bbc2145349b8d0b77c0d4e5f3b228418807b1ee27cefb11f69ed3d233b", + "sha256:f1be258c4d3dc609e654a1dc59d37b17d7fef05df912c01fc2e15eb43a9735f3", + "sha256:f9ced82717c7ec65a67667bb05865ffe38af0e835cdd78728f1209c8fffe0cad", + "sha256:fe17d10b97fdf58155f858606bddb4e037b805a60ae023c009f760d8361a4eb8", + "sha256:fe749b052bb7233fe5d072fcb549221a8cb1a16725c47c37e42b0b9cb3ff2c3f" ], "index": "pypi", - "version": "==4.9.0" + "version": "==4.9.1" }, "markupsafe": { "hashes": [ @@ -516,13 +497,18 @@ "sha256:f121a1420d4e173a5d96e47e9a0c0dcff965afdf1626d28de1460815f7c4ee7a", "sha256:fc7b548b17d238737688817ab67deebb30e8073c95749d55538ed473130ec0c7" ], + "markers": "python_version >= '3.7'", "version": "==2.1.1" }, "murmurhash": { "hashes": [ + "sha256:0548657e1e3088543d2339be13ba006e010b39fd9eeeb6609ba29a5a7622af9d", "sha256:0b317021f38505d48a9ab89ce32e3a06d7f3f32b06b16ceda8bb93c82eb6aea8", + "sha256:0feb951a169808c0468c66395cd7f7efb3a966fb1acd523ee4ad7442b3a4b9ce", "sha256:13723aeb2b0f5ebc96bbcf133410481b28bfc7341ce65ae86fc32f02b54a68c1", "sha256:2165e2d0e8fa806e5aacc7fd9e7e79c633618b23c11aa724192fad5dda6599ef", + "sha256:237e2217fa86834039574097f1d01aeb2fc6083891b554cda2ee3c33fd1da8d1", + "sha256:274b540360f2201b19614e9a8aa7600b1c00b4480685f767d09a6a7c3053639c", "sha256:2f0ef0c80b590b4ad5cd474771f0bed81ecdb1942c549319d6895fa985d96dc3", "sha256:3d2cc5e8ff2ee28b124bf32a944e31e5e164000233b772d72281f6b62568dc86", "sha256:400c6a0a96f4fee3a3a384091044acb54f509af8b582d339de329d82ac4585f6", @@ -535,6 +521,7 @@ "sha256:8797bc59cc5c0c6aa5019951d503be5329328ade5bc61d96348bcb8201ac6e52", "sha256:92bdc94f5c898f68ae2e334dd7385d801d666d3ea31d5fb64bb2654af8445cfc", "sha256:966d2efec6e01aa32c5774c44906724efca00da3507f06faa11acafb47ea1230", + "sha256:ce446c71105276008914e0a23a0f9e52aa5473ff23c086fd13ba39c90abce921", "sha256:dee39a6f4067cdfefb2156374de230f49405850bc3280eb787e8f6c8daefeb8d", "sha256:f53f16ef143f93127e9aa920a30cda11a799e172b28508c32fb538a82b487a0c", "sha256:f7da66418c84982eca3494528b54ded4185d10a6b3231d53e1a2c83751e701e6", @@ -547,40 +534,43 @@ "sha256:6397602efb3c2d7baebd2166ed85731ae1c1d475abca22090b7141ff5034b3e1", "sha256:9c9f243fcec7f410f138cb12c21c84c64fde4195481a30c9bfb05b5f003adfed" ], + "markers": "python_version >= '3.5' and python_version < '4.0'", "version": "==1.45.1" }, "numpy": { "hashes": [ - "sha256:020218fc82390f1d537cb193d6f1449a919ec97df69b5a64c0a1d017486e0032", - "sha256:04e4dbe6b777e977813e7ff5f43aa030ef4f6f75cbc1a4504d3135942b5c12fe", - "sha256:1c881827ff0ad7d607047c19a075a7d7c7125cc103fb969a9200bad26175fb9d", - "sha256:1f22f6f3cb7094ad77c8d352e4bfd2c1db1c38bc08d0b6c74e9b46343c53b052", - "sha256:279dce16b143bc50d49bab52dc279d6ab5b0edc7f4d2cc7edaf6a547586bda7e", - "sha256:2e66decdea13ae8091ba480209dd5ce31261fa3b021ec06b30bd2f4a304861b7", - "sha256:548f4d86aa259a448f2da0c07df070bf1f71b68c1f84b1356d4a2ed832598758", - "sha256:6fbd492bead87ab83240c56b3490ac301595ab1399ace3e3c1b7c130e3529358", - "sha256:785d6520f7bf10ff188762bc460579d6a31c11f960976b2a29efc383b0346572", - "sha256:804293d9bdf33f9c9fb0b4a753f9e84114bb0ad538d184fc579b30782326c827", - "sha256:82e69890c394a4e1cbcaf12b47d8477bbac4635866fc46a77670abbe4bb4085d", - "sha256:8e8a88657c028b8b77f3df6f266a5e6ffb4419cbc3dfb525cbbb80ba710f5da2", - "sha256:9793feff4758c68502f7652fab08e5ec427d9973d26014767cc15c1b1d885f56", - "sha256:9810b840a751b6f0c73c21fb2a50e306d7d0be4114cded4c7d069e142ce488cf", - "sha256:ae7e8801b93124a6b0becedc06285ddbaca2daab2d30e35ea413d3bec252717a", - "sha256:b7be00b0a76384490845395714e62f597e64bc6dc8f8a14be0e96034dde3667c", - "sha256:c308afc8ec782badd073999385a6c93c27ee68e6c0991697394d4fd56566af1f", - "sha256:c335800064f04e0b474b64779ab234ae23c0a5b2f5a06284bb07d297d73692bd", - "sha256:d17f7feb2cca596daa4b3dae86b611a13e9ace061e6583a8db21841f529ca891", - "sha256:dbc987d14f46ae4c476068543d3ad2a20e7ebcb06b211eb4292224dc136eb01d", - "sha256:f5a1c7c45ff29db501f9e38a360aedd833e355c14c75155ba2bd46ee3799e30a", - "sha256:fde47931544086a648b12ee7c9ccf30edd6c6db776005fb07e4a019a04980042" + "sha256:092f5e6025813e64ad6d1b52b519165d08c730d099c114a9247c9bb635a2a450", + "sha256:196cd074c3f97c4121601790955f915187736f9cf458d3ee1f1b46aff2b1ade0", + "sha256:1c29b44905af288b3919803aceb6ec7fec77406d8b08aaa2e8b9e63d0fe2f160", + "sha256:2b2da66582f3a69c8ce25ed7921dcd8010d05e59ac8d89d126a299be60421171", + "sha256:5043bcd71fcc458dfb8a0fc5509bbc979da0131b9d08e3d5f50fb0bbb36f169a", + "sha256:58bfd40eb478f54ff7a5710dd61c8097e169bc36cc68333d00a9bcd8def53b38", + "sha256:79a506cacf2be3a74ead5467aee97b81fca00c9c4c8b3ba16dbab488cd99ba10", + "sha256:94b170b4fa0168cd6be4becf37cb5b127bd12a795123984385b8cd4aca9857e5", + "sha256:97a76604d9b0e79f59baeca16593c711fddb44936e40310f78bfef79ee9a835f", + "sha256:98e8e0d8d69ff4d3fa63e6c61e8cfe2d03c29b16b58dbef1f9baa175bbed7860", + "sha256:ac86f407873b952679f5f9e6c0612687e51547af0e14ddea1eedfcb22466babd", + "sha256:ae8adff4172692ce56233db04b7ce5792186f179c415c37d539c25de7298d25d", + "sha256:bd3fa4fe2e38533d5336e1272fc4e765cabbbde144309ccee8675509d5cd7b05", + "sha256:d0d2094e8f4d760500394d77b383a1b06d3663e8892cdf5df3c592f55f3bff66", + "sha256:d54b3b828d618a19779a84c3ad952e96e2c2311b16384e973e671aa5be1f6187", + "sha256:d6ca8dabe696c2785d0c8c9b0d8a9b6e5fdbe4f922bde70d57fa1a2848134f95", + "sha256:d8cc87bed09de55477dba9da370c1679bd534df9baa171dd01accbb09687dac3", + "sha256:f0f18804df7370571fb65db9b98bf1378172bd4e962482b857e612d1fec0f53e", + "sha256:f1d88ef79e0a7fa631bb2c3dda1ea46b32b1fe614e10fedd611d3d5398447f2f", + "sha256:f9c3fc2adf67762c9fe1849c859942d23f8d3e0bee7b5ed3d4a9c3eeb50a2f07", + "sha256:fc431493df245f3c627c0c05c2bd134535e7929dbe2e602b80e42bf52ff760bc", + "sha256:fe8b9683eb26d2c4d5db32cd29b38fdcf8381324ab48313b5b69088e0e355379" ], - "version": "==1.23.0rc2" + "markers": "python_version >= '3.8'", + "version": "==1.23.0" }, "oauthlib": { "hashes": [ "sha256:23a8208d75b902797ea29fd31fa80a15ed9dc2c6c16fe73f5d346f83f6fa27a2", "sha256:6db33440354787f9b7f3a6dbd4febf5d0f93758354060e802f6c06cb493022fe" ], + "markers": "python_version >= '3.6'", "version": "==3.2.0" }, "ocrd-pyexiftool": { @@ -597,63 +587,116 @@ "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb", "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522" ], + "markers": "python_version >= '3.6'", "version": "==21.3" }, + "pandas": { + "hashes": [ + "sha256:07238a58d7cbc8a004855ade7b75bbd22c0db4b0ffccc721556bab8a095515f6", + "sha256:0daf876dba6c622154b2e6741f29e87161f844e64f84801554f879d27ba63c0d", + "sha256:16ad23db55efcc93fa878f7837267973b61ea85d244fc5ff0ccbcfa5638706c5", + "sha256:1d9382f72a4f0e93909feece6fef5500e838ce1c355a581b3d8f259839f2ea76", + "sha256:24ea75f47bbd5574675dae21d51779a4948715416413b30614c1e8b480909f81", + "sha256:2893e923472a5e090c2d5e8db83e8f907364ec048572084c7d10ef93546be6d1", + "sha256:2ff7788468e75917574f080cd4681b27e1a7bf36461fe968b49a87b5a54d007c", + "sha256:41fc406e374590a3d492325b889a2686b31e7a7780bec83db2512988550dadbf", + "sha256:48350592665ea3cbcd07efc8c12ff12d89be09cd47231c7925e3b8afada9d50d", + "sha256:605d572126eb4ab2eadf5c59d5d69f0608df2bf7bcad5c5880a47a20a0699e3e", + "sha256:6dfbf16b1ea4f4d0ee11084d9c026340514d1d30270eaa82a9f1297b6c8ecbf0", + "sha256:6f803320c9da732cc79210d7e8cc5c8019aad512589c910c66529eb1b1818230", + "sha256:721a3dd2f06ef942f83a819c0f3f6a648b2830b191a72bbe9451bcd49c3bd42e", + "sha256:755679c49460bd0d2f837ab99f0a26948e68fa0718b7e42afbabd074d945bf84", + "sha256:78b00429161ccb0da252229bcda8010b445c4bf924e721265bec5a6e96a92e92", + "sha256:958a0588149190c22cdebbc0797e01972950c927a11a900fe6c2296f207b1d6f", + "sha256:a3924692160e3d847e18702bb048dc38e0e13411d2b503fecb1adf0fcf950ba4", + "sha256:d51674ed8e2551ef7773820ef5dab9322be0828629f2cbf8d1fc31a0c4fed640", + "sha256:d5ebc990bd34f4ac3c73a2724c2dcc9ee7bf1ce6cf08e87bb25c6ad33507e318", + "sha256:d6c0106415ff1a10c326c49bc5dd9ea8b9897a6ca0c8688eb9c30ddec49535ef", + "sha256:e48fbb64165cda451c06a0f9e4c7a16b534fcabd32546d531b3c240ce2844112" + ], + "markers": "python_version >= '3.8'", + "version": "==1.4.3" + }, "pathy": { "hashes": [ - "sha256:25fd04cec6393661113086730ce69c789d121bea83ab1aa18452e8fd42faf29a", - "sha256:838624441f799a06b446a657e4ecc9ebc3fdd05234397e044a7c87e8f6e76b1c" + "sha256:3178215bdadf3741107d987020be0fb5b59888f60f96de43cce5fe45d9d4b64a", + "sha256:a7aa9794fade161bb4c28a33c5bc2c6bf41f61ec5eee51cfa8914f0a433447e1" ], - "version": "==0.6.1" + "markers": "python_version >= '3.6'", + "version": "==0.6.2" }, "pillow": { "hashes": [ - "sha256:088df396b047477dd1bbc7de6e22f58400dae2f21310d9e2ec2933b2ef7dfa4f", - "sha256:09e67ef6e430f90caa093528bd758b0616f8165e57ed8d8ce014ae32df6a831d", - "sha256:0b4d5ad2cd3a1f0d1df882d926b37dbb2ab6c823ae21d041b46910c8f8cd844b", - "sha256:0b525a356680022b0af53385944026d3486fc8c013638cf9900eb87c866afb4c", - "sha256:1d4331aeb12f6b3791911a6da82de72257a99ad99726ed6b63f481c0184b6fb9", - "sha256:20d514c989fa28e73a5adbddd7a171afa5824710d0ab06d4e1234195d2a2e546", - "sha256:2b291cab8a888658d72b575a03e340509b6b050b62db1f5539dd5cd18fd50578", - "sha256:3f6c1716c473ebd1649663bf3b42702d0d53e27af8b64642be0dd3598c761fb1", - "sha256:42dfefbef90eb67c10c45a73a9bc1599d4dac920f7dfcbf4ec6b80cb620757fe", - "sha256:488f3383cf5159907d48d32957ac6f9ea85ccdcc296c14eca1a4e396ecc32098", - "sha256:4d45dbe4b21a9679c3e8b3f7f4f42a45a7d3ddff8a4a16109dff0e1da30a35b2", - "sha256:53c27bd452e0f1bc4bfed07ceb235663a1df7c74df08e37fd6b03eb89454946a", - "sha256:55e74faf8359ddda43fee01bffbc5bd99d96ea508d8a08c527099e84eb708f45", - "sha256:59789a7d06c742e9d13b883d5e3569188c16acb02eeed2510fd3bfdbc1bd1530", - "sha256:5b650dbbc0969a4e226d98a0b440c2f07a850896aed9266b6fedc0f7e7834108", - "sha256:66daa16952d5bf0c9d5389c5e9df562922a59bd16d77e2a276e575d32e38afd1", - "sha256:6e760cf01259a1c0a50f3c845f9cad1af30577fd8b670339b1659c6d0e7a41dd", - "sha256:7502539939b53d7565f3d11d87c78e7ec900d3c72945d4ee0e2f250d598309a0", - "sha256:769a7f131a2f43752455cc72f9f7a093c3ff3856bf976c5fb53a59d0ccc704f6", - "sha256:7c150dbbb4a94ea4825d1e5f2c5501af7141ea95825fadd7829f9b11c97aaf6c", - "sha256:8844217cdf66eabe39567118f229e275f0727e9195635a15e0e4b9227458daaf", - "sha256:8a66fe50386162df2da701b3722781cbe90ce043e7d53c1fd6bd801bca6b48d4", - "sha256:9370d6744d379f2de5d7fa95cdbd3a4d92f0b0ef29609b4b1687f16bc197063d", - "sha256:937a54e5694684f74dcbf6e24cc453bfc5b33940216ddd8f4cd8f0f79167f765", - "sha256:9c857532c719fb30fafabd2371ce9b7031812ff3889d75273827633bca0c4602", - "sha256:a4165205a13b16a29e1ac57efeee6be2dfd5b5408122d59ef2145bc3239fa340", - "sha256:b3fe2ff1e1715d4475d7e2c3e8dabd7c025f4410f79513b4ff2de3d51ce0fa9c", - "sha256:b6617221ff08fbd3b7a811950b5c3f9367f6e941b86259843eab77c8e3d2b56b", - "sha256:b761727ed7d593e49671d1827044b942dd2f4caae6e51bab144d4accf8244a84", - "sha256:baf3be0b9446a4083cc0c5bb9f9c964034be5374b5bc09757be89f5d2fa247b8", - "sha256:c17770a62a71718a74b7548098a74cd6880be16bcfff5f937f900ead90ca8e92", - "sha256:c67db410508b9de9c4694c57ed754b65a460e4812126e87f5052ecf23a011a54", - "sha256:d78ca526a559fb84faaaf84da2dd4addef5edb109db8b81677c0bb1aad342601", - "sha256:e9ed59d1b6ee837f4515b9584f3d26cf0388b742a11ecdae0d9237a94505d03a", - "sha256:f054b020c4d7e9786ae0404278ea318768eb123403b18453e28e47cdb7a0a4bf", - "sha256:f372d0f08eff1475ef426344efe42493f71f377ec52237bf153c5713de987251", - "sha256:f3f6a6034140e9e17e9abc175fc7a266a6e63652028e157750bd98e804a8ed9a", - "sha256:ffde4c6fabb52891d81606411cbfaf77756e3b561b566efd270b3ed3791fde4e" + "sha256:0030fdbd926fb85844b8b92e2f9449ba89607231d3dd597a21ae72dc7fe26927", + "sha256:030e3460861488e249731c3e7ab59b07c7853838ff3b8e16aac9561bb345da14", + "sha256:0ed2c4ef2451de908c90436d6e8092e13a43992f1860275b4d8082667fbb2ffc", + "sha256:136659638f61a251e8ed3b331fc6ccd124590eeff539de57c5f80ef3a9594e58", + "sha256:13b725463f32df1bfeacbf3dd197fb358ae8ebcd8c5548faa75126ea425ccb60", + "sha256:1536ad017a9f789430fb6b8be8bf99d2f214c76502becc196c6f2d9a75b01b76", + "sha256:15928f824870535c85dbf949c09d6ae7d3d6ac2d6efec80f3227f73eefba741c", + "sha256:17d4cafe22f050b46d983b71c707162d63d796a1235cdf8b9d7a112e97b15bac", + "sha256:1802f34298f5ba11d55e5bb09c31997dc0c6aed919658dfdf0198a2fe75d5490", + "sha256:1cc1d2451e8a3b4bfdb9caf745b58e6c7a77d2e469159b0d527a4554d73694d1", + "sha256:1fd6f5e3c0e4697fa7eb45b6e93996299f3feee73a3175fa451f49a74d092b9f", + "sha256:254164c57bab4b459f14c64e93df11eff5ded575192c294a0c49270f22c5d93d", + "sha256:2ad0d4df0f5ef2247e27fc790d5c9b5a0af8ade9ba340db4a73bb1a4a3e5fb4f", + "sha256:2c58b24e3a63efd22554c676d81b0e57f80e0a7d3a5874a7e14ce90ec40d3069", + "sha256:2d33a11f601213dcd5718109c09a52c2a1c893e7461f0be2d6febc2879ec2402", + "sha256:337a74fd2f291c607d220c793a8135273c4c2ab001b03e601c36766005f36885", + "sha256:37ff6b522a26d0538b753f0b4e8e164fdada12db6c6f00f62145d732d8a3152e", + "sha256:3d1f14f5f691f55e1b47f824ca4fdcb4b19b4323fe43cc7bb105988cad7496be", + "sha256:408673ed75594933714482501fe97e055a42996087eeca7e5d06e33218d05aa8", + "sha256:4134d3f1ba5f15027ff5c04296f13328fecd46921424084516bdb1b2548e66ff", + "sha256:4ad2f835e0ad81d1689f1b7e3fbac7b01bb8777d5a985c8962bedee0cc6d43da", + "sha256:50dff9cc21826d2977ef2d2a205504034e3a4563ca6f5db739b0d1026658e004", + "sha256:510cef4a3f401c246cfd8227b300828715dd055463cdca6176c2e4036df8bd4f", + "sha256:5aed7dde98403cd91d86a1115c78d8145c83078e864c1de1064f52e6feb61b20", + "sha256:69bd1a15d7ba3694631e00df8de65a8cb031911ca11f44929c97fe05eb9b6c1d", + "sha256:6bf088c1ce160f50ea40764f825ec9b72ed9da25346216b91361eef8ad1b8f8c", + "sha256:6e8c66f70fb539301e064f6478d7453e820d8a2c631da948a23384865cd95544", + "sha256:727dd1389bc5cb9827cbd1f9d40d2c2a1a0c9b32dd2261db522d22a604a6eec9", + "sha256:74a04183e6e64930b667d321524e3c5361094bb4af9083db5c301db64cd341f3", + "sha256:75e636fd3e0fb872693f23ccb8a5ff2cd578801251f3a4f6854c6a5d437d3c04", + "sha256:7761afe0126d046974a01e030ae7529ed0ca6a196de3ec6937c11df0df1bc91c", + "sha256:7888310f6214f19ab2b6df90f3f06afa3df7ef7355fc025e78a3044737fab1f5", + "sha256:7b0554af24df2bf96618dac71ddada02420f946be943b181108cac55a7a2dcd4", + "sha256:7c7b502bc34f6e32ba022b4a209638f9e097d7a9098104ae420eb8186217ebbb", + "sha256:808add66ea764ed97d44dda1ac4f2cfec4c1867d9efb16a33d158be79f32b8a4", + "sha256:831e648102c82f152e14c1a0938689dbb22480c548c8d4b8b248b3e50967b88c", + "sha256:93689632949aff41199090eff5474f3990b6823404e45d66a5d44304e9cdc467", + "sha256:96b5e6874431df16aee0c1ba237574cb6dff1dcb173798faa6a9d8b399a05d0e", + "sha256:9a54614049a18a2d6fe156e68e188da02a046a4a93cf24f373bffd977e943421", + "sha256:a138441e95562b3c078746a22f8fca8ff1c22c014f856278bdbdd89ca36cff1b", + "sha256:a647c0d4478b995c5e54615a2e5360ccedd2f85e70ab57fbe817ca613d5e63b8", + "sha256:a9c9bc489f8ab30906d7a85afac4b4944a572a7432e00698a7239f44a44e6efb", + "sha256:ad2277b185ebce47a63f4dc6302e30f05762b688f8dc3de55dbae4651872cdf3", + "sha256:b6d5e92df2b77665e07ddb2e4dbd6d644b78e4c0d2e9272a852627cdba0d75cf", + "sha256:bc431b065722a5ad1dfb4df354fb9333b7a582a5ee39a90e6ffff688d72f27a1", + "sha256:bdd0de2d64688ecae88dd8935012c4a72681e5df632af903a1dca8c5e7aa871a", + "sha256:c79698d4cd9318d9481d89a77e2d3fcaeff5486be641e60a4b49f3d2ecca4e28", + "sha256:cb6259196a589123d755380b65127ddc60f4c64b21fc3bb46ce3a6ea663659b0", + "sha256:d5b87da55a08acb586bad5c3aa3b86505f559b84f39035b233d5bf844b0834b1", + "sha256:dcd7b9c7139dc8258d164b55696ecd16c04607f1cc33ba7af86613881ffe4ac8", + "sha256:dfe4c1fedfde4e2fbc009d5ad420647f7730d719786388b7de0999bf32c0d9fd", + "sha256:ea98f633d45f7e815db648fd7ff0f19e328302ac36427343e4432c84432e7ff4", + "sha256:ec52c351b35ca269cb1f8069d610fc45c5bd38c3e91f9ab4cbbf0aebc136d9c8", + "sha256:eef7592281f7c174d3d6cbfbb7ee5984a671fcd77e3fc78e973d492e9bf0eb3f", + "sha256:f07f1f00e22b231dd3d9b9208692042e29792d6bd4f6639415d2f23158a80013", + "sha256:f3fac744f9b540148fa7715a435d2283b71f68bfb6d4aae24482a890aed18b59", + "sha256:fa768eff5f9f958270b081bb33581b4b569faabf8774726b283edb06617101dc", + "sha256:fac2d65901fb0fdf20363fbd345c01958a742f2dc62a8dd4495af66e3ff502a4" ], - "version": "==9.1.1" + "markers": "python_version >= '3.7'", + "version": "==9.2.0" }, "polyphemus": { - "git": "https://github.com/bellingcat/polyphemus" + "git": "https://github.com/bellingcat/polyphemus", + "ref": "b18e5591fa4f903e5506742c2e3f17d45bb88755" }, "preshed": { "hashes": [ + "sha256:08e10bc02a92cdf95dbab7af386f7d995aaaf7715c15869c7dae0d6b162215fb", + "sha256:1ba02155edf20b16de5469f6c9d918d002520397ea458b43e80689070d984112", "sha256:3af09f4cfcdaca085fd87dac8107617c4e2bb0ad1458f953841b71e9728287f5", "sha256:58661bea8d0d63a648588511407285e43d43627e27f836e30819801fb3c75d70", "sha256:5f99837e7353ce1fa81f0074d4b15f36e0af5af60a2a54d4d11e13cb09768a9e", @@ -669,8 +712,11 @@ "sha256:ab8b5232255ebf7ee96e3cb4f1bedaace6ae0925d1113d4ede9d44c78f088ef2", "sha256:cfe1495fcfc7f479de840ddc4f426dbb55351e218ae5c8712c1269183a4d0060", "sha256:e03ae3eee961106a517fcd827b5a7c51f7317236b3e665c989054ab8dc381d28", + "sha256:ea00900c6fbbfa332055251e10dcf7f79ee1997d207fddb22787e0812f5022a5", "sha256:ea8aa9610837e907e8442e79300df0a861bfdb4dcaf026a5d9642a688ad04815", "sha256:eaffbc71fdb8625f9aac4fe7e19e20bf318d1421ea05903bebe3e6ffef27b587", + "sha256:eb5ea5b423daa2d5516ea0ac5e52745e44f8cb23bf55f80a38e81fcacfde8ba5", + "sha256:f2640258a158a83bc5c59944a42cf9998f7f8ba87b85009236818abfd04b2cc6", "sha256:f92e752a868ea2690e1b38c4b775251a145e0fce36b9bdd972539e8271b7a23a", "sha256:fb3b7588a3a0f2f2f1bf3fe403361b2b031212b73a37025aea1df7215af3772a" ], @@ -737,35 +783,39 @@ }, "pycryptodomex": { "hashes": [ - "sha256:1ca8e1b4c62038bb2da55451385246f51f412c5f5eabd64812c01766a5989b4a", - "sha256:298c00ea41a81a491d5b244d295d18369e5aac4b61b77b2de5b249ca61cd6659", - "sha256:2aa887683eee493e015545bd69d3d21ac8d5ad582674ec98f4af84511e353e45", - "sha256:2ce76ed0081fd6ac8c74edc75b9d14eca2064173af79843c24fa62573263c1f2", - "sha256:3da13c2535b7aea94cc2a6d1b1b37746814c74b6e80790daddd55ca5c120a489", - "sha256:406ec8cfe0c098fadb18d597dc2ee6de4428d640c0ccafa453f3d9b2e58d29e2", - "sha256:4d0db8df9ffae36f416897ad184608d9d7a8c2b46c4612c6bc759b26c073f750", - "sha256:530756d2faa40af4c1f74123e1d889bd07feae45bac2fd32f259a35f7aa74151", - "sha256:77931df40bb5ce5e13f4de2bfc982b2ddc0198971fbd947776c8bb5050896eb2", - "sha256:797a36bd1f69df9e2798e33edb4bd04e5a30478efc08f9428c087f17f65a7045", - "sha256:8085bd0ad2034352eee4d4f3e2da985c2749cb7344b939f4d95ead38c2520859", - "sha256:8536bc08d130cae6dcba1ea689f2913dfd332d06113904d171f2f56da6228e89", - "sha256:a4d412eba5679ede84b41dbe48b1bed8f33131ab9db06c238a235334733acc5e", - "sha256:aebecde2adc4a6847094d3bd6a8a9538ef3438a5ea84ac1983fcb167db614461", - "sha256:b276cc4deb4a80f9dfd47a41ebb464b1fe91efd8b1b8620cf5ccf8b824b850d6", - "sha256:b5a185ae79f899b01ca49f365bdf15a45d78d9856f09b0de1a41b92afce1a07f", - "sha256:c4d8977ccda886d88dc3ca789de2f1adc714df912ff3934b3d0a3f3d777deafb", - "sha256:c5dd3ffa663c982d7f1be9eb494a8924f6d40e2e2f7d1d27384cfab1b2ac0662", - "sha256:ca88f2f7020002638276439a01ffbb0355634907d1aa5ca91f3dc0c2e44e8f3b", - "sha256:d2cce1c82a7845d7e2e8a0956c6b7ed3f1661c9acf18eb120fc71e098ab5c6fe", - "sha256:d709572d64825d8d59ea112e11cc7faf6007f294e9951324b7574af4251e4de8", - "sha256:da8db8374295fb532b4b0c467e66800ef17d100e4d5faa2bbbd6df35502da125", - "sha256:e36c7e3b5382cd5669cf199c4a04a0279a43b2a3bdd77627e9b89778ac9ec08c", - "sha256:e95a4a6c54d27a84a4624d2af8bb9ee178111604653194ca6880c98dcad92f48", - "sha256:ee835def05622e0c8b1435a906491760a43d0c462f065ec9143ec4b8d79f8bff", - "sha256:f75009715dcf4a3d680c2338ab19dac5498f8121173a929872950f4fb3a48fbf", - "sha256:f8524b8bc89470cec7ac51734907818d3620fb1637f8f8b542d650ebec42a126" + "sha256:04cc393045a8f19dd110c975e30f38ed7ab3faf21ede415ea67afebd95a22380", + "sha256:0776bfaf2c48154ab54ea45392847c1283d2fcf64e232e85565f858baedfc1fa", + "sha256:0fadb9f7fa3150577800eef35f62a8a24b9ddf1563ff060d9bd3af22d3952c8c", + "sha256:18e2ab4813883ae63396c0ffe50b13554b32bb69ec56f0afaf052e7a7ae0d55b", + "sha256:191e73bc84a8064ad1874dba0ebadedd7cce4dedee998549518f2c74a003b2e1", + "sha256:35a8f7afe1867118330e2e0e0bf759c409e28557fb1fc2fbb1c6c937297dbe9a", + "sha256:3709f13ca3852b0b07fc04a2c03b379189232b24007c466be0f605dd4723e9d4", + "sha256:4540904c09704b6f831059c0dfb38584acb82cb97b0125cd52688c1f1e3fffa6", + "sha256:463119d7d22d0fc04a0f9122e9d3e6121c6648bcb12a052b51bd1eed1b996aa2", + "sha256:46b3f05f2f7ac7841053da4e0f69616929ca3c42f238c405f6c3df7759ad2780", + "sha256:48697790203909fab02a33226fda546604f4e2653f9d47bc5d3eb40879fa7c64", + "sha256:5676a132169a1c1a3712edf25250722ebc8c9102aa9abd814df063ca8362454f", + "sha256:65204412d0c6a8e3c41e21e93a5e6054a74fea501afa03046a388cf042e3377a", + "sha256:67e1e6a92151023ccdfcfbc0afb3314ad30080793b4c27956ea06ab1fb9bcd8a", + "sha256:6f5b6ba8aefd624834bc177a2ac292734996bb030f9d1b388e7504103b6fcddf", + "sha256:7341f1bb2dadb0d1a0047f34c3a58208a92423cdbd3244d998e4b28df5eac0ed", + "sha256:78d9621cf0ea35abf2d38fa2ca6d0634eab6c991a78373498ab149953787e5e5", + "sha256:8eecdf9cdc7343001d047f951b9cc805cd68cb6cd77b20ea46af5bffc5bd3dfb", + "sha256:94c7b60e1f52e1a87715571327baea0733708ab4723346598beca4a3b6879794", + "sha256:996e1ba717077ce1e6d4849af7a1426f38b07b3d173b879e27d5e26d2e958beb", + "sha256:a07a64709e366c2041cd5cfbca592b43998bf4df88f7b0ca73dca37071ccf1bd", + "sha256:b6306403228edde6e289f626a3908a2f7f67c344e712cf7c0a508bab3ad9e381", + "sha256:b9279adc16e4b0f590ceff581f53a80179b02cba9056010d733eb4196134a870", + "sha256:c4cb9cb492ea7dcdf222a8d19a1d09002798ea516aeae8877245206d27326d86", + "sha256:dd452a5af7014e866206d41751886c9b4bf379a339fdf2dbfc7dd16c0fb4f8e0", + "sha256:e2b12968522a0358b8917fc7b28865acac002f02f4c4c6020fcb264d76bfd06d", + "sha256:e3164a18348bd53c69b4435ebfb4ac8a4076291ffa2a70b54f0c4b80c7834b1d", + "sha256:e47bf8776a7e15576887f04314f5228c6527b99946e6638cf2f16da56d260cab", + "sha256:f8be976cec59b11f011f790b88aca67b4ea2bd286578d0bd3e31bcd19afcd3e4", + "sha256:fc9bc7a9b79fe5c750fc81a307052f8daabb709bdaabb0fb18fb136b66b653b5" ], - "version": "==3.14.1" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==3.15.0" }, "pydantic": { "hashes": [ @@ -792,6 +842,7 @@ "sha256:ea5cb40a3b23b3265f6325727ddfc45141b08ed665458be8c6285e7b85bd73a1", "sha256:fec866a0b59f372b7e776f2d7308511784dace622e0992a0b59ea3ccee0ae833" ], + "markers": "python_full_version >= '3.6.1'", "version": "==1.8.2" }, "pyparsing": { @@ -799,8 +850,17 @@ "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb", "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc" ], + "markers": "python_full_version >= '3.6.8'", "version": "==3.0.9" }, + "pysocks": { + "hashes": [ + "sha256:08e69f092cc6dbe92a0fdd16eeb9b9ffbc13cadfe5ca4c7bd92ffb078b293299", + "sha256:2725bd0a9925919b9b51739eea5f9e2bae91e83288108a9ad338b2e3a4435ee5", + "sha256:3f8804571ebe159c380ac6de37643bb4685970655d3bba243530d6558b799aa0" + ], + "version": "==1.7.1" + }, "pytesseract": { "hashes": [ "sha256:7e2bafc7f48d1bb71443ce4633a56f5e21925a98f220a36c336297edcd1956d0", @@ -814,6 +874,7 @@ "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.8.2" }, "pytz": { @@ -829,6 +890,7 @@ "sha256:8314c9692a636c8eb3bda879b9f119e350e93223ae83e70e80c31675a0fdc1a6", "sha256:af097bae1b616dde5c5744441e2ddc69e74dfdcb0c263129610d85b87445a59d" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", "version": "==0.1.0.post0" }, "ratelimit": { @@ -915,21 +977,26 @@ "sha256:fbc88d3ba402b5d041d204ec2449c4078898f89c4a6e6f0ed1c1a510ef1e221d", "sha256:fbd3fe37353c62fd0eb19fb76f78aa693716262bcd5f9c14bb9e5aca4b3f0dc4" ], + "markers": "python_version >= '3.6'", "version": "==2022.3.2" }, "requests": { + "extras": [ + "socks" + ], "hashes": [ - "sha256:bc7861137fbce630f17b03d3ad02ad0bf978c844f3536d0edda6499dafce2b6f", - "sha256:d568723a7ebd25875d8d1eaf5dfa068cd2fc8194b2e483d7b1f7c81918dbec6b" + "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983", + "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349" ], "index": "pypi", - "version": "==2.28.0" + "version": "==2.28.1" }, "requests-oauthlib": { "hashes": [ "sha256:2577c501a2fb8d05a304c09d090d6e47c306fef15809d102b327cf8364bddab5", "sha256:75beac4a47881eeb94d5ea5d6ad31ef88856affe2332b9aafb52c6452ccf0d7a" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.3.1" }, "rsa": { @@ -937,6 +1004,7 @@ "sha256:5c6bd9dc7a543b7fe4304a631f8a8a3b674e2bbfc49c2ae96200cdbe55df6b17", "sha256:95c5d300c4e879ee69708c428ba566c59478fd653cc3a22243eeb8ed846950bb" ], + "markers": "python_version >= '3.6' and python_version < '4.0'", "version": "==4.8" }, "s3transfer": { @@ -944,13 +1012,23 @@ "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd", "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947" ], + "markers": "python_version >= '3.7'", "version": "==0.6.0" }, + "setuptools": { + "hashes": [ + "sha256:16923d366ced322712c71ccb97164d07472abeecd13f3a6c283f6d5d26722793", + "sha256:db3b8e2f922b2a910a29804776c643ea609badb6a32c4bcc226fd4fd902cce65" + ], + "markers": "python_version >= '3.7'", + "version": "==63.1.0" + }, "six": { "hashes": [ "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.16.0" }, "smart-open": { @@ -958,16 +1036,19 @@ "sha256:71d14489da58b60ce12fc3ecb823facc59a8b23cd1b58edb97175640350d3a62", "sha256:75abf758717a92a8f53aa96953f0c245c8cedf8e1e4184903db3659b419d4c17" ], + "markers": "python_version >= '3.6' and python_version < '4.0'", "version": "==5.2.1" }, "snscrape": { - "git": "https://github.com/bellingcat/snscrape" + "git": "https://github.com/bellingcat/snscrape", + "ref": "0822a9c3548c4d0736a98f617d823b8475d24fda" }, "soupsieve": { "hashes": [ "sha256:3b2503d3c7084a42b1ebd08116e5f81aadfaea95863628c80a3b774a11b7c759", "sha256:fc53893b3da2c33de295667a0e19f078c14bf86544af307354de5fcf12a3f30d" ], + "markers": "python_version >= '3.6'", "version": "==2.3.2.post1" }, "spacy": { @@ -976,16 +1057,21 @@ "sha256:090e684eec551b5b7d56d9242cea18742515a706191ad158e32e16e8f2fe15ac", "sha256:0e9a98999b0fce03d4f483112837ac7111378449ace069c7cd050908f0fa5d9f", "sha256:2053cb78bcf4eec38aa266890a5700167a284d1a26197f851710d29f3d7071b3", + "sha256:35be636d02b25984b94922d8851aaf9dc61cc20770a39193a08c59c647991235", "sha256:36e9ef5a32834383d37bbd27fca49388e31e9b53f77c91ba8ccbf19af10e3aef", "sha256:3e4c6f298d54044582daca1142b082ee38831bb3d7bb931d2ee601e8b8dce64f", "sha256:400af3490c36c1b6c895de526ec06f6c7655af5ca595743c07e09e9bc8f378ea", "sha256:6db861f69f18ba5e00d0bd44744cf1662e00cc3b564d17a1ccdc4625ec3d5c3d", + "sha256:79df903edd757767f3bed8d87d81da81f22636b701a9b79e6e0d9d2cbe5eb4c5", "sha256:7e20c63ba47eaa33ebd4b2cc6eefa3e8906505273799138ad8ab231b146d8875", "sha256:87bd072ccacedbf8bc5a692fea1d5c320abd26821c63af157a7c95baa47dc36d", "sha256:89be328ff378e4cdcfb4dcf38ca2fad740f87213825ed10e8ce9f54b822277b8", + "sha256:955539fb34230457a5e555fc7692b3b9287b02e92655c285c6960dbf2269d765", "sha256:ac2288e87de1066ad65676e930f53978d6ee97c34044dca4d24f64a24e2a88b6", + "sha256:ace1353902dd47cf7db57bde9b35e6900d6e400bda7e9191e35e7f625226ceed", "sha256:bc9184973c9052e1bb9eeb975801e6906aacbe0c009533ec0c34f443832473fd", "sha256:cff47cdaa824802cd38ae94fe98af9cde6810d86334cd283659c868e0011831a", + "sha256:e6857d54b8de6f58b3091af1e3bd0838787c2a036872fabffe73ac153df7432d", "sha256:e759e27da39e469b6367b82281a10eb4e50de04260ba49d42091cbdfe2d99633", "sha256:ed29278fc89f07c1999ceca5f6702b379589c8e884a57816bdaeb05a1a7b2bbb" ], @@ -997,6 +1083,7 @@ "sha256:4f7dcbc4e6c8e8cb4eadbb009f9c0a1a2a67442e0032c8d6776c9470c3759903", "sha256:dfd58b0cc65b3596cb06f7b95e7bf4fff34668297c59eb179eb050db07b199df" ], + "markers": "python_version >= '3.6'", "version": "==3.0.9" }, "spacy-loggers": { @@ -1004,49 +1091,50 @@ "sha256:d48c9313a577ad1818da961cf6db71a73fd1e556ae47e6e68d7e28b541d11e18", "sha256:e75d44f4cf99e6763d7132ca7c8c420e0a92790222a08bc8eb9e24ea2c13536e" ], + "markers": "python_version >= '3.6'", "version": "==1.0.2" }, "sqlalchemy": { "hashes": [ - "sha256:06ec11a5e6a4b6428167d3ce33b5bd455c020c867dabe3e6951fa98836e0741d", - "sha256:0e7fd52e48e933771f177c2a1a484b06ea03774fc7741651ebdf19985a34037c", - "sha256:139c50b9384e6d32a74fc4dcd0e9717f343ed38f95dbacf832c782c68e3862f3", - "sha256:17417327b87a0f703c9a20180f75e953315207d048159aff51822052f3e33e69", - "sha256:29a742c29fea12259f1d2a9ee2eb7fe4694a85d904a4ac66d15e01177b17ad7f", - "sha256:2aac2a685feb9882d09f457f4e5586c885d578af4e97a2b759e91e8c457cbce5", - "sha256:3197441772dc3b1c6419f13304402f2418a18d7fe78000aa5a026e7100836739", - "sha256:3688f92c62db6c5df268e2264891078f17ecb91e3141b400f2e28d0f75796dea", - "sha256:3862a069a24f354145e01a76c7c720c263d62405fe5bed038c46a7ce900f5dd6", - "sha256:4a17c1a1152ca4c29d992714aa9df3054da3af1598e02134f2e7314a32ef69d8", - "sha256:4c1d9fb3931e27d59166bb5c4dcc911400fee51082cfba66ceb19ac954ade068", - "sha256:4e8706919829d455a9fa687c6bbd1b048e36fec3919a59f2d366247c2bfdbd9c", - "sha256:50c8eaf44c3fed5ba6758d375de25f163e46137c39fda3a72b9ee1d1bb327dfc", - "sha256:5e4e517ce72fad35cce364a01aff165f524449e9c959f1837dc71088afa2824c", - "sha256:6629c79967a6c92e33fad811599adf9bc5cee6e504a1027bbf9cc1b6fb2d276d", - "sha256:78363f400fbda80f866e8e91d37d36fe6313ff847ded08674e272873c1377ea5", - "sha256:7a44683cf97744a405103ef8fdd31199e9d7fc41b4a67e9044523b29541662b0", - "sha256:7e579d6e281cc937bdb59917017ab98e618502067e04efb1d24ac168925e1d2a", - "sha256:7ee34c85cbda7779d66abac392c306ec78c13f5c73a1f01b8b767916d4895d23", - "sha256:8b38e088659b30c2ca0af63e5d139fad1779a7925d75075a08717a21c406c0f6", - "sha256:9785d6f962d2c925aeb06a7539ac9d16608877da6aeaaf341984b3693ae80a02", - "sha256:a91d0668cada27352432f15b92ac3d43e34d8f30973fa8b86f5e9fddee928f3b", - "sha256:a940c551cfbd2e1e646ceea2777944425f5c3edff914bc808fe734d9e66f8d71", - "sha256:aaa0e90e527066409c2ea5676282cf4afb4a40bb9dce0f56c8ec2768bff22a6e", - "sha256:b4c92823889cf9846b972ee6db30c0e3a92c0ddfc76c6060a6cda467aa5fb694", - "sha256:b55932fd0e81b43f4aff397c8ad0b3c038f540af37930423ab8f47a20b117e4c", - "sha256:c37885f83b59e248bebe2b35beabfbea398cb40960cdc6d3a76eac863d4e1938", - "sha256:caca6acf3f90893d7712ae2c6616ecfeac3581b4cc677c928a330ce6fbad4319", - "sha256:cffc67cdd07f0e109a1fc83e333972ae423ea5ad414585b63275b66b870ea62b", - "sha256:d4c3b009c9220ae6e33f17b45f43fb46b9a1d281d76118405af13e26376f2e11", - "sha256:d58f2d9d1a4b1459e8956a0153a4119da80f54ee5a9ea623cd568e99459a3ef1", - "sha256:d6927c9e3965b194acf75c8e0fb270b4d54512db171f65faae15ef418721996e", - "sha256:d9050b0c4a7f5538650c74aaba5c80cd64450e41c206f43ea6d194ae6d060ff9", - "sha256:eec39a17bab3f69c44c9df4e0ed87c7306f2d2bf1eca3070af644927ec4199fa", - "sha256:f9940528bf9c4df9e3c3872d23078b6b2da6431c19565637c09f1b88a427a684", - "sha256:ffe487570f47536b96eff5ef2b84034a8ba4e19aab5ab7647e677d94a119ea55" + "sha256:047ef5ccd8860f6147b8ac6c45a4bc573d4e030267b45d9a1c47b55962ff0e6f", + "sha256:05a05771617bfa723ba4cef58d5b25ac028b0d68f28f403edebed5b8243b3a87", + "sha256:0ec54460475f0c42512895c99c63d90dd2d9cbd0c13491a184182e85074b04c5", + "sha256:107df519eb33d7f8e0d0d052128af2f25066c1a0f6b648fd1a9612ab66800b86", + "sha256:14ea8ff2d33c48f8e6c3c472111d893b9e356284d1482102da9678195e5a8eac", + "sha256:1745987ada1890b0e7978abdb22c133eca2e89ab98dc17939042240063e1ef21", + "sha256:1962dfee37b7fb17d3d4889bf84c4ea08b1c36707194c578f61e6e06d12ab90f", + "sha256:20bf65bcce65c538e68d5df27402b39341fabeecf01de7e0e72b9d9836c13c52", + "sha256:26146c59576dfe9c546c9f45397a7c7c4a90c25679492ff610a7500afc7d03a6", + "sha256:365b75938049ae31cf2176efd3d598213ddb9eb883fbc82086efa019a5f649df", + "sha256:4770eb3ba69ec5fa41c681a75e53e0e342ac24c1f9220d883458b5596888e43a", + "sha256:50e7569637e2e02253295527ff34666706dbb2bc5f6c61a5a7f44b9610c9bb09", + "sha256:5c2d19bfb33262bf987ef0062345efd0f54c4189c2d95159c72995457bf4a359", + "sha256:621f050e72cc7dfd9ad4594ff0abeaad954d6e4a2891545e8f1a53dcdfbef445", + "sha256:6d81de54e45f1d756785405c9d06cd17918c2eecc2d4262dc2d276ca612c2f61", + "sha256:6f95706da857e6e79b54c33c1214f5467aab10600aa508ddd1239d5df271986e", + "sha256:752ef2e8dbaa3c5d419f322e3632f00ba6b1c3230f65bc97c2ff5c5c6c08f441", + "sha256:7b2785dd2a0c044a36836857ac27310dc7a99166253551ee8f5408930958cc60", + "sha256:7f13644b15665f7322f9e0635129e0ef2098409484df67fcd225d954c5861559", + "sha256:8194896038753b46b08a0b0ae89a5d80c897fb601dd51e243ed5720f1f155d27", + "sha256:864d4f89f054819cb95e93100b7d251e4d114d1c60bc7576db07b046432af280", + "sha256:8b773c9974c272aae0fa7e95b576d98d17ee65f69d8644f9b6ffc90ee96b4d19", + "sha256:8f901be74f00a13bf375241a778455ee864c2c21c79154aad196b7a994e1144f", + "sha256:91d2b89bb0c302f89e753bea008936acfa4e18c156fb264fe41eb6bbb2bbcdeb", + "sha256:b0538b66f959771c56ff996d828081908a6a52a47c5548faed4a3d0a027a5368", + "sha256:b30e70f1594ee3c8902978fd71900d7312453922827c4ce0012fa6a8278d6df4", + "sha256:b71be98ef6e180217d1797185c75507060a57ab9cd835653e0112db16a710f0d", + "sha256:c6d00cb9da8d0cbfaba18cad046e94b06de6d4d0ffd9d4095a3ad1838af22528", + "sha256:d1f665e50592caf4cad3caed3ed86f93227bffe0680218ccbb293bd5a6734ca8", + "sha256:e6e2c8581c6620136b9530137954a8376efffd57fe19802182c7561b0ab48b48", + "sha256:e7a7667d928ba6ee361a3176e1bef6847c1062b37726b33505cc84136f657e0d", + "sha256:ec3985c883d6d217cf2013028afc6e3c82b8907192ba6195d6e49885bfc4b19d", + "sha256:ede13a472caa85a13abe5095e71676af985d7690eaa8461aeac5c74f6600b6c0", + "sha256:f24d4d6ec301688c59b0c4bb1c1c94c5d0bff4ecad33bb8f5d9efdfb8d8bc925", + "sha256:f2a42acc01568b9701665e85562bbff78ec3e21981c7d51d56717c22e5d3d58b", + "sha256:fbc076f79d830ae4c9d49926180a1140b49fa675d0f0d555b44c9a15b29f4c80" ], "index": "pypi", - "version": "==1.4.37" + "version": "==1.4.39" }, "srsly": { "hashes": [ @@ -1055,21 +1143,27 @@ "sha256:27b3f693296d8a24c306aacd5df38a565ec43214f2aeb51a38170af5dc8b48bc", "sha256:2c986766d83cf8f508ef2296da5263d47f68766122bbb0306d8bfbd83f596a6e", "sha256:2d0236feafe3805b384532221596e6749a54d0ff10ba022b333dc1de7aa1b2f7", + "sha256:451e269fd3c4735d160fb7001a9f405ab870389cc40eb70b620e128e1ddd921e", + "sha256:4940a841079d1e1d1b7c02da274970adef355c0d0e03a4e01dd43185ee2b51ac", "sha256:61e31a72370238387a8ff2a4cebea402227215a1450648b852cad9e511a8b59e", "sha256:62630dbf20e240610fa64b6717545fcc28d9f18a6085ee93656be000678592a6", "sha256:82cbf1ec388ed0c16f8062fee30dc54ba8513bd51aae0602570143c6d9218e4c", "sha256:97a67c8f86ce3207e5e810b998a94ea49d439139adc21d9aadbd0bfab9faa64b", + "sha256:98c0a3d66c2ffed495cae0320fa0c0cc576660534ff79a6b722c92468c6615fa", "sha256:a906c9b1f62c109ddcfaeaf242b19b2ebc5d2f865eb38ef4af35959027c5185b", "sha256:ab31586fd89e5e5fe6f38664209577b03e85fb834f238c928c15ed3c80ab9c73", "sha256:acbb14546da9bdf287dfefa0883e793ac563c7868eca32cd65504463980022fa", + "sha256:b36385c0c38c03735cfbb3bfda253ce39e14a06c13d06e9e4bd191cb39d79232", "sha256:cffec31143c6e1c783ead11245c08938cae859115d4cb0f4cf423e2895707b74", "sha256:d1d13dc2133d5a83d30774793adb2c3fd9be905da339e2d54e2c79d55248c1a5", "sha256:d3b93531f086c516a26f729beac9b052c2ad0528d72e80f9d193de26aa2202be", "sha256:dbe91f6dd4aea9e819493628356dc715bd9c606486297bb7ca5748e6e003841c", + "sha256:e4441f95c47cb8d586d05fdb42d757070ae06bb0479e7d43d26b7552bc4b5664", "sha256:f5ddcc5f36eb318d011c6f142e826c1ca15cb34bd5beab2f21fee62d4ae4d590", "sha256:f96af9fde9f58d5923091fa723fa0fed58a83781b98e143a5d1fac5e738b9f0d", "sha256:fb08416fd6ef04c51fdeefd6d28592b64563b2853243c571a9b0d67403b5be7f" ], + "markers": "python_version >= '3.6'", "version": "==2.4.3" }, "telethon": { @@ -1084,12 +1178,17 @@ "hashes": [ "sha256:042c518aa799a38bec22a7a0bf28df80ce617eb7de32bc049798707c0a36167f", "sha256:0f86b8ee3c2605f0de9925dd6f0fc2c1c2392c8ca4fbc93511eee97299c09260", + "sha256:2455798db0709d1b7eb156db9d44849c3a90e1f532727a890f40ca3a206f0701", "sha256:2d1b481a2c88796988ac6a2c755059aace586d2494bb186f709aa3981ead96b2", + "sha256:384da6b8806c709f8781236659f740b65f3d50c25382d722e046378706aef4d6", "sha256:3eb3e5c897cbd5501048666ef30e4fac5921941b735a91e6803a7cf714dacd92", "sha256:4737feae51aef428f2b7be03cabc28d4e4116968ec1419cf0508c460ee8a059d", "sha256:515e86cead73d42828db140efa36f6e1b826ac4401426236aa9fca5eb3e6f068", "sha256:55780308cae6119c75a7b6cfee154fded0a03692858c308032151151b37d1571", "sha256:73d454210e9cf11537887635ddeea4b2aad607886a6d4360524df10d57ff8272", + "sha256:73d7a0ffbc1f9d1fd5b34972ec0384a6f06e5edca5a065c23ff39bdae9be8fa3", + "sha256:9795cdaea647af8aedbb9b898ed0f53ae9a0abd6c6b31d61031e6bc013280c88", + "sha256:980004dc14923e762df69be26a6c094bc4a79d1e5b8d681cfd06ffbd2c23ec56", "sha256:9c42240d19bf7f02837fb5fe395c19b8e7ae8b5539dec7d4373555e1c940ab49", "sha256:b104ff10bb968d1625b9a81ef56f947a25e24b46069bbaf35fc3ea4562c92cbc", "sha256:c4a6e7e4c00ae560b799d269e7cf2b87379eaf15350a312405d93bd8c7076ce8", @@ -1102,6 +1201,7 @@ "sha256:eba973fe229e7fa86b99f2c5e2724f7f19040ac75a8ef7c8b23b434dac1eadea", "sha256:fd2d49a80a6c95be4eb0f8370a22eef903ecad10b65762d39c9b192abf905f7c" ], + "markers": "python_version >= '3.6'", "version": "==8.0.17" }, "tqdm": { @@ -1114,17 +1214,19 @@ }, "typer": { "hashes": [ - "sha256:5646aef0d936b2c761a10393f0384ee6b5c7fe0bb3e5cd710b17134ca1d99cff", - "sha256:e8467f0ebac0c81366c2168d6ad9f888efdfb6d4e1d3d5b4a004f46fa444b5c3" + "sha256:023bae00d1baf358a6cc7cea45851639360bb716de687b42b0a4641cd99173f1", + "sha256:b8261c6c0152dd73478b5ba96ba677e5d6948c715c310f7c91079f311f62ec03" ], - "version": "==0.4.1" + "markers": "python_version >= '3.6'", + "version": "==0.4.2" }, "typing-extensions": { "hashes": [ - "sha256:6657594ee297170d19f67d55c05852a874e7eb634f4f753dbd667855e07c1708", - "sha256:f1c24655a0da0d1b67f07e17a5e6b2a105894e6824b92096378bb3668ef02376" + "sha256:25642c956049920a5aa49edcdd6ab1e06d7e5d467fc00e0506c44ac86fbfca02", + "sha256:e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6" ], - "version": "==4.2.0" + "markers": "python_version >= '3.7'", + "version": "==4.3.0" }, "tzdata": { "hashes": [ @@ -1139,6 +1241,7 @@ "sha256:89885494684c929d9191c57aa27502afc87a579be5cdd3225c77c463ea043745", "sha256:ee5842fa3a795f023514ac2d801c4a81d1743bbe642e3940143326b3a00addd7" ], + "markers": "python_version >= '3.6'", "version": "==4.2" }, "urllib3": { @@ -1146,6 +1249,7 @@ "sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14", "sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4.0'", "version": "==1.26.9" }, "wasabi": { @@ -1206,15 +1310,16 @@ "sha256:fab7c640815812ed5f10fbee7abbf58788d602046b7bb3af9b1ac753a6d5e916", "sha256:fc06cc8073c8e87072138ba1e431300e2d408f054b27047d047b549455066ff4" ], + "markers": "python_version >= '3.7'", "version": "==10.3" }, "yt-dlp": { "hashes": [ - "sha256:3a7b59d2fb4b39ce8ba8e0b9c5a37fe20e5624f46a2346b4ae66ab1320e35134", - "sha256:deec1009442312c1e2ee5298966842194d0e950b433f0d4fc844ef464b9c32a7" + "sha256:5fbfac72fd035d11bc2693e5d1cd6933b1bc0712f742f5082a261703810bb5c9", + "sha256:a688f5cbc4a824456983774ccdd4a12befd379f6c92e25074fa85e7b8ce31704" ], "index": "pypi", - "version": "==2022.5.18" + "version": "==2022.6.29" } }, "develop": { @@ -1230,66 +1335,74 @@ "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4", "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==21.4.0" }, "babel": { "hashes": [ - "sha256:3f349e85ad3154559ac4930c3918247d319f21910d5ce4b25d439ed8693b98d2", - "sha256:98aeaca086133efb3e1e2aad0396987490c8425929ddbcfe0550184fdc54cd13" + "sha256:7614553711ee97490f732126dc077f8d0ae084ebc6a96e23db1482afabdb2c51", + "sha256:ff56f4892c1c4bf0d814575ea23471c230d544203c7748e8c68f0089478d48eb" ], - "version": "==2.10.1" + "markers": "python_version >= '3.6'", + "version": "==2.10.3" }, "black": { "hashes": [ - "sha256:06f9d8846f2340dfac80ceb20200ea5d1b3f181dd0556b47af4e8e0b24fa0a6b", - "sha256:10dbe6e6d2988049b4655b2b739f98785a884d4d6b85bc35133a8fb9a2233176", - "sha256:2497f9c2386572e28921fa8bec7be3e51de6801f7459dffd6e62492531c47e09", - "sha256:30d78ba6bf080eeaf0b7b875d924b15cd46fec5fd044ddfbad38c8ea9171043a", - "sha256:328efc0cc70ccb23429d6be184a15ce613f676bdfc85e5fe8ea2a9354b4e9015", - "sha256:35020b8886c022ced9282b51b5a875b6d1ab0c387b31a065b84db7c33085ca79", - "sha256:5795a0375eb87bfe902e80e0c8cfaedf8af4d49694d69161e5bd3206c18618bb", - "sha256:5891ef8abc06576985de8fa88e95ab70641de6c1fca97e2a15820a9b69e51b20", - "sha256:637a4014c63fbf42a692d22b55d8ad6968a946b4a6ebc385c5505d9625b6a464", - "sha256:67c8301ec94e3bcc8906740fe071391bce40a862b7be0b86fb5382beefecd968", - "sha256:6d2fc92002d44746d3e7db7cf9313cf4452f43e9ea77a2c939defce3b10b5c82", - "sha256:6ee227b696ca60dd1c507be80a6bc849a5a6ab57ac7352aad1ffec9e8b805f21", - "sha256:863714200ada56cbc366dc9ae5291ceb936573155f8bf8e9de92aef51f3ad0f0", - "sha256:9b542ced1ec0ceeff5b37d69838106a6348e60db7b8fdd245294dc1d26136265", - "sha256:a6342964b43a99dbc72f72812bf88cad8f0217ae9acb47c0d4f141a6416d2d7b", - "sha256:ad4efa5fad66b903b4a5f96d91461d90b9507a812b3c5de657d544215bb7877a", - "sha256:bc58025940a896d7e5356952228b68f793cf5fcb342be703c3a2669a1488cb72", - "sha256:cc1e1de68c8e5444e8f94c3670bb48a2beef0e91dddfd4fcc29595ebd90bb9ce", - "sha256:cee3e11161dde1b2a33a904b850b0899e0424cc331b7295f2a9698e79f9a69a0", - "sha256:e3556168e2e5c49629f7b0f377070240bd5511e45e25a4497bb0073d9dda776a", - "sha256:e8477ec6bbfe0312c128e74644ac8a02ca06bcdb8982d4ee06f209be28cdf163", - "sha256:ee8f1f7228cce7dffc2b464f07ce769f478968bfb3dd1254a4c2eeed84928aad", - "sha256:fd57160949179ec517d32ac2ac898b5f20d68ed1a9c977346efbac9c2f1e779d" + "sha256:074458dc2f6e0d3dab7928d4417bb6957bb834434516f21514138437accdbe90", + "sha256:187d96c5e713f441a5829e77120c269b6514418f4513a390b0499b0987f2ff1c", + "sha256:2ea29072e954a4d55a2ff58971b83365eba5d3d357352a07a7a4df0d95f51c78", + "sha256:4af5bc0e1f96be5ae9bd7aaec219c901a94d6caa2484c21983d043371c733fc4", + "sha256:560558527e52ce8afba936fcce93a7411ab40c7d5fe8c2463e279e843c0328ee", + "sha256:568ac3c465b1c8b34b61cd7a4e349e93f91abf0f9371eda1cf87194663ab684e", + "sha256:6797f58943fceb1c461fb572edbe828d811e719c24e03375fd25170ada53825e", + "sha256:6c1734ab264b8f7929cef8ae5f900b85d579e6cbfde09d7387da8f04771b51c6", + "sha256:6c6d39e28aed379aec40da1c65434c77d75e65bb59a1e1c283de545fb4e7c6c9", + "sha256:7ba9be198ecca5031cd78745780d65a3f75a34b2ff9be5837045dce55db83d1c", + "sha256:94783f636bca89f11eb5d50437e8e17fbc6a929a628d82304c80fa9cd945f256", + "sha256:a218d7e5856f91d20f04e931b6f16d15356db1c846ee55f01bac297a705ca24f", + "sha256:a3db5b6409b96d9bd543323b23ef32a1a2b06416d525d27e0f67e74f1446c8f2", + "sha256:ac609cf8ef5e7115ddd07d85d988d074ed00e10fbc3445aee393e70164a2219c", + "sha256:b154e6bbde1e79ea3260c4b40c0b7b3109ffcdf7bc4ebf8859169a6af72cd70b", + "sha256:b270a168d69edb8b7ed32c193ef10fd27844e5c60852039599f9184460ce0807", + "sha256:b9fd45787ba8aa3f5e0a0a98920c1012c884622c6c920dbe98dbd05bc7c70fbf", + "sha256:c85928b9d5f83b23cee7d0efcb310172412fbf7cb9d9ce963bd67fd141781def", + "sha256:c9a3ac16efe9ec7d7381ddebcc022119794872abce99475345c5a61aa18c45ad", + "sha256:cfaf3895a9634e882bf9d2363fed5af8888802d670f58b279b0bece00e9a872d", + "sha256:e439798f819d49ba1c0bd9664427a05aab79bfba777a6db94fd4e56fae0cb849", + "sha256:f586c26118bc6e714ec58c09df0157fe2d9ee195c764f630eb0d8e7ccce72e69", + "sha256:f6fe02afde060bbeef044af7996f335fbe90b039ccf3f5eb8f16df8b20f77666" ], "index": "pypi", - "version": "==22.3.0" + "version": "==22.6.0" }, "certifi": { "hashes": [ - "sha256:9c5705e395cd70084351dd8ad5c41e65655e08ce46f2ec9cf6c2c08390f71eb7", - "sha256:f1d53542ee8cbedbe2118b5686372fb33c297fcd6379b050cca0ef13a597382a" + "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d", + "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412" ], - "version": "==2022.5.18.1" + "markers": "python_version >= '3.6'", + "version": "==2022.6.15" }, "charset-normalizer": { "hashes": [ - "sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597", - "sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df" + "sha256:5189b6f22b01957427f35b6a08d9a0bc45b46d3788ef5a92e978433c7a35f8a5", + "sha256:575e708016ff3a5e3681541cb9d79312c416835686d054a23accb873b254f413" ], - "version": "==2.0.12" + "markers": "python_version >= '3.6'", + "version": "==2.1.0" }, "click": { "hashes": [ "sha256:6a7a62563bbfabfda3a38f3023a1db4a35978c0abd76f6c9605ecd6554d6d9b1", "sha256:8458d7b1287c5fb128c90e23381cf99dcde74beaf6c7ff6384ce84d6fe090adb" ], + "markers": "python_version >= '3.6'", "version": "==8.0.4" }, "coverage": { + "extras": [ + "toml" + ], "hashes": [ "sha256:01c5615d13f3dd3aa8543afc069e5319cfa0c7d712f6e04b920431e5c564a749", "sha256:106c16dfe494de3193ec55cac9640dd039b66e196e4641fa8ac396181578b982", @@ -1333,6 +1446,7 @@ "sha256:fdb6f7bd51c2d1714cea40718f6149ad9be6a2ee7d93b19e9f00934c0f2a74d9", "sha256:ffa9297c3a453fba4717d06df579af42ab9a28022444cae7fa605af4df612d54" ], + "markers": "python_version >= '3.7'", "version": "==6.4.1" }, "docutils": { @@ -1340,6 +1454,7 @@ "sha256:686577d2e4c32380bb50cbb22f575ed742d58168cee37e99117a854bcd88f125", "sha256:cf316c8370a737a022b72b56874f6602acf974a37a9fba42ec2876387549fc61" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==0.17.1" }, "idna": { @@ -1347,22 +1462,24 @@ "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff", "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d" ], + "markers": "python_version >= '3.5'", "version": "==3.3" }, "imagesize": { "hashes": [ - "sha256:1db2f82529e53c3e929e8926a1fa9235aa82d0bd0c580359c67ec31b2fddaa8c", - "sha256:cd1750d452385ca327479d45b64d9c7729ecf0b3969a58148298c77092261f9d" + "sha256:0d8d18d08f840c19d0ee7ca1fd82490fdc3729b7ac93f49870406ddde8ef8d8b", + "sha256:69150444affb9cb0d5cc5a92b3676f0b2fb7cd9ae39e947a5e11a36b4497cd4a" ], - "version": "==1.3.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.4.1" }, "importlib-metadata": { "hashes": [ - "sha256:5d26852efe48c0a32b0509ffbc583fda1a2266545a78d104a6f4aff3db17d700", - "sha256:c58c8eb8a762858f49e18436ff552e83914778e50e9d2f1660535ffb364552ec" + "sha256:637245b8bab2b6502fcbc752cc4b7a6f6243bb02b31c5c26156ad103d3d45670", + "sha256:7401a975809ea1fdc658c3aa4f78cc2195a0e019c5cbc4c06122884e9ae80c23" ], "markers": "python_version < '3.10'", - "version": "==4.11.4" + "version": "==4.12.0" }, "iniconfig": { "hashes": [ @@ -1376,6 +1493,7 @@ "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852", "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61" ], + "markers": "python_version >= '3.7'", "version": "==3.1.2" }, "markupsafe": { @@ -1421,6 +1539,7 @@ "sha256:f121a1420d4e173a5d96e47e9a0c0dcff965afdf1626d28de1460815f7c4ee7a", "sha256:fc7b548b17d238737688817ab67deebb30e8073c95749d55538ed473130ec0c7" ], + "markers": "python_version >= '3.7'", "version": "==2.1.1" }, "mypy-extensions": { @@ -1435,6 +1554,7 @@ "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb", "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522" ], + "markers": "python_version >= '3.6'", "version": "==21.3" }, "pathspec": { @@ -1449,6 +1569,7 @@ "sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788", "sha256:58c8abb07dcb441e6ee4b11d8df0ac856038f944ab98b7be6b27b2a3c7feef19" ], + "markers": "python_version >= '3.7'", "version": "==2.5.2" }, "pluggy": { @@ -1456,6 +1577,7 @@ "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159", "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3" ], + "markers": "python_version >= '3.6'", "version": "==1.0.0" }, "py": { @@ -1463,6 +1585,7 @@ "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719", "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==1.11.0" }, "pygments": { @@ -1470,6 +1593,7 @@ "sha256:5eb116118f9612ff1ee89ac96437bb6b49e8f04d8a13b514ba26f620208e26eb", "sha256:dc9c10fb40944260f6ed4c688ece0cd2048414940f1cea51b8b226318411c519" ], + "markers": "python_version >= '3.6'", "version": "==2.12.0" }, "pyparsing": { @@ -1477,6 +1601,7 @@ "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb", "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc" ], + "markers": "python_full_version >= '3.6.8'", "version": "==3.0.9" }, "pytest": { @@ -1520,12 +1645,15 @@ "version": "==2022.1" }, "requests": { + "extras": [ + "socks" + ], "hashes": [ - "sha256:bc7861137fbce630f17b03d3ad02ad0bf978c844f3536d0edda6499dafce2b6f", - "sha256:d568723a7ebd25875d8d1eaf5dfa068cd2fc8194b2e483d7b1f7c81918dbec6b" + "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983", + "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349" ], "index": "pypi", - "version": "==2.28.0" + "version": "==2.28.1" }, "snowballstemmer": { "hashes": [ @@ -1536,11 +1664,11 @@ }, "sphinx": { "hashes": [ - "sha256:36aa2a3c2f6d5230be94585bc5d74badd5f9ed8f3388b8eedc1726fe45b1ad30", - "sha256:f4da1187785a5bc7312cc271b0e867a93946c319d106363e102936a3d9857306" + "sha256:b18e978ea7565720f26019c702cd85c84376e948370f1cd43d60265010e1c7b0", + "sha256:d3e57663eed1d7c5c50895d191fdeda0b54ded6f44d5621b50709466c338d1e8" ], "index": "pypi", - "version": "==5.0.1" + "version": "==5.0.2" }, "sphinx-rtd-theme": { "hashes": [ @@ -1555,6 +1683,7 @@ "sha256:806111e5e962be97c29ec4c1e7fe277bfd19e9652fb1a4392105b43e01af885a", "sha256:a072735ec80e7675e3f432fcae8610ecf509c5f1869d17e2eecff44389cdbc58" ], + "markers": "python_version >= '3.5'", "version": "==1.0.2" }, "sphinxcontrib-devhelp": { @@ -1562,6 +1691,7 @@ "sha256:8165223f9a335cc1af7ffe1ed31d2871f325254c0423bc0c4c7cd1c1e4734a2e", "sha256:ff7f1afa7b9642e7060379360a67e9c41e8f3121f2ce9164266f61b9f4b338e4" ], + "markers": "python_version >= '3.5'", "version": "==1.0.2" }, "sphinxcontrib-htmlhelp": { @@ -1569,6 +1699,7 @@ "sha256:d412243dfb797ae3ec2b59eca0e52dac12e75a241bf0e4eb861e450d06c6ed07", "sha256:f5f8bb2d0d629f398bf47d0d69c07bc13b65f75a81ad9e2f71a63d4b7a2f6db2" ], + "markers": "python_version >= '3.6'", "version": "==2.0.0" }, "sphinxcontrib-jsmath": { @@ -1576,6 +1707,7 @@ "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178", "sha256:a9925e4a4587247ed2191a22df5f6970656cb8ca2bd6284309578f2153e0c4b8" ], + "markers": "python_version >= '3.5'", "version": "==1.0.1" }, "sphinxcontrib-qthelp": { @@ -1583,6 +1715,7 @@ "sha256:4c33767ee058b70dba89a6fc5c1892c0d57a54be67ddd3e7875a18d14cba5a72", "sha256:bd9fc24bcb748a8d51fd4ecaade681350aa63009a347a8c14e637895444dfab6" ], + "markers": "python_version >= '3.5'", "version": "==1.0.3" }, "sphinxcontrib-serializinghtml": { @@ -1590,6 +1723,7 @@ "sha256:352a9a00ae864471d3a7ead8d7d79f5fc0b57e8b3f95e9867eb9eb28999b92fd", "sha256:aa5f6de5dfdf809ef505c4895e51ef5c9eac17d0f287933eb49ec495280b6952" ], + "markers": "python_version >= '3.5'", "version": "==1.1.5" }, "tomli": { @@ -1597,20 +1731,23 @@ "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f" ], + "markers": "python_version >= '3.7'", "version": "==2.0.1" }, "typing-extensions": { "hashes": [ - "sha256:6657594ee297170d19f67d55c05852a874e7eb634f4f753dbd667855e07c1708", - "sha256:f1c24655a0da0d1b67f07e17a5e6b2a105894e6824b92096378bb3668ef02376" + "sha256:25642c956049920a5aa49edcdd6ab1e06d7e5d467fc00e0506c44ac86fbfca02", + "sha256:e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6" ], - "version": "==4.2.0" + "markers": "python_version >= '3.7'", + "version": "==4.3.0" }, "urllib3": { "hashes": [ "sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14", "sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4.0'", "version": "==1.26.9" }, "zipp": { @@ -1618,6 +1755,7 @@ "sha256:56bf8aadb83c24db6c4b577e13de374ccfb67da2078beba1d037c17980bf43ad", "sha256:c4f6e5bbf48e74f7a38e7cc5b0480ff42b0ae5178957d564d18932525d5cf099" ], + "markers": "python_version >= '3.7'", "version": "==3.8.0" } } diff --git a/app.py b/app.py index 9fc0cb4..b51f007 100644 --- a/app.py +++ b/app.py @@ -100,6 +100,12 @@ def transform_info(args): controller = get_transformer_controller() controller.transform_all_untransformed_info() +def transform_media(args): + logger.info(f"Transforming untransformed channel media") + + controller = get_transformer_controller() + controller.transform_all_untransformed_media() + def init_db(): engine = create_engine(os.environ["DB"]) mapper_registry.metadata.create_all(bind=engine) @@ -144,5 +150,8 @@ if __name__ == "__main__": elif args.command == "transform-info": logger.add("logs/transform-info.log", level="TRACE", rotation="100 MB") transform_info(args) + elif args.command == "transform-media": + logger.add("logs/transform-media.log", level="TRACE", rotation="100 MB") + transform_media(args) else: logger.error(f"Unrecognized command {args.command}") diff --git a/cisticola/base.py b/cisticola/base.py index 28ff9f8..2a0cb9e 100644 --- a/cisticola/base.py +++ b/cisticola/base.py @@ -326,6 +326,24 @@ class Media: #: Original URL of the media from the the original post. original_url: str + #: String specifying name and version of scraper used to generate result, e.g. ``"TwitterScraper 0.0.1"``. + scraper: str + + #: String specifying name and version of transformer used to tranform result, e.g. ``"TwitterTransformer 0.0.1"``. + transformer: str + + #: Name of platform from which result was scraped, e.g. ``"Twitter"``. + platform: str + + #: Datetime (relative to UTC) that the scraped post was created at. + date: datetime + + #: Datetime (relative to UTC) that the scraped post was archived at. + date_archived: datetime + + #: Datetime (UTC) that the scraped post was transformed at. + date_transformed: datetime + #: JSON dump of the dict containing metadata information for the media file. exif: str = None @@ -384,7 +402,14 @@ class Image(Media): @dataclass class Video(Media): - """Class for organizing information about an image file. + """Class for organizing information about an video file. + """ + + pass + +@dataclass +class Audio(Media): + """Class for organizing information about an audio file. """ pass @@ -490,7 +515,13 @@ media_table = Table('media', mapper_registry.metadata, Column('url', String), Column('original_url', String), Column('exif', String), - Column('ocr', String)) + Column('ocr', String), + Column('date', DateTime, index=True), + Column('date_archived', DateTime, index=True), + Column('date_transformed', DateTime, index=True), + Column('scraper', String), + Column('transformer', String) + ) mapper_registry.map_imperatively(Post, post_table) mapper_registry.map_imperatively(Channel, channel_table) @@ -499,4 +530,5 @@ mapper_registry.map_imperatively(RawChannelInfo, raw_channel_info_table) mapper_registry.map_imperatively(ChannelInfo, channel_info_table) mapper_registry.map_imperatively(Media, media_table, polymorphic_on='type', polymorphic_identity='media') mapper_registry.map_imperatively(Image, media_table, inherits=Media, polymorphic_on='type', polymorphic_identity='image') -mapper_registry.map_imperatively(Video, media_table, inherits=Media, polymorphic_on='type', polymorphic_identity='video') \ No newline at end of file +mapper_registry.map_imperatively(Video, media_table, inherits=Media, polymorphic_on='type', polymorphic_identity='video') +mapper_registry.map_imperatively(Audio, media_table, inherits=Media, polymorphic_on='type', polymorphic_identity='audio') diff --git a/cisticola/scraper/base.py b/cisticola/scraper/base.py index 973fc55..43e7352 100644 --- a/cisticola/scraper/base.py +++ b/cisticola/scraper/base.py @@ -421,6 +421,9 @@ class ScraperController: session.commit() added += 1 + if added > 100: + break + session.commit() logger.info( f"{scraper} found {added} new posts from {channel}") diff --git a/cisticola/transformer/base.py b/cisticola/transformer/base.py index 20ca1e1..d42f24b 100644 --- a/cisticola/transformer/base.py +++ b/cisticola/transformer/base.py @@ -1,12 +1,13 @@ from typing import List, Generator, Union, Callable from loguru import logger +from sqlalchemy import cast, String from sqlalchemy.orm import sessionmaker, make_transient from sqlalchemy.engine.base import Engine from sqlalchemy.sql.expression import func from collections import defaultdict -from datetime import datetime +from datetime import datetime, timezone -from cisticola.base import RawChannelInfo, ChannelInfo, ScraperResult, Post, Media, Channel, mapper_registry +from cisticola.base import RawChannelInfo, ChannelInfo, ScraperResult, Post, Media, Channel, mapper_registry, Image, Video, Audio class Transformer: @@ -50,6 +51,24 @@ class Transformer: pass + def transform_media(self, data: ScraperResult, transformed: Post, insert: Callable): + '''Transform media''' + for k in data.archived_urls: + if data.archived_urls[k]: + archived_url = data.archived_urls[k] + filename = archived_url.split('/')[-1] + ext = None if '.' not in filename else filename.split('.')[-1].lower() + + if ext == 'mp4' or ext == 'mov' or ext == 'avi' or ext =='mkv': + insert(Video(url=archived_url, post=transformed.id, raw_id=data.id, original_url=k, date=data.date, date_archived=data.date_archived, date_transformed=datetime.now(timezone.utc), transformer=self.__version__, scraper=data.scraper, platform=data.platform)) + elif ext == 'oga' or ext == 'mp3' or ext == "wav" or ext == 'aif' or ext == 'aiff' or ext == 'aac': + insert(Audio(url=archived_url, post=transformed.id, raw_id=data.id, original_url=k, date=data.date, date_archived=data.date_archived, date_transformed=datetime.now(timezone.utc), transformer=self.__version__, scraper=data.scraper, platform=data.platform)) + elif ext == 'jpg' or ext == 'jpeg' or ext == 'png' or ext == 'gif' or ext == 'bmp' or ext == 'heic' or ext == 'tiff': + insert(Image(url=archived_url, post=transformed.id, raw_id=data.id, original_url=k, date=data.date, date_archived=data.date_archived, date_transformed=datetime.now(timezone.utc), transformer=self.__version__, scraper=data.scraper, platform=data.platform)) + else: + logger.warning(f"Unknown file extension {ext}") + insert(Media(url=archived_url, post=transformed.id, raw_id=data.id, original_url=k, date=data.date, date_archived=data.date_archived, date_transformed=datetime.now(timezone.utc), transformer=self.__version__, scraper=data.scraper, platform=data.platform)) + class ETLController: """An ETLController will transform raw scraped data (ScrapedResult objects) into a more detailed format @@ -103,33 +122,35 @@ class ETLController: # instance = session.query(Post).filter_by(platform=obj.platform, platform_id=obj.platform_id).first() elif issubclass(type(obj), Media): - instance = session.query(type(obj)).filter_by(original_url=obj.original_url, post=obj.post).first() - if instance: - logger.info(f"Found matching DB entry for {obj}: {instance}") - return instance + instance = None + # instance = session.query(type(obj)).filter_by(original_url=obj.original_url, post=obj.post).first() + # if instance: + # logger.info(f"Found matching DB entry for {obj}: {instance}") + # return instance - instance = session.query(type(obj)).filter_by(original_url=obj.original_url).first() + # instance = session.query(type(obj)).filter_by(original_url=obj.original_url).first() - # For Media objects we want to duplicate the entry to preserve the relationship with the post. - # However, we also want to avoid rehydration, hence the code below: - if instance: - logger.info(f"Found matching media record, duplicating and inserting for new post") + # # For Media objects we want to duplicate the entry to preserve the relationship with the post. + # # However, we also want to avoid rehydration, hence the code below: + # if instance: + # logger.info(f"Found matching media record, duplicating and inserting for new post") - session.expunge(instance) - make_transient(instance) - instance.id = None - instance.post = obj.post - instance.raw_id = obj.raw_id + # session.expunge(instance) + # make_transient(instance) + # instance.id = None + # instance.post = obj.post + # instance.raw_id = obj.raw_id - session.add(instance) - session.flush() - return instance + # session.add(instance) + # session.flush() + # return instance if instance: logger.info(f"Found matching DB entry for {obj}: {instance}") return instance - if hydrate: + # Don't hydrate videos, because they can be quite large and this is time consuming + if hydrate and type(obj) != Video: obj.hydrate() session.add(obj) @@ -274,3 +295,77 @@ class ETLController: logger.info(f"Found {len(batch)} info items to ETL ({offset} already processed)") self.transform_info(batch) + + @logger.catch(reraise=True) + def transform_media(self, results: List, hydrate: bool = True): + """Transforms raw ScraperResults objects into Post objects and + Media objects. Then, adds them to the database. + + Parameters + ---------- + results : List[ScraperResult] + A list of ScraperResult objects to be transformed + hydrate : bool + Whether or not to fully hydrate transformed media. Default True. + """ + if self.session is None: + logger.error("No DB session") + return + + session = self.session() + + for total_result in results: + result = total_result.ScraperResult + if result.scraper is not None and result.platform is not None: + for transformer in self.transformers: + handled = False + + if transformer.can_handle(result): + logger.trace(f"{transformer} is handling result {result.id} ({result.date})") + handled = True + + transformer.transform_media(result, total_result.Post, lambda obj: self.insert_or_select(obj, session, hydrate)) + + session.commit() + break + + if handled == False: + logger.warning(f"No Transformer could handle ID {result.id} with platform {result.platform} ({result.date})") + + @logger.catch(reraise=True) + def transform_all_untransformed_media(self, hydrate=True): + """Transform all ScraperResult objects in the database that do not have an + equivalent Post object stored. + + Parameters + ---------- + hydrate : bool + Whether or not to fully hydrate transformed media. Default True. + """ + + if self.session is None: + logger.error("No DB session") + return + + session = self.session() + + BATCH_SIZE = 5000 + offset = 0 + batch = [] + + query = (session.query(ScraperResult, Post) + .join(Post) + .join(Media, isouter=True) + .filter((ScraperResult.media_archived != None) & (cast(ScraperResult.archived_urls, String) != '{}') & (Media.id == None)) + .order_by(ScraperResult.date.asc()) + ) + + while len(batch) > 0 or offset == 0: + logger.info(f"Fetching untransformed post media batch of {BATCH_SIZE}, offset {offset}") + + batch = query.slice(offset, offset + BATCH_SIZE).all() + offset += BATCH_SIZE + + logger.info(f"Found {len(batch)} items to ETL ({offset} already processed)") + + self.transform_media(batch, hydrate=hydrate) diff --git a/cisticola/transformer/bitchute.py b/cisticola/transformer/bitchute.py index c8dff36..3619f75 100644 --- a/cisticola/transformer/bitchute.py +++ b/cisticola/transformer/bitchute.py @@ -21,13 +21,13 @@ class BitchuteTransformer(Transformer): return False - def transform_media(self, data: ScraperResult, insert: Callable, transformed: Post) -> Generator[Media, None, None]: + def transform_media(self, data: ScraperResult, transformed: Post, insert: Callable) -> Generator[Media, None, None]: raw = json.loads(data.raw_data) orig = raw['video_url'] new = data.archived_urls[orig] - m = Video(url=new, post=transformed.id, raw_id=data.id, original_url=orig) + m = Video(url=new, post=transformed.id, raw_id=data.id, original_url=orig, date=data.date, date_archived=data.date_archived, date_transformed=datetime.now(timezone.utc), transformer=self.__version__, scraper=data.scraper, platform=data.platform) insert(m) diff --git a/cisticola/transformer/telegram_telethon.py b/cisticola/transformer/telegram_telethon.py index b5326e2..169d2f2 100644 --- a/cisticola/transformer/telegram_telethon.py +++ b/cisticola/transformer/telegram_telethon.py @@ -16,7 +16,7 @@ from datetime import datetime, timezone from sqlalchemy import func from cisticola.transformer.base import Transformer -from cisticola.base import RawChannelInfo, ChannelInfo, ScraperResult, Post, Image, Video, Media, Channel +from cisticola.base import RawChannelInfo, ChannelInfo, ScraperResult, Post, Image, Video, Audio, Media, Channel class TelegramTelethonTransformer(Transformer): @@ -227,16 +227,6 @@ class TelegramTelethonTransformer(Transformer): transformed = insert(transformed) - # for k in data.archived_urls: - # if data.archived_urls[k]: - # archived_url = data.archived_urls[k] - # ext = archived_url.split('.')[-1] - - # if ext == 'mp4' or ext == 'mov' or ext == 'avi' or ext =='mkv': - # insert(Video(url=archived_url, post=transformed.id, raw_id=data.id, original_url=k)) - # else: - # insert(Image(url=archived_url, post=transformed.id, raw_id=data.id, original_url=k)) - def add_markdown_links(raw_post): global_offset = 0 @@ -257,4 +247,4 @@ def add_markdown_links(raw_post): transformed_content = before_link + link_text + link_href + trailing_whitespace + after_link global_offset += (4 + len(url)) - return transformed_content \ No newline at end of file + return transformed_content diff --git a/spacy_setup.sh b/spacy_setup.sh old mode 100644 new mode 100755