mirror of
https://github.com/bellingcat/auto-archiver.git
synced 2026-06-08 03:18:28 +03:00
telethon join channels working
This commit is contained in:
msramalho
3
.gitignore
vendored
3
.gitignore
vendored
@@ -23,4 +23,5 @@ secrets/*
|
||||
browsertrix/*
|
||||
browsertrix-tmp/*
|
||||
instaloader/*
|
||||
instaloader.session
|
||||
instaloader.session
|
||||
orchestration.yaml
|
||||
1
Pipfile
1
Pipfile
@@ -26,6 +26,7 @@ dateparser = "*"
|
||||
vk-url-scraper = "*"
|
||||
python-twitter-v2 = "*"
|
||||
instaloader = "*"
|
||||
tqdm = "*"
|
||||
|
||||
[requires]
|
||||
python_version = "3.9"
|
||||
|
||||
457
Pipfile.lock
generated
457
Pipfile.lock
generated
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"_meta": {
|
||||
"hash": {
|
||||
"sha256": "bd987e7237c7e32d2dffb295db633f5a022ce1a718435d11d8ac303c9e37a4d3"
|
||||
"sha256": "60b8f39d7a466e194c98a3fb6a03f74f03b108f5fac4cce8657c5ffdf6a02962"
|
||||
},
|
||||
"pipfile-spec": 6,
|
||||
"requires": {
|
||||
@@ -57,22 +57,24 @@
|
||||
},
|
||||
"boto3": {
|
||||
"hashes": [
|
||||
"sha256:3b0fa19390895e664045713f2e47e63ad29c9f98b7bee6836dec7124953e48b8",
|
||||
"sha256:9feb98e045736f943c2099d955415cfe44133e03d8e2d7581d2e5dc74d0ed064"
|
||||
"sha256:53badfc5f145b8a3f9117512b41bc5a64db1cce1b549061d8edba68909e63fdf",
|
||||
"sha256:548081a0f8854bb2eea1e368ab29945478105f56989546f653c75528dcb07d88"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==1.26.1"
|
||||
"version": "==1.26.28"
|
||||
},
|
||||
"botocore": {
|
||||
"hashes": [
|
||||
"sha256:75c65130ffab527d0a3d948c6d87eb8eac210e079e1ff2768c66484be57bb77c",
|
||||
"sha256:e38b7cdce927cefabe45608dde61660b76458fba6624240dcdb6c4b8453d17f7"
|
||||
"sha256:982732e7ed65cb6ed11ea3ce0e32dff2bcd465836c32376154f0802aa0a112c7",
|
||||
"sha256:f0b8bb976e368dea20a960b47169e31fc0828feb6f0b9f59f1e5be8d08919b10"
|
||||
],
|
||||
"markers": "python_version >= '3.7'",
|
||||
"version": "==1.29.1"
|
||||
"version": "==1.29.28"
|
||||
},
|
||||
"brotli": {
|
||||
"hashes": [
|
||||
"sha256:02177603aaca36e1fd21b091cb742bb3b305a569e2402f1ca38af471777fb019",
|
||||
"sha256:11d3283d89af7033236fa4e73ec2cbe743d4f6a81d41bd234f24bf63dde979df",
|
||||
"sha256:12effe280b8ebfd389022aa65114e30407540ccb89b177d3fbc9a4f177c4bd5d",
|
||||
"sha256:160c78292e98d21e73a4cc7f76a234390e516afcd982fa17e1422f7c6a9ce9c8",
|
||||
"sha256:16d528a45c2e1909c2798f27f7bf0a3feec1dc9e50948e738b961618e38b6a7b",
|
||||
@@ -83,9 +85,15 @@
|
||||
"sha256:26d168aac4aaec9a4394221240e8a5436b5634adc3cd1cdf637f6645cecbf181",
|
||||
"sha256:29d1d350178e5225397e28ea1b7aca3648fcbab546d20e7475805437bfb0a130",
|
||||
"sha256:2aad0e0baa04517741c9bb5b07586c642302e5fb3e75319cb62087bd0995ab19",
|
||||
"sha256:3148362937217b7072cf80a2dcc007f09bb5ecb96dae4617316638194113d5be",
|
||||
"sha256:330e3f10cd01da535c70d09c4283ba2df5fb78e915bea0a28becad6e2ac010be",
|
||||
"sha256:336b40348269f9b91268378de5ff44dc6fbaa2268194f85177b53463d313842a",
|
||||
"sha256:3496fc835370da351d37cada4cf744039616a6db7d13c430035e901443a34daa",
|
||||
"sha256:35a3edbe18e876e596553c4007a087f8bcfd538f19bc116917b3c7522fca0429",
|
||||
"sha256:3b78a24b5fd13c03ee2b7b86290ed20efdc95da75a3557cc06811764d5ad1126",
|
||||
"sha256:3b8b09a16a1950b9ef495a0f8b9d0a87599a9d1f179e2d4ac014b2ec831f87e7",
|
||||
"sha256:3c1306004d49b84bd0c4f90457c6f57ad109f5cc6067a9664e12b7b79a9948ad",
|
||||
"sha256:3ffaadcaeafe9d30a7e4e1e97ad727e4f5610b9fa2f7551998471e3736738679",
|
||||
"sha256:40d15c79f42e0a2c72892bf407979febd9cf91f36f495ffb333d1d04cebb34e4",
|
||||
"sha256:44bb8ff420c1d19d91d79d8c3574b8954288bdff0273bf788954064d260d7ab0",
|
||||
"sha256:4688c1e42968ba52e57d8670ad2306fe92e0169c6f3af0089be75bbac0c64a3b",
|
||||
@@ -95,6 +103,7 @@
|
||||
"sha256:56d027eace784738457437df7331965473f2c0da2c70e1a1f6fdbae5402e0389",
|
||||
"sha256:5913a1177fc36e30fcf6dc868ce23b0453952c78c04c266d3149b3d39e1410d6",
|
||||
"sha256:5b6ef7d9f9c38292df3690fe3e302b5b530999fa90014853dcd0d6902fb59f26",
|
||||
"sha256:5bf37a08493232fbb0f8229f1824b366c2fc1d02d64e7e918af40acd15f3e337",
|
||||
"sha256:5cb1e18167792d7d21e21365d7650b72d5081ed476123ff7b8cac7f45189c0c7",
|
||||
"sha256:61a7ee1f13ab913897dac7da44a73c6d44d48a4adff42a5701e3239791c96e14",
|
||||
"sha256:622a231b08899c864eb87e85f81c75e7b9ce05b001e59bbfbf43d4a71f5f32b2",
|
||||
@@ -102,6 +111,7 @@
|
||||
"sha256:6b2ae9f5f67f89aade1fab0f7fd8f2832501311c363a21579d02defa844d9296",
|
||||
"sha256:6c772d6c0a79ac0f414a9f8947cc407e119b8598de7621f39cacadae3cf57d12",
|
||||
"sha256:6d847b14f7ea89f6ad3c9e3901d1bc4835f6b390a9c71df999b0162d9bb1e20f",
|
||||
"sha256:73fd30d4ce0ea48010564ccee1a26bfe39323fde05cb34b5863455629db61dc7",
|
||||
"sha256:76ffebb907bec09ff511bb3acc077695e2c32bc2142819491579a695f77ffd4d",
|
||||
"sha256:7bbff90b63328013e1e8cb50650ae0b9bac54ffb4be6104378490193cd60f85a",
|
||||
"sha256:7cb81373984cc0e4682f31bc3d6be9026006d96eecd07ea49aafb06897746452",
|
||||
@@ -111,6 +121,7 @@
|
||||
"sha256:87fdccbb6bb589095f413b1e05734ba492c962b4a45a13ff3408fa44ffe6479b",
|
||||
"sha256:88c63a1b55f352b02c6ffd24b15ead9fc0e8bf781dbe070213039324922a2eea",
|
||||
"sha256:8a674ac10e0a87b683f4fa2b6fa41090edfd686a6524bd8dedbd6138b309175c",
|
||||
"sha256:8ed6a5b3d23ecc00ea02e1ed8e0ff9a08f4fc87a1f58a2530e71c0f48adf882f",
|
||||
"sha256:93130612b837103e15ac3f9cbacb4613f9e348b58b3aad53721d92e57f96d46a",
|
||||
"sha256:9744a863b489c79a73aba014df554b0e7a0fc44ef3f8a0ef2a52919c7d155031",
|
||||
"sha256:9749a124280a0ada4187a6cfd1ffd35c350fb3af79c706589d98e088c5044267",
|
||||
@@ -120,15 +131,24 @@
|
||||
"sha256:9ed4c92a0665002ff8ea852353aeb60d9141eb04109e88928026d3c8a9e5433c",
|
||||
"sha256:a72661af47119a80d82fa583b554095308d6a4c356b2a554fdc2799bc19f2a43",
|
||||
"sha256:afde17ae04d90fbe53afb628f7f2d4ca022797aa093e809de5c3cf276f61bbfa",
|
||||
"sha256:b1375b5d17d6145c798661b67e4ae9d5496920d9265e2f00f1c2c0b5ae91fbde",
|
||||
"sha256:b336c5e9cf03c7be40c47b5fd694c43c9f1358a80ba384a21969e0b4e66a9b17",
|
||||
"sha256:b3523f51818e8f16599613edddb1ff924eeb4b53ab7e7197f85cbc321cdca32f",
|
||||
"sha256:b43775532a5904bc938f9c15b77c613cb6ad6fb30990f3b0afaea82797a402d8",
|
||||
"sha256:b663f1e02de5d0573610756398e44c130add0eb9a3fc912a09665332942a2efb",
|
||||
"sha256:b83bb06a0192cccf1eb8d0a28672a1b79c74c3a8a5f2619625aeb6f28b3a82bb",
|
||||
"sha256:ba72d37e2a924717990f4d7482e8ac88e2ef43fb95491eb6e0d124d77d2a150d",
|
||||
"sha256:c2415d9d082152460f2bd4e382a1e85aed233abc92db5a3880da2257dc7daf7b",
|
||||
"sha256:c83aa123d56f2e060644427a882a36b3c12db93727ad7a7b9efd7d7f3e9cc2c4",
|
||||
"sha256:c8e521a0ce7cf690ca84b8cc2272ddaf9d8a50294fd086da67e517439614c755",
|
||||
"sha256:cab1b5964b39607a66adbba01f1c12df2e55ac36c81ec6ed44f2fca44178bf1a",
|
||||
"sha256:cb02ed34557afde2d2da68194d12f5719ee96cfb2eacc886352cb73e3808fc5d",
|
||||
"sha256:cc0283a406774f465fb45ec7efb66857c09ffefbe49ec20b7882eff6d3c86d3a",
|
||||
"sha256:cfc391f4429ee0a9370aa93d812a52e1fee0f37a81861f4fdd1f4fb28e8547c3",
|
||||
"sha256:db844eb158a87ccab83e868a762ea8024ae27337fc7ddcbfcddd157f841fdfe7",
|
||||
"sha256:defed7ea5f218a9f2336301e6fd379f55c655bea65ba2476346340a0ce6f74a1",
|
||||
"sha256:e16eb9541f3dd1a3e92b89005e37b1257b157b7256df0e36bd7b33b50be73bcb",
|
||||
"sha256:e1abbeef02962596548382e393f56e4c94acd286bd0c5afba756cffc33670e8a",
|
||||
"sha256:e23281b9a08ec338469268f98f194658abfb13658ee98e2b7f85ee9dd06caa91",
|
||||
"sha256:e2d9e1cbc1b25e22000328702b014227737756f4b5bf5c485ac1d8091ada078b",
|
||||
"sha256:e48f4234f2469ed012a98f4b7874e7f7e173c167bed4934912a29e03167cf6b1",
|
||||
@@ -249,10 +269,10 @@
|
||||
},
|
||||
"cloudscraper": {
|
||||
"hashes": [
|
||||
"sha256:152fa9f9db5f19f4ada7e75623e93f45d05bfd3fb29d9cae84f29173a2591530",
|
||||
"sha256:59d964acded1a63336b3ce4daf3f2dfed3de7c88f6bf4d904c661b0b4e1b5f5e"
|
||||
"sha256:5f0cde23774270e8a092de68e0fbd68e17854c767fc2d4042a91bda9e4816871",
|
||||
"sha256:ec30da6cee60d0a95e898d9b3aaf09291a0d8b6cf751e86c6f3420b699a00091"
|
||||
],
|
||||
"version": "==1.2.64"
|
||||
"version": "==1.2.66"
|
||||
},
|
||||
"commonmark": {
|
||||
"hashes": [
|
||||
@@ -263,35 +283,35 @@
|
||||
},
|
||||
"cryptography": {
|
||||
"hashes": [
|
||||
"sha256:068147f32fa662c81aebab95c74679b401b12b57494872886eb5c1139250ec5d",
|
||||
"sha256:06fc3cc7b6f6cca87bd56ec80a580c88f1da5306f505876a71c8cfa7050257dd",
|
||||
"sha256:25c1d1f19729fb09d42e06b4bf9895212292cb27bb50229f5aa64d039ab29146",
|
||||
"sha256:402852a0aea73833d982cabb6d0c3bb582c15483d29fb7085ef2c42bfa7e38d7",
|
||||
"sha256:4e269dcd9b102c5a3d72be3c45d8ce20377b8076a43cbed6f660a1afe365e436",
|
||||
"sha256:5419a127426084933076132d317911e3c6eb77568a1ce23c3ac1e12d111e61e0",
|
||||
"sha256:554bec92ee7d1e9d10ded2f7e92a5d70c1f74ba9524947c0ba0c850c7b011828",
|
||||
"sha256:5e89468fbd2fcd733b5899333bc54d0d06c80e04cd23d8c6f3e0542358c6060b",
|
||||
"sha256:65535bc550b70bd6271984d9863a37741352b4aad6fb1b3344a54e6950249b55",
|
||||
"sha256:6ab9516b85bebe7aa83f309bacc5f44a61eeb90d0b4ec125d2d003ce41932d36",
|
||||
"sha256:6addc3b6d593cd980989261dc1cce38263c76954d758c3c94de51f1e010c9a50",
|
||||
"sha256:728f2694fa743a996d7784a6194da430f197d5c58e2f4e278612b359f455e4a2",
|
||||
"sha256:785e4056b5a8b28f05a533fab69febf5004458e20dad7e2e13a3120d8ecec75a",
|
||||
"sha256:78cf5eefac2b52c10398a42765bfa981ce2372cbc0457e6bf9658f41ec3c41d8",
|
||||
"sha256:7f836217000342d448e1c9a342e9163149e45d5b5eca76a30e84503a5a96cab0",
|
||||
"sha256:8d41a46251bf0634e21fac50ffd643216ccecfaf3701a063257fe0b2be1b6548",
|
||||
"sha256:984fe150f350a3c91e84de405fe49e688aa6092b3525f407a18b9646f6612320",
|
||||
"sha256:9b24bcff7853ed18a63cfb0c2b008936a9554af24af2fb146e16d8e1aed75748",
|
||||
"sha256:b1b35d9d3a65542ed2e9d90115dfd16bbc027b3f07ee3304fc83580f26e43249",
|
||||
"sha256:b1b52c9e5f8aa2b802d48bd693190341fae201ea51c7a167d69fc48b60e8a959",
|
||||
"sha256:bbf203f1a814007ce24bd4d51362991d5cb90ba0c177a9c08825f2cc304d871f",
|
||||
"sha256:be243c7e2bfcf6cc4cb350c0d5cdf15ca6383bbcb2a8ef51d3c9411a9d4386f0",
|
||||
"sha256:bfbe6ee19615b07a98b1d2287d6a6073f734735b49ee45b11324d85efc4d5cbd",
|
||||
"sha256:c46837ea467ed1efea562bbeb543994c2d1f6e800785bd5a2c98bc096f5cb220",
|
||||
"sha256:dfb4f4dd568de1b6af9f4cda334adf7d72cf5bc052516e1b2608b683375dd95c",
|
||||
"sha256:ed7b00096790213e09eb11c97cc6e2b757f15f3d2f85833cd2d3ec3fe37c1722"
|
||||
"sha256:0e70da4bdff7601b0ef48e6348339e490ebfb0cbe638e083c9c41fb49f00c8bd",
|
||||
"sha256:10652dd7282de17990b88679cb82f832752c4e8237f0c714be518044269415db",
|
||||
"sha256:175c1a818b87c9ac80bb7377f5520b7f31b3ef2a0004e2420319beadedb67290",
|
||||
"sha256:1d7e632804a248103b60b16fb145e8df0bc60eed790ece0d12efe8cd3f3e7744",
|
||||
"sha256:1f13ddda26a04c06eb57119caf27a524ccae20533729f4b1e4a69b54e07035eb",
|
||||
"sha256:2ec2a8714dd005949d4019195d72abed84198d877112abb5a27740e217e0ea8d",
|
||||
"sha256:2fa36a7b2cc0998a3a4d5af26ccb6273f3df133d61da2ba13b3286261e7efb70",
|
||||
"sha256:2fb481682873035600b5502f0015b664abc26466153fab5c6bc92c1ea69d478b",
|
||||
"sha256:3178d46f363d4549b9a76264f41c6948752183b3f587666aff0555ac50fd7876",
|
||||
"sha256:4367da5705922cf7070462e964f66e4ac24162e22ab0a2e9d31f1b270dd78083",
|
||||
"sha256:4eb85075437f0b1fd8cd66c688469a0c4119e0ba855e3fef86691971b887caf6",
|
||||
"sha256:50a1494ed0c3f5b4d07650a68cd6ca62efe8b596ce743a5c94403e6f11bf06c1",
|
||||
"sha256:53049f3379ef05182864d13bb9686657659407148f901f3f1eee57a733fb4b00",
|
||||
"sha256:6391e59ebe7c62d9902c24a4d8bcbc79a68e7c4ab65863536127c8a9cd94043b",
|
||||
"sha256:67461b5ebca2e4c2ab991733f8ab637a7265bb582f07c7c88914b5afb88cb95b",
|
||||
"sha256:78e47e28ddc4ace41dd38c42e6feecfdadf9c3be2af389abbfeef1ff06822285",
|
||||
"sha256:80ca53981ceeb3241998443c4964a387771588c4e4a5d92735a493af868294f9",
|
||||
"sha256:8a4b2bdb68a447fadebfd7d24855758fe2d6fecc7fed0b78d190b1af39a8e3b0",
|
||||
"sha256:8e45653fb97eb2f20b8c96f9cd2b3a0654d742b47d638cf2897afbd97f80fa6d",
|
||||
"sha256:998cd19189d8a747b226d24c0207fdaa1e6658a1d3f2494541cb9dfbf7dcb6d2",
|
||||
"sha256:a10498349d4c8eab7357a8f9aa3463791292845b79597ad1b98a543686fb1ec8",
|
||||
"sha256:b4cad0cea995af760f82820ab4ca54e5471fc782f70a007f31531957f43e9dee",
|
||||
"sha256:bfe6472507986613dc6cc00b3d492b2f7564b02b3b3682d25ca7f40fa3fd321b",
|
||||
"sha256:c9e0d79ee4c56d841bd4ac6e7697c8ff3c8d6da67379057f29e66acffcd1e9a7",
|
||||
"sha256:ca57eb3ddaccd1112c18fc80abe41db443cc2e9dcb1917078e02dfa010a4f353",
|
||||
"sha256:ce127dd0a6a0811c251a6cddd014d292728484e530d80e872ad9806cfb1c5b3c"
|
||||
],
|
||||
"markers": "python_version >= '3.6'",
|
||||
"version": "==38.0.3"
|
||||
"version": "==38.0.4"
|
||||
},
|
||||
"dataclasses-json": {
|
||||
"hashes": [
|
||||
@@ -303,19 +323,19 @@
|
||||
},
|
||||
"dateparser": {
|
||||
"hashes": [
|
||||
"sha256:711f7eef6d431225bec56c00e386af3f6a47083276253375bdae1ae6c8d23d4a",
|
||||
"sha256:ae7a7de30f26983d09fff802c1f9d35d54e1c11d7ab52ae904a1f3fc037ecba5"
|
||||
"sha256:4431159799b63d8acec5d7d844c5e06edf3d1b0eb2bda6d4cac87134ddddd01c",
|
||||
"sha256:73ec6e44a133c54076ecf9f9dc0fbe3dd4831f154f977ff06f53114d57c5425e"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==1.1.3"
|
||||
"version": "==1.1.4"
|
||||
},
|
||||
"exceptiongroup": {
|
||||
"hashes": [
|
||||
"sha256:2ac84b496be68464a2da60da518af3785fff8b7ec0d090a581604bc870bdee41",
|
||||
"sha256:affbabf13fb6e98988c38d9c5650e701569fe3c1de3233cfb61c5f33774690ad"
|
||||
"sha256:542adf9dea4055530d6e1279602fa5cb11dab2395fa650b8674eaec35fc4a828",
|
||||
"sha256:bd14967b79cd9bdb54d97323216f8fdf533e278df937aa2a90089e7d6e06e5ec"
|
||||
],
|
||||
"markers": "python_version < '3.11'",
|
||||
"version": "==1.0.0"
|
||||
"version": "==1.0.4"
|
||||
},
|
||||
"ffmpeg-python": {
|
||||
"hashes": [
|
||||
@@ -327,11 +347,11 @@
|
||||
},
|
||||
"filelock": {
|
||||
"hashes": [
|
||||
"sha256:55447caa666f2198c5b6b13a26d2084d26fa5b115c00d065664b2124680c4edc",
|
||||
"sha256:617eb4e5eedc82fc5f47b6d61e4d11cb837c56cb4544e39081099fa17ad109d4"
|
||||
"sha256:7565f628ea56bfcd8e54e42bdc55da899c85c1abfe1b5bcfd147e9188cebb3b2",
|
||||
"sha256:8df285554452285f79c035efb0c861eb33a4bcfa5b7a137016e32e6a90f9792c"
|
||||
],
|
||||
"markers": "python_version >= '3.7'",
|
||||
"version": "==3.8.0"
|
||||
"version": "==3.8.2"
|
||||
},
|
||||
"flask": {
|
||||
"hashes": [
|
||||
@@ -350,27 +370,27 @@
|
||||
},
|
||||
"google-api-core": {
|
||||
"hashes": [
|
||||
"sha256:10c06f7739fe57781f87523375e8e1a3a4674bf6392cd6131a3222182b971320",
|
||||
"sha256:34f24bd1d5f72a8c4519773d99ca6bf080a6c4e041b4e9f024fe230191dda62e"
|
||||
"sha256:4b9bb5d5a380a0befa0573b302651b8a9a89262c1730e37bf423cec511804c22",
|
||||
"sha256:ce222e27b0de0d7bc63eb043b956996d6dccab14cc3b690aaea91c9cc99dc16e"
|
||||
],
|
||||
"markers": "python_version >= '3.7'",
|
||||
"version": "==2.10.2"
|
||||
"version": "==2.11.0"
|
||||
},
|
||||
"google-api-python-client": {
|
||||
"hashes": [
|
||||
"sha256:2c6611530308b3f931dcf1360713aa3a20cf465d0bf2bac65f2ec99e8c9860de",
|
||||
"sha256:b8a0ca8454ad57bc65199044717d3d214197ae1e2d666426bbcd4021b36762e0"
|
||||
"sha256:03624a28b5ba94f3c3d44761081f5dbf8cabaa20c5c3a96c046457c5713efb9b",
|
||||
"sha256:bc2447a7479006d98927fb20faa74d892d3758ff68e99b621367632bc42b8af8"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==2.65.0"
|
||||
"version": "==2.69.0"
|
||||
},
|
||||
"google-auth": {
|
||||
"hashes": [
|
||||
"sha256:1ad5b0e6eba5f69645971abb3d2c197537d5914070a8c6d30299dfdb07c5c700",
|
||||
"sha256:cf24817855d874ede2efd071aa22125445f555de1685b739a9782fcf408c2a3d"
|
||||
"sha256:6897b93556d8d807ad70701bb89f000183aea366ca7ed94680828b37437a4994",
|
||||
"sha256:72f12a6cfc968d754d7bdab369c5c5c16032106e52d32c6dfd8484e4c01a6d1f"
|
||||
],
|
||||
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
|
||||
"version": "==2.14.0"
|
||||
"version": "==2.15.0"
|
||||
},
|
||||
"google-auth-httplib2": {
|
||||
"hashes": [
|
||||
@@ -382,27 +402,27 @@
|
||||
},
|
||||
"google-auth-oauthlib": {
|
||||
"hashes": [
|
||||
"sha256:53019edbde83e08ff0740eefc5bded7e26a289941d12e7ae1f0f5bacf2faa031",
|
||||
"sha256:db11bce4b3effc99b518ec22a2903470e0853c0c92be57694e3684e738d22513"
|
||||
"sha256:40cc612a13c3336d5433e94e2adb42a0c88f6feb6c55769e44500fc70043a576",
|
||||
"sha256:81056a310fb1c4a3e5a7e1a443e1eb96593c6bbc55b26c0261e4d3295d3e6593"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==0.7.0"
|
||||
"version": "==0.8.0"
|
||||
},
|
||||
"googleapis-common-protos": {
|
||||
"hashes": [
|
||||
"sha256:8eb2cbc91b69feaf23e32452a7ae60e791e09967d81d4fcc7fc388182d1bd394",
|
||||
"sha256:c25873c47279387cfdcbdafa36149887901d36202cb645a0e4f29686bf6e4417"
|
||||
"sha256:27a849d6205838fb6cc3c1c21cb9800707a661bb21c6ce7fb13e99eb1f8a0c46",
|
||||
"sha256:a9f4a1d7f6d9809657b7f1316a1aa527f6664891531bcfcc13b6696e685f443c"
|
||||
],
|
||||
"markers": "python_version >= '3.7'",
|
||||
"version": "==1.56.4"
|
||||
"version": "==1.57.0"
|
||||
},
|
||||
"gspread": {
|
||||
"hashes": [
|
||||
"sha256:41f7a416425f1ec5a1b677f49b8fbf599102766c27ed7be6601a58c9a1550ebc",
|
||||
"sha256:d3bbff4b7aad0fc2c986458e148537a02fe7b46e7162f41f3a42392bfa2adb89"
|
||||
"sha256:ce76f9c16b88ccb792350142224a59afa8e69f7463f3d3417148cbe892efc7cb",
|
||||
"sha256:dbeedd08c6a7f7b0bfc1a54e17c29205362250c77bf98e11125c5d99fd7f4ba7"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==5.6.2"
|
||||
"version": "==5.7.2"
|
||||
},
|
||||
"h11": {
|
||||
"hashes": [
|
||||
@@ -591,11 +611,11 @@
|
||||
},
|
||||
"marshmallow": {
|
||||
"hashes": [
|
||||
"sha256:35e02a3a06899c9119b785c12a22f4cda361745d66a71ab691fd7610202ae104",
|
||||
"sha256:6804c16114f7fce1f5b4dadc31f4674af23317fcc7f075da21e35c1a35d781f7"
|
||||
"sha256:90032c0fd650ce94b6ec6dc8dfeb0e3ff50c144586462c389b81a07205bedb78",
|
||||
"sha256:93f0958568da045b0021ec6aeb7ac37c81bfcccbb9a0e7ed8559885070b3a19b"
|
||||
],
|
||||
"markers": "python_version >= '3.7'",
|
||||
"version": "==3.18.0"
|
||||
"version": "==3.19.0"
|
||||
},
|
||||
"marshmallow-enum": {
|
||||
"hashes": [
|
||||
@@ -645,31 +665,31 @@
|
||||
},
|
||||
"packaging": {
|
||||
"hashes": [
|
||||
"sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb",
|
||||
"sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522"
|
||||
"sha256:2198ec20bd4c017b8f9717e00f0c8714076fc2fd93816750ab48e2c41de2cfd3",
|
||||
"sha256:957e2148ba0e1a3b282772e791ef1d8083648bc131c8ab0c1feba110ce1146c3"
|
||||
],
|
||||
"markers": "python_version >= '3.6'",
|
||||
"version": "==21.3"
|
||||
"markers": "python_version >= '3.7'",
|
||||
"version": "==22.0"
|
||||
},
|
||||
"protobuf": {
|
||||
"hashes": [
|
||||
"sha256:2c9c2ed7466ad565f18668aa4731c535511c5d9a40c6da39524bccf43e441719",
|
||||
"sha256:48e2cd6b88c6ed3d5877a3ea40df79d08374088e89bedc32557348848dff250b",
|
||||
"sha256:5b0834e61fb38f34ba8840d7dcb2e5a2f03de0c714e0293b3963b79db26de8ce",
|
||||
"sha256:61f21493d96d2a77f9ca84fefa105872550ab5ef71d21c458eb80edcf4885a99",
|
||||
"sha256:6e0be9f09bf9b6cf497b27425487706fa48c6d1632ddd94dab1a5fe11a422392",
|
||||
"sha256:6e312e280fbe3c74ea9e080d9e6080b636798b5e3939242298b591064470b06b",
|
||||
"sha256:7eb8f2cc41a34e9c956c256e3ac766cf4e1a4c9c925dc757a41a01be3e852965",
|
||||
"sha256:84ea107016244dfc1eecae7684f7ce13c788b9a644cd3fca5b77871366556444",
|
||||
"sha256:9227c14010acd9ae7702d6467b4625b6fe853175a6b150e539b21d2b2f2b409c",
|
||||
"sha256:a419cc95fca8694804709b8c4f2326266d29659b126a93befe210f5bbc772536",
|
||||
"sha256:a7d0ea43949d45b836234f4ebb5ba0b22e7432d065394b532cdca8f98415e3cf",
|
||||
"sha256:b5ab0b8918c136345ff045d4b3d5f719b505b7c8af45092d7f45e304f55e50a1",
|
||||
"sha256:e575c57dc8b5b2b2caa436c16d44ef6981f2235eb7179bfc847557886376d740",
|
||||
"sha256:f9eae277dd240ae19bb06ff4e2346e771252b0e619421965504bd1b1bba7c5fa"
|
||||
"sha256:25266bf373ee06d5d66f9eb1ec9d434b243dccce5c32faf151054cfa6f9dcbf1",
|
||||
"sha256:260e346927fd4e6fbb49ab545137b19610c24a1d853dc5f29ddf777ab1987211",
|
||||
"sha256:2c6a4d13732d9b094db31b3841986c38b17ac61a3fe05ee26a779d94c4c3fb43",
|
||||
"sha256:4922e3320ed70e81f05060822da36923d09fd9e04e17f411f2d8d8d0070f9f5c",
|
||||
"sha256:4b75c947289a2e9c1f37d21c593f1ef6fb4fed33977dfb2ac84f799eb29a8ff4",
|
||||
"sha256:4d01ef83517c181d60ea1c6d0b2f644be250ade740d6554a2f5a021b1ad622e3",
|
||||
"sha256:553e35c0878f6855e55f01a14561e6bce6df79b6636a5acf83b9d9ac7eab7922",
|
||||
"sha256:85ccb4753ee21de7dc81a7a68a051f25dbe133ffa01a639ac998427d0b223387",
|
||||
"sha256:a5a14b907a191319e7a58b38c583bbf50deb21e002f723a912c5e4f6969a778e",
|
||||
"sha256:a944dc9550baae276afc7dc8193191d4c2ad660270a1e5ed5a71539817ebe2e2",
|
||||
"sha256:bab4b21a986ded225b9392c07ce21c35d790951f51e1ebfd32e4d443b05c3726",
|
||||
"sha256:c3b9e329b4c247dc3ba5c50f60915a84e08278eb6d9e3fa674d0d04ff816bfd7",
|
||||
"sha256:d91a47c77b33580024b0271b65bb820c4e0264c25eb49151ad01e691de8fa0b6",
|
||||
"sha256:efb16b16fd3eef25357f84d516062753014b76279ce4e0ec4880badd2fba7370"
|
||||
],
|
||||
"markers": "python_version >= '3.7'",
|
||||
"version": "==4.21.9"
|
||||
"version": "==4.21.11"
|
||||
},
|
||||
"pyaes": {
|
||||
"hashes": [
|
||||
@@ -722,39 +742,35 @@
|
||||
},
|
||||
"pycryptodomex": {
|
||||
"hashes": [
|
||||
"sha256:04cc393045a8f19dd110c975e30f38ed7ab3faf21ede415ea67afebd95a22380",
|
||||
"sha256:0776bfaf2c48154ab54ea45392847c1283d2fcf64e232e85565f858baedfc1fa",
|
||||
"sha256:0fadb9f7fa3150577800eef35f62a8a24b9ddf1563ff060d9bd3af22d3952c8c",
|
||||
"sha256:18e2ab4813883ae63396c0ffe50b13554b32bb69ec56f0afaf052e7a7ae0d55b",
|
||||
"sha256:191e73bc84a8064ad1874dba0ebadedd7cce4dedee998549518f2c74a003b2e1",
|
||||
"sha256:35a8f7afe1867118330e2e0e0bf759c409e28557fb1fc2fbb1c6c937297dbe9a",
|
||||
"sha256:3709f13ca3852b0b07fc04a2c03b379189232b24007c466be0f605dd4723e9d4",
|
||||
"sha256:4540904c09704b6f831059c0dfb38584acb82cb97b0125cd52688c1f1e3fffa6",
|
||||
"sha256:463119d7d22d0fc04a0f9122e9d3e6121c6648bcb12a052b51bd1eed1b996aa2",
|
||||
"sha256:46b3f05f2f7ac7841053da4e0f69616929ca3c42f238c405f6c3df7759ad2780",
|
||||
"sha256:48697790203909fab02a33226fda546604f4e2653f9d47bc5d3eb40879fa7c64",
|
||||
"sha256:5676a132169a1c1a3712edf25250722ebc8c9102aa9abd814df063ca8362454f",
|
||||
"sha256:65204412d0c6a8e3c41e21e93a5e6054a74fea501afa03046a388cf042e3377a",
|
||||
"sha256:67e1e6a92151023ccdfcfbc0afb3314ad30080793b4c27956ea06ab1fb9bcd8a",
|
||||
"sha256:6f5b6ba8aefd624834bc177a2ac292734996bb030f9d1b388e7504103b6fcddf",
|
||||
"sha256:7341f1bb2dadb0d1a0047f34c3a58208a92423cdbd3244d998e4b28df5eac0ed",
|
||||
"sha256:78d9621cf0ea35abf2d38fa2ca6d0634eab6c991a78373498ab149953787e5e5",
|
||||
"sha256:8eecdf9cdc7343001d047f951b9cc805cd68cb6cd77b20ea46af5bffc5bd3dfb",
|
||||
"sha256:94c7b60e1f52e1a87715571327baea0733708ab4723346598beca4a3b6879794",
|
||||
"sha256:996e1ba717077ce1e6d4849af7a1426f38b07b3d173b879e27d5e26d2e958beb",
|
||||
"sha256:a07a64709e366c2041cd5cfbca592b43998bf4df88f7b0ca73dca37071ccf1bd",
|
||||
"sha256:b6306403228edde6e289f626a3908a2f7f67c344e712cf7c0a508bab3ad9e381",
|
||||
"sha256:b9279adc16e4b0f590ceff581f53a80179b02cba9056010d733eb4196134a870",
|
||||
"sha256:c4cb9cb492ea7dcdf222a8d19a1d09002798ea516aeae8877245206d27326d86",
|
||||
"sha256:dd452a5af7014e866206d41751886c9b4bf379a339fdf2dbfc7dd16c0fb4f8e0",
|
||||
"sha256:e2b12968522a0358b8917fc7b28865acac002f02f4c4c6020fcb264d76bfd06d",
|
||||
"sha256:e3164a18348bd53c69b4435ebfb4ac8a4076291ffa2a70b54f0c4b80c7834b1d",
|
||||
"sha256:e47bf8776a7e15576887f04314f5228c6527b99946e6638cf2f16da56d260cab",
|
||||
"sha256:f8be976cec59b11f011f790b88aca67b4ea2bd286578d0bd3e31bcd19afcd3e4",
|
||||
"sha256:fc9bc7a9b79fe5c750fc81a307052f8daabb709bdaabb0fb18fb136b66b653b5"
|
||||
"sha256:04610536921c1ec7adba158ef570348550c9f3a40bc24be9f8da2ef7ab387981",
|
||||
"sha256:0ba28aa97cdd3ff5ed1a4f2b7f5cd04e721166bd75bd2b929e2734433882b583",
|
||||
"sha256:0da835af786fdd1c9930994c78b23e88d816dc3f99aa977284a21bbc26d19735",
|
||||
"sha256:1619087fb5b31510b0b0b058a54f001a5ffd91e6ffee220d9913064519c6a69d",
|
||||
"sha256:1cda60207be8c1cf0b84b9138f9e3ca29335013d2b690774a5e94678ff29659a",
|
||||
"sha256:22aed0868622d95179217c298e37ed7410025c7b29dac236d3230617d1e4ed56",
|
||||
"sha256:231dc8008cbdd1ae0e34645d4523da2dbc7a88c325f0d4a59635a86ee25b41dd",
|
||||
"sha256:2ad9bb86b355b6104796567dd44c215b3dc953ef2fae5e0bdfb8516731df92cf",
|
||||
"sha256:4dbbe18cc232b5980c7633972ae5417d0df76fe89e7db246eefd17ef4d8e6d7a",
|
||||
"sha256:6a465e4f856d2a4f2a311807030c89166529ccf7ccc65bef398de045d49144b6",
|
||||
"sha256:70288d9bfe16b2fd0d20b6c365db614428f1bcde7b20d56e74cf88ade905d9eb",
|
||||
"sha256:7993d26dae4d83b8f4ce605bb0aecb8bee330bb3c95475ef06f3694403621e71",
|
||||
"sha256:8851585ff19871e5d69e1790f4ca5f6fd1699d6b8b14413b472a4c0dbc7ea780",
|
||||
"sha256:893f8a97d533c66cc3a56e60dd3ed40a3494ddb4aafa7e026429a08772f8a849",
|
||||
"sha256:8dd2d9e3c617d0712ed781a77efd84ea579e76c5f9b2a4bc0b684ebeddf868b2",
|
||||
"sha256:a1c0ae7123448ecb034c75c713189cb00ebe2d415b11682865b6c54d200d9c93",
|
||||
"sha256:b0789a8490114a2936ed77c87792cfe77582c829cb43a6d86ede0f9624ba8aa3",
|
||||
"sha256:b3d04c00d777c36972b539fb79958790126847d84ec0129fce1efef250bfe3ce",
|
||||
"sha256:ba57ac7861fd2c837cdb33daf822f2a052ff57dd769a2107807f52a36d0e8d38",
|
||||
"sha256:ce338a9703f54b2305a408fc9890eb966b727ce72b69f225898bb4e9d9ed3f1f",
|
||||
"sha256:daa67f5ebb6fbf1ee9c90decaa06ca7fc88a548864e5e484d52b0920a57fe8a5",
|
||||
"sha256:e2453162f473c1eae4826eb10cd7bce19b5facac86d17fb5f29a570fde145abd",
|
||||
"sha256:e25a2f5667d91795f9417cb856f6df724ccdb0cdd5cbadb212ee9bf43946e9f8",
|
||||
"sha256:e5a670919076b71522c7d567a9043f66f14b202414a63c3a078b5831ae342c03",
|
||||
"sha256:e9ba9d8ed638733c9e95664470b71d624a6def149e2db6cc52c1aca5a6a2df1d",
|
||||
"sha256:f2b971a7b877348a27dcfd0e772a0343fb818df00b74078e91c008632284137d"
|
||||
],
|
||||
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
|
||||
"version": "==3.15.0"
|
||||
"version": "==3.16.0"
|
||||
},
|
||||
"pygments": {
|
||||
"hashes": [
|
||||
@@ -798,19 +814,19 @@
|
||||
},
|
||||
"python-slugify": {
|
||||
"hashes": [
|
||||
"sha256:272d106cb31ab99b3496ba085e3fea0e9e76dcde967b5e9992500d1f785ce4e1",
|
||||
"sha256:7b2c274c308b62f4269a9ba701aa69a797e9bca41aeee5b3a9e79e36b6656927"
|
||||
"sha256:003aee64f9fd955d111549f96c4b58a3f40b9319383c70fad6277a4974bbf570",
|
||||
"sha256:7a0f21a39fa6c1c4bf2e5984c9b9ae944483fd10b54804cb0e23a3ccd4954f0b"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==6.1.2"
|
||||
"version": "==7.0.0"
|
||||
},
|
||||
"python-twitter-v2": {
|
||||
"hashes": [
|
||||
"sha256:04349e74ec6ebaa3c71d02dc82610acd3b6b346a0060adf4bad2379fd3f46701",
|
||||
"sha256:1b17b3243108a7d8d1af0b71a3e87f28d105b5fe61cfd09944e28a7903769c81"
|
||||
"sha256:18c14853da8b499775a11a3f5e1d0692a7017fa41eca91ac5afa73f35b935a90",
|
||||
"sha256:fbe582ae7c6b33f6055b97e23dd106874e6650091d257fe67bfd024b96ebf8d6"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==0.7.9"
|
||||
"version": "==0.8.0"
|
||||
},
|
||||
"pytz": {
|
||||
"hashes": [
|
||||
@@ -875,83 +891,97 @@
|
||||
},
|
||||
"regex": {
|
||||
"hashes": [
|
||||
"sha256:0008650041531d0eadecc96a73d37c2dc4821cf51b0766e374cb4f1ddc4e1c14",
|
||||
"sha256:03299b0bcaa7824eb7c0ebd7ef1e3663302d1b533653bfe9dc7e595d453e2ae9",
|
||||
"sha256:06b1df01cf2aef3a9790858af524ae2588762c8a90e784ba00d003f045306204",
|
||||
"sha256:09b4b6ccc61d4119342b26246ddd5a04accdeebe36bdfe865ad87a0784efd77f",
|
||||
"sha256:0be0c34a39e5d04a62fd5342f0886d0e57592a4f4993b3f9d257c1f688b19737",
|
||||
"sha256:0d96eec8550fd2fd26f8e675f6d8b61b159482ad8ffa26991b894ed5ee19038b",
|
||||
"sha256:0eb0e2845e81bdea92b8281a3969632686502565abf4a0b9e4ab1471c863d8f3",
|
||||
"sha256:13bbf0c9453c6d16e5867bda7f6c0c7cff1decf96c5498318bb87f8136d2abd4",
|
||||
"sha256:17e51ad1e6131c496b58d317bc9abec71f44eb1957d32629d06013a21bc99cac",
|
||||
"sha256:1977bb64264815d3ef016625adc9df90e6d0e27e76260280c63eca993e3f455f",
|
||||
"sha256:1e30762ddddb22f7f14c4f59c34d3addabc789216d813b0f3e2788d7bcf0cf29",
|
||||
"sha256:1e73652057473ad3e6934944af090852a02590c349357b79182c1b681da2c772",
|
||||
"sha256:20e6a27959f162f979165e496add0d7d56d7038237092d1aba20b46de79158f1",
|
||||
"sha256:286ff9ec2709d56ae7517040be0d6c502642517ce9937ab6d89b1e7d0904f863",
|
||||
"sha256:297c42ede2c81f0cb6f34ea60b5cf6dc965d97fa6936c11fc3286019231f0d66",
|
||||
"sha256:320c2f4106962ecea0f33d8d31b985d3c185757c49c1fb735501515f963715ed",
|
||||
"sha256:35ed2f3c918a00b109157428abfc4e8d1ffabc37c8f9abc5939ebd1e95dabc47",
|
||||
"sha256:3d146e5591cb67c5e836229a04723a30af795ef9b70a0bbd913572e14b7b940f",
|
||||
"sha256:42bb37e2b2d25d958c25903f6125a41aaaa1ed49ca62c103331f24b8a459142f",
|
||||
"sha256:42d6007722d46bd2c95cce700181570b56edc0dcbadbfe7855ec26c3f2d7e008",
|
||||
"sha256:43eba5c46208deedec833663201752e865feddc840433285fbadee07b84b464d",
|
||||
"sha256:452519bc4c973e961b1620c815ea6dd8944a12d68e71002be5a7aff0a8361571",
|
||||
"sha256:4b9c16a807b17b17c4fa3a1d8c242467237be67ba92ad24ff51425329e7ae3d0",
|
||||
"sha256:5510932596a0f33399b7fff1bd61c59c977f2b8ee987b36539ba97eb3513584a",
|
||||
"sha256:55820bc631684172b9b56a991d217ec7c2e580d956591dc2144985113980f5a3",
|
||||
"sha256:57484d39447f94967e83e56db1b1108c68918c44ab519b8ecfc34b790ca52bf7",
|
||||
"sha256:58ba41e462653eaf68fc4a84ec4d350b26a98d030be1ab24aba1adcc78ffe447",
|
||||
"sha256:5bc5f921be39ccb65fdda741e04b2555917a4bced24b4df14eddc7569be3b493",
|
||||
"sha256:5dcc4168536c8f68654f014a3db49b6b4a26b226f735708be2054314ed4964f4",
|
||||
"sha256:5f92a7cdc6a0ae2abd184e8dfd6ef2279989d24c85d2c85d0423206284103ede",
|
||||
"sha256:67250b36edfa714ba62dc62d3f238e86db1065fccb538278804790f578253640",
|
||||
"sha256:6df070a986fc064d865c381aecf0aaff914178fdf6874da2f2387e82d93cc5bd",
|
||||
"sha256:729aa8ca624c42f309397c5fc9e21db90bf7e2fdd872461aabdbada33de9063c",
|
||||
"sha256:72bc3a5effa5974be6d965ed8301ac1e869bc18425c8a8fac179fbe7876e3aee",
|
||||
"sha256:74d86e8924835f863c34e646392ef39039405f6ce52956d8af16497af4064a30",
|
||||
"sha256:79e5af1ff258bc0fe0bdd6f69bc4ae33935a898e3cbefbbccf22e88a27fa053b",
|
||||
"sha256:7b103dffb9f6a47ed7ffdf352b78cfe058b1777617371226c1894e1be443afec",
|
||||
"sha256:83f03f0bd88c12e63ca2d024adeee75234d69808b341e88343b0232329e1f1a1",
|
||||
"sha256:86d7a68fa53688e1f612c3246044157117403c7ce19ebab7d02daf45bd63913e",
|
||||
"sha256:878c626cbca3b649e14e972c14539a01191d79e58934e3f3ef4a9e17f90277f8",
|
||||
"sha256:878f5d649ba1db9f52cc4ef491f7dba2d061cdc48dd444c54260eebc0b1729b9",
|
||||
"sha256:87bc01226cd288f0bd9a4f9f07bf6827134dc97a96c22e2d28628e824c8de231",
|
||||
"sha256:8babb2b5751105dc0aef2a2e539f4ba391e738c62038d8cb331c710f6b0f3da7",
|
||||
"sha256:91e0f7e7be77250b808a5f46d90bf0032527d3c032b2131b63dee54753a4d729",
|
||||
"sha256:9557545c10d52c845f270b665b52a6a972884725aa5cf12777374e18f2ea8960",
|
||||
"sha256:9ccb0a4ab926016867260c24c192d9df9586e834f5db83dfa2c8fffb3a6e5056",
|
||||
"sha256:9d828c5987d543d052b53c579a01a52d96b86f937b1777bbfe11ef2728929357",
|
||||
"sha256:9efa41d1527b366c88f265a227b20bcec65bda879962e3fc8a2aee11e81266d7",
|
||||
"sha256:aaf5317c961d93c1a200b9370fb1c6b6836cc7144fef3e5a951326912bf1f5a3",
|
||||
"sha256:ab69b4fe09e296261377d209068d52402fb85ef89dc78a9ac4a29a895f4e24a7",
|
||||
"sha256:ad397bc7d51d69cb07ef89e44243f971a04ce1dca9bf24c992c362406c0c6573",
|
||||
"sha256:ae17fc8103f3b63345709d3e9654a274eee1c6072592aec32b026efd401931d0",
|
||||
"sha256:af4d8cc28e4c7a2f6a9fed544228c567340f8258b6d7ea815b62a72817bbd178",
|
||||
"sha256:b22ff939a8856a44f4822da38ef4868bd3a9ade22bb6d9062b36957c850e404f",
|
||||
"sha256:b549d851f91a4efb3e65498bd4249b1447ab6035a9972f7fc215eb1f59328834",
|
||||
"sha256:be319f4eb400ee567b722e9ea63d5b2bb31464e3cf1b016502e3ee2de4f86f5c",
|
||||
"sha256:c0446b2871335d5a5e9fcf1462f954586b09a845832263db95059dcd01442015",
|
||||
"sha256:c68d2c04f7701a418ec2e5631b7f3552efc32f6bcc1739369c6eeb1af55f62e0",
|
||||
"sha256:c87ac58b9baaf50b6c1b81a18d20eda7e2883aa9a4fb4f1ca70f2e443bfcdc57",
|
||||
"sha256:caa2734ada16a44ae57b229d45091f06e30a9a52ace76d7574546ab23008c635",
|
||||
"sha256:cb34c2d66355fb70ae47b5595aafd7218e59bb9c00ad8cc3abd1406ca5874f07",
|
||||
"sha256:cb3652bbe6720786b9137862205986f3ae54a09dec8499a995ed58292bdf77c2",
|
||||
"sha256:cf668f26604e9f7aee9f8eaae4ca07a948168af90b96be97a4b7fa902a6d2ac1",
|
||||
"sha256:d326ff80ed531bf2507cba93011c30fff2dd51454c85f55df0f59f2030b1687b",
|
||||
"sha256:d6c2441538e4fadd4291c8420853431a229fcbefc1bf521810fbc2629d8ae8c2",
|
||||
"sha256:d6ecfd1970b3380a569d7b3ecc5dd70dba295897418ed9e31ec3c16a5ab099a5",
|
||||
"sha256:e5602a9b5074dcacc113bba4d2f011d2748f50e3201c8139ac5b68cf2a76bd8b",
|
||||
"sha256:ef806f684f17dbd6263d72a54ad4073af42b42effa3eb42b877e750c24c76f86",
|
||||
"sha256:f3356afbb301ec34a500b8ba8b47cba0b44ed4641c306e1dd981a08b416170b5",
|
||||
"sha256:f6f7ee2289176cb1d2c59a24f50900f8b9580259fa9f1a739432242e7d254f93",
|
||||
"sha256:f7e8f1ee28e0a05831c92dc1c0c1c94af5289963b7cf09eca5b5e3ce4f8c91b0",
|
||||
"sha256:f8169ec628880bdbca67082a9196e2106060a4a5cbd486ac51881a4df805a36f",
|
||||
"sha256:fbc88d3ba402b5d041d204ec2449c4078898f89c4a6e6f0ed1c1a510ef1e221d",
|
||||
"sha256:fbd3fe37353c62fd0eb19fb76f78aa693716262bcd5f9c14bb9e5aca4b3f0dc4"
|
||||
"sha256:052b670fafbe30966bbe5d025e90b2a491f85dfe5b2583a163b5e60a85a321ad",
|
||||
"sha256:0653d012b3bf45f194e5e6a41df9258811ac8fc395579fa82958a8b76286bea4",
|
||||
"sha256:0a069c8483466806ab94ea9068c34b200b8bfc66b6762f45a831c4baaa9e8cdd",
|
||||
"sha256:0cf0da36a212978be2c2e2e2d04bdff46f850108fccc1851332bcae51c8907cc",
|
||||
"sha256:131d4be09bea7ce2577f9623e415cab287a3c8e0624f778c1d955ec7c281bd4d",
|
||||
"sha256:144486e029793a733e43b2e37df16a16df4ceb62102636ff3db6033994711066",
|
||||
"sha256:1ddf14031a3882f684b8642cb74eea3af93a2be68893901b2b387c5fd92a03ec",
|
||||
"sha256:1eba476b1b242620c266edf6325b443a2e22b633217a9835a52d8da2b5c051f9",
|
||||
"sha256:20f61c9944f0be2dc2b75689ba409938c14876c19d02f7585af4460b6a21403e",
|
||||
"sha256:22960019a842777a9fa5134c2364efaed5fbf9610ddc5c904bd3a400973b0eb8",
|
||||
"sha256:22e7ebc231d28393dfdc19b185d97e14a0f178bedd78e85aad660e93b646604e",
|
||||
"sha256:23cbb932cc53a86ebde0fb72e7e645f9a5eec1a5af7aa9ce333e46286caef783",
|
||||
"sha256:29c04741b9ae13d1e94cf93fca257730b97ce6ea64cfe1eba11cf9ac4e85afb6",
|
||||
"sha256:2bde29cc44fa81c0a0c8686992c3080b37c488df167a371500b2a43ce9f026d1",
|
||||
"sha256:2cdc55ca07b4e70dda898d2ab7150ecf17c990076d3acd7a5f3b25cb23a69f1c",
|
||||
"sha256:370f6e97d02bf2dd20d7468ce4f38e173a124e769762d00beadec3bc2f4b3bc4",
|
||||
"sha256:395161bbdbd04a8333b9ff9763a05e9ceb4fe210e3c7690f5e68cedd3d65d8e1",
|
||||
"sha256:44136355e2f5e06bf6b23d337a75386371ba742ffa771440b85bed367c1318d1",
|
||||
"sha256:44a6c2f6374e0033873e9ed577a54a3602b4f609867794c1a3ebba65e4c93ee7",
|
||||
"sha256:4919899577ba37f505aaebdf6e7dc812d55e8f097331312db7f1aab18767cce8",
|
||||
"sha256:4b4b1fe58cd102d75ef0552cf17242705ce0759f9695334a56644ad2d83903fe",
|
||||
"sha256:4bdd56ee719a8f751cf5a593476a441c4e56c9b64dc1f0f30902858c4ef8771d",
|
||||
"sha256:4bf41b8b0a80708f7e0384519795e80dcb44d7199a35d52c15cc674d10b3081b",
|
||||
"sha256:4cac3405d8dda8bc6ed499557625585544dd5cbf32072dcc72b5a176cb1271c8",
|
||||
"sha256:4fe7fda2fe7c8890d454f2cbc91d6c01baf206fbc96d89a80241a02985118c0c",
|
||||
"sha256:50921c140561d3db2ab9f5b11c5184846cde686bb5a9dc64cae442926e86f3af",
|
||||
"sha256:5217c25229b6a85049416a5c1e6451e9060a1edcf988641e309dbe3ab26d3e49",
|
||||
"sha256:5352bea8a8f84b89d45ccc503f390a6be77917932b1c98c4cdc3565137acc714",
|
||||
"sha256:542e3e306d1669b25936b64917285cdffcd4f5c6f0247636fec037187bd93542",
|
||||
"sha256:543883e3496c8b6d58bd036c99486c3c8387c2fc01f7a342b760c1ea3158a318",
|
||||
"sha256:586b36ebda81e6c1a9c5a5d0bfdc236399ba6595e1397842fd4a45648c30f35e",
|
||||
"sha256:597f899f4ed42a38df7b0e46714880fb4e19a25c2f66e5c908805466721760f5",
|
||||
"sha256:5a260758454580f11dd8743fa98319bb046037dfab4f7828008909d0aa5292bc",
|
||||
"sha256:5aefb84a301327ad115e9d346c8e2760009131d9d4b4c6b213648d02e2abe144",
|
||||
"sha256:5e6a5567078b3eaed93558842346c9d678e116ab0135e22eb72db8325e90b453",
|
||||
"sha256:5ff525698de226c0ca743bfa71fc6b378cda2ddcf0d22d7c37b1cc925c9650a5",
|
||||
"sha256:61edbca89aa3f5ef7ecac8c23d975fe7261c12665f1d90a6b1af527bba86ce61",
|
||||
"sha256:659175b2144d199560d99a8d13b2228b85e6019b6e09e556209dfb8c37b78a11",
|
||||
"sha256:6a9a19bea8495bb419dc5d38c4519567781cd8d571c72efc6aa959473d10221a",
|
||||
"sha256:6b30bddd61d2a3261f025ad0f9ee2586988c6a00c780a2fb0a92cea2aa702c54",
|
||||
"sha256:6ffd55b5aedc6f25fd8d9f905c9376ca44fcf768673ffb9d160dd6f409bfda73",
|
||||
"sha256:702d8fc6f25bbf412ee706bd73019da5e44a8400861dfff7ff31eb5b4a1276dc",
|
||||
"sha256:74bcab50a13960f2a610cdcd066e25f1fd59e23b69637c92ad470784a51b1347",
|
||||
"sha256:75f591b2055523fc02a4bbe598aa867df9e953255f0b7f7715d2a36a9c30065c",
|
||||
"sha256:763b64853b0a8f4f9cfb41a76a4a85a9bcda7fdda5cb057016e7706fde928e66",
|
||||
"sha256:76c598ca73ec73a2f568e2a72ba46c3b6c8690ad9a07092b18e48ceb936e9f0c",
|
||||
"sha256:78d680ef3e4d405f36f0d6d1ea54e740366f061645930072d39bca16a10d8c93",
|
||||
"sha256:7b280948d00bd3973c1998f92e22aa3ecb76682e3a4255f33e1020bd32adf443",
|
||||
"sha256:7db345956ecce0c99b97b042b4ca7326feeec6b75facd8390af73b18e2650ffc",
|
||||
"sha256:7dbdce0c534bbf52274b94768b3498abdf675a691fec5f751b6057b3030f34c1",
|
||||
"sha256:7ef6b5942e6bfc5706301a18a62300c60db9af7f6368042227ccb7eeb22d0892",
|
||||
"sha256:7f5a3ffc731494f1a57bd91c47dc483a1e10048131ffb52d901bfe2beb6102e8",
|
||||
"sha256:8a45b6514861916c429e6059a55cf7db74670eaed2052a648e3e4d04f070e001",
|
||||
"sha256:8ad241da7fac963d7573cc67a064c57c58766b62a9a20c452ca1f21050868dfa",
|
||||
"sha256:8b0886885f7323beea6f552c28bff62cbe0983b9fbb94126531693ea6c5ebb90",
|
||||
"sha256:8ca88da1bd78990b536c4a7765f719803eb4f8f9971cc22d6ca965c10a7f2c4c",
|
||||
"sha256:8e0caeff18b96ea90fc0eb6e3bdb2b10ab5b01a95128dfeccb64a7238decf5f0",
|
||||
"sha256:957403a978e10fb3ca42572a23e6f7badff39aa1ce2f4ade68ee452dc6807692",
|
||||
"sha256:9af69f6746120998cd9c355e9c3c6aec7dff70d47247188feb4f829502be8ab4",
|
||||
"sha256:9c94f7cc91ab16b36ba5ce476f1904c91d6c92441f01cd61a8e2729442d6fcf5",
|
||||
"sha256:a37d51fa9a00d265cf73f3de3930fa9c41548177ba4f0faf76e61d512c774690",
|
||||
"sha256:a3a98921da9a1bf8457aeee6a551948a83601689e5ecdd736894ea9bbec77e83",
|
||||
"sha256:a3c1ebd4ed8e76e886507c9eddb1a891673686c813adf889b864a17fafcf6d66",
|
||||
"sha256:a5f9505efd574d1e5b4a76ac9dd92a12acb2b309551e9aa874c13c11caefbe4f",
|
||||
"sha256:a8ff454ef0bb061e37df03557afda9d785c905dab15584860f982e88be73015f",
|
||||
"sha256:a9d0b68ac1743964755ae2d89772c7e6fb0118acd4d0b7464eaf3921c6b49dd4",
|
||||
"sha256:aa62a07ac93b7cb6b7d0389d8ef57ffc321d78f60c037b19dfa78d6b17c928ee",
|
||||
"sha256:ac741bf78b9bb432e2d314439275235f41656e189856b11fb4e774d9f7246d81",
|
||||
"sha256:ae1e96785696b543394a4e3f15f3f225d44f3c55dafe3f206493031419fedf95",
|
||||
"sha256:b683e5fd7f74fb66e89a1ed16076dbab3f8e9f34c18b1979ded614fe10cdc4d9",
|
||||
"sha256:b7a8b43ee64ca8f4befa2bea4083f7c52c92864d8518244bfa6e88c751fa8fff",
|
||||
"sha256:b8e38472739028e5f2c3a4aded0ab7eadc447f0d84f310c7a8bb697ec417229e",
|
||||
"sha256:bfff48c7bd23c6e2aec6454aaf6edc44444b229e94743b34bdcdda2e35126cf5",
|
||||
"sha256:c14b63c9d7bab795d17392c7c1f9aaabbffd4cf4387725a0ac69109fb3b550c6",
|
||||
"sha256:c27cc1e4b197092e50ddbf0118c788d9977f3f8f35bfbbd3e76c1846a3443df7",
|
||||
"sha256:c28d3309ebd6d6b2cf82969b5179bed5fefe6142c70f354ece94324fa11bf6a1",
|
||||
"sha256:c670f4773f2f6f1957ff8a3962c7dd12e4be54d05839b216cb7fd70b5a1df394",
|
||||
"sha256:ce6910b56b700bea7be82c54ddf2e0ed792a577dfaa4a76b9af07d550af435c6",
|
||||
"sha256:d0213671691e341f6849bf33cd9fad21f7b1cb88b89e024f33370733fec58742",
|
||||
"sha256:d03fe67b2325cb3f09be029fd5da8df9e6974f0cde2c2ac6a79d2634e791dd57",
|
||||
"sha256:d0e5af9a9effb88535a472e19169e09ce750c3d442fb222254a276d77808620b",
|
||||
"sha256:d243b36fbf3d73c25e48014961e83c19c9cc92530516ce3c43050ea6276a2ab7",
|
||||
"sha256:d26166acf62f731f50bdd885b04b38828436d74e8e362bfcb8df221d868b5d9b",
|
||||
"sha256:d403d781b0e06d2922435ce3b8d2376579f0c217ae491e273bab8d092727d244",
|
||||
"sha256:d8716f82502997b3d0895d1c64c3b834181b1eaca28f3f6336a71777e437c2af",
|
||||
"sha256:e4f781ffedd17b0b834c8731b75cce2639d5a8afe961c1e58ee7f1f20b3af185",
|
||||
"sha256:e613a98ead2005c4ce037c7b061f2409a1a4e45099edb0ef3200ee26ed2a69a8",
|
||||
"sha256:ef4163770525257876f10e8ece1cf25b71468316f61451ded1a6f44273eedeb5"
|
||||
],
|
||||
"markers": "python_version >= '3.6'",
|
||||
"version": "==2022.3.2"
|
||||
"version": "==2022.10.31"
|
||||
},
|
||||
"requests": {
|
||||
"hashes": [
|
||||
@@ -1003,10 +1033,11 @@
|
||||
},
|
||||
"selenium": {
|
||||
"hashes": [
|
||||
"sha256:a733dd77d3171b846893f4d51b18967d809313f547a10974e26579f9ce797462"
|
||||
"sha256:06a1c7d9f313130b21c3218ddd8852070d0e7419afdd31f96160cd576555a5ce",
|
||||
"sha256:3aefa14a28a42e520550c1cd0f29cf1d566328186ea63aa9a3e01fb265b5894d"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==4.5.0"
|
||||
"version": "==4.7.2"
|
||||
},
|
||||
"six": {
|
||||
"hashes": [
|
||||
@@ -1049,11 +1080,11 @@
|
||||
},
|
||||
"telethon": {
|
||||
"hashes": [
|
||||
"sha256:8df802aad2d11f7198f1d5b1d84c7498ef19c28e160041dcb8aaf0814f91115b",
|
||||
"sha256:a085348801bd62db79ad75c9a67c5c8312507b113f0228b92e2dd4397edc7c1d"
|
||||
"sha256:148ac8c27908853d5d8a116d55ce947e9ba167bb697c75226ae95645b2e5a504",
|
||||
"sha256:de7a1619110a2c06390fb5340839c6503c6b108b5f1a2f3bbe1ef60f02cecacb"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==1.25.4"
|
||||
"version": "==1.26.0"
|
||||
},
|
||||
"text-unidecode": {
|
||||
"hashes": [
|
||||
@@ -1074,7 +1105,7 @@
|
||||
"sha256:5f4f682a004951c1b450bc753c710e9280c5746ce6ffedee253ddbcbf54cf1e4",
|
||||
"sha256:6fee160d6ffcd1b1c68c65f14c829c22832bc401726335ce92c52d395944a6a1"
|
||||
],
|
||||
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
|
||||
"index": "pypi",
|
||||
"version": "==4.64.1"
|
||||
},
|
||||
"trio": {
|
||||
@@ -1110,11 +1141,11 @@
|
||||
},
|
||||
"tzdata": {
|
||||
"hashes": [
|
||||
"sha256:04a680bdc5b15750c39c12a448885a51134a27ec9af83667663f0b3a1bf3f342",
|
||||
"sha256:91f11db4503385928c15598c98573e3af07e7229181bee5375bd30f1695ddcae"
|
||||
"sha256:2b88858b0e3120792a3c0635c23daf36a7d7eeeca657c323da299d2094402a0d",
|
||||
"sha256:fe5f866eddd8b96e9fcba978f8e503c909b19ea7efda11e52e39494bad3a7bfa"
|
||||
],
|
||||
"markers": "python_version >= '3.6'",
|
||||
"version": "==2022.6"
|
||||
"version": "==2022.7"
|
||||
},
|
||||
"tzlocal": {
|
||||
"hashes": [
|
||||
@@ -1149,11 +1180,11 @@
|
||||
},
|
||||
"vk-url-scraper": {
|
||||
"hashes": [
|
||||
"sha256:3718a569e431c9c2bc7e92e9156e25b7112dc0b9b461c8001fa481a00ccbd3bc",
|
||||
"sha256:baebe32bb29d6f188d849f38ecc43d04d5b5bad05db7f31dfdbe450f684042f0"
|
||||
"sha256:1cd6daad89a1f920902cb68c5952c5ab5e80ba2bf4a8c3657c781b5b0f9d406b",
|
||||
"sha256:d430de947575e321cedceecfdf198b8bd14db3026038b924547e8b1c7c6a09ed"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==0.3.8"
|
||||
"version": "==0.3.10"
|
||||
},
|
||||
"websockets": {
|
||||
"hashes": [
|
||||
@@ -1266,11 +1297,11 @@
|
||||
},
|
||||
"pycodestyle": {
|
||||
"hashes": [
|
||||
"sha256:2c9607871d58c76354b697b42f5d57e1ada7d261c261efac224b664affdc5785",
|
||||
"sha256:d1735fc58b418fd7c5f658d28d943854f8a849b01a5d0a1e6f3f3fdd0166804b"
|
||||
"sha256:347187bdb476329d98f695c213d7295a846d1152ff4fe9bacb8a9590b8ee7053",
|
||||
"sha256:8a4eaf0d0495c7395bdab3589ac2db602797d76207242c17d470186815706610"
|
||||
],
|
||||
"markers": "python_version >= '3.6'",
|
||||
"version": "==2.9.1"
|
||||
"version": "==2.10.0"
|
||||
},
|
||||
"tomli": {
|
||||
"hashes": [
|
||||
|
||||
@@ -22,6 +22,12 @@ Use this to make sure you help making sure you did all the required steps:
|
||||
* [ ] (optional for instagram) `instaloader.session` file which appears after the 1st run and login in telegram
|
||||
* [ ] (optional for browsertrix) `profile.tar.gz` file
|
||||
|
||||
### Private telegram channels
|
||||
* Cannot use bot token
|
||||
* Should have one with bot token, one without
|
||||
* Setup join all private invite links at the start
|
||||
*
|
||||
|
||||
## Setup
|
||||
### Always required
|
||||
1. [A Google Service account is necessary for use with `gspread`.](https://gspread.readthedocs.io/en/latest/oauth2.html#for-bots-using-service-account) Credentials for this account should be stored in `service_account.json`, in the same directory as the script.
|
||||
|
||||
@@ -8,8 +8,8 @@ steps:
|
||||
# - twitter
|
||||
# - instagram
|
||||
# - webarchive # this way it runs as a failsafe only
|
||||
enrichers:
|
||||
- screenshot
|
||||
# enrichers:
|
||||
# - screenshot
|
||||
# - wacz
|
||||
# - webarchive # this way it runs for every case, webarchive extends archiver and enrichment
|
||||
# - thumbnails
|
||||
@@ -29,11 +29,11 @@ configurations:
|
||||
global:
|
||||
- save_logs: False
|
||||
gsheets_feeder:
|
||||
sheet: auto-archiver-test
|
||||
sheet: my-auto-archiver
|
||||
header: 2 # defaults to 1 in GSheetsFeeder
|
||||
service_account: "secrets/service_account.json"
|
||||
allow_worksheets: "aa-refactor-tests"
|
||||
block_worksheets: "blocked,test-cases-008"
|
||||
# allow_worksheets: "allowed"
|
||||
# block_worksheets: "blocked1,blocked2"
|
||||
columns:
|
||||
'url': 'link'
|
||||
'status': 'archive status'
|
||||
@@ -49,6 +49,16 @@ configurations:
|
||||
'hash': 'hash'
|
||||
'wacz': 'wacz'
|
||||
'replaywebpage': 'replaywebpage'
|
||||
telethon:
|
||||
api_id: "1234567"
|
||||
api_hash: "examplehash"
|
||||
session_file: "secrets/anon"
|
||||
channel_invites:
|
||||
- invite: https://t.me/+XXXXXXXXXXXXXX
|
||||
id: 1000000000
|
||||
- invite: https://t.me/joinchat/XXXXXXXXXXXXXX
|
||||
id: 1000000001
|
||||
|
||||
tiktok:
|
||||
api_keys:
|
||||
- username: 1
|
||||
@@ -60,7 +70,7 @@ configurations:
|
||||
token: "here"
|
||||
screenshot:
|
||||
width: 1280
|
||||
height: 720000
|
||||
height: 4600
|
||||
wacz:
|
||||
profile: secrets/profile.tar.gz
|
||||
webarchive:
|
||||
@@ -1,5 +1,6 @@
|
||||
# we need to explicitly expose the available imports here
|
||||
from .base_archiver import Archiver, ArchiveResult
|
||||
from .archiver import Archiverv2
|
||||
from .telegram_archiver import TelegramArchiver
|
||||
from .telethon_archiver import TelethonArchiver
|
||||
from .tiktok_archiver import TiktokArchiver
|
||||
@@ -8,4 +9,6 @@ from .youtubedl_archiver import YoutubeDLArchiver
|
||||
from .twitter_archiver import TwitterArchiver
|
||||
from .vk_archiver import VkArchiver
|
||||
from .twitter_api_archiver import TwitterApiArchiver
|
||||
from .instagram_archiver import InstagramArchiver
|
||||
from .instagram_archiver import InstagramArchiver
|
||||
|
||||
from .telethon_archiverv2 import TelethonArchiver
|
||||
26
src/archivers/archiver.py
Normal file
26
src/archivers/archiver.py
Normal file
@@ -0,0 +1,26 @@
|
||||
from __future__ import annotations
|
||||
from abc import abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from metadata import Metadata
|
||||
from steps.step import Step
|
||||
|
||||
|
||||
@dataclass
|
||||
class Archiverv2(Step):
|
||||
name = "archiver"
|
||||
|
||||
def __init__(self, config: dict) -> None:
|
||||
# without this STEP.__init__ is not called
|
||||
super().__init__(config)
|
||||
# self.setup()
|
||||
|
||||
# only for typing...
|
||||
def init(name: str, config: dict) -> Archiverv2:
|
||||
return Step.init(name, config, Archiverv2)
|
||||
|
||||
def setup(self) -> None:
|
||||
# used when archivers need to login or do other one-time setup
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def download(self, item: Metadata) -> Metadata: pass
|
||||
114
src/archivers/telethon_archiverv2.py
Normal file
114
src/archivers/telethon_archiverv2.py
Normal file
@@ -0,0 +1,114 @@
|
||||
from archivers import Archiverv2
|
||||
from metadata import Metadata
|
||||
from telethon.sync import TelegramClient
|
||||
from telethon.errors import ChannelInvalidError
|
||||
from telethon.tl.types import PeerUser, PeerChat, PeerChannel
|
||||
from telethon.tl.functions.messages import ImportChatInviteRequest
|
||||
from telethon.errors.rpcerrorlist import UserAlreadyParticipantError, FloodWaitError, InviteRequestSentError, InviteHashExpiredError
|
||||
from loguru import logger
|
||||
from tqdm import tqdm
|
||||
import re, time, json
|
||||
|
||||
|
||||
|
||||
class TelethonArchiver(Archiverv2):
|
||||
name = "telethon"
|
||||
link_pattern = re.compile(r"https:\/\/t\.me(\/c){0,1}\/(.+)\/(\d+)")
|
||||
invite_pattern = re.compile(r"t.me(\/joinchat){0,1}\/\+?(.+)")
|
||||
|
||||
def __init__(self, config: dict) -> None:
|
||||
super().__init__(config)
|
||||
assert self.api_id is not None and type(self.api_id) == str and len(self.api_id) > 0, f"invalid telethon api_id value ({self.api_id}) should be a valid string"
|
||||
assert self.api_hash is not None and type(self.api_hash) == str and len(self.api_hash) > 0, f"invalid telethon api_hash value ({self.api_hash}) should be a valid string"
|
||||
|
||||
self.client = TelegramClient(self.session_file, self.api_id, self.api_hash)
|
||||
|
||||
@staticmethod
|
||||
def configs() -> dict:
|
||||
return {
|
||||
"api_id": {"default": None, "help": "telegram API_ID value, go to https://my.telegram.org/apps"},
|
||||
"api_hash": {"default": None, "help": "telegram API_HASH value, go to https://my.telegram.org/apps"},
|
||||
# "bot_token": {"default": None, "help": "optional, but allows access to more content such as large videos, talk to @botfather"},
|
||||
"session_file": {"default": "secrets/anon", "help": "optional, records the telegram login session for future usage"},
|
||||
"channel_invites": {
|
||||
"default": {},
|
||||
"help": "(JSON string) private channel invite links (format: t.me/joinchat/HASH OR t.me/+HASH) and (optional but important to avoid hanging for minutes on startup) channel id (format: CHANNEL_ID taken from a post url like https://t.me/c/CHANNEL_ID/1), the telegram account will join any new channels on setup",
|
||||
"cli_set": lambda cli_val, cur_val: dict(cur_val, **json.loads(cli_val))
|
||||
}
|
||||
}
|
||||
|
||||
def setup(self) -> None:
|
||||
logger.info(f"SETUP {self.name} checking login...")
|
||||
with self.client.start(): pass
|
||||
|
||||
if len(self.channel_invites):
|
||||
logger.info(f"SETUP {self.name} joining channels...")
|
||||
with self.client.start():
|
||||
# get currently joined channels
|
||||
# https://docs.telethon.dev/en/stable/modules/custom.html#module-telethon.tl.custom.dialog
|
||||
joined_channel_ids = [c.id for c in self.client.get_dialogs() if c.is_channel]
|
||||
logger.info(f"already part of {len(joined_channel_ids)} channels")
|
||||
|
||||
i = 0
|
||||
pbar = tqdm(desc=f"joining {len(self.channel_invites)} invite links", total=len(self.channel_invites))
|
||||
while i < len(self.channel_invites):
|
||||
channel_invite = self.channel_invites[i]
|
||||
channel_id = channel_invite.get("id", False)
|
||||
invite = channel_invite["invite"]
|
||||
if (match := self.invite_pattern.search(invite)):
|
||||
try:
|
||||
if channel_id:
|
||||
ent = self.client.get_entity(int(channel_id)) # fails if not a member
|
||||
else:
|
||||
ent = self.client.get_entity(invite) # fails if not a member
|
||||
logger.warning(f"please add the property id='{ent.id}' to the 'channel_invites' configuration where {invite=}, not doing so can lead to a minutes-long setup time due to telegram's rate limiting.")
|
||||
except ValueError as e:
|
||||
logger.info(f"joining new channel {invite=}")
|
||||
try:
|
||||
self.client(ImportChatInviteRequest(match.group(2)))
|
||||
except UserAlreadyParticipantError as e:
|
||||
logger.info(f"already joined {invite=}")
|
||||
except InviteRequestSentError:
|
||||
logger.warning(f"already sent a join request with {invite} still no answer")
|
||||
except InviteHashExpiredError:
|
||||
logger.warning(f"{invite=} has expired please find a more recent one")
|
||||
except Exception as e:
|
||||
logger.error(f"could not join channel with {invite=} due to {e}")
|
||||
except FloodWaitError as e:
|
||||
logger.warning(f"got a flood error, need to wait {e.seconds} seconds")
|
||||
time.sleep(e.seconds)
|
||||
continue
|
||||
else:
|
||||
logger.warning(f"Invalid invite link {invite}")
|
||||
i+=1
|
||||
pbar.update()
|
||||
|
||||
|
||||
def download(self, item: Metadata) -> Metadata:
|
||||
url = self.get_url(item)
|
||||
print(f"downloading {url=}")
|
||||
# detect URLs that we definitely cannot handle
|
||||
match = self.link_pattern.search(url)
|
||||
if not match: return False
|
||||
|
||||
# app will ask (stall for user input!) for phone number and auth code if anon.session not found
|
||||
# TODO: not using bot_token since then private channels cannot be archived
|
||||
# with self.client.start(bot_token=self.bot_token):
|
||||
with self.client.start():
|
||||
# self.client(ImportChatInviteRequest('4kAkN49IKJBhZDk6'))
|
||||
is_private = match.group(1) == "/c"
|
||||
print(f"{is_private=}")
|
||||
chat = int(match.group(2)) if is_private else match.group(2)
|
||||
post_id = int(match.group(3))
|
||||
|
||||
try:
|
||||
post = self.client.get_messages(chat, ids=post_id)
|
||||
except ValueError as e:
|
||||
logger.error(f"Could not fetch telegram {url} possibly it's private: {e}")
|
||||
return False
|
||||
except ChannelInvalidError as e:
|
||||
logger.error(f"Could not fetch telegram {url}. This error can be fixed if you setup a bot_token in addition to api_id and api_hash: {e}")
|
||||
return False
|
||||
|
||||
if post is None: return False
|
||||
print(post)
|
||||
@@ -3,9 +3,9 @@
|
||||
import argparse, yaml
|
||||
from dataclasses import dataclass, field
|
||||
from typing import List
|
||||
from feeders.feeder import Feeder
|
||||
from archivers import Archiverv2
|
||||
from feeders import Feeder
|
||||
from steps.step import Step
|
||||
from utils import Util
|
||||
from enrichers import Enricher
|
||||
from collections import defaultdict
|
||||
|
||||
@@ -16,10 +16,11 @@ class ConfigV2:
|
||||
configurable_parents = [
|
||||
Feeder,
|
||||
Enricher,
|
||||
Archiverv2,
|
||||
# Util
|
||||
]
|
||||
feeder: Step # TODO:= BaseFeeder
|
||||
archivers: List[Step] = field(default_factory=[]) # TODO: fix type
|
||||
archivers: List[Archiverv2] = field(default_factory=[]) # TODO: fix type
|
||||
enrichers: List[Enricher] = field(default_factory=[])
|
||||
formatters: List[Step] = field(default_factory=[]) # TODO: fix type
|
||||
storages: List[Step] = field(default_factory=[]) # TODO: fix type
|
||||
@@ -48,7 +49,7 @@ class ConfigV2:
|
||||
assert "." not in child.name, f"class prop name cannot contain dots('.'): {child.name}"
|
||||
assert "." not in config, f"config property cannot contain dots('.'): {config}"
|
||||
config_path = f"{child.name}.{config}"
|
||||
parser.add_argument(f'--{config_path}', action='store', dest=config_path, help=details['help'])
|
||||
parser.add_argument(f'--{config_path}', action='store', dest=config_path, help=f"{details['help']} (defaults to {details['default']})")
|
||||
self.defaults[config_path] = details["default"]
|
||||
if "cli_set" in details:
|
||||
self.cli_ops[config_path] = details["cli_set"]
|
||||
@@ -82,9 +83,11 @@ class ConfigV2:
|
||||
|
||||
self.feeder = Feeder.init(steps.get("feeder", "cli_feeder"), self.config)
|
||||
self.enrichers = [Enricher.init(e, self.config) for e in steps.get("enrichers", [])]
|
||||
self.archivers = [Archiverv2.init(e, self.config) for e in steps.get("archivers", [])]
|
||||
|
||||
print("feeder", self.feeder)
|
||||
print("enrichers", [e for e in self.enrichers])
|
||||
print("archivers", [e for e in self.archivers])
|
||||
|
||||
def validate(self):
|
||||
pass
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
from utils import Webdriver
|
||||
from . import Enricher
|
||||
from metadata import Metadata
|
||||
from loguru import logger
|
||||
from selenium.common.exceptions import TimeoutException
|
||||
import time
|
||||
|
||||
|
||||
class ScreenshotEnricher(Enricher):
|
||||
@@ -11,43 +14,19 @@ class ScreenshotEnricher(Enricher):
|
||||
return {
|
||||
"width": {"default": 1280, "help": "width of the screenshots"},
|
||||
"height": {"default": 720, "help": "height of the screenshots"},
|
||||
"timeout": {"default": 60, "help": "timeout for taking the screenshot"}
|
||||
}
|
||||
|
||||
def enrich(self, item: Metadata) -> Metadata:
|
||||
url = self.get_url(item)
|
||||
print("enrich")
|
||||
# driver = config.webdriver
|
||||
# with driver as Webdriver(): # TODO: make a util
|
||||
# #TODO: take screenshot
|
||||
# pass
|
||||
print(f"enriching {url=}")
|
||||
with Webdriver(self.width, self.height, self.timeout, 'facebook.com' in url) as driver: # TODO: make a util
|
||||
try:
|
||||
driver.get(url)
|
||||
time.sleep(2)
|
||||
except TimeoutException:
|
||||
logger.info("TimeoutException loading page for screenshot")
|
||||
|
||||
# logger.debug(f"getting screenshot for {url=}")
|
||||
# key = self._get_key_from_url(url, ".png", append_datetime=True)
|
||||
# filename = os.path.join(Storage.TMP_FOLDER, key)
|
||||
|
||||
# # Accept cookies popup dismiss for ytdlp video
|
||||
# if 'facebook.com' in url:
|
||||
# try:
|
||||
# logger.debug(f'Trying fb click accept cookie popup for {url}')
|
||||
# self.driver.get("http://www.facebook.com")
|
||||
# foo = self.driver.find_element(By.XPATH, "//button[@data-cookiebanner='accept_only_essential_button']")
|
||||
# foo.click()
|
||||
# logger.debug(f'fb click worked')
|
||||
# # linux server needs a sleep otherwise facebook cookie won't have worked and we'll get a popup on next page
|
||||
# time.sleep(2)
|
||||
# except:
|
||||
# logger.warning(f'Failed on fb accept cookies for url {url}')
|
||||
|
||||
# try:
|
||||
# self.driver.get(url)
|
||||
# time.sleep(6)
|
||||
# except TimeoutException:
|
||||
# logger.info("TimeoutException loading page for screenshot")
|
||||
|
||||
# self.driver.save_screenshot(filename)
|
||||
# self.storage.upload(filename, key, extra_args={'ACL': 'public-read', 'ContentType': 'image/png'})
|
||||
|
||||
# cdn_url = self.storage.get_cdn_url(key)
|
||||
# self.add_to_media(cdn_url, key)
|
||||
|
||||
# return cdn_url
|
||||
#TODO: return saved object
|
||||
driver.save_screenshot("TODO-HASH_OR_UUID.png")
|
||||
return None
|
||||
|
||||
@@ -4,7 +4,7 @@ import json, gspread
|
||||
from loguru import logger
|
||||
|
||||
# from . import Enricher
|
||||
from feeders.feeder import Feeder
|
||||
from feeders import Feeder
|
||||
from steps.gsheet import Gsheets
|
||||
from utils import GWorksheet
|
||||
|
||||
@@ -30,7 +30,7 @@ class GsheetsFeeder(Gsheets, Feeder):
|
||||
},
|
||||
"block_worksheets": {
|
||||
"default": set(),
|
||||
"help": "(CSV) explicitly block some worksheets from being processed, defaults to empty",
|
||||
"help": "(CSV) explicitly block some worksheets from being processed",
|
||||
"cli_set": lambda cli_val, cur_val: set(cli_val.split(","))
|
||||
}
|
||||
})
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
|
||||
from __future__ import annotations
|
||||
from typing import Union, Dict
|
||||
from typing import Any, Union, Dict
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@@ -12,18 +12,28 @@ class Metadata:
|
||||
# title: str
|
||||
# url: str
|
||||
# hash: str
|
||||
metadata: Dict[str, Metadata]
|
||||
metadata: Dict[str, Any]
|
||||
|
||||
@staticmethod
|
||||
def merge(left: Metadata, right: Metadata, overwrite_left=True) -> Metadata:
|
||||
# TODO: remove and use default?
|
||||
def __init__(self) -> None:
|
||||
self.status = ""
|
||||
self.metadata = {}
|
||||
|
||||
# @staticmethod
|
||||
def merge(self: Metadata, right: Metadata, overwrite_left=True) -> Metadata:
|
||||
# should return a merged version of the Metadata
|
||||
# will work for archived() and enriched()
|
||||
# what if 2 metadatas contain the same keys? only one can remain! : overwrite_left
|
||||
pass
|
||||
|
||||
def get(self, key: str) -> Union[Metadata, str]:
|
||||
# TODO: setters?
|
||||
def set(self, key: str, val: Any) -> Union[Metadata, str]:
|
||||
# goes through metadata and returns the Metadata available
|
||||
pass
|
||||
self.metadata[key] = val
|
||||
|
||||
def get(self, key: str, default: Any = None) -> Union[Metadata, str]:
|
||||
# goes through metadata and returns the Metadata available
|
||||
return self.metadata.get(key, default)
|
||||
|
||||
def as_json(self) -> str:
|
||||
# converts all metadata and data into JSON
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
from __future__ import annotations
|
||||
from ast import List
|
||||
from typing import Union, Dict
|
||||
from dataclasses import dataclass
|
||||
from archivers.archiver import Archiverv2
|
||||
|
||||
from enrichers.enricher import Enricher
|
||||
from metadata import Metadata
|
||||
|
||||
"""
|
||||
how not to couple the different pieces of logic
|
||||
@@ -108,12 +111,13 @@ Once an archiver returns a link to a local file (for eg to a storage), how do we
|
||||
The context metadata should include a temporary folder (maybe a LocalStorage instance?)
|
||||
"""
|
||||
|
||||
|
||||
class ArchivingOrchestrator:
|
||||
def __init__(self, config) -> None:
|
||||
# in config.py we should test that the archivers exist and log mismatches (blocking execution)
|
||||
# identify each formatter, storage, database, etc
|
||||
# self.feeder = Feeder.init(config.feeder, config.get(config.feeder))
|
||||
|
||||
|
||||
# Is it possible to overwrite config.yaml values? it could be useful: share config file and modify gsheets_feeder.sheet via CLI
|
||||
# where does that update/processing happen? in config.py
|
||||
# reflection for Archiver to know wihch child classes it has? use Archiver.__subclasses__
|
||||
@@ -123,7 +127,13 @@ class ArchivingOrchestrator:
|
||||
# ]
|
||||
self.feeder = config.feeder
|
||||
self.enrichers = config.enrichers
|
||||
self.archivers: List[Archiverv2] = config.archivers
|
||||
|
||||
for a in self.archivers: a.setup()
|
||||
|
||||
self.formatters = []
|
||||
self.storages = []
|
||||
self.databases = []
|
||||
# self.formatters = [
|
||||
# Formatter.init(f, config)
|
||||
# for f in config.formatters
|
||||
@@ -145,30 +155,33 @@ class ArchivingOrchestrator:
|
||||
def feed(self) -> list(ArchiveResult):
|
||||
for url in self.feeder:
|
||||
print("ARCHIVING", url)
|
||||
# self.archive(url)
|
||||
self.archive(url)
|
||||
# how does this handle the parameters like folder which can be different for each archiver?
|
||||
# the storage needs to know where to archive!!
|
||||
# solution: feeders have context: extra metadata that they can read or ignore,
|
||||
# solution: feeders have context: extra metadata that they can read or ignore,
|
||||
# all of it should have sensible defaults (eg: folder)
|
||||
# default feeder is a list with 1 element
|
||||
|
||||
def archive(self, url) -> Union[ArchiveResult, None]:
|
||||
url = clear_url(url)
|
||||
result = Metadata(url=url)
|
||||
|
||||
# TODO:
|
||||
# url = clear_url(url)
|
||||
# result = Metadata(url=url)
|
||||
result = Metadata()
|
||||
result.set("url", url)
|
||||
|
||||
should_archive = True
|
||||
for d in databases: should_archive &= d.should_process(url)
|
||||
for d in self.databases: should_archive &= d.should_process(url)
|
||||
# should storages also be able to check?
|
||||
for s in storages: should_archive &= s.should_process(url)
|
||||
for s in self.storages: should_archive &= s.should_process(url)
|
||||
|
||||
if not should_archive:
|
||||
print("skipping")
|
||||
return "skipping"
|
||||
|
||||
# signal to DB that archiving has started
|
||||
for d in databases:
|
||||
for d in self.databases:
|
||||
# are the databases to decide whether to archive?
|
||||
# they can simply return True by default, otherwise they can avoid duplicates. should this logic be more granular, for example on the archiver level: a tweet will not need be scraped twice, whereas an instagram profile might. the archiver could not decide from the link which parts to archive,
|
||||
# they can simply return True by default, otherwise they can avoid duplicates. should this logic be more granular, for example on the archiver level: a tweet will not need be scraped twice, whereas an instagram profile might. the archiver could not decide from the link which parts to archive,
|
||||
# instagram profile example: it would always re-archive everything
|
||||
# maybe the database/storage could use a hash/key to decide if there's a need to re-archive
|
||||
if d.should_process(url):
|
||||
@@ -180,15 +193,15 @@ class ArchivingOrchestrator:
|
||||
return
|
||||
|
||||
# vk, telethon, ...
|
||||
for a in archivers:
|
||||
for a in self.archivers:
|
||||
# with automatic try/catch in download + archived (+ the other ops below)
|
||||
# should the archivers come with the config already? are there configs which change at runtime?
|
||||
# should the archivers come with the config already? are there configs which change at runtime?
|
||||
# think not, so no need to pass config as parameter
|
||||
# do they need to be refreshed with every execution?
|
||||
# do they need to be refreshed with every execution?
|
||||
# this is where the Hashes come from, the place with access to all content
|
||||
# the archiver does not have access to storage
|
||||
result.update(a.download(url))
|
||||
if result.is_success(): break
|
||||
result.merge(a.download(result))
|
||||
if True or result.is_success(): break
|
||||
|
||||
# what if an archiver returns multiple entries and one is to be part of HTMLgenerator?
|
||||
# should it call the HTMLgenerator as if it's not an enrichment?
|
||||
@@ -196,20 +209,20 @@ class ArchivingOrchestrator:
|
||||
# then how to execute it last? should there also be post-processors? are there other examples?
|
||||
# maybe as a PDF? or a Markdown file
|
||||
# side captures: screenshot, wacz, webarchive, thumbnails, HTMLgenerator
|
||||
for e in enrichers:
|
||||
result.update(e.enrich(result))
|
||||
for e in self.enrichers:
|
||||
result.merge(e.enrich(result))
|
||||
|
||||
# formatters, enrichers, and storages will sometimes look for specific properties: eg <li>Screenshot: <img src="{res.get("screenshot")}"> </li>
|
||||
for p in formatter:
|
||||
result.update(p.process(result))
|
||||
for f in self.formatters:
|
||||
result.merge(f.format(result))
|
||||
|
||||
# storages
|
||||
for s in storages:
|
||||
for s in self.storages:
|
||||
for m in result.media:
|
||||
m.update(s.store(m))
|
||||
m.merge(s.store(m))
|
||||
|
||||
# signal completion to databases (DBs, Google Sheets, CSV, ...)
|
||||
# a hash registration service could be one database: forensic archiving
|
||||
for d in databases: d.done( result)
|
||||
for d in self.databases: d.done(result)
|
||||
|
||||
return result
|
||||
return result
|
||||
|
||||
@@ -36,7 +36,7 @@ class Gsheets(Step):
|
||||
'wacz': 'wacz',
|
||||
'replaywebpage': 'replaywebpage',
|
||||
},
|
||||
"help": "names of columns in the google sheet",
|
||||
"help": "names of columns in the google sheet (stringified JSON object)",
|
||||
"cli_set": lambda cli_val, cur_val: dict(cur_val, **json.loads(cli_val))
|
||||
},
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
# we need to explicitly expose the available imports here
|
||||
from .gworksheet import GWorksheet
|
||||
from .misc import *
|
||||
from .util import Util
|
||||
from .util import Util
|
||||
from .webdriver import Webdriver
|
||||
@@ -1,11 +1,12 @@
|
||||
from __future__ import annotations
|
||||
from abc import abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from abc import abstractmethod, ABC
|
||||
from metadata import Metadata
|
||||
from steps.step import Step
|
||||
|
||||
#TODO: likely unused
|
||||
@dataclass
|
||||
class Util(Step, ABC):
|
||||
class Util(Step):
|
||||
name = "util"
|
||||
|
||||
def __init__(self, config: dict) -> None:
|
||||
|
||||
45
src/utils/webdriver.py
Normal file
45
src/utils/webdriver.py
Normal file
@@ -0,0 +1,45 @@
|
||||
from __future__ import annotations
|
||||
from selenium import webdriver
|
||||
from selenium.common.exceptions import TimeoutException
|
||||
from loguru import logger
|
||||
from selenium.webdriver.common.by import By
|
||||
import time
|
||||
|
||||
|
||||
class Webdriver:
|
||||
def __init__(self, width: int, height: int, timeout_seconds: int, facebook_accept_cookies: bool = False) -> webdriver:
|
||||
self.width = width
|
||||
self.height = height
|
||||
self.timeout_seconds = timeout_seconds
|
||||
self.facebook_accept_cookies = facebook_accept_cookies
|
||||
|
||||
def __enter__(self) -> webdriver:
|
||||
options = webdriver.FirefoxOptions()
|
||||
options.headless = True
|
||||
options.set_preference('network.protocol-handler.external.tg', False)
|
||||
try:
|
||||
self.driver = webdriver.Firefox(options=options)
|
||||
self.driver.set_window_size(self.width, self.height)
|
||||
self.driver.set_page_load_timeout(self.timeout_seconds)
|
||||
except TimeoutException as e:
|
||||
logger.error(f"failed to get new webdriver, possibly due to insufficient system resources or timeout settings: {e}")
|
||||
|
||||
if self.facebook_accept_cookies:
|
||||
try:
|
||||
logger.debug(f'Trying fb click accept cookie popup.')
|
||||
self.driver.get("http://www.facebook.com")
|
||||
foo = self.driver.find_element(By.XPATH, "//button[@data-cookiebanner='accept_only_essential_button']")
|
||||
foo.click()
|
||||
logger.debug(f'fb click worked')
|
||||
# linux server needs a sleep otherwise facebook cookie won't have worked and we'll get a popup on next page
|
||||
time.sleep(2)
|
||||
except:
|
||||
logger.warning(f'Failed on fb accept cookies.')
|
||||
|
||||
return self.driver
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
self.driver.close()
|
||||
self.driver.quit()
|
||||
del self.driver
|
||||
return True
|
||||
Reference in New Issue
Block a user