mirror of
https://github.com/bellingcat/auto-archiver.git
synced 2026-06-07 19:08:30 +03:00
Refactor archivers to make it easier to add support for new types of URLs
This commit is contained in:
Logan Williams
1
.gitignore
vendored
1
.gitignore
vendored
@@ -4,3 +4,4 @@ tmp/
|
||||
expmt/
|
||||
service_account.json
|
||||
__pycache__/
|
||||
._*
|
||||
|
||||
8
Pipfile
8
Pipfile
@@ -11,8 +11,14 @@ youtube_dl = "*"
|
||||
argparse = "*"
|
||||
ffmpeg-python = "*"
|
||||
beautifulsoup4 = "*"
|
||||
nordvpn-switcher = "*"
|
||||
tiktok-downloader = {git = "https://github.com/msramalho/tiktok-downloader"}
|
||||
telethon = "*"
|
||||
ffmpeg = "*"
|
||||
bs4 = "*"
|
||||
loguru = "*"
|
||||
|
||||
[dev-packages]
|
||||
|
||||
[requires]
|
||||
python_version = "3.8"
|
||||
python_version = "3.9"
|
||||
|
||||
421
Pipfile.lock
generated
421
Pipfile.lock
generated
@@ -1,11 +1,11 @@
|
||||
{
|
||||
"_meta": {
|
||||
"hash": {
|
||||
"sha256": "2aa6e5f9d7cda1a459444bf812fb2f7a4acfe547e7c65a975ab41530f9213da5"
|
||||
"sha256": "420a5d5c155830dac792fe2f037bebce97c30f4271301bb5950a288254798660"
|
||||
},
|
||||
"pipfile-spec": 6,
|
||||
"requires": {
|
||||
"python_version": "3.8"
|
||||
"python_version": "3.9"
|
||||
},
|
||||
"sources": [
|
||||
{
|
||||
@@ -26,49 +26,79 @@
|
||||
},
|
||||
"beautifulsoup4": {
|
||||
"hashes": [
|
||||
"sha256:4c98143716ef1cb40bf7f39a8e3eec8f8b009509e74904ba3a7b315431577e35",
|
||||
"sha256:84729e322ad1d5b4d25f805bfa05b902dd96450f43842c4e99067d5e1369eb25",
|
||||
"sha256:fff47e031e34ec82bf17e00da8f592fe7de69aeea38be00523c04623c04fb666"
|
||||
"sha256:9a315ce70049920ea4572a4055bc4bd700c940521d36fc858205ad4fcde149bf",
|
||||
"sha256:c23ad23c521d818955a4151a67d81580319d4bf548d3d49f4223ae041ff98891"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==4.9.3"
|
||||
"version": "==4.10.0"
|
||||
},
|
||||
"boto3": {
|
||||
"hashes": [
|
||||
"sha256:7209b79833bdf13753aa24f76bf533890ffed2cc4fe1fe08619d223c209bbd11",
|
||||
"sha256:f46c93d09acd4d4bfc6b9522ed852fecbdc508e0365f29ddfb3c146aae784b4e"
|
||||
"sha256:aa00024cc1f3d24b2318dae4d5dbaa173c8da8bc6f9d12f0b2e67467ec460989",
|
||||
"sha256:ab4ab2392f7520c01ce6e40e6df4b5b65a575ee6bd9fb78db0239cb2a06de557"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==1.18.27"
|
||||
"version": "==1.21.3"
|
||||
},
|
||||
"botocore": {
|
||||
"hashes": [
|
||||
"sha256:8c99abd7093ab11ce8d09c68732aeeb6065a53d2fe371568452e99291817fff5",
|
||||
"sha256:b9e2c90bad164d111c229102f58f995c28576e719dd116b446965e1b786f8fa5"
|
||||
"sha256:979e5c5e826ff115f4903fe9887b191f3809229f694a747f910e1221fe63efc7",
|
||||
"sha256:ca33f747c67cd0e109fab9398d39c38c1a2df352c1e1f9823839df8f1db58046"
|
||||
],
|
||||
"version": "==1.21.27"
|
||||
"markers": "python_version >= '3.6'",
|
||||
"version": "==1.24.3"
|
||||
},
|
||||
"bs4": {
|
||||
"hashes": [
|
||||
"sha256:36ecea1fd7cc5c0c6e4a1ff075df26d50da647b75376626cc186e2212886dd3a"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==0.0.1"
|
||||
},
|
||||
"cachetools": {
|
||||
"hashes": [
|
||||
"sha256:2cc0b89715337ab6dbba85b5b50effe2b0c74e035d83ee8ed637cf52f12ae001",
|
||||
"sha256:61b5ed1e22a0924aed1d23b478f37e8d52549ff8a961de2909c69bf950020cff"
|
||||
"sha256:486471dfa8799eb7ec503a8059e263db000cdda20075ce5e48903087f79d5fd6",
|
||||
"sha256:8fecd4203a38af17928be7b90689d8083603073622229ca7077b72d8e5a976e4"
|
||||
],
|
||||
"version": "==4.2.2"
|
||||
"markers": "python_version ~= '3.7'",
|
||||
"version": "==5.0.0"
|
||||
},
|
||||
"certifi": {
|
||||
"hashes": [
|
||||
"sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee",
|
||||
"sha256:50b1e4f8446b06f41be7dd6338db18e0990601dce795c2b1686458aa7e8fa7d8"
|
||||
"sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872",
|
||||
"sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569"
|
||||
],
|
||||
"version": "==2021.5.30"
|
||||
"version": "==2021.10.8"
|
||||
},
|
||||
"charset-normalizer": {
|
||||
"hashes": [
|
||||
"sha256:0c8911edd15d19223366a194a513099a302055a962bca2cec0f54b8b63175d8b",
|
||||
"sha256:f23667ebe1084be45f6ae0538e4a5a865206544097e4e8bbcacf42cd02a348f3"
|
||||
"sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597",
|
||||
"sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df"
|
||||
],
|
||||
"markers": "python_version >= '3'",
|
||||
"version": "==2.0.4"
|
||||
"version": "==2.0.12"
|
||||
},
|
||||
"click": {
|
||||
"hashes": [
|
||||
"sha256:6a7a62563bbfabfda3a38f3023a1db4a35978c0abd76f6c9605ecd6554d6d9b1",
|
||||
"sha256:8458d7b1287c5fb128c90e23381cf99dcde74beaf6c7ff6384ce84d6fe090adb"
|
||||
],
|
||||
"markers": "python_version >= '3.6'",
|
||||
"version": "==8.0.4"
|
||||
},
|
||||
"cloudscraper": {
|
||||
"hashes": [
|
||||
"sha256:674fd739f9412188aae8d6614e3e6316939fc0670ef5646abd3d316f1a59d3c2",
|
||||
"sha256:dda29028c5628b5ba3e4dc43816ed38fd46bd945ef938c420f185586a6d8dff2"
|
||||
],
|
||||
"version": "==1.2.58"
|
||||
},
|
||||
"ffmpeg": {
|
||||
"hashes": [
|
||||
"sha256:6931692c890ff21d39938433c2189747815dca0c60ddc7f9bb97f199dba0b5b9"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==1.4"
|
||||
},
|
||||
"ffmpeg-python": {
|
||||
"hashes": [
|
||||
@@ -78,55 +108,272 @@
|
||||
"index": "pypi",
|
||||
"version": "==0.2.0"
|
||||
},
|
||||
"flask": {
|
||||
"hashes": [
|
||||
"sha256:59da8a3170004800a2837844bfa84d49b022550616070f7cb1a659682b2e7c9f",
|
||||
"sha256:e1120c228ca2f553b470df4a5fa927ab66258467526069981b3eb0a91902687d"
|
||||
],
|
||||
"markers": "python_version >= '3.6'",
|
||||
"version": "==2.0.3"
|
||||
},
|
||||
"future": {
|
||||
"hashes": [
|
||||
"sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d"
|
||||
],
|
||||
"markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
|
||||
"version": "==0.18.2"
|
||||
},
|
||||
"google-auth": {
|
||||
"hashes": [
|
||||
"sha256:c012c8be7c442c8309ca8fa0876fef33f5fd977c467be1e1c1c2f721e8ebd73c",
|
||||
"sha256:ea1af050b3e06eb73e4470f704d23007307bc0e87c13e015f6b90460f1407bd3"
|
||||
"sha256:218ca03d7744ca0c8b6697b6083334be7df49b7bf76a69d555962fd1a7657b5f",
|
||||
"sha256:ad160fc1ea8f19e331a16a14a79f3d643d813a69534ba9611d2c80dc10439dad"
|
||||
],
|
||||
"version": "==2.0.1"
|
||||
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
|
||||
"version": "==2.6.0"
|
||||
},
|
||||
"google-auth-oauthlib": {
|
||||
"hashes": [
|
||||
"sha256:4ab58e6c3dc6ccf112f921fcced40e5426fba266768986ea502228488276eaba",
|
||||
"sha256:b5a1ce7c617d247ccb2dfbba9d4bfc734b41096803d854a2c52592ae80150a67"
|
||||
"sha256:3f2a6e802eebbb6fb736a370fbf3b055edcb6b52878bf2f26330b5e041316c73",
|
||||
"sha256:a90a072f6993f2c327067bf65270046384cda5a8ecb20b94ea9a687f1f233a7a"
|
||||
],
|
||||
"version": "==0.4.5"
|
||||
"markers": "python_version >= '3.6'",
|
||||
"version": "==0.4.6"
|
||||
},
|
||||
"gspread": {
|
||||
"hashes": [
|
||||
"sha256:236a0f24e3724b49bae4cbd5144ed036b0ae6feaf5828ad033eb2824bf05e5be",
|
||||
"sha256:4933c3e2359e82698c0990f3b0e312627fcbf8fecc8bc81d26713f5860e20b48"
|
||||
"sha256:d9db8c43d552f541ea072d4727d1e955bc2368b095dd86c5429a845c9d8aed8f",
|
||||
"sha256:ffba57786e27519fb97125e3de37a0f062134a396506681f5baacaf47a9febe3"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==4.0.1"
|
||||
"version": "==5.1.1"
|
||||
},
|
||||
"idna": {
|
||||
"hashes": [
|
||||
"sha256:14475042e284991034cb48e06f6851428fb14c4dc953acd9be9a5e95c7b6dd7a",
|
||||
"sha256:467fbad99067910785144ce333826c71fb0e63a425657295239737f7ecd125f3"
|
||||
"sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff",
|
||||
"sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"
|
||||
],
|
||||
"markers": "python_version >= '3'",
|
||||
"version": "==3.2"
|
||||
"version": "==3.3"
|
||||
},
|
||||
"itsdangerous": {
|
||||
"hashes": [
|
||||
"sha256:29285842166554469a56d427addc0843914172343784cb909695fdbe90a3e129",
|
||||
"sha256:d848fcb8bc7d507c4546b448574e8a44fc4ea2ba84ebf8d783290d53e81992f5"
|
||||
],
|
||||
"markers": "python_version >= '3.7'",
|
||||
"version": "==2.1.0"
|
||||
},
|
||||
"jinja2": {
|
||||
"hashes": [
|
||||
"sha256:077ce6014f7b40d03b47d1f1ca4b0fc8328a692bd284016f806ed0eaca390ad8",
|
||||
"sha256:611bb273cd68f3b993fabdc4064fc858c5b47a973cb5aa7999ec1ba405c87cd7"
|
||||
],
|
||||
"markers": "python_version >= '3.6'",
|
||||
"version": "==3.0.3"
|
||||
},
|
||||
"jmespath": {
|
||||
"hashes": [
|
||||
"sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9",
|
||||
"sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f"
|
||||
],
|
||||
"markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
|
||||
"version": "==0.10.0"
|
||||
},
|
||||
"loguru": {
|
||||
"hashes": [
|
||||
"sha256:066bd06758d0a513e9836fd9c6b5a75bfb3fd36841f4b996bc60b547a309d41c",
|
||||
"sha256:4e2414d534a2ab57573365b3e6d0234dfb1d84b68b7f3b948e6fb743860a77c3"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==0.6.0"
|
||||
},
|
||||
"lxml": {
|
||||
"hashes": [
|
||||
"sha256:078306d19a33920004addeb5f4630781aaeabb6a8d01398045fcde085091a169",
|
||||
"sha256:0c1978ff1fd81ed9dcbba4f91cf09faf1f8082c9d72eb122e92294716c605428",
|
||||
"sha256:1010042bfcac2b2dc6098260a2ed022968dbdfaf285fc65a3acf8e4eb1ffd1bc",
|
||||
"sha256:1d650812b52d98679ed6c6b3b55cbb8fe5a5460a0aef29aeb08dc0b44577df85",
|
||||
"sha256:20b8a746a026017acf07da39fdb10aa80ad9877046c9182442bf80c84a1c4696",
|
||||
"sha256:2403a6d6fb61c285969b71f4a3527873fe93fd0abe0832d858a17fe68c8fa507",
|
||||
"sha256:24f5c5ae618395ed871b3d8ebfcbb36e3f1091fd847bf54c4de623f9107942f3",
|
||||
"sha256:28d1af847786f68bec57961f31221125c29d6f52d9187c01cd34dc14e2b29430",
|
||||
"sha256:31499847fc5f73ee17dbe1b8e24c6dafc4e8d5b48803d17d22988976b0171f03",
|
||||
"sha256:31ba2cbc64516dcdd6c24418daa7abff989ddf3ba6d3ea6f6ce6f2ed6e754ec9",
|
||||
"sha256:330bff92c26d4aee79c5bc4d9967858bdbe73fdbdbacb5daf623a03a914fe05b",
|
||||
"sha256:5045ee1ccd45a89c4daec1160217d363fcd23811e26734688007c26f28c9e9e7",
|
||||
"sha256:52cbf2ff155b19dc4d4100f7442f6a697938bf4493f8d3b0c51d45568d5666b5",
|
||||
"sha256:530f278849031b0eb12f46cca0e5db01cfe5177ab13bd6878c6e739319bae654",
|
||||
"sha256:545bd39c9481f2e3f2727c78c169425efbfb3fbba6e7db4f46a80ebb249819ca",
|
||||
"sha256:5804e04feb4e61babf3911c2a974a5b86f66ee227cc5006230b00ac6d285b3a9",
|
||||
"sha256:5a58d0b12f5053e270510bf12f753a76aaf3d74c453c00942ed7d2c804ca845c",
|
||||
"sha256:5f148b0c6133fb928503cfcdfdba395010f997aa44bcf6474fcdd0c5398d9b63",
|
||||
"sha256:5f7d7d9afc7b293147e2d506a4596641d60181a35279ef3aa5778d0d9d9123fe",
|
||||
"sha256:60d2f60bd5a2a979df28ab309352cdcf8181bda0cca4529769a945f09aba06f9",
|
||||
"sha256:6259b511b0f2527e6d55ad87acc1c07b3cbffc3d5e050d7e7bcfa151b8202df9",
|
||||
"sha256:6268e27873a3d191849204d00d03f65c0e343b3bcb518a6eaae05677c95621d1",
|
||||
"sha256:627e79894770783c129cc5e89b947e52aa26e8e0557c7e205368a809da4b7939",
|
||||
"sha256:62f93eac69ec0f4be98d1b96f4d6b964855b8255c345c17ff12c20b93f247b68",
|
||||
"sha256:6d6483b1229470e1d8835e52e0ff3c6973b9b97b24cd1c116dca90b57a2cc613",
|
||||
"sha256:6f7b82934c08e28a2d537d870293236b1000d94d0b4583825ab9649aef7ddf63",
|
||||
"sha256:6fe4ef4402df0250b75ba876c3795510d782def5c1e63890bde02d622570d39e",
|
||||
"sha256:719544565c2937c21a6f76d520e6e52b726d132815adb3447ccffbe9f44203c4",
|
||||
"sha256:730766072fd5dcb219dd2b95c4c49752a54f00157f322bc6d71f7d2a31fecd79",
|
||||
"sha256:74eb65ec61e3c7c019d7169387d1b6ffcfea1b9ec5894d116a9a903636e4a0b1",
|
||||
"sha256:7993232bd4044392c47779a3c7e8889fea6883be46281d45a81451acfd704d7e",
|
||||
"sha256:80bbaddf2baab7e6de4bc47405e34948e694a9efe0861c61cdc23aa774fcb141",
|
||||
"sha256:86545e351e879d0b72b620db6a3b96346921fa87b3d366d6c074e5a9a0b8dadb",
|
||||
"sha256:891dc8f522d7059ff0024cd3ae79fd224752676447f9c678f2a5c14b84d9a939",
|
||||
"sha256:8a31f24e2a0b6317f33aafbb2f0895c0bce772980ae60c2c640d82caac49628a",
|
||||
"sha256:8b99ec73073b37f9ebe8caf399001848fced9c08064effdbfc4da2b5a8d07b93",
|
||||
"sha256:986b7a96228c9b4942ec420eff37556c5777bfba6758edcb95421e4a614b57f9",
|
||||
"sha256:a1547ff4b8a833511eeaceacbcd17b043214fcdb385148f9c1bc5556ca9623e2",
|
||||
"sha256:a2bfc7e2a0601b475477c954bf167dee6d0f55cb167e3f3e7cefad906e7759f6",
|
||||
"sha256:a3c5f1a719aa11866ffc530d54ad965063a8cbbecae6515acbd5f0fae8f48eaa",
|
||||
"sha256:a9f1c3489736ff8e1c7652e9dc39f80cff820f23624f23d9eab6e122ac99b150",
|
||||
"sha256:aa0cf4922da7a3c905d000b35065df6184c0dc1d866dd3b86fd961905bbad2ea",
|
||||
"sha256:ad4332a532e2d5acb231a2e5d33f943750091ee435daffca3fec0a53224e7e33",
|
||||
"sha256:b2582b238e1658c4061ebe1b4df53c435190d22457642377fd0cb30685cdfb76",
|
||||
"sha256:b6fc2e2fb6f532cf48b5fed57567ef286addcef38c28874458a41b7837a57807",
|
||||
"sha256:b92d40121dcbd74831b690a75533da703750f7041b4bf951befc657c37e5695a",
|
||||
"sha256:bbab6faf6568484707acc052f4dfc3802bdb0cafe079383fbaa23f1cdae9ecd4",
|
||||
"sha256:c0b88ed1ae66777a798dc54f627e32d3b81c8009967c63993c450ee4cbcbec15",
|
||||
"sha256:ce13d6291a5f47c1c8dbd375baa78551053bc6b5e5c0e9bb8e39c0a8359fd52f",
|
||||
"sha256:db3535733f59e5605a88a706824dfcb9bd06725e709ecb017e165fc1d6e7d429",
|
||||
"sha256:dd10383f1d6b7edf247d0960a3db274c07e96cf3a3fc7c41c8448f93eac3fb1c",
|
||||
"sha256:e01f9531ba5420838c801c21c1b0f45dbc9607cb22ea2cf132844453bec863a5",
|
||||
"sha256:e11527dc23d5ef44d76fef11213215c34f36af1608074561fcc561d983aeb870",
|
||||
"sha256:e1ab2fac607842ac36864e358c42feb0960ae62c34aa4caaf12ada0a1fb5d99b",
|
||||
"sha256:e1fd7d2fe11f1cb63d3336d147c852f6d07de0d0020d704c6031b46a30b02ca8",
|
||||
"sha256:e9f84ed9f4d50b74fbc77298ee5c870f67cb7e91dcdc1a6915cb1ff6a317476c",
|
||||
"sha256:ec4b4e75fc68da9dc0ed73dcdb431c25c57775383fec325d23a770a64e7ebc87",
|
||||
"sha256:f10ce66fcdeb3543df51d423ede7e238be98412232fca5daec3e54bcd16b8da0",
|
||||
"sha256:f63f62fc60e6228a4ca9abae28228f35e1bd3ce675013d1dfb828688d50c6e23",
|
||||
"sha256:fa56bb08b3dd8eac3a8c5b7d075c94e74f755fd9d8a04543ae8d37b1612dd170",
|
||||
"sha256:fa9b7c450be85bfc6cd39f6df8c5b8cbd76b5d6fc1f69efec80203f9894b885f"
|
||||
],
|
||||
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
|
||||
"version": "==4.8.0"
|
||||
},
|
||||
"markupsafe": {
|
||||
"hashes": [
|
||||
"sha256:023af8c54fe63530545f70dd2a2a7eed18d07a9a77b94e8bf1e2ff7f252db9a3",
|
||||
"sha256:09c86c9643cceb1d87ca08cdc30160d1b7ab49a8a21564868921959bd16441b8",
|
||||
"sha256:142119fb14a1ef6d758912b25c4e803c3ff66920635c44078666fe7cc3f8f759",
|
||||
"sha256:1d1fb9b2eec3c9714dd936860850300b51dbaa37404209c8d4cb66547884b7ed",
|
||||
"sha256:204730fd5fe2fe3b1e9ccadb2bd18ba8712b111dcabce185af0b3b5285a7c989",
|
||||
"sha256:24c3be29abb6b34052fd26fc7a8e0a49b1ee9d282e3665e8ad09a0a68faee5b3",
|
||||
"sha256:290b02bab3c9e216da57c1d11d2ba73a9f73a614bbdcc027d299a60cdfabb11a",
|
||||
"sha256:3028252424c72b2602a323f70fbf50aa80a5d3aa616ea6add4ba21ae9cc9da4c",
|
||||
"sha256:30c653fde75a6e5eb814d2a0a89378f83d1d3f502ab710904ee585c38888816c",
|
||||
"sha256:3cace1837bc84e63b3fd2dfce37f08f8c18aeb81ef5cf6bb9b51f625cb4e6cd8",
|
||||
"sha256:4056f752015dfa9828dce3140dbadd543b555afb3252507348c493def166d454",
|
||||
"sha256:454ffc1cbb75227d15667c09f164a0099159da0c1f3d2636aa648f12675491ad",
|
||||
"sha256:598b65d74615c021423bd45c2bc5e9b59539c875a9bdb7e5f2a6b92dfcfc268d",
|
||||
"sha256:599941da468f2cf22bf90a84f6e2a65524e87be2fce844f96f2dd9a6c9d1e635",
|
||||
"sha256:5ddea4c352a488b5e1069069f2f501006b1a4362cb906bee9a193ef1245a7a61",
|
||||
"sha256:62c0285e91414f5c8f621a17b69fc0088394ccdaa961ef469e833dbff64bd5ea",
|
||||
"sha256:679cbb78914ab212c49c67ba2c7396dc599a8479de51b9a87b174700abd9ea49",
|
||||
"sha256:6e104c0c2b4cd765b4e83909cde7ec61a1e313f8a75775897db321450e928cce",
|
||||
"sha256:736895a020e31b428b3382a7887bfea96102c529530299f426bf2e636aacec9e",
|
||||
"sha256:75bb36f134883fdbe13d8e63b8675f5f12b80bb6627f7714c7d6c5becf22719f",
|
||||
"sha256:7d2f5d97fcbd004c03df8d8fe2b973fe2b14e7bfeb2cfa012eaa8759ce9a762f",
|
||||
"sha256:80beaf63ddfbc64a0452b841d8036ca0611e049650e20afcb882f5d3c266d65f",
|
||||
"sha256:84ad5e29bf8bab3ad70fd707d3c05524862bddc54dc040982b0dbcff36481de7",
|
||||
"sha256:8da5924cb1f9064589767b0f3fc39d03e3d0fb5aa29e0cb21d43106519bd624a",
|
||||
"sha256:961eb86e5be7d0973789f30ebcf6caab60b844203f4396ece27310295a6082c7",
|
||||
"sha256:96de1932237abe0a13ba68b63e94113678c379dca45afa040a17b6e1ad7ed076",
|
||||
"sha256:a0a0abef2ca47b33fb615b491ce31b055ef2430de52c5b3fb19a4042dbc5cadb",
|
||||
"sha256:b2a5a856019d2833c56a3dcac1b80fe795c95f401818ea963594b345929dffa7",
|
||||
"sha256:b8811d48078d1cf2a6863dafb896e68406c5f513048451cd2ded0473133473c7",
|
||||
"sha256:c532d5ab79be0199fa2658e24a02fce8542df196e60665dd322409a03db6a52c",
|
||||
"sha256:d3b64c65328cb4cd252c94f83e66e3d7acf8891e60ebf588d7b493a55a1dbf26",
|
||||
"sha256:d4e702eea4a2903441f2735799d217f4ac1b55f7d8ad96ab7d4e25417cb0827c",
|
||||
"sha256:d5653619b3eb5cbd35bfba3c12d575db2a74d15e0e1c08bf1db788069d410ce8",
|
||||
"sha256:d66624f04de4af8bbf1c7f21cc06649c1c69a7f84109179add573ce35e46d448",
|
||||
"sha256:e67ec74fada3841b8c5f4c4f197bea916025cb9aa3fe5abf7d52b655d042f956",
|
||||
"sha256:e6f7f3f41faffaea6596da86ecc2389672fa949bd035251eab26dc6697451d05",
|
||||
"sha256:f02cf7221d5cd915d7fa58ab64f7ee6dd0f6cddbb48683debf5d04ae9b1c2cc1",
|
||||
"sha256:f0eddfcabd6936558ec020130f932d479930581171368fd728efcfb6ef0dd357",
|
||||
"sha256:fabbe18087c3d33c5824cb145ffca52eccd053061df1d79d4b66dafa5ad2a5ea",
|
||||
"sha256:fc3150f85e2dbcf99e65238c842d1cfe69d3e7649b19864c1cc043213d9cd730"
|
||||
],
|
||||
"markers": "python_version >= '3.7'",
|
||||
"version": "==2.1.0"
|
||||
},
|
||||
"nordvpn-switcher": {
|
||||
"hashes": [
|
||||
"sha256:764db054715d949af0f836da5e46c4053afe92282a0d4b2cfc6b8cfe8c3045de",
|
||||
"sha256:9788c2c3113d0d7b00894dae3ea19bed14f3b38d111d7223c126f001b1729a3b"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==0.2.9"
|
||||
},
|
||||
"oauthlib": {
|
||||
"hashes": [
|
||||
"sha256:42bf6354c2ed8c6acb54d971fce6f88193d97297e18602a3a886603f9d7730cc",
|
||||
"sha256:8f0215fcc533dd8dd1bee6f4c412d4f0cd7297307d43ac61666389e3bc3198a3"
|
||||
"sha256:23a8208d75b902797ea29fd31fa80a15ed9dc2c6c16fe73f5d346f83f6fa27a2",
|
||||
"sha256:6db33440354787f9b7f3a6dbd4febf5d0f93758354060e802f6c06cb493022fe"
|
||||
],
|
||||
"version": "==3.1.1"
|
||||
"markers": "python_version >= '3.6'",
|
||||
"version": "==3.2.0"
|
||||
},
|
||||
"pathlib": {
|
||||
"hashes": [
|
||||
"sha256:6940718dfc3eff4258203ad5021090933e5c04707d5ca8cc9e73c94a7894ea9f"
|
||||
],
|
||||
"version": "==1.0.1"
|
||||
},
|
||||
"psutil": {
|
||||
"hashes": [
|
||||
"sha256:072664401ae6e7c1bfb878c65d7282d4b4391f1bc9a56d5e03b5a490403271b5",
|
||||
"sha256:1070a9b287846a21a5d572d6dddd369517510b68710fca56b0e9e02fd24bed9a",
|
||||
"sha256:1d7b433519b9a38192dfda962dd8f44446668c009833e1429a52424624f408b4",
|
||||
"sha256:3151a58f0fbd8942ba94f7c31c7e6b310d2989f4da74fcbf28b934374e9bf841",
|
||||
"sha256:32acf55cb9a8cbfb29167cd005951df81b567099295291bcfd1027365b36591d",
|
||||
"sha256:3611e87eea393f779a35b192b46a164b1d01167c9d323dda9b1e527ea69d697d",
|
||||
"sha256:3d00a664e31921009a84367266b35ba0aac04a2a6cad09c550a89041034d19a0",
|
||||
"sha256:4e2fb92e3aeae3ec3b7b66c528981fd327fb93fd906a77215200404444ec1845",
|
||||
"sha256:539e429da49c5d27d5a58e3563886057f8fc3868a5547b4f1876d9c0f007bccf",
|
||||
"sha256:55ce319452e3d139e25d6c3f85a1acf12d1607ddedea5e35fb47a552c051161b",
|
||||
"sha256:58c7d923dc209225600aec73aa2c4ae8ea33b1ab31bc11ef8a5933b027476f07",
|
||||
"sha256:7336292a13a80eb93c21f36bde4328aa748a04b68c13d01dfddd67fc13fd0618",
|
||||
"sha256:742c34fff804f34f62659279ed5c5b723bb0195e9d7bd9907591de9f8f6558e2",
|
||||
"sha256:7641300de73e4909e5d148e90cc3142fb890079e1525a840cf0dfd39195239fd",
|
||||
"sha256:76cebf84aac1d6da5b63df11fe0d377b46b7b500d892284068bacccf12f20666",
|
||||
"sha256:7779be4025c540d1d65a2de3f30caeacc49ae7a2152108adeaf42c7534a115ce",
|
||||
"sha256:7d190ee2eaef7831163f254dc58f6d2e2a22e27382b936aab51c835fc080c3d3",
|
||||
"sha256:8293942e4ce0c5689821f65ce6522ce4786d02af57f13c0195b40e1edb1db61d",
|
||||
"sha256:869842dbd66bb80c3217158e629d6fceaecc3a3166d3d1faee515b05dd26ca25",
|
||||
"sha256:90a58b9fcae2dbfe4ba852b57bd4a1dded6b990a33d6428c7614b7d48eccb492",
|
||||
"sha256:9b51917c1af3fa35a3f2dabd7ba96a2a4f19df3dec911da73875e1edaf22a40b",
|
||||
"sha256:b2237f35c4bbae932ee98902a08050a27821f8f6dfa880a47195e5993af4702d",
|
||||
"sha256:c3400cae15bdb449d518545cbd5b649117de54e3596ded84aacabfbb3297ead2",
|
||||
"sha256:c51f1af02334e4b516ec221ee26b8fdf105032418ca5a5ab9737e8c87dafe203",
|
||||
"sha256:cb8d10461c1ceee0c25a64f2dd54872b70b89c26419e147a05a10b753ad36ec2",
|
||||
"sha256:d62a2796e08dd024b8179bd441cb714e0f81226c352c802fca0fd3f89eeacd94",
|
||||
"sha256:df2c8bd48fb83a8408c8390b143c6a6fa10cb1a674ca664954de193fdcab36a9",
|
||||
"sha256:e5c783d0b1ad6ca8a5d3e7b680468c9c926b804be83a3a8e95141b05c39c9f64",
|
||||
"sha256:e9805fed4f2a81de98ae5fe38b75a74c6e6ad2df8a5c479594c7629a1fe35f56",
|
||||
"sha256:ea42d747c5f71b5ccaa6897b216a7dadb9f52c72a0fe2b872ef7d3e1eacf3ba3",
|
||||
"sha256:ef216cc9feb60634bda2f341a9559ac594e2eeaadd0ba187a4c2eb5b5d40b91c",
|
||||
"sha256:ff0d41f8b3e9ebb6b6110057e40019a432e96aae2008951121ba4e56040b84f3"
|
||||
],
|
||||
"markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
|
||||
"version": "==5.9.0"
|
||||
},
|
||||
"py-mini-racer": {
|
||||
"hashes": [
|
||||
"sha256:346e73bb89a2024888244d487834be24a121089ceb0641dd0200cb96c4e24b57",
|
||||
"sha256:42896c24968481dd953eeeb11de331f6870917811961c9b26ba09071e07180e2",
|
||||
"sha256:97cab31bbf63ce462ba4cd6e978c572c916d8b15586156c7c5e0b2e42c10baab",
|
||||
"sha256:f71e36b643d947ba698c57cd9bd2232c83ca997b0802fc2f7f79582377040c11"
|
||||
],
|
||||
"version": "==0.6.0"
|
||||
},
|
||||
"pyaes": {
|
||||
"hashes": [
|
||||
"sha256:02c1b1405c38d3c370b085fb952dd8bea3fadcee6411ad99f312cc129c536d8f"
|
||||
],
|
||||
"version": "==1.6.1"
|
||||
},
|
||||
"pyasn1": {
|
||||
"hashes": [
|
||||
@@ -164,80 +411,128 @@
|
||||
],
|
||||
"version": "==0.2.8"
|
||||
},
|
||||
"pyparsing": {
|
||||
"hashes": [
|
||||
"sha256:18ee9022775d270c55187733956460083db60b37d0d0fb357445f3094eed3eea",
|
||||
"sha256:a6c06a88f252e6c322f65faf8f418b16213b51bdfaece0524c1c1bc30c63c484"
|
||||
],
|
||||
"markers": "python_version >= '3.6'",
|
||||
"version": "==3.0.7"
|
||||
},
|
||||
"python-dateutil": {
|
||||
"hashes": [
|
||||
"sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86",
|
||||
"sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"
|
||||
],
|
||||
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
|
||||
"version": "==2.8.2"
|
||||
},
|
||||
"python-dotenv": {
|
||||
"hashes": [
|
||||
"sha256:aae25dc1ebe97c420f50b81fb0e5c949659af713f31fdb63c749ca68748f34b1",
|
||||
"sha256:f521bc2ac9a8e03c736f62911605c5d83970021e3fa95b37d769e2bbbe9b6172"
|
||||
"sha256:32b2bdc1873fd3a3c346da1c6db83d0053c3c62f28f1f38516070c4c8971b1d3",
|
||||
"sha256:a5de49a31e953b45ff2d2fd434bbc2670e8db5273606c1e737cc6b93eff3655f"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==0.19.0"
|
||||
"version": "==0.19.2"
|
||||
},
|
||||
"random-user-agent": {
|
||||
"hashes": [
|
||||
"sha256:535636a55fb63fe3d74fd0260d854c241d9f2946447026464e578e68eac17dac",
|
||||
"sha256:8f8ca26ec8cb1d24ad1758d8b8f700d154064d641dbe9a255cfec42960fbd012"
|
||||
],
|
||||
"version": "==1.0.1"
|
||||
},
|
||||
"requests": {
|
||||
"hashes": [
|
||||
"sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24",
|
||||
"sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7"
|
||||
"sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61",
|
||||
"sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d"
|
||||
],
|
||||
"version": "==2.26.0"
|
||||
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
|
||||
"version": "==2.27.1"
|
||||
},
|
||||
"requests-oauthlib": {
|
||||
"hashes": [
|
||||
"sha256:7f71572defaecd16372f9006f33c2ec8c077c3cfa6f5911a9a90202beb513f3d",
|
||||
"sha256:b4261601a71fd721a8bd6d7aa1cc1d6a8a93b4a9f5e96626f8e4d91e8beeaa6a",
|
||||
"sha256:fa6c47b933f01060936d87ae9327fead68768b69c6c9ea2109c48be30f2d4dbc"
|
||||
"sha256:2577c501a2fb8d05a304c09d090d6e47c306fef15809d102b327cf8364bddab5",
|
||||
"sha256:75beac4a47881eeb94d5ea5d6ad31ef88856affe2332b9aafb52c6452ccf0d7a"
|
||||
],
|
||||
"version": "==1.3.0"
|
||||
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
|
||||
"version": "==1.3.1"
|
||||
},
|
||||
"requests-toolbelt": {
|
||||
"hashes": [
|
||||
"sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f",
|
||||
"sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0"
|
||||
],
|
||||
"version": "==0.9.1"
|
||||
},
|
||||
"rsa": {
|
||||
"hashes": [
|
||||
"sha256:78f9a9bf4e7be0c5ded4583326e7461e3a3c5aae24073648b4bdfa797d78c9d2",
|
||||
"sha256:9d689e6ca1b3038bc82bf8d23e944b6b6037bc02301a574935b2dd946e0353b9"
|
||||
"sha256:5c6bd9dc7a543b7fe4304a631f8a8a3b674e2bbfc49c2ae96200cdbe55df6b17",
|
||||
"sha256:95c5d300c4e879ee69708c428ba566c59478fd653cc3a22243eeb8ed846950bb"
|
||||
],
|
||||
"version": "==4.7.2"
|
||||
"markers": "python_version >= '3.6'",
|
||||
"version": "==4.8"
|
||||
},
|
||||
"s3transfer": {
|
||||
"hashes": [
|
||||
"sha256:50ed823e1dc5868ad40c8dc92072f757aa0e653a192845c94a3b676f4a62da4c",
|
||||
"sha256:9c1dc369814391a6bda20ebbf4b70a0f34630592c9aa520856bf384916af2803"
|
||||
"sha256:25c140f5c66aa79e1ac60be50dcd45ddc59e83895f062a3aab263b870102911f",
|
||||
"sha256:69d264d3e760e569b78aaa0f22c97e955891cd22e32b10c51f784eeda4d9d10a"
|
||||
],
|
||||
"version": "==0.5.0"
|
||||
"markers": "python_version >= '3.6'",
|
||||
"version": "==0.5.1"
|
||||
},
|
||||
"six": {
|
||||
"hashes": [
|
||||
"sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
|
||||
"sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
|
||||
],
|
||||
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
|
||||
"version": "==1.16.0"
|
||||
},
|
||||
"soupsieve": {
|
||||
"hashes": [
|
||||
"sha256:052774848f448cf19c7e959adf5566904d525f33a3f8b6ba6f6f8f26ec7de0cc",
|
||||
"sha256:c2c1c2d44f158cdbddab7824a9af8c4f83c76b1e23e049479aa432feb6c4c23b"
|
||||
"sha256:1a3cca2617c6b38c0343ed661b1fa5de5637f257d4fe22bd9f1338010a1efefb",
|
||||
"sha256:b8d49b1cd4f037c7082a9683dfa1801aa2597fb11c3a1155b7a5b94829b4f1f9"
|
||||
],
|
||||
"markers": "python_version >= '3.0'",
|
||||
"version": "==2.2.1"
|
||||
"markers": "python_version >= '3.6'",
|
||||
"version": "==2.3.1"
|
||||
},
|
||||
"telethon": {
|
||||
"hashes": [
|
||||
"sha256:04fdc5fa4ed3e886e6ecf4bad79205ab8880c6aefbd42c29c89c689a502aa816",
|
||||
"sha256:818cb61281ed3f75ba4da9b68cb69486bed9474d2db4e0aa16e482053117452c"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==1.24.0"
|
||||
},
|
||||
"tiktok-downloader": {
|
||||
"git": "https://github.com/msramalho/tiktok-downloader",
|
||||
"ref": "81c6ea1f959b2cc5620961043272592bd1bfc2e2"
|
||||
},
|
||||
"urllib3": {
|
||||
"hashes": [
|
||||
"sha256:39fb8672126159acb139a7718dd10806104dec1e2f0f6c88aab05d17df10c8d4",
|
||||
"sha256:f57b4c16c62fa2760b7e3d97c35b255512fb6b59a259730f36ba32ce9f8e342f"
|
||||
"sha256:000ca7f471a233c2251c6c7023ee85305721bfdf18621ebff4fd17a8653427ed",
|
||||
"sha256:0e7c33d9a63e7ddfcb86780aac87befc2fbddf46c58dbb487e0855f7ceec283c"
|
||||
],
|
||||
"version": "==1.26.6"
|
||||
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
|
||||
"version": "==1.26.8"
|
||||
},
|
||||
"werkzeug": {
|
||||
"hashes": [
|
||||
"sha256:1421ebfc7648a39a5c58c601b154165d05cf47a3cd0ccb70857cbdacf6c8f2b8",
|
||||
"sha256:b863f8ff057c522164b6067c9e28b041161b4be5ba4d0daceeaa50a163822d3c"
|
||||
],
|
||||
"markers": "python_version >= '3.6'",
|
||||
"version": "==2.0.3"
|
||||
},
|
||||
"youtube-dl": {
|
||||
"hashes": [
|
||||
"sha256:263e04d53fb8ba3dfbd246ad09b7d388e896c132a20cc770c26ee7684de050ac",
|
||||
"sha256:cb2d3ee002158ede783e97a82c95f3817594df54367ea6a77ce5ceea4772f0ab"
|
||||
"sha256:bc59e86c5d15d887ac590454511f08ce2c47698d5a82c27bfe27b5d814bbaed2",
|
||||
"sha256:f1336d5de68647e0364a47b3c0712578e59ec76f02048ff5c50ef1c69d79cd55"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==2021.6.6"
|
||||
"version": "==2021.12.17"
|
||||
}
|
||||
},
|
||||
"develop": {}
|
||||
}
|
||||
}
|
||||
390
archivers.py
Normal file
390
archivers.py
Normal file
@@ -0,0 +1,390 @@
|
||||
from dataclasses import dataclass
|
||||
import youtube_dl
|
||||
from bs4 import BeautifulSoup
|
||||
import requests
|
||||
import tiktok_downloader
|
||||
from loguru import logger
|
||||
import os
|
||||
import datetime
|
||||
import ffmpeg
|
||||
from botocore.errorfactory import ClientError
|
||||
import time
|
||||
import traceback
|
||||
|
||||
# TODO There should be a better way of generating keys, that adds the following info:
|
||||
# - name of sheet that it is being archived from
|
||||
# (this means we might archive the same media twice on different sheets, but that's OK I think)
|
||||
# - name of archiver/platform that the video comes from
|
||||
# This should make it easier to maintain and clean the archive later
|
||||
|
||||
# TODO "check_if_exists" has lots of repeated code across the archivers. Can this be
|
||||
# cleaned up? Difficult is we don't know the filename until the archivers start working.
|
||||
|
||||
|
||||
def get_cdn_url(key):
|
||||
return 'https://{}.{}.cdn.digitaloceanspaces.com/{}'.format(
|
||||
os.getenv('DO_BUCKET'), os.getenv('DO_SPACES_REGION'), key)
|
||||
|
||||
|
||||
def do_s3_upload(s3_client, f, key):
|
||||
s3_client.upload_fileobj(f, Bucket=os.getenv(
|
||||
'DO_BUCKET'), Key=key, ExtraArgs={'ACL': 'public-read'})
|
||||
|
||||
|
||||
def get_key(filename):
|
||||
key = filename.split('/')[1]
|
||||
if 'unknown_video' in key:
|
||||
key = key.replace('unknown_video', 'jpg')
|
||||
return key
|
||||
|
||||
|
||||
def get_thumbnails(filename, s3_client, duration=None):
|
||||
if not os.path.exists(filename.split('.')[0]):
|
||||
os.mkdir(filename.split('.')[0])
|
||||
|
||||
fps = 0.5
|
||||
if duration is not None:
|
||||
duration = float(duration)
|
||||
|
||||
if duration < 60:
|
||||
fps = 10.0 / duration
|
||||
elif duration < 120:
|
||||
fps = 20.0 / duration
|
||||
else:
|
||||
fps = 40.0 / duration
|
||||
|
||||
stream = ffmpeg.input(filename)
|
||||
stream = ffmpeg.filter(stream, 'fps', fps=fps).filter('scale', 512, -1)
|
||||
stream.output(filename.split('.')[0] + '/out%d.jpg').run()
|
||||
|
||||
thumbnails = os.listdir(filename.split('.')[0] + '/')
|
||||
cdn_urls = []
|
||||
|
||||
for fname in thumbnails:
|
||||
if fname[-3:] == 'jpg':
|
||||
thumbnail_filename = filename.split('.')[0] + '/' + fname
|
||||
key = filename.split('/')[1].split('.')[0] + '/' + fname
|
||||
|
||||
cdn_url = get_cdn_url(key)
|
||||
|
||||
with open(thumbnail_filename, 'rb') as f:
|
||||
do_s3_upload(s3_client, f, key)
|
||||
|
||||
cdn_urls.append(cdn_url)
|
||||
os.remove(thumbnail_filename)
|
||||
|
||||
if len(cdn_urls) == 0:
|
||||
return ('None', 'None')
|
||||
|
||||
key_thumb = cdn_urls[int(len(cdn_urls)*0.1)]
|
||||
|
||||
index_page = f'''<html><head><title>{filename}</title></head>
|
||||
<body>'''
|
||||
|
||||
for t in cdn_urls:
|
||||
index_page += f'<img src="{t}" />'
|
||||
|
||||
index_page += f"</body></html>"
|
||||
index_fname = filename.split('.')[0] + '/index.html'
|
||||
|
||||
with open(index_fname, 'w') as f:
|
||||
f.write(index_page)
|
||||
|
||||
thumb_index = filename.split('/')[1].split('.')[0] + '/index.html'
|
||||
|
||||
s3_client.upload_fileobj(open(index_fname, 'rb'), Bucket=os.getenv(
|
||||
'DO_BUCKET'), Key=thumb_index, ExtraArgs={'ACL': 'public-read', 'ContentType': 'text/html'})
|
||||
|
||||
thumb_index_cdn_url = get_cdn_url(thumb_index)
|
||||
|
||||
return (key_thumb, thumb_index_cdn_url)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ArchiveResult:
|
||||
status: str
|
||||
cdn_url: str = None
|
||||
thumbnail: str = None
|
||||
thumbnail_index: str = None
|
||||
duration: float = None
|
||||
title: str = None
|
||||
timestamp: datetime.datetime = None
|
||||
|
||||
|
||||
class Archiver:
|
||||
def __init__(self, s3_client):
|
||||
self.s3 = s3_client
|
||||
|
||||
def download(self, url):
|
||||
pass
|
||||
|
||||
|
||||
class TelegramArchiver(Archiver):
|
||||
def download(self, url, check_if_exists=False):
|
||||
# detect URLs that we definitely cannot handle
|
||||
if 'http://t.me/' not in url and 'https://t.me/' not in url:
|
||||
return False
|
||||
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36'}
|
||||
status = "success"
|
||||
|
||||
original_url = url
|
||||
|
||||
if url[-8:] != "?embed=1":
|
||||
url += "?embed=1"
|
||||
|
||||
t = requests.get(url, headers=headers)
|
||||
s = BeautifulSoup(t.content, 'html.parser')
|
||||
video = s.find("video")
|
||||
|
||||
if video is None:
|
||||
return False # could not find video
|
||||
|
||||
video_url = video.get('src')
|
||||
key = video_url.split('/')[-1].split('?')[0]
|
||||
filename = 'tmp/' + key
|
||||
|
||||
if check_if_exists:
|
||||
try:
|
||||
self.s3.head_object(Bucket=os.getenv('DO_BUCKET'), Key=key)
|
||||
|
||||
# file exists
|
||||
cdn_url = get_cdn_url(key)
|
||||
|
||||
status = 'already archived'
|
||||
|
||||
except ClientError:
|
||||
pass
|
||||
|
||||
v = requests.get(video_url, headers=headers)
|
||||
|
||||
with open(filename, 'wb') as f:
|
||||
f.write(v.content)
|
||||
|
||||
if status != 'already archived':
|
||||
cdn_url = get_cdn_url(key)
|
||||
|
||||
with open(filename, 'rb') as f:
|
||||
do_s3_upload(self.s3, f, key)
|
||||
|
||||
# extract duration from HTML
|
||||
duration = s.find_all('time')[0].contents[0]
|
||||
if ':' in duration:
|
||||
duration = float(duration.split(
|
||||
':')[0])*60 + float(duration.split(':')[1])
|
||||
else:
|
||||
duration = float(duration)
|
||||
|
||||
# process thumbnails
|
||||
key_thumb, thumb_index = get_thumbnails(
|
||||
filename, self.s3, duration=duration)
|
||||
os.remove(filename)
|
||||
|
||||
return ArchiveResult(status=status, cdn_url=cdn_url, thumbnail=key_thumb, thumbnail_index=thumb_index,
|
||||
duration=duration, title=original_url, timestamp=s.find_all('time')[1].get('datetime'))
|
||||
|
||||
|
||||
class YoutubeDLArchiver(Archiver):
|
||||
def download(self, url, check_if_exists=False):
|
||||
ydl_opts = {'outtmpl': 'tmp/%(id)s.%(ext)s', 'quiet': False}
|
||||
if (url[0:21] == 'https://facebook.com/' or url[0:25] == 'https://wwww.facebook.com/') and os.getenv('FB_COOKIE'):
|
||||
logger.info('Using Facebook cookie')
|
||||
youtube_dl.utils.std_headers['cookie'] = os.getenv('FB_COOKIE')
|
||||
|
||||
ydl = youtube_dl.YoutubeDL(ydl_opts)
|
||||
cdn_url = None
|
||||
status = 'success'
|
||||
|
||||
try:
|
||||
info = ydl.extract_info(url, download=False)
|
||||
except youtube_dl.utils.DownloadError:
|
||||
# no video here
|
||||
return False
|
||||
|
||||
if 'is_live' in info and info['is_live']:
|
||||
logger.warning("Live streaming media, not archiving now")
|
||||
return ArchiveResult(status="Streaming media")
|
||||
|
||||
if check_if_exists:
|
||||
if 'entries' in info:
|
||||
if len(info['entries']) > 1:
|
||||
logger.warning(
|
||||
'YoutubeDLArchiver cannot archive channels or pages with multiple videos')
|
||||
return False
|
||||
|
||||
filename = ydl.prepare_filename(info['entries'][0])
|
||||
else:
|
||||
filename = ydl.prepare_filename(info)
|
||||
|
||||
key = get_key(filename)
|
||||
|
||||
try:
|
||||
self.s3.head_object(Bucket=os.getenv('DO_BUCKET'), Key=key)
|
||||
|
||||
# file exists
|
||||
cdn_url = get_cdn_url(key)
|
||||
|
||||
status = 'already archived'
|
||||
|
||||
except ClientError:
|
||||
pass
|
||||
|
||||
# sometimes this results in a different filename, so do this again
|
||||
info = ydl.extract_info(url, download=True)
|
||||
|
||||
if 'entries' in info:
|
||||
if len(info['entries']) > 1:
|
||||
logger.warning(
|
||||
'YoutubeDLArchiver cannot archive channels or pages with multiple videos')
|
||||
return False
|
||||
else:
|
||||
info = info['entries'][0]
|
||||
|
||||
filename = ydl.prepare_filename(info)
|
||||
|
||||
if not os.path.exists(filename):
|
||||
filename = filename.split('.')[0] + '.mkv'
|
||||
|
||||
if status != 'already archived':
|
||||
key = get_key(filename)
|
||||
cdn_url = get_cdn_url(key)
|
||||
|
||||
with open(filename, 'rb') as f:
|
||||
do_s3_upload(self.s3, f, key)
|
||||
|
||||
# get duration
|
||||
duration = info['duration'] if 'duration' in info else None
|
||||
|
||||
# get thumbnails
|
||||
key_thumb, thumb_index = get_thumbnails(
|
||||
filename, self.s3, duration=duration)
|
||||
os.remove(filename)
|
||||
|
||||
return ArchiveResult(status=status, cdn_url=cdn_url, thumbnail=key_thumb, thumbnail_index=thumb_index, duration=duration,
|
||||
title=info['title'] if 'title' in info else None,
|
||||
timestamp=info['timestamp'] if 'timestamp' in info else datetime.datetime.strptime(info['upload_date'], '%Y%m%d').timestamp() if 'upload_date' in info else None)
|
||||
|
||||
|
||||
class WaybackArchiver(Archiver):
|
||||
def __init__(self, s3_client):
|
||||
self.s3 = s3_client
|
||||
self.seen_urls = {}
|
||||
|
||||
def download(self, url, check_if_exists=False):
|
||||
if check_if_exists and url in self.seen_urls:
|
||||
return self.seen_urls[url]
|
||||
|
||||
ia_headers = {
|
||||
"Accept": "application/json",
|
||||
"Authorization": "LOW " + os.getenv('INTERNET_ARCHIVE_S3_KEY') + ":" + os.getenv('INTERNET_ARCHIVE_S3_SECRET')
|
||||
}
|
||||
|
||||
r = requests.post(
|
||||
'https://web.archive.org/save/', headers=ia_headers, data={'url': url})
|
||||
|
||||
if r.status_code != 200:
|
||||
return ArchiveResult(status="Internet archive failed")
|
||||
|
||||
job_id = r.json()['job_id']
|
||||
|
||||
status_r = requests.get(
|
||||
'https://web.archive.org/save/status/' + job_id, headers=ia_headers)
|
||||
|
||||
retries = 0
|
||||
|
||||
# wait 90-120 seconds for the archive job to finish
|
||||
while (status_r.status_code != 200 or status_r.json()['status'] == 'pending') and retries < 30:
|
||||
time.sleep(3)
|
||||
|
||||
try:
|
||||
status_r = requests.get(
|
||||
'https://web.archive.org/save/status/' + job_id, headers=ia_headers)
|
||||
except:
|
||||
time.sleep(1)
|
||||
|
||||
retries += 1
|
||||
|
||||
if status_r.status_code != 200:
|
||||
return ArchiveResult(status="Internet archive failed")
|
||||
|
||||
status_json = status_r.json()
|
||||
|
||||
if status_json['status'] != 'success':
|
||||
return ArchiveResult(status='Internet Archive failed: ' + status_json['message'])
|
||||
|
||||
archive_url = 'https://web.archive.org/web/' + \
|
||||
status_json['timestamp'] + '/' + status_json['original_url']
|
||||
|
||||
try:
|
||||
r = requests.get(archive_url)
|
||||
|
||||
parsed = BeautifulSoup(
|
||||
r.content, 'html.parser')
|
||||
|
||||
title = parsed.find_all('title')[
|
||||
0].text
|
||||
except:
|
||||
title = "Could not get title"
|
||||
|
||||
result = ArchiveResult(
|
||||
status='Internet Archive fallback', cdn_url=archive_url, title=title)
|
||||
self.seen_urls[url] = result
|
||||
return result
|
||||
|
||||
|
||||
class TiktokArchiver(Archiver):
|
||||
def download(self, url, check_if_exists=False):
|
||||
if 'tiktok.com' not in url:
|
||||
return False
|
||||
|
||||
status = 'success'
|
||||
|
||||
try:
|
||||
info = tiktok_downloader.info_post(url)
|
||||
key = 'tiktok_' + str(info.id) + '.mp4'
|
||||
filename = 'tmp/' + key
|
||||
|
||||
if check_if_exists:
|
||||
try:
|
||||
self.s3.head_object(Bucket=os.getenv('DO_BUCKET'), Key=key)
|
||||
|
||||
# file exists
|
||||
cdn_url = get_cdn_url(key)
|
||||
|
||||
status = 'already archived'
|
||||
|
||||
except ClientError:
|
||||
pass
|
||||
|
||||
if status != 'already archived':
|
||||
media = tiktok_downloader.snaptik(url).get_media()
|
||||
if len(media) > 0:
|
||||
media[0].download(filename)
|
||||
with open(filename, 'rb') as f:
|
||||
do_s3_upload(self.s3, f, key)
|
||||
|
||||
cdn_url = get_cdn_url(key)
|
||||
else:
|
||||
status = 'could not download media'
|
||||
|
||||
try:
|
||||
key_thumb, thumb_index = get_thumbnails(
|
||||
filename, self.s3, duration=info.duration)
|
||||
except:
|
||||
key_thumb = ''
|
||||
thumb_index = 'error creating thumbnails'
|
||||
|
||||
os.remove(filename)
|
||||
|
||||
return ArchiveResult(status=status, cdn_url=cdn_url, thumbnail=key_thumb,
|
||||
thumbnail_index=thumb_index, duration=info.duration, title=info.caption, timestamp=info.create.isoformat())
|
||||
|
||||
except tiktok_downloader.Except.InvalidUrl:
|
||||
status = 'Invalid URL'
|
||||
return ArchiveResult(status=status)
|
||||
|
||||
except:
|
||||
error = traceback.format_exc()
|
||||
status = 'Other Tiktok error: ' + str(error)
|
||||
return ArchiveResult(status=status)
|
||||
391
auto_archive.py
391
auto_archive.py
@@ -1,19 +1,15 @@
|
||||
from dataclasses import dataclass
|
||||
import gspread
|
||||
import youtube_dl
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import datetime
|
||||
import boto3
|
||||
import os
|
||||
from dotenv import load_dotenv
|
||||
from botocore.errorfactory import ClientError
|
||||
import argparse
|
||||
import math
|
||||
import ffmpeg
|
||||
import threading
|
||||
import time
|
||||
from bs4 import BeautifulSoup
|
||||
import requests
|
||||
from loguru import logger
|
||||
import archivers
|
||||
|
||||
load_dotenv()
|
||||
|
||||
@@ -46,328 +42,64 @@ def index_to_col(index):
|
||||
else:
|
||||
return alphabet[index]
|
||||
|
||||
def get_cdn_url(key):
|
||||
return 'https://{}.{}.cdn.digitaloceanspaces.com/{}'.format(
|
||||
os.getenv('DO_BUCKET'), os.getenv('DO_SPACES_REGION'), key)
|
||||
|
||||
def do_s3_upload(s3_client, f, key):
|
||||
s3_client.upload_fileobj(f, Bucket=os.getenv(
|
||||
'DO_BUCKET'), Key=key, ExtraArgs={'ACL': 'public-read'})
|
||||
|
||||
|
||||
def get_thumbnails(filename, s3_client, duration = None):
|
||||
if not os.path.exists(filename.split('.')[0]):
|
||||
os.mkdir(filename.split('.')[0])
|
||||
|
||||
fps = 0.5
|
||||
if duration is not None:
|
||||
duration = float(duration)
|
||||
|
||||
if duration < 60:
|
||||
fps = 10.0 / duration
|
||||
elif duration < 120:
|
||||
fps = 20.0 / duration
|
||||
else:
|
||||
fps = 40.0 / duration
|
||||
|
||||
|
||||
stream = ffmpeg.input(filename)
|
||||
stream = ffmpeg.filter(stream, 'fps', fps=fps).filter('scale', 512, -1)
|
||||
stream.output(filename.split('.')[0] + '/out%d.jpg').run()
|
||||
|
||||
thumbnails = os.listdir(filename.split('.')[0] + '/')
|
||||
cdn_urls = []
|
||||
|
||||
for fname in thumbnails:
|
||||
if fname[-3:] == 'jpg':
|
||||
thumbnail_filename = filename.split('.')[0] + '/' + fname
|
||||
key = filename.split('/')[1].split('.')[0] + '/' + fname
|
||||
|
||||
cdn_url = get_cdn_url(key)
|
||||
|
||||
with open(thumbnail_filename, 'rb') as f:
|
||||
do_s3_upload(s3_client, f, key)
|
||||
|
||||
cdn_urls.append(cdn_url)
|
||||
os.remove(thumbnail_filename)
|
||||
|
||||
if len(cdn_urls) == 0:
|
||||
return ('None', 'None')
|
||||
|
||||
key_thumb = cdn_urls[int(len(cdn_urls)*0.1)]
|
||||
|
||||
index_page = f'''<html><head><title>{filename}</title></head>
|
||||
<body>'''
|
||||
|
||||
for t in cdn_urls:
|
||||
index_page += f'<img src="{t}" />'
|
||||
|
||||
index_page += f"</body></html>"
|
||||
index_fname = filename.split('.')[0] + '/index.html'
|
||||
|
||||
with open(index_fname, 'w') as f:
|
||||
f.write(index_page)
|
||||
|
||||
thumb_index = filename.split('/')[1].split('.')[0] + '/index.html'
|
||||
|
||||
s3_client.upload_fileobj(open(index_fname, 'rb'), Bucket=os.getenv(
|
||||
'DO_BUCKET'), Key=thumb_index, ExtraArgs={'ACL': 'public-read', 'ContentType': 'text/html'})
|
||||
|
||||
thumb_index_cdn_url = get_cdn_url(thumb_index)
|
||||
|
||||
return (key_thumb, thumb_index_cdn_url)
|
||||
|
||||
|
||||
def download_telegram_video(url, s3_client, check_if_exists=False):
|
||||
status = 'success'
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36'}
|
||||
|
||||
original_url = url
|
||||
|
||||
if url[-8:] != "?embed=1":
|
||||
url += "?embed=1"
|
||||
|
||||
t = requests.get(url, headers=headers)
|
||||
s = BeautifulSoup(t.content, 'html.parser')
|
||||
video = s.find("video")
|
||||
|
||||
if video is None:
|
||||
return ({}, 'No telegram video found')
|
||||
else:
|
||||
video_url = video.get('src')
|
||||
key = video_url.split('/')[-1].split('?')[0]
|
||||
filename = 'tmp/' + key
|
||||
|
||||
if check_if_exists:
|
||||
try:
|
||||
s3_client.head_object(Bucket=os.getenv('DO_BUCKET'), Key=key)
|
||||
|
||||
# file exists
|
||||
cdn_url = get_cdn_url(key)
|
||||
|
||||
status = 'already archived'
|
||||
|
||||
except ClientError:
|
||||
pass
|
||||
|
||||
v = requests.get(video_url, headers=headers)
|
||||
|
||||
with open(filename, 'wb') as f:
|
||||
f.write(v.content)
|
||||
|
||||
if status != 'already archived':
|
||||
cdn_url = get_cdn_url(key)
|
||||
|
||||
with open(filename, 'rb') as f:
|
||||
do_s3_upload(s3_client, f, key)
|
||||
|
||||
duration = s.find_all('time')[0].contents[0]
|
||||
if ':' in duration:
|
||||
duration = float(duration.split(':')[0])*60 + float(duration.split(':')[1])
|
||||
else:
|
||||
duration = float(duration)
|
||||
|
||||
key_thumb, thumb_index = get_thumbnails(filename, s3_client, duration=duration)
|
||||
os.remove(filename)
|
||||
|
||||
video_data = {
|
||||
'cdn_url': cdn_url,
|
||||
'thumbnail': key_thumb,
|
||||
'thumbnail_index': thumb_index,
|
||||
'duration': duration,
|
||||
'title': original_url,
|
||||
'timestamp': s.find_all('time')[1].get('datetime')
|
||||
}
|
||||
|
||||
return (video_data, status)
|
||||
|
||||
|
||||
def internet_archive(url, s3_client):
|
||||
|
||||
|
||||
ia_headers = {
|
||||
"Accept": "application/json",
|
||||
"Authorization": "LOW " + os.getenv('INTERNET_ARCHIVE_S3_KEY') + ":" + os.getenv('INTERNET_ARCHIVE_S3_SECRET')
|
||||
}
|
||||
|
||||
r = requests.post(
|
||||
'https://web.archive.org/save/', headers=ia_headers, data={'url': url})
|
||||
|
||||
if r.status_code != 200:
|
||||
return ({}, 'Internet archive failed')
|
||||
else:
|
||||
job_id = r.json()['job_id']
|
||||
|
||||
status_r = requests.get(
|
||||
'https://web.archive.org/save/status/' + job_id, headers=ia_headers)
|
||||
|
||||
retries = 0
|
||||
|
||||
while status_r.json()['status'] == 'pending' and retries < 40:
|
||||
time.sleep(5)
|
||||
|
||||
try:
|
||||
status_r = requests.get(
|
||||
'https://web.archive.org/save/status/' + job_id, headers=ia_headers)
|
||||
except:
|
||||
time.sleep(1)
|
||||
|
||||
retries += 1
|
||||
|
||||
status_json = status_r.json()
|
||||
|
||||
if status_json['status'] == 'success':
|
||||
url = 'https://web.archive.org/web/' + \
|
||||
status_json['timestamp'] + '/' + status_json['original_url']
|
||||
|
||||
r = requests.get(url)
|
||||
|
||||
parsed = BeautifulSoup(
|
||||
r.content, 'html.parser')
|
||||
title = parsed.find_all('title')[
|
||||
0].text
|
||||
|
||||
return ({'cdn_url': url, 'title': title}, 'Internet Archive fallback')
|
||||
else:
|
||||
return ({}, 'Internet Archive failed: ' + status_json['message'])
|
||||
|
||||
def get_key(filename):
|
||||
key = filename.split('/')[1]
|
||||
if 'unknown_video' in key:
|
||||
key = key.replace('unknown_video', 'jpg')
|
||||
return key
|
||||
|
||||
|
||||
def download_vid(url, s3_client, check_if_exists=False):
|
||||
ydl_opts = {'outtmpl': 'tmp/%(id)s.%(ext)s', 'quiet': False}
|
||||
if (url[0:21] == 'https://facebook.com/' or url[0:25] == 'https://wwww.facebook.com/') and os.getenv('FB_COOKIE'):
|
||||
print('Using cookie')
|
||||
youtube_dl.utils.std_headers['cookie'] = os.getenv('FB_COOKIE')
|
||||
ydl = youtube_dl.YoutubeDL(ydl_opts)
|
||||
cdn_url = None
|
||||
status = 'success'
|
||||
|
||||
if check_if_exists:
|
||||
info = ydl.extract_info(url, download=False)
|
||||
|
||||
if 'entries' in info:
|
||||
if len(info['entries']) > 1:
|
||||
raise Exception(
|
||||
'ERROR: Cannot archive channels or pages with multiple videos')
|
||||
|
||||
filename = ydl.prepare_filename(info['entries'][0])
|
||||
else:
|
||||
filename = ydl.prepare_filename(info)
|
||||
|
||||
key = get_key(filename)
|
||||
|
||||
try:
|
||||
s3_client.head_object(Bucket=os.getenv('DO_BUCKET'), Key=key)
|
||||
|
||||
# file exists
|
||||
cdn_url = get_cdn_url(os, key)
|
||||
|
||||
status = 'already archived'
|
||||
|
||||
except ClientError:
|
||||
pass
|
||||
|
||||
# sometimes this results in a different filename, so do this again
|
||||
info = ydl.extract_info(url, download=True)
|
||||
|
||||
if 'entries' in info:
|
||||
if len(info['entries']) > 1:
|
||||
raise Exception(
|
||||
'ERROR: Cannot archive channels or pages with multiple videos')
|
||||
else:
|
||||
info = info['entries'][0]
|
||||
|
||||
filename = ydl.prepare_filename(info)
|
||||
|
||||
if not os.path.exists(filename):
|
||||
filename = filename.split('.')[0] + '.mkv'
|
||||
|
||||
if status != 'already archived':
|
||||
key = get_key(filename)
|
||||
cdn_url = get_cdn_url(os, key)
|
||||
|
||||
with open(filename, 'rb') as f:
|
||||
do_s3_upload(s3_client, f, key)
|
||||
|
||||
duration = info['duration'] if 'duration' in info else None
|
||||
key_thumb, thumb_index = get_thumbnails(filename, s3_client, duration=duration)
|
||||
os.remove(filename)
|
||||
|
||||
video_data = {
|
||||
'cdn_url': cdn_url,
|
||||
'thumbnail': key_thumb,
|
||||
'thumbnail_index': thumb_index,
|
||||
'duration': duration,
|
||||
'title': info['title'] if 'title' in info else None,
|
||||
'timestamp': info['timestamp'] if 'timestamp' in info else datetime.datetime.strptime(info['upload_date'], '%Y%m%d').timestamp() if 'upload_date' in info else None,
|
||||
}
|
||||
|
||||
return (video_data, status)
|
||||
|
||||
|
||||
def update_sheet(wks, row, status, video_data, columns, v):
|
||||
def update_sheet(wks, row, result : archivers.ArchiveResult, columns, v):
|
||||
update = []
|
||||
|
||||
if columns['status'] is not None:
|
||||
update += [{
|
||||
'range': columns['status'] + str(row),
|
||||
'values': [[status]]
|
||||
'values': [[result.status]]
|
||||
}]
|
||||
|
||||
if 'cdn_url' in video_data and video_data['cdn_url'] is not None and columns['archive'] is not None and v[col_to_index(columns['archive'])] == '':
|
||||
if result.cdn_url and columns['archive'] is not None and v[col_to_index(columns['archive'])] == '':
|
||||
update += [{
|
||||
'range': columns['archive'] + str(row),
|
||||
'values': [[video_data['cdn_url']]]
|
||||
'values': [[result.cdn_url]]
|
||||
}]
|
||||
|
||||
if 'date' in video_data and columns['date'] is not None and v[col_to_index(columns['date'])] == '':
|
||||
if columns['date'] is not None and v[col_to_index(columns['date'])] == '':
|
||||
update += [{
|
||||
'range': columns['date'] + str(row),
|
||||
'values': [[datetime.datetime.now().isoformat()]]
|
||||
}]
|
||||
|
||||
if 'thumbnail' in video_data and columns['thumbnail'] is not None and v[col_to_index(columns['thumbnail'])] == '':
|
||||
if result.thumbnail and columns['thumbnail'] is not None and v[col_to_index(columns['thumbnail'])] == '':
|
||||
update += [{
|
||||
'range': columns['thumbnail'] + str(row),
|
||||
'values': [['=IMAGE("' + video_data['thumbnail'] + '")']]
|
||||
'values': [['=IMAGE("' + result.thumbnail + '")']]
|
||||
}]
|
||||
|
||||
if 'thumbnail_index' in video_data and columns['thumbnail_index'] is not None and v[col_to_index(columns['thumbnail_index'])] == '':
|
||||
if result.thumbnail_index and columns['thumbnail_index'] is not None and v[col_to_index(columns['thumbnail_index'])] == '':
|
||||
update += [{
|
||||
'range': columns['thumbnail_index'] + str(row),
|
||||
'values': [[video_data['thumbnail_index']]]
|
||||
'values': [[result.thumbnail_index]]
|
||||
}]
|
||||
|
||||
if 'timestamp' in video_data and columns['timestamp'] is not None and video_data['timestamp'] is not None and v[col_to_index(columns['timestamp'])] == '':
|
||||
if result.timestamp and columns['timestamp'] is not None and v[col_to_index(columns['timestamp'])] == '':
|
||||
update += [{
|
||||
'range': columns['timestamp'] + str(row),
|
||||
'values': [[video_data['timestamp']]] if type(video_data['timestamp']) == str else [[datetime.datetime.fromtimestamp(video_data['timestamp']).isoformat()]]
|
||||
'values': [[result.timestamp]] if type(result.timestamp) == str else [[datetime.datetime.fromtimestamp(result.timestamp).isoformat()]]
|
||||
}]
|
||||
|
||||
if 'title' in video_data and columns['title'] is not None and video_data['title'] is not None and v[col_to_index(columns['title'])] == '':
|
||||
if result.title and columns['title'] is not None and v[col_to_index(columns['title'])] == '':
|
||||
update += [{
|
||||
'range': columns['title'] + str(row),
|
||||
'values': [[video_data['title']]]
|
||||
'values': [[result.title]]
|
||||
}]
|
||||
|
||||
if 'duration' in video_data and columns['duration'] is not None and video_data['duration'] is not None and v[col_to_index(columns['duration'])] == '':
|
||||
if result.duration and columns['duration'] is not None and v[col_to_index(columns['duration'])] == '':
|
||||
update += [{
|
||||
'range': columns['duration'] + str(row),
|
||||
'values': [[str(video_data['duration'])]]
|
||||
'values': [[str(result.duration)]]
|
||||
}]
|
||||
|
||||
wks.batch_update(update, value_input_option='USER_ENTERED')
|
||||
|
||||
|
||||
def record_stream(url, s3_client, wks, i, columns, v):
|
||||
video_data, status = download_vid(url, s3_client)
|
||||
update_sheet(wks, i, status, video_data, columns, v)
|
||||
# def record_stream(url, s3_client, wks, i, columns, v):
|
||||
# video_data, status = download_vid(url, s3_client)
|
||||
# update_sheet(wks, i, status, video_data, columns, v)
|
||||
|
||||
|
||||
def process_sheet(sheet):
|
||||
@@ -384,7 +116,7 @@ def process_sheet(sheet):
|
||||
|
||||
# loop through worksheets to check
|
||||
for ii in range(n_worksheets):
|
||||
print("Opening worksheet " + str(ii))
|
||||
logger.info("Opening worksheet " + str(ii))
|
||||
wks = sh.get_worksheet(ii)
|
||||
values = wks.get_all_values()
|
||||
|
||||
@@ -396,7 +128,7 @@ def process_sheet(sheet):
|
||||
'source url')) if 'source url' in headers else None
|
||||
|
||||
if columns['url'] is None:
|
||||
print("No 'Media URL' column found, skipping")
|
||||
logger.warning("No 'Media URL' column found, skipping")
|
||||
continue
|
||||
|
||||
url_index = col_to_index(columns['url'])
|
||||
@@ -409,7 +141,7 @@ def process_sheet(sheet):
|
||||
'archive status')) if 'archive status' in headers else None
|
||||
|
||||
if columns['status'] is None:
|
||||
print("No 'Archive status' column found, skipping")
|
||||
logger.warning("No 'Archive status' column found, skipping")
|
||||
continue
|
||||
|
||||
columns['thumbnail'] = index_to_col(headers.index(
|
||||
@@ -423,6 +155,15 @@ def process_sheet(sheet):
|
||||
columns['duration'] = index_to_col(headers.index(
|
||||
'duration')) if 'duration' in headers else None
|
||||
|
||||
|
||||
active_archivers = [
|
||||
archivers.TelegramArchiver(s3_client),
|
||||
archivers.TiktokArchiver(s3_client),
|
||||
archivers.YoutubeDLArchiver(s3_client),
|
||||
archivers.WaybackArchiver(s3_client)
|
||||
]
|
||||
|
||||
|
||||
# loop through rows in worksheet
|
||||
for i in range(2, len(values)+1):
|
||||
v = values[i-1]
|
||||
@@ -434,61 +175,25 @@ def process_sheet(sheet):
|
||||
# check so we don't step on each others' toes
|
||||
if latest_val == '' or latest_val is None:
|
||||
wks.update(
|
||||
columns['status'] + str(i), 'Archive in progress')
|
||||
columns['status'] + str(i), 'Archive in progress')
|
||||
|
||||
if 'http://t.me/' in v[url_index] or 'https://t.me/' in v[url_index]:
|
||||
video_data, status = download_telegram_video(
|
||||
v[url_index], s3_client, check_if_exists=True)
|
||||
|
||||
if status == 'No telegram video found':
|
||||
print("Trying Internet Archive fallback")
|
||||
for archiver in active_archivers:
|
||||
logger.debug(f"Trying {archiver} on row {i}")
|
||||
result = archiver.download(v[url_index], check_if_exists=True)
|
||||
if result:
|
||||
logger.info(f"{archiver} succeeded on row {i}")
|
||||
break
|
||||
|
||||
video_data, status = internet_archive(
|
||||
v[url_index], s3_client)
|
||||
|
||||
update_sheet(wks, i, status, video_data, columns, v)
|
||||
if result:
|
||||
update_sheet(wks, i, result, columns, v)
|
||||
|
||||
else:
|
||||
try:
|
||||
ydl_opts = {
|
||||
'outtmpl': 'tmp/%(id)s.%(ext)s', 'quiet': False}
|
||||
if (v[url_index][0:21] == 'https://facebook.com/' or v[url_index][0:25] == 'https://www.facebook.com/') and os.getenv('FB_COOKIE'):
|
||||
print('Using cookie')
|
||||
youtube_dl.utils.std_headers['cookie'] = os.getenv(
|
||||
'FB_COOKIE')
|
||||
ydl = youtube_dl.YoutubeDL(ydl_opts)
|
||||
info = ydl.extract_info(
|
||||
v[url_index], download=False)
|
||||
|
||||
if 'is_live' in info and info['is_live']:
|
||||
wks.update(columns['status'] +
|
||||
str(i), 'Recording stream')
|
||||
t = threading.Thread(target=record_stream, args=(
|
||||
v[url_index], s3_client, wks, i, columns, v))
|
||||
t.start()
|
||||
continue
|
||||
elif 'is_live' not in info or not info['is_live']:
|
||||
video_data, status = download_vid(
|
||||
v[url_index], s3_client, check_if_exists=True)
|
||||
update_sheet(wks, i, status,
|
||||
video_data, columns, v)
|
||||
|
||||
except:
|
||||
# i'm sure there's a better way to handle this than nested try/catch blocks
|
||||
try:
|
||||
print("Trying Internet Archive fallback")
|
||||
# except:
|
||||
# if any unexpected errors occured, log these into the Google Sheet
|
||||
# t, value, traceback = sys.exc_info()
|
||||
|
||||
video_data, status = internet_archive(
|
||||
v[url_index], s3_client)
|
||||
update_sheet(wks, i, status,
|
||||
video_data, columns, v)
|
||||
|
||||
except:
|
||||
# if any unexpected errors occured, log these into the Google Sheet
|
||||
t, value, traceback = sys.exc_info()
|
||||
|
||||
update_sheet(wks, i, str(
|
||||
value), {}, columns, v)
|
||||
# update_sheet(wks, i, str(
|
||||
# value), {}, columns, v)
|
||||
|
||||
|
||||
def main():
|
||||
@@ -497,7 +202,7 @@ def main():
|
||||
parser.add_argument("--sheet", action="store", dest="sheet")
|
||||
args = parser.parse_args()
|
||||
|
||||
print("Opening document " + args.sheet)
|
||||
logger.info("Opening document " + args.sheet)
|
||||
|
||||
process_sheet(args.sheet)
|
||||
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
import gspread
|
||||
import subprocess
|
||||
import argparse
|
||||
import auto_archive
|
||||
import datetime
|
||||
from loguru import logger
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
@@ -11,8 +10,7 @@ def main():
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
print(datetime.datetime.now())
|
||||
print("Opening document " + args.sheet)
|
||||
logger.info("Opening document " + args.sheet)
|
||||
|
||||
gc = gspread.service_account(filename='service_account.json')
|
||||
sh = gc.open(args.sheet)
|
||||
@@ -23,7 +21,7 @@ def main():
|
||||
for i in range(11, len(values)):
|
||||
sheet_name = values[i][0]
|
||||
|
||||
print("Processing " + sheet_name)
|
||||
logger.info("Processing " + sheet_name)
|
||||
|
||||
auto_archive.process_sheet(sheet_name)
|
||||
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
gspread
|
||||
youtube_dl
|
||||
boto3
|
||||
python-dotenv
|
||||
|
||||
Reference in New Issue
Block a user