From cb18289e4f0211512d6874c8ba43ba244a8ecc2d Mon Sep 17 00:00:00 2001 From: Dave Mateer Date: Mon, 9 May 2022 10:55:38 +0100 Subject: [PATCH 01/10] Get Twitter original size image quality --- archivers/twitter_archiver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/archivers/twitter_archiver.py b/archivers/twitter_archiver.py index 49ab5ae..099d279 100644 --- a/archivers/twitter_archiver.py +++ b/archivers/twitter_archiver.py @@ -41,7 +41,7 @@ class TwitterArchiver(Archiver): elif type(media) == Gif: urls.append(media.variants[0].url) elif type(media) == Photo: - urls.append(media.fullUrl) + urls.append(media.fullUrl.replace('name=large', 'name=orig')) else: logger.warning(f"Could not get media URL of {media}") From bd235347ac905147582b70f68253e88b4ea9bd86 Mon Sep 17 00:00:00 2001 From: Dave Mateer Date: Mon, 9 May 2022 11:23:42 +0100 Subject: [PATCH 02/10] Added catch in youtubedl_archiver for twitter.com to see if a linked video is in there eg vk.com --- archivers/youtubedl_archiver.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/archivers/youtubedl_archiver.py b/archivers/youtubedl_archiver.py index 6e626b5..1ea194c 100644 --- a/archivers/youtubedl_archiver.py +++ b/archivers/youtubedl_archiver.py @@ -30,6 +30,13 @@ class YoutubeDLArchiver(Archiver): if info.get('is_live', False): logger.warning("Live streaming media, not archiving now") return ArchiveResult(status="Streaming media") + if 'twitter.com' in netloc: + if 'https://twitter.com/' in info['webpage_url']: + logger.info('Found https://twitter.com/ in the download url from Twitter') + else: + logger.info('Found a linked video probably in a link in a tweet - not getting that video as there may be images in the tweet') + return False + if check_if_exists: if 'entries' in info: From 7ae6e0c6f89b1fcfa537bfdd5add85f2e91c35d5 Mon Sep 17 00:00:00 2001 From: Dave Mateer Date: Mon, 9 May 2022 11:38:08 +0100 Subject: [PATCH 03/10] fb cooke in ytd --- archivers/youtubedl_archiver.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/archivers/youtubedl_archiver.py b/archivers/youtubedl_archiver.py index 6e626b5..b710393 100644 --- a/archivers/youtubedl_archiver.py +++ b/archivers/youtubedl_archiver.py @@ -5,17 +5,21 @@ import yt_dlp from loguru import logger from .base_archiver import Archiver, ArchiveResult - +from storages import Storage class YoutubeDLArchiver(Archiver): name = "youtube_dl" ydl_opts = {'outtmpl': 'tmp/%(id)s.%(ext)s', 'quiet': False} + def __init__(self, storage: Storage, driver, fb_cookie): + super().__init__(storage, driver) + self.fb_cookie = fb_cookie + def download(self, url, check_if_exists=False): netloc = self.get_netloc(url) - if netloc in ['facebook.com', 'www.facebook.com'] and os.getenv('FB_COOKIE'): - logger.info('Using Facebook cookie') - yt_dlp.utils.std_headers['cookie'] = os.getenv('FB_COOKIE') + if netloc in ['facebook.com', 'www.facebook.com']: + logger.debug('Using Facebook cookie') + yt_dlp.utils.std_headers['cookie'] = self.fb_cookie ydl = yt_dlp.YoutubeDL(YoutubeDLArchiver.ydl_opts) cdn_url = None From 51f635ce50895d9952ab2e7faff96be0f8216015 Mon Sep 17 00:00:00 2001 From: Dave Mateer Date: Mon, 9 May 2022 11:44:25 +0100 Subject: [PATCH 04/10] get env variable FACEBOOK_COOKIE patch through from auto_archive --- auto_archive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auto_archive.py b/auto_archive.py index d3db9a2..a7bde8b 100644 --- a/auto_archive.py +++ b/auto_archive.py @@ -108,7 +108,7 @@ def process_sheet(sheet, header=1, columns=GWorksheet.COLUMN_NAMES): archivers.TelethonArchiver(s3_client, driver, telegram_config), archivers.TelegramArchiver(s3_client, driver), archivers.TiktokArchiver(s3_client, driver), - archivers.YoutubeDLArchiver(s3_client, driver), + archivers.YoutubeDLArchiver(s3_client, driver, os.getenv('FACEBOOK_COOKIE')), archivers.TwitterArchiver(s3_client, driver), archivers.WaybackArchiver(s3_client, driver) ] From 07b82f0e82862c109086001cc21fb7950c4b4289 Mon Sep 17 00:00:00 2001 From: Dave Mateer Date: Mon, 9 May 2022 11:46:19 +0100 Subject: [PATCH 05/10] update .example.env --- .example.env | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.example.env b/.example.env index deb01a2..4a200cf 100644 --- a/.example.env +++ b/.example.env @@ -5,4 +5,6 @@ DO_BUCKET= INTERNET_ARCHIVE_S3_KEY= INTERNET_ARCHIVE_S3_SECRET= TELEGRAM_API_ID= -TELEGRAM_API_HASH= \ No newline at end of file +TELEGRAM_API_HASH= + +FACEBOOK_COOKIE=cookie: datr= xxxx \ No newline at end of file From e18a9779dbcebe248c1856695f7663a5c45e50c4 Mon Sep 17 00:00:00 2001 From: Dave Mateer Date: Mon, 9 May 2022 11:55:10 +0100 Subject: [PATCH 06/10] added log diretory and file creation --- auto_archive.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/auto_archive.py b/auto_archive.py index d3db9a2..9172b5d 100644 --- a/auto_archive.py +++ b/auto_archive.py @@ -5,15 +5,20 @@ import requests import shutil import gspread from loguru import logger -from dotenv import load_dotenv +# from dotenv import load_dotenv from selenium import webdriver import traceback import archivers from storages import S3Storage, S3Config from utils import GWorksheet, mkdir_if_not_exists +import sys -load_dotenv() +logger.add("logs/1trace.log", level="TRACE") +logger.add("logs/2info.log", level="INFO") +logger.add("logs/3success.log", level="SUCCESS") +logger.add("logs/4warning.log", level="WARNING") +logger.add("logs/5error.log", level="ERROR") def update_sheet(gw, row, result: archivers.ArchiveResult): @@ -86,7 +91,7 @@ def process_sheet(sheet, header=1, columns=GWorksheet.COLUMN_NAMES): # loop through worksheets to check for ii, wks in enumerate(sh.worksheets()): - logger.info(f'Opening worksheet {ii}: "{wks.title}" header={header}') + logger.info(f'Opening worksheet ii={ii}: {wks.title} header={header}') gw = GWorksheet(wks, header_row=header, columns=columns) if not gw.col_exists('url'): @@ -151,8 +156,9 @@ def process_sheet(sheet, header=1, columns=GWorksheet.COLUMN_NAMES): logger.success(f'Finshed worksheet {wks.title}') driver.quit() - +@logger.catch def main(): + logger.debug(f'Passed args:{sys.argv}') parser = argparse.ArgumentParser( description='Automatically archive social media videos from a Google Sheets document') parser.add_argument('--sheet', action='store', dest='sheet', help='the name of the google sheets document', required=True) From e3c0ae1d45e214a5fd3443ec6005bf24a382c206 Mon Sep 17 00:00:00 2001 From: Dave Mateer Date: Mon, 9 May 2022 11:57:54 +0100 Subject: [PATCH 07/10] dotenv --- auto_archive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auto_archive.py b/auto_archive.py index 9172b5d..1342285 100644 --- a/auto_archive.py +++ b/auto_archive.py @@ -5,7 +5,7 @@ import requests import shutil import gspread from loguru import logger -# from dotenv import load_dotenv +from dotenv import load_dotenv from selenium import webdriver import traceback From f52d8cdef80c67e2017ee3bef0db9f3403d66d4f Mon Sep 17 00:00:00 2001 From: Dave Mateer Date: Mon, 9 May 2022 12:02:43 +0100 Subject: [PATCH 08/10] add back in d_dotenv() --- auto_archive.py | 1 + 1 file changed, 1 insertion(+) diff --git a/auto_archive.py b/auto_archive.py index 1342285..a529a82 100644 --- a/auto_archive.py +++ b/auto_archive.py @@ -20,6 +20,7 @@ logger.add("logs/3success.log", level="SUCCESS") logger.add("logs/4warning.log", level="WARNING") logger.add("logs/5error.log", level="ERROR") +load_dotenv() def update_sheet(gw, row, result: archivers.ArchiveResult): cell_updates = [] From bb599f702d9ec49b26ace026e7c80f386b307e4c Mon Sep 17 00:00:00 2001 From: Dave Mateer Date: Mon, 9 May 2022 12:16:18 +0100 Subject: [PATCH 09/10] Reload firefox driver on every spreadsheet row --- auto_archive.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/auto_archive.py b/auto_archive.py index d3db9a2..25b2d13 100644 --- a/auto_archive.py +++ b/auto_archive.py @@ -76,13 +76,7 @@ def process_sheet(sheet, header=1, columns=GWorksheet.COLUMN_NAMES): api_hash=os.getenv('TELEGRAM_API_HASH') ) - options = webdriver.FirefoxOptions() - options.headless = True - options.set_preference('network.protocol-handler.external.tg', False) - - driver = webdriver.Firefox(options=options) - driver.set_window_size(1400, 2000) - driver.set_page_load_timeout(10) + # loop through worksheets to check for ii, wks in enumerate(sh.worksheets()): @@ -122,7 +116,17 @@ def process_sheet(sheet, header=1, columns=GWorksheet.COLUMN_NAMES): gw.set_cell(row, 'status', 'Archive in progress') url = expand_url(url) + + # make a new driver so each spreadsheet row is idempotent + options = webdriver.FirefoxOptions() + options.headless = True + options.set_preference('network.protocol-handler.external.tg', False) + + driver = webdriver.Firefox(options=options) + driver.set_window_size(1400, 2000) + # in seconds, telegram screenshots catch which don't come back + driver.set_page_load_timeout(120) for archiver in active_archivers: logger.debug(f'Trying {archiver} on row {row}') @@ -143,13 +147,13 @@ def process_sheet(sheet, header=1, columns=GWorksheet.COLUMN_NAMES): f'{archiver} did not succeed on row {row}, final status: {result.status}') result.status = archiver.name + \ ": " + str(result.status) - + # get rid of driver so can reload on next row + driver.quit() if result: update_sheet(gw, row, result) else: gw.set_cell(row, 'status', 'failed: no archiver') logger.success(f'Finshed worksheet {wks.title}') - driver.quit() def main(): From 6e8eccefd82e48ceb095b8460ce34ac5beb9f6ab Mon Sep 17 00:00:00 2001 From: Miguel Sozinho Ramalho <19508417+msramalho@users.noreply.github.com> Date: Mon, 9 May 2022 15:59:35 +0100 Subject: [PATCH 10/10] self-documenting info message --- auto_archive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auto_archive.py b/auto_archive.py index a529a82..322b069 100644 --- a/auto_archive.py +++ b/auto_archive.py @@ -92,7 +92,7 @@ def process_sheet(sheet, header=1, columns=GWorksheet.COLUMN_NAMES): # loop through worksheets to check for ii, wks in enumerate(sh.worksheets()): - logger.info(f'Opening worksheet ii={ii}: {wks.title} header={header}') + logger.info(f'Opening worksheet {ii=}: {wks.title=} {header=}') gw = GWorksheet(wks, header_row=header, columns=columns) if not gw.col_exists('url'):