mirror of
https://github.com/bellingcat/auto-archiver.git
synced 2026-06-11 12:48:28 +03:00
124 lines
4.0 KiB
Python
124 lines
4.0 KiB
Python
import os
|
|
import datetime
|
|
import argparse
|
|
import requests
|
|
import gspread
|
|
from loguru import logger
|
|
from dotenv import load_dotenv
|
|
|
|
import archivers
|
|
from storages import S3Storage, S3Config
|
|
from gworksheet import GWorksheet
|
|
|
|
load_dotenv()
|
|
|
|
|
|
def update_sheet(gw, row, result: archivers.ArchiveResult):
|
|
update = []
|
|
|
|
def batch_if_valid(col, val, final_value=None):
|
|
final_value = final_value or val
|
|
if val and gw.col_exists(col) and gw.cell(row, col) == '':
|
|
update.append((row, col, final_value))
|
|
|
|
update.append((row, 'status', result.status))
|
|
|
|
batch_if_valid('archive', result.cdn_url)
|
|
batch_if_valid('archive', True, datetime.datetime.now().isoformat())
|
|
batch_if_valid('thumbnail', result.thumbnail, f'=IMAGE("{result.thumbnail}")')
|
|
batch_if_valid('thumbnail_index', result.thumbnail_index)
|
|
batch_if_valid('title', result.title)
|
|
batch_if_valid('duration', result.duration, str(result.duration))
|
|
|
|
if result.timestamp and type(result.timestamp) != str:
|
|
result.timestamp = datetime.datetime.fromtimestamp(result.timestamp).isoformat()
|
|
batch_if_valid('timestamp', result.timestamp)
|
|
|
|
gw.update_batch(update)
|
|
|
|
|
|
|
|
def process_sheet(sheet):
|
|
gc = gspread.service_account(filename='service_account.json')
|
|
sh = gc.open(sheet)
|
|
|
|
s3_config = S3Config(
|
|
bucket=os.getenv('DO_BUCKET'),
|
|
region=os.getenv('DO_SPACES_REGION'),
|
|
key=os.getenv('DO_SPACES_KEY'),
|
|
secret=os.getenv('DO_SPACES_SECRET')
|
|
)
|
|
|
|
# loop through worksheets to check
|
|
for ii, wks in enumerate(sh.worksheets()):
|
|
logger.info(f'Opening worksheet {ii}: "{wks.title}"')
|
|
gw = GWorksheet(wks)
|
|
|
|
if not gw.col_exists("url"):
|
|
logger.warning(f'No "Media URL" column found, skipping worksheet {wks.title}')
|
|
continue
|
|
|
|
if not gw.col_exists("status"):
|
|
logger.warning("No 'Archive status' column found, skipping")
|
|
continue
|
|
|
|
# archives will be in a folder 'doc_name/worksheet_name'
|
|
s3_config.folder = f'{sheet}/{wks.title}/'
|
|
s3_client = S3Storage(s3_config)
|
|
|
|
# order matters, first to succeed excludes remaining
|
|
active_archivers = [
|
|
archivers.TelegramArchiver(s3_client),
|
|
archivers.TiktokArchiver(s3_client),
|
|
archivers.YoutubeDLArchiver(s3_client),
|
|
archivers.WaybackArchiver(s3_client)
|
|
]
|
|
|
|
# loop through rows in worksheet
|
|
for i in range(2, gw.count_rows() + 1):
|
|
row = gw.get_row(i)
|
|
url = gw.cell(row, 'url')
|
|
status = gw.cell(row, 'status')
|
|
if url != '' and status in ['', None]:
|
|
gw.update(i, 'status', 'Archive in progress')
|
|
|
|
# expand short URL links
|
|
if 'https://t.co/' in url:
|
|
r = requests.get(url)
|
|
url = r.url
|
|
|
|
for archiver in active_archivers:
|
|
logger.debug(f'Trying {archiver} on row {i}')
|
|
result = archiver.download(url, check_if_exists=True)
|
|
|
|
if result:
|
|
logger.success(f'{archiver} succeeded on row {i}')
|
|
break
|
|
|
|
if result:
|
|
update_sheet(gw, i, result)
|
|
else:
|
|
gw.update(i, 'status', 'failed: no archiver')
|
|
|
|
# # except:
|
|
# # if any unexpected errors occured, log these into the Google Sheet
|
|
# # t, value, traceback = sys.exc_info()
|
|
|
|
# # update_sheet(wks, i, str(
|
|
# # value), {}, columns, v)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(
|
|
description="Automatically archive social media videos from a Google Sheets document")
|
|
parser.add_argument("--sheet", action="store", dest="sheet")
|
|
args = parser.parse_args()
|
|
|
|
logger.info("Opening document " + args.sheet)
|
|
|
|
process_sheet(args.sheet)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|