mirror of
https://github.com/bellingcat/auto-archiver.git
synced 2026-06-13 05:38:29 +03:00
Add auto-auto archiver
This commit is contained in:
Logan Williams
317
auto_archive.py
Normal file
317
auto_archive.py
Normal file
@@ -0,0 +1,317 @@
|
||||
import gspread
|
||||
import youtube_dl
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import datetime
|
||||
import boto3
|
||||
import os
|
||||
from dotenv import load_dotenv
|
||||
from botocore.errorfactory import ClientError
|
||||
import argparse
|
||||
import math
|
||||
import ffmpeg
|
||||
import threading
|
||||
import time
|
||||
|
||||
load_dotenv()
|
||||
|
||||
def col_to_index(col):
|
||||
col = list(col)
|
||||
ndigits = len(col)
|
||||
alphabet = ' ABCDEFGHIJKLMNOPQRSTUVWXYZ'
|
||||
v = 0
|
||||
i = ndigits - 1
|
||||
|
||||
for digit in col:
|
||||
index = alphabet.find(digit)
|
||||
v += (26 ** i) * index
|
||||
i -= 1
|
||||
|
||||
return v - 1
|
||||
|
||||
def index_to_col(index):
|
||||
alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
|
||||
|
||||
if index > 25:
|
||||
t = index
|
||||
dig = 0
|
||||
while t > 25:
|
||||
t = math.floor(t / 26)
|
||||
dig += 1
|
||||
return alphabet[t - 1] + index_to_col(index - t * int(math.pow(26, dig)))
|
||||
else:
|
||||
return alphabet[index]
|
||||
|
||||
def get_thumbnails(filename, s3_client):
|
||||
if not os.path.exists(filename.split('.')[0]):
|
||||
os.mkdir(filename.split('.')[0])
|
||||
|
||||
stream = ffmpeg.input(filename)
|
||||
stream = ffmpeg.filter(stream, 'fps', fps=0.5).filter('scale', 512, -1)
|
||||
stream.output(filename.split('.')[0] + '/out%d.jpg').run()
|
||||
|
||||
thumbnails = os.listdir(filename.split('.')[0] + '/')
|
||||
cdn_urls = []
|
||||
|
||||
for fname in thumbnails:
|
||||
thumbnail_filename = filename.split('.')[0] + '/' + fname
|
||||
key = filename.split('/')[1].split('.')[0] + '/' + fname
|
||||
|
||||
cdn_url = 'https://{}.{}.cdn.digitaloceanspaces.com/{}'.format(
|
||||
os.getenv('DO_BUCKET'), os.getenv('DO_SPACES_REGION'), key)
|
||||
|
||||
with open(thumbnail_filename, 'rb') as f:
|
||||
s3_client.upload_fileobj(f, Bucket=os.getenv(
|
||||
'DO_BUCKET'), Key=key, ExtraArgs={'ACL': 'public-read'})
|
||||
|
||||
cdn_urls.append(cdn_url)
|
||||
os.remove(thumbnail_filename)
|
||||
|
||||
key_thumb = cdn_urls[int(len(cdn_urls)*0.25)]
|
||||
|
||||
index_page = f'''<html><head><title>{filename}</title></head>
|
||||
<body>'''
|
||||
|
||||
for t in cdn_urls:
|
||||
index_page += f'<img src="{t}" />'
|
||||
|
||||
index_page += f"</body></html>"
|
||||
index_fname = filename.split('.')[0] + '/index.html'
|
||||
|
||||
with open(index_fname, 'w') as f:
|
||||
f.write(index_page)
|
||||
|
||||
thumb_index = filename.split('/')[1].split('.')[0] + '/index.html'
|
||||
|
||||
s3_client.upload_fileobj(open(index_fname, 'rb'), Bucket=os.getenv(
|
||||
'DO_BUCKET'), Key=thumb_index, ExtraArgs={'ACL': 'public-read', 'ContentType': 'text/html'})
|
||||
|
||||
thumb_index_cdn_url = 'https://{}.{}.cdn.digitaloceanspaces.com/{}'.format(
|
||||
os.getenv('DO_BUCKET'), os.getenv('DO_SPACES_REGION'), thumb_index)
|
||||
|
||||
return (key_thumb, thumb_index_cdn_url)
|
||||
|
||||
|
||||
def download_vid(url, s3_client, check_if_exists=False):
|
||||
ydl_opts = {'outtmpl': 'tmp/%(id)s.%(ext)s', 'quiet': False}
|
||||
ydl = youtube_dl.YoutubeDL(ydl_opts)
|
||||
cdn_url = None
|
||||
status = 'success'
|
||||
|
||||
if check_if_exists:
|
||||
info = ydl.extract_info(url, download=False)
|
||||
|
||||
if 'entries' in info:
|
||||
if len(info['entries']) > 1:
|
||||
raise Exception(
|
||||
'ERROR: Cannot archive channels or pages with multiple videos')
|
||||
|
||||
filename = ydl.prepare_filename(info['entries'][0])
|
||||
else:
|
||||
filename = ydl.prepare_filename(info)
|
||||
|
||||
key = filename.split('/')[1]
|
||||
|
||||
try:
|
||||
s3_client.head_object(Bucket=os.getenv('DO_BUCKET'), Key=key)
|
||||
|
||||
# file exists
|
||||
cdn_url = 'https://{}.{}.cdn.digitaloceanspaces.com/{}'.format(
|
||||
os.getenv('DO_BUCKET'), os.getenv('DO_SPACES_REGION'), key)
|
||||
|
||||
status = 'already archived'
|
||||
|
||||
except ClientError:
|
||||
pass
|
||||
|
||||
# sometimes this results in a different filename, so do this again
|
||||
info = ydl.extract_info(url, download=True)
|
||||
|
||||
if 'entries' in info:
|
||||
if len(info['entries']) > 1:
|
||||
raise Exception(
|
||||
'ERROR: Cannot archive channels or pages with multiple videos')
|
||||
|
||||
filename = ydl.prepare_filename(info['entries'][0])
|
||||
else:
|
||||
filename = ydl.prepare_filename(info)
|
||||
|
||||
if not os.path.exists(filename):
|
||||
filename = filename.split('.')[0] + '.mkv'
|
||||
|
||||
if status != 'already archived':
|
||||
key = filename.split('/')[1]
|
||||
cdn_url = 'https://{}.{}.cdn.digitaloceanspaces.com/{}'.format(
|
||||
os.getenv('DO_BUCKET'), os.getenv('DO_SPACES_REGION'), key)
|
||||
|
||||
with open(filename, 'rb') as f:
|
||||
s3_client.upload_fileobj(f, Bucket=os.getenv(
|
||||
'DO_BUCKET'), Key=key, ExtraArgs={'ACL': 'public-read'})
|
||||
|
||||
key_thumb, thumb_index = get_thumbnails(filename, s3_client)
|
||||
os.remove(filename)
|
||||
|
||||
video_data = {
|
||||
'cdn_url': cdn_url,
|
||||
'thumbnail': key_thumb,
|
||||
'thumbnail_index': thumb_index,
|
||||
'duration': info['duration'] if 'duration' in info else None,
|
||||
'title': info['title'] if 'title' in info else None,
|
||||
'timestamp': info['timestamp'] if 'timestamp' in info else datetime.datetime.strptime(info['upload_date'], '%Y%m%d').timestamp() if 'upload_date' in info else None,
|
||||
}
|
||||
|
||||
return (video_data, status)
|
||||
|
||||
|
||||
def update_sheet(wks, row, status, video_data, columns, v):
|
||||
update = []
|
||||
|
||||
if columns['status'] is not None:
|
||||
update += [{
|
||||
'range': columns['status'] + str(row),
|
||||
'values': [[status]]
|
||||
}]
|
||||
|
||||
if 'cdn_url' in video_data and video_data['cdn_url'] is not None and columns['archive'] is not None and v[col_to_index(columns['archive'])] == '':
|
||||
update += [{
|
||||
'range': columns['archive'] + str(row),
|
||||
'values': [[video_data['cdn_url']]]
|
||||
}]
|
||||
|
||||
if 'date' in video_data and columns['date'] is not None and v[col_to_index(columns['date'])] == '':
|
||||
update += [{
|
||||
'range': columns['date'] + str(row),
|
||||
'values': [[datetime.datetime.now().isoformat()]]
|
||||
}]
|
||||
|
||||
if 'thumbnail' in video_data and columns['thumbnail'] is not None and v[col_to_index(columns['thumbnail'])] == '':
|
||||
update += [{
|
||||
'range': columns['thumbnail'] + str(row),
|
||||
'values': [['=IMAGE("' + video_data['thumbnail'] + '")']]
|
||||
}]
|
||||
|
||||
if 'thumbnail_index' in video_data and columns['thumbnail_index'] is not None and v[col_to_index(columns['thumbnail_index'])] == '':
|
||||
update += [{
|
||||
'range': columns['thumbnail_index'] + str(row),
|
||||
'values': [[video_data['thumbnail_index']]]
|
||||
}]
|
||||
|
||||
if 'timestamp' in video_data and columns['timestamp'] is not None and video_data['timestamp'] is not None and v[col_to_index(columns['timestamp'])] == '':
|
||||
update += [{
|
||||
'range': columns['timestamp'] + str(row),
|
||||
'values': [[datetime.datetime.fromtimestamp(video_data['timestamp']).isoformat()]]
|
||||
}]
|
||||
|
||||
if 'title' in video_data and columns['title'] is not None and video_data['title'] is not None and v[col_to_index(columns['title'])] == '':
|
||||
update += [{
|
||||
'range': columns['title'] + str(row),
|
||||
'values': [[video_data['title']]]
|
||||
}]
|
||||
|
||||
if 'duration' in video_data and columns['duration'] is not None and video_data['duration'] is not None and v[col_to_index(columns['duration'])] == '':
|
||||
update += [{
|
||||
'range': columns['duration'] + str(row),
|
||||
'values': [[str(video_data['duration'])]]
|
||||
}]
|
||||
|
||||
wks.batch_update(update, value_input_option='USER_ENTERED')
|
||||
|
||||
def record_stream(url, s3_client, wks, i, columns, v):
|
||||
video_data, status = download_vid(url, s3_client)
|
||||
update_sheet(wks, i, status, video_data, columns, v)
|
||||
|
||||
def process_sheet(sheet):
|
||||
gc = gspread.service_account()
|
||||
sh = gc.open(sheet)
|
||||
n_worksheets = len(sh.worksheets())
|
||||
|
||||
s3_client = boto3.client('s3',
|
||||
region_name=os.getenv('DO_SPACES_REGION'),
|
||||
endpoint_url='https://{}.digitaloceanspaces.com'.format(
|
||||
os.getenv('DO_SPACES_REGION')),
|
||||
aws_access_key_id=os.getenv('DO_SPACES_KEY'),
|
||||
aws_secret_access_key=os.getenv('DO_SPACES_SECRET'))
|
||||
|
||||
# loop through worksheets to check
|
||||
for ii in range(n_worksheets):
|
||||
print("Opening worksheet " + str(ii))
|
||||
wks = sh.get_worksheet(ii)
|
||||
values = wks.get_all_values()
|
||||
|
||||
headers = values[0]
|
||||
columns = {}
|
||||
|
||||
columns['url'] = index_to_col(headers.index(
|
||||
'Media URL')) if 'Media URL' in headers else None
|
||||
|
||||
if columns['url'] is None:
|
||||
print("No 'Media URL' column found, skipping")
|
||||
continue
|
||||
|
||||
url_index = col_to_index(columns['url'])
|
||||
|
||||
columns['archive'] = index_to_col(headers.index(
|
||||
'Archive location')) if 'Archive location' in headers else None
|
||||
columns['date'] = index_to_col(headers.index(
|
||||
'Archive date')) if 'Archive date' in headers else None
|
||||
columns['status'] = index_to_col(headers.index(
|
||||
'Archive status')) if 'Archive status' in headers else None
|
||||
|
||||
if columns['status'] is None:
|
||||
print("No 'Archive status' column found, skipping")
|
||||
continue
|
||||
|
||||
columns['thumbnail'] = index_to_col(headers.index(
|
||||
'Thumbnail')) if 'Thumbnail' in headers else None
|
||||
columns['thumbnail_index'] = index_to_col(headers.index(
|
||||
'Thumbnail index')) if 'Thumbnail index' in headers else None
|
||||
columns['timestamp'] = index_to_col(headers.index(
|
||||
'Upload timestamp')) if 'Upload timestamp' in headers else None
|
||||
columns['title'] = index_to_col(headers.index(
|
||||
'Upload title')) if 'Upload title' in headers else None
|
||||
columns['duration'] = index_to_col(headers.index(
|
||||
'Duration')) if 'Duration' in headers else None
|
||||
|
||||
# loop through rows in worksheet
|
||||
for i in range(2, len(values)+1):
|
||||
v = values[i-1]
|
||||
|
||||
if v[url_index] != "" and v[col_to_index(columns['status'])] == "":
|
||||
try:
|
||||
ydl_opts = {
|
||||
'outtmpl': 'tmp/%(id)s.%(ext)s', 'quiet': False}
|
||||
ydl = youtube_dl.YoutubeDL(ydl_opts)
|
||||
info = ydl.extract_info(v[url_index], download=False)
|
||||
|
||||
if 'is_live' in info and info['is_live']:
|
||||
wks.update(columns['status'] + str(i), 'Recording stream')
|
||||
t = threading.Thread(target=record_stream, args=(v[url_index], s3_client, wks, i, columns, v))
|
||||
t.start()
|
||||
continue
|
||||
elif 'is_live' not in info or not info['is_live']:
|
||||
latest_val = wks.acell(columns['status'] + str(i)).value
|
||||
|
||||
# check so we don't step on each others' toes
|
||||
if latest_val == '' or latest_val is None:
|
||||
wks.update(columns['status'] + str(i), 'Archive in progress')
|
||||
video_data, status = download_vid(
|
||||
v[url_index], s3_client, check_if_exists=True)
|
||||
update_sheet(wks, i, status, video_data, columns, v)
|
||||
|
||||
except:
|
||||
# if any unexpected errors occured, log these into the Google Sheet
|
||||
t, value, traceback = sys.exc_info()
|
||||
update_sheet(wks, i, str(value), {}, columns, v)
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Automatically use youtube-dl to download media from a Google Sheet")
|
||||
parser.add_argument("--sheet", action="store", dest="sheet")
|
||||
args = parser.parse_args()
|
||||
|
||||
print("Opening document " + args.sheet)
|
||||
|
||||
process_sheet(args.sheet)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user