mirror of
https://github.com/bellingcat/auto-archiver.git
synced 2026-06-08 03:18:28 +03:00
92 lines
2.9 KiB
Python
92 lines
2.9 KiB
Python
import gspread
|
|
import youtube_dl
|
|
from pathlib import Path
|
|
import sys
|
|
import datetime
|
|
import boto3
|
|
import os
|
|
from dotenv import load_dotenv
|
|
from botocore.errorfactory import ClientError
|
|
|
|
load_dotenv()
|
|
|
|
gc = gspread.service_account()
|
|
sh = gc.open("Bellingcat media archiver")
|
|
wks = sh.sheet1
|
|
values = wks.get_all_values()
|
|
|
|
ydl_opts = {'outtmpl': 'tmp/%(id)s.%(ext)s', 'quiet': True}
|
|
ydl = youtube_dl.YoutubeDL(ydl_opts)
|
|
|
|
s3_client = boto3.client('s3',
|
|
region_name=os.getenv('DO_SPACES_REGION'),
|
|
endpoint_url='https://{}.digitaloceanspaces.com'.format(os.getenv('DO_SPACES_REGION')),
|
|
aws_access_key_id=os.getenv('DO_SPACES_KEY'),
|
|
aws_secret_access_key=os.getenv('DO_SPACES_SECRET'))
|
|
|
|
for i in range(2, len(values)+1):
|
|
v = values[i-1]
|
|
|
|
if v[2] == "":
|
|
print(v[0])
|
|
|
|
try:
|
|
info = ydl.extract_info(v[0], download=False)
|
|
filename = ydl.prepare_filename(info)
|
|
key = filename.split('/')[1]
|
|
cdn_url = 'https://{}.{}.cdn.digitaloceanspaces.com/{}'.format(os.getenv('DO_BUCKET'), os.getenv('DO_SPACES_REGION'), key)
|
|
|
|
try:
|
|
s3_client.head_object(Bucket=os.getenv('DO_BUCKET'), Key=key)
|
|
|
|
# file exists
|
|
|
|
update = [{
|
|
'range': 'C' + str(i),
|
|
'values': [['already archived']]
|
|
}, {
|
|
'range': 'D' + str(i),
|
|
'values': [[cdn_url]]
|
|
}]
|
|
|
|
wks.batch_update(update)
|
|
|
|
except ClientError:
|
|
# Not found
|
|
|
|
# sometimes this results in a different filename, so do this again
|
|
info = ydl.extract_info(v[0], download=True)
|
|
filename = ydl.prepare_filename(info)
|
|
key = filename.split('/')[1]
|
|
cdn_url = 'https://{}.{}.cdn.digitaloceanspaces.com/{}'.format(os.getenv('DO_BUCKET'), os.getenv('DO_SPACES_REGION'), key)
|
|
|
|
with open(filename, 'rb') as f:
|
|
s3_client.upload_fileobj(f, Bucket=os.getenv('DO_BUCKET'), Key=key, ExtraArgs={'ACL': 'public-read'})
|
|
|
|
os.remove(filename)
|
|
|
|
update = [{
|
|
'range': 'C' + str(i),
|
|
'values': [['successful']]
|
|
}, {
|
|
'range': 'B' + str(i),
|
|
'values': [[datetime.datetime.now().isoformat()]]
|
|
}, {
|
|
'range': 'D' + str(i),
|
|
'values': [[cdn_url]]
|
|
}]
|
|
|
|
wks.batch_update(update)
|
|
except:
|
|
t, value, traceback = sys.exc_info()
|
|
|
|
update = [{
|
|
'range': 'C' + str(i),
|
|
'values': [[str(value)]]
|
|
}, {
|
|
'range': 'B' + str(i),
|
|
'values': [[datetime.datetime.now().isoformat()]]
|
|
}]
|
|
|
|
wks.batch_update(update)
|