bellingcat/auto-archiver
1
0
Fork 0
mirror of https://github.com/bellingcat/auto-archiver.git synced 2026-06-08 03:18:28 +03:00
Code Issues Packages Projects Releases Wiki Activity

pep8 format

Browse Source
This commit is contained in:
Logan Williams
2021-03-15 10:08:02 +01:00
parent ad883c9232
commit a0180ccaaa
2 changed files with 82 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@@ -1,4 +1,5 @@
tmp/
.env
.DS_Store
expmt/

118
auto-archive.py
View File

@@ -8,9 +8,11 @@ import os
from dotenv import load_dotenv
from botocore.errorfactory import ClientError
import argparse
import math
load_dotenv()
def col_to_index(col):
col = list(col)
ndigits = len(col)
@@ -25,6 +27,17 @@ def col_to_index(col):
return v - 1
def index_to_col(index):
alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'
if index > 25:
t = math.floor(index / 26)
return alphabet[t - 1] + index_to_col(index - t * 26)
else:
return alphabet[index]
def download_vid(url, s3_client, check_if_exists=False):
ydl_opts = {'outtmpl': 'tmp/%(id)s.%(ext)s', 'quiet': False}
ydl = youtube_dl.YoutubeDL(ydl_opts)
@@ -34,18 +47,20 @@ def download_vid(url, s3_client, check_if_exists=False):
if 'entries' in info:
if len(info['entries']) > 1:
raise Exception('ERROR: Cannot archive channels or pages with multiple videos')
raise Exception(
'ERROR: Cannot archive channels or pages with multiple videos')
filename = ydl.prepare_filename(info['entries'][0])
else:
filename = ydl.prepare_filename(info)
key = filename.split('/')[1]
cdn_url = 'https://{}.{}.cdn.digitaloceanspaces.com/{}'.format(os.getenv('DO_BUCKET'), os.getenv('DO_SPACES_REGION'), key)
cdn_url = 'https://{}.{}.cdn.digitaloceanspaces.com/{}'.format(
os.getenv('DO_BUCKET'), os.getenv('DO_SPACES_REGION'), key)
try:
s3_client.head_object(Bucket=os.getenv('DO_BUCKET'), Key=key)
# file exists
return (cdn_url, 'already archived')
@@ -57,7 +72,8 @@ def download_vid(url, s3_client, check_if_exists=False):
if 'entries' in info:
if len(info['entries']) > 1:
raise Exception('ERROR: Cannot archive channels or pages with multiple videos')
raise Exception(
'ERROR: Cannot archive channels or pages with multiple videos')
filename = ydl.prepare_filename(info['entries'][0])
else:
@@ -67,42 +83,54 @@ def download_vid(url, s3_client, check_if_exists=False):
filename = filename.split('.')[0] + '.mkv'
key = filename.split('/')[1]
cdn_url = 'https://{}.{}.cdn.digitaloceanspaces.com/{}'.format(os.getenv('DO_BUCKET'), os.getenv('DO_SPACES_REGION'), key)
cdn_url = 'https://{}.{}.cdn.digitaloceanspaces.com/{}'.format(
os.getenv('DO_BUCKET'), os.getenv('DO_SPACES_REGION'), key)
with open(filename, 'rb') as f:
s3_client.upload_fileobj(f, Bucket=os.getenv('DO_BUCKET'), Key=key, ExtraArgs={'ACL': 'public-read'})
s3_client.upload_fileobj(f, Bucket=os.getenv(
'DO_BUCKET'), Key=key, ExtraArgs={'ACL': 'public-read'})
os.remove(filename)
return (cdn_url, 'success')
def main():
parser = argparse.ArgumentParser(description="Automatically use youtube-dl to download media from a Google Sheet")
parser.add_argument("--sheet", action="store", dest="sheet")
parser.add_argument('--streaming', dest='streaming', action='store_true')
parser.add_argument('--all-worksheets', dest='all_worksheets', action='store_true')
parser.add_argument('--url-col', dest='url', action='store')
parser.add_argument('--archive-col', dest='archive', action='store')
parser.add_argument('--date-col', dest='date', action='store')
parser.add_argument('--status-col', dest='status', action='store')
args = parser.parse_args()
def update_sheet(wks, row, status, url):
update = [{
'range': args.status + str(row),
'values': [[status]]
}, {
'range': args.date + str(row),
'values': [[datetime.datetime.now().isoformat()]]
}, {
def update_sheet(wks, row, status, url, columns):
update = []
if url is not None and columns['archive'] is not None:
update += [{
'range': args.archive + str(row),
'values': [[url]]
}]
if url is None:
update = update[:-1]
if columns['status'] is not None:
update += [{
'range': args.status + str(row),
'values': [[status]]
}]
wks.batch_update(update)
if columns['date'] is not None:
update += [{
'range': args.date + str(row),
'values': [[datetime.datetime.now().isoformat()]]
}]
wks.batch_update(update)
def main():
parser = argparse.ArgumentParser(
description="Automatically use youtube-dl to download media from a Google Sheet")
parser.add_argument("--sheet", action="store", dest="sheet")
parser.add_argument('--streaming', dest='streaming', action='store_true')
parser.add_argument('--all-worksheets',
dest='all_worksheets', action='store_true')
# parser.add_argument('--url-col', dest='url', action='store')
# parser.add_argument('--archive-col', dest='archive', action='store')
# parser.add_argument('--date-col', dest='date', action='store')
# parser.add_argument('--status-col', dest='status', action='store')
args = parser.parse_args()
print("Opening document " + args.sheet)
@@ -111,10 +139,11 @@ def main():
n_worksheets = len(sh.worksheets()) if args.all_worksheets else 1
s3_client = boto3.client('s3',
region_name=os.getenv('DO_SPACES_REGION'),
endpoint_url='https://{}.digitaloceanspaces.com'.format(os.getenv('DO_SPACES_REGION')),
aws_access_key_id=os.getenv('DO_SPACES_KEY'),
aws_secret_access_key=os.getenv('DO_SPACES_SECRET'))
region_name=os.getenv('DO_SPACES_REGION'),
endpoint_url='https://{}.digitaloceanspaces.com'.format(
os.getenv('DO_SPACES_REGION')),
aws_access_key_id=os.getenv('DO_SPACES_KEY'),
aws_secret_access_key=os.getenv('DO_SPACES_SECRET'))
# loop through worksheets to check
for ii in range(n_worksheets):
@@ -122,6 +151,18 @@ def main():
wks = sh.get_worksheet(ii)
values = wks.get_all_values()
headers = values[0]
columns = {}
columns['url'] = index_to_col(headers.index(
'Media URL')) if 'Media URL' in headers else None
columns['archive'] = index_to_col(headers.index(
'Archive location')) if 'Archive location' in headers else None
columns['date'] = index_to_col(headers.index(
'Archive date')) if 'Archive date' in headers else None
columns['status'] = index_to_col(headers.index(
'Archive status')) if 'Archive status' in headers else None
# loop through rows in worksheet
for i in range(2, len(values)+1):
v = values[i-1]
@@ -132,23 +173,26 @@ def main():
print(v[col_to_index(args.url)])
try:
ydl_opts = {'outtmpl': 'tmp/%(id)s.%(ext)s', 'quiet': False}
ydl_opts = {
'outtmpl': 'tmp/%(id)s.%(ext)s', 'quiet': False}
ydl = youtube_dl.YoutubeDL(ydl_opts)
info = ydl.extract_info(v[url_index], download=False)
if args.streaming and 'is_live' in info and info['is_live']:
wks.update(args.status + str(i), 'Recording stream')
cdn_url, status = download_vid(v[url_index], s3_client)
update_sheet(wks, i, status, cdn_url)
update_sheet(wks, i, status, cdn_url, columns)
sys.exit()
elif not args.streaming and ('is_live' not in info or not info['is_live']):
cdn_url, status = download_vid(v[url_index], s3_client, check_if_exists=True)
update_sheet(wks, i, status, cdn_url)
cdn_url, status = download_vid(
v[url_index], s3_client, check_if_exists=True)
update_sheet(wks, i, status, cdn_url, columns)
except:
# if any unexpected errors occured, log these into the Google Sheet
t, value, traceback = sys.exc_info()
update_sheet(wks, i, str(value), None)
if __name__ == "__main__":
main()