bellingcat/auto-archiver
1
0
Fork 0
mirror of https://github.com/bellingcat/auto-archiver.git synced 2026-06-12 21:28:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

Synchronize with remote version

Browse Source
This commit is contained in:
Logan Williams
2021-02-08 11:21:42 +00:00
parent e55a4d55dc
commit 853e018ace
2 changed files with 156 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

106
update.py
View File

@@ -13,6 +13,7 @@ load_dotenv()
gc = gspread.service_account()
sh = gc.open("Media Sheet (January 16-20 + People)")
# loop through worksheets to check
for ii in range(5):
wks = sh.get_worksheet(ii)
values = wks.get_all_values()
@@ -26,6 +27,7 @@ for ii in range(5):
aws_access_key_id=os.getenv('DO_SPACES_KEY'),
aws_secret_access_key=os.getenv('DO_SPACES_SECRET'))
# loop through rows in worksheet
for i in range(2, len(values)+1):
v = values[i-1]
@@ -35,68 +37,70 @@ for ii in range(5):
try:
info = ydl.extract_info(v[1], download=False)
if 'entries' in info:
if len(info['entries']) > 1:
raise Exception('ERROR: Cannot archive channels or pages with multiple videos')
filename = ydl.prepare_filename(info['entries'][0])
else:
filename = ydl.prepare_filename(info)
print(filename)
key = filename.split('/')[1]
cdn_url = 'https://{}.{}.cdn.digitaloceanspaces.com/{}'.format(os.getenv('DO_BUCKET'), os.getenv('DO_SPACES_REGION'), key)
try:
s3_client.head_object(Bucket=os.getenv('DO_BUCKET'), Key=key)
# file exists
update = [{
'range': 'K' + str(i),
'values': [['already archived']]
}, {
'range': 'M' + str(i),
'values': [[cdn_url]]
}]
wks.batch_update(update)
except ClientError:
# Not found
# sometimes this results in a different filename, so do this again
info = ydl.extract_info(v[1], download=True)
# skip if live
if 'is_live' not in info or not info['is_live']:
if 'entries' in info:
if len(info['entries']) > 1:
raise Exception('ERROR: Cannot archive channels or pages with multiple videos')
filename = ydl.prepare_filename(info['entries'][0])
else:
filename = ydl.prepare_filename(info)
if not os.path.exists(filename):
filename = filename.split('.')[0] + '.mkv'
print(filename)
key = filename.split('/')[1]
cdn_url = 'https://{}.{}.cdn.digitaloceanspaces.com/{}'.format(os.getenv('DO_BUCKET'), os.getenv('DO_SPACES_REGION'), key)
with open(filename, 'rb') as f:
s3_client.upload_fileobj(f, Bucket=os.getenv('DO_BUCKET'), Key=key, ExtraArgs={'ACL': 'public-read'})
try:
s3_client.head_object(Bucket=os.getenv('DO_BUCKET'), Key=key)
os.remove(filename)
# file exists
update = [{
'range': 'K' + str(i),
'values': [['successful']]
}, {
'range': 'L' + str(i),
'values': [[datetime.datetime.now().isoformat()]]
}, {
'range': 'M' + str(i),
'values': [[cdn_url]]
}]
update = [{
'range': 'K' + str(i),
'values': [['already archived']]
}, {
'range': 'M' + str(i),
'values': [[cdn_url]]
}]
wks.batch_update(update)
wks.batch_update(update)
except ClientError:
# Not found
# sometimes this results in a different filename, so do this again
info = ydl.extract_info(v[1], download=True)
if 'entries' in info:
filename = ydl.prepare_filename(info['entries'][0])
else:
filename = ydl.prepare_filename(info)
if not os.path.exists(filename):
filename = filename.split('.')[0] + '.mkv'
print(filename)
key = filename.split('/')[1]
cdn_url = 'https://{}.{}.cdn.digitaloceanspaces.com/{}'.format(os.getenv('DO_BUCKET'), os.getenv('DO_SPACES_REGION'), key)
with open(filename, 'rb') as f:
s3_client.upload_fileobj(f, Bucket=os.getenv('DO_BUCKET'), Key=key, ExtraArgs={'ACL': 'public-read'})
os.remove(filename)
update = [{
'range': 'K' + str(i),
'values': [['successful']]
}, {
'range': 'L' + str(i),
'values': [[datetime.datetime.now().isoformat()]]
}, {
'range': 'M' + str(i),
'values': [[cdn_url]]
}]
wks.batch_update(update)
except:
t, value, traceback = sys.exc_info()