diff --git a/.gitignore b/.gitignore index 1a8432c..8156f6a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ tmp/ .env .DS_Store +expmt/ diff --git a/auto-archive.py b/auto-archive.py index 4057eab..026447e 100644 --- a/auto-archive.py +++ b/auto-archive.py @@ -8,9 +8,11 @@ import os from dotenv import load_dotenv from botocore.errorfactory import ClientError import argparse +import math load_dotenv() + def col_to_index(col): col = list(col) ndigits = len(col) @@ -25,6 +27,17 @@ def col_to_index(col): return v - 1 + +def index_to_col(index): + alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' + + if index > 25: + t = math.floor(index / 26) + return alphabet[t - 1] + index_to_col(index - t * 26) + else: + return alphabet[index] + + def download_vid(url, s3_client, check_if_exists=False): ydl_opts = {'outtmpl': 'tmp/%(id)s.%(ext)s', 'quiet': False} ydl = youtube_dl.YoutubeDL(ydl_opts) @@ -34,18 +47,20 @@ def download_vid(url, s3_client, check_if_exists=False): if 'entries' in info: if len(info['entries']) > 1: - raise Exception('ERROR: Cannot archive channels or pages with multiple videos') + raise Exception( + 'ERROR: Cannot archive channels or pages with multiple videos') filename = ydl.prepare_filename(info['entries'][0]) else: filename = ydl.prepare_filename(info) - + key = filename.split('/')[1] - cdn_url = 'https://{}.{}.cdn.digitaloceanspaces.com/{}'.format(os.getenv('DO_BUCKET'), os.getenv('DO_SPACES_REGION'), key) + cdn_url = 'https://{}.{}.cdn.digitaloceanspaces.com/{}'.format( + os.getenv('DO_BUCKET'), os.getenv('DO_SPACES_REGION'), key) try: s3_client.head_object(Bucket=os.getenv('DO_BUCKET'), Key=key) - + # file exists return (cdn_url, 'already archived') @@ -57,7 +72,8 @@ def download_vid(url, s3_client, check_if_exists=False): if 'entries' in info: if len(info['entries']) > 1: - raise Exception('ERROR: Cannot archive channels or pages with multiple videos') + raise Exception( + 'ERROR: Cannot archive channels or pages with multiple videos') filename = ydl.prepare_filename(info['entries'][0]) else: @@ -67,42 +83,54 @@ def download_vid(url, s3_client, check_if_exists=False): filename = filename.split('.')[0] + '.mkv' key = filename.split('/')[1] - cdn_url = 'https://{}.{}.cdn.digitaloceanspaces.com/{}'.format(os.getenv('DO_BUCKET'), os.getenv('DO_SPACES_REGION'), key) + cdn_url = 'https://{}.{}.cdn.digitaloceanspaces.com/{}'.format( + os.getenv('DO_BUCKET'), os.getenv('DO_SPACES_REGION'), key) with open(filename, 'rb') as f: - s3_client.upload_fileobj(f, Bucket=os.getenv('DO_BUCKET'), Key=key, ExtraArgs={'ACL': 'public-read'}) + s3_client.upload_fileobj(f, Bucket=os.getenv( + 'DO_BUCKET'), Key=key, ExtraArgs={'ACL': 'public-read'}) os.remove(filename) return (cdn_url, 'success') -def main(): - parser = argparse.ArgumentParser(description="Automatically use youtube-dl to download media from a Google Sheet") - parser.add_argument("--sheet", action="store", dest="sheet") - parser.add_argument('--streaming', dest='streaming', action='store_true') - parser.add_argument('--all-worksheets', dest='all_worksheets', action='store_true') - parser.add_argument('--url-col', dest='url', action='store') - parser.add_argument('--archive-col', dest='archive', action='store') - parser.add_argument('--date-col', dest='date', action='store') - parser.add_argument('--status-col', dest='status', action='store') - args = parser.parse_args() - def update_sheet(wks, row, status, url): - update = [{ - 'range': args.status + str(row), - 'values': [[status]] - }, { - 'range': args.date + str(row), - 'values': [[datetime.datetime.now().isoformat()]] - }, { +def update_sheet(wks, row, status, url, columns): + update = [] + + if url is not None and columns['archive'] is not None: + update += [{ 'range': args.archive + str(row), 'values': [[url]] }] - if url is None: - update = update[:-1] + if columns['status'] is not None: + update += [{ + 'range': args.status + str(row), + 'values': [[status]] + }] - wks.batch_update(update) + if columns['date'] is not None: + update += [{ + 'range': args.date + str(row), + 'values': [[datetime.datetime.now().isoformat()]] + }] + + wks.batch_update(update) + + +def main(): + parser = argparse.ArgumentParser( + description="Automatically use youtube-dl to download media from a Google Sheet") + parser.add_argument("--sheet", action="store", dest="sheet") + parser.add_argument('--streaming', dest='streaming', action='store_true') + parser.add_argument('--all-worksheets', + dest='all_worksheets', action='store_true') + # parser.add_argument('--url-col', dest='url', action='store') + # parser.add_argument('--archive-col', dest='archive', action='store') + # parser.add_argument('--date-col', dest='date', action='store') + # parser.add_argument('--status-col', dest='status', action='store') + args = parser.parse_args() print("Opening document " + args.sheet) @@ -111,10 +139,11 @@ def main(): n_worksheets = len(sh.worksheets()) if args.all_worksheets else 1 s3_client = boto3.client('s3', - region_name=os.getenv('DO_SPACES_REGION'), - endpoint_url='https://{}.digitaloceanspaces.com'.format(os.getenv('DO_SPACES_REGION')), - aws_access_key_id=os.getenv('DO_SPACES_KEY'), - aws_secret_access_key=os.getenv('DO_SPACES_SECRET')) + region_name=os.getenv('DO_SPACES_REGION'), + endpoint_url='https://{}.digitaloceanspaces.com'.format( + os.getenv('DO_SPACES_REGION')), + aws_access_key_id=os.getenv('DO_SPACES_KEY'), + aws_secret_access_key=os.getenv('DO_SPACES_SECRET')) # loop through worksheets to check for ii in range(n_worksheets): @@ -122,6 +151,18 @@ def main(): wks = sh.get_worksheet(ii) values = wks.get_all_values() + headers = values[0] + columns = {} + + columns['url'] = index_to_col(headers.index( + 'Media URL')) if 'Media URL' in headers else None + columns['archive'] = index_to_col(headers.index( + 'Archive location')) if 'Archive location' in headers else None + columns['date'] = index_to_col(headers.index( + 'Archive date')) if 'Archive date' in headers else None + columns['status'] = index_to_col(headers.index( + 'Archive status')) if 'Archive status' in headers else None + # loop through rows in worksheet for i in range(2, len(values)+1): v = values[i-1] @@ -132,23 +173,26 @@ def main(): print(v[col_to_index(args.url)]) try: - ydl_opts = {'outtmpl': 'tmp/%(id)s.%(ext)s', 'quiet': False} + ydl_opts = { + 'outtmpl': 'tmp/%(id)s.%(ext)s', 'quiet': False} ydl = youtube_dl.YoutubeDL(ydl_opts) info = ydl.extract_info(v[url_index], download=False) if args.streaming and 'is_live' in info and info['is_live']: wks.update(args.status + str(i), 'Recording stream') cdn_url, status = download_vid(v[url_index], s3_client) - update_sheet(wks, i, status, cdn_url) + update_sheet(wks, i, status, cdn_url, columns) sys.exit() elif not args.streaming and ('is_live' not in info or not info['is_live']): - cdn_url, status = download_vid(v[url_index], s3_client, check_if_exists=True) - update_sheet(wks, i, status, cdn_url) - + cdn_url, status = download_vid( + v[url_index], s3_client, check_if_exists=True) + update_sheet(wks, i, status, cdn_url, columns) + except: # if any unexpected errors occured, log these into the Google Sheet t, value, traceback = sys.exc_info() update_sheet(wks, i, str(value), None) + if __name__ == "__main__": main()