From 9070689d95f5be438076e4d5b52a915a19f75a2d Mon Sep 17 00:00:00 2001 From: Logan Williams Date: Thu, 18 Mar 2021 11:03:13 +0100 Subject: [PATCH] Thumbnail and metadata extraction --- Pipfile | 1 + Pipfile.lock | 65 ++++++++++++-------- auto-archive.py | 156 +++++++++++++++++++++++++++++++++++++++--------- 3 files changed, 170 insertions(+), 52 deletions(-) diff --git a/Pipfile b/Pipfile index e07801e..561857b 100644 --- a/Pipfile +++ b/Pipfile @@ -9,6 +9,7 @@ boto3 = "*" python-dotenv = "*" youtube_dl = "*" argparse = "*" +ffmpeg-python = "*" [dev-packages] diff --git a/Pipfile.lock b/Pipfile.lock index f84b429..b28382a 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "f8d5bda536d37e5fca14f05a2b6102b002ee5846e84298175ba319d20bd41d60" + "sha256": "a5308ff7514ddcff08e39ae06fe8f96e63ea3eecfbf4c106c907c8aa6b76b8dd" }, "pipfile-spec": 6, "requires": { @@ -26,19 +26,19 @@ }, "boto3": { "hashes": [ - "sha256:65514427f5f849245c9a272fa06a5a014ae3945333f4f407489d034fb99dc61f", - "sha256:af87efaa772f95de67f72ed91aed2feef63593b5290696f669799202bc484b99" + "sha256:d39c04b51e60197f5503f8489f043bc904981567cc8431d389367767dc3fd5ae", + "sha256:fe1898c5b10035528207995c9931b78f2f50bb70cf93bac353152aea47c04780" ], "index": "pypi", - "version": "==1.17.4" + "version": "==1.17.30" }, "botocore": { "hashes": [ - "sha256:61657a1e4b3cdda9627084184bdf9dca4637c1523daead31a36974be0d51686d", - "sha256:96f9e0920ac91b6caae3039e5de09b80648ad57b4a97fc7d81a369afae34fb10" + "sha256:63951595a736dfc9759f57e33bec6eaea4f09c4800626ef5309437060b263e48", + "sha256:98ff1eb210d394a1ffe736b33c8a7be68f30f0a03550b559c5bb6fdf0c29328d" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", - "version": "==1.20.4" + "version": "==1.20.30" }, "cachetools": { "hashes": [ @@ -63,29 +63,44 @@ "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", "version": "==4.0.0" }, + "ffmpeg-python": { + "hashes": [ + "sha256:65225db34627c578ef0e11c8b1eb528bb35e024752f6f10b78c011f6f64c4127", + "sha256:ac441a0404e053f8b6a1113a77c0f452f1cfc62f6344a769475ffdc0f56c23c5" + ], + "index": "pypi", + "version": "==0.2.0" + }, + "future": { + "hashes": [ + "sha256:b1bead90b70cf6ec3f0710ae53a525360fa360d306a86583adc6bf83a4db537d" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.18.2" + }, "google-auth": { "hashes": [ - "sha256:008e23ed080674f69f9d2d7d80db4c2591b9bb307d136cea7b3bc129771d211d", - "sha256:514e39f4190ca972200ba33876da5a8857c5665f2b4ccc36c8b8ee21228aae80" + "sha256:9bd436d19ab047001a1340720d2b629eb96dd503258c524921ec2af3ee88a80e", + "sha256:dcaba3aa9d4e0e96fd945bf25a86b6f878fcb05770b67adbeb50a63ca4d28a5e" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", - "version": "==1.25.0" + "version": "==1.28.0" }, "google-auth-oauthlib": { "hashes": [ - "sha256:65b65bc39ad8cab15039b35e5898455d3d66296d0584d96fe0e79d67d04c51d9", - "sha256:d4d98c831ea21d574699978827490a41b94f05d565c617fe1b420e88f1fc8d8d" + "sha256:54431535309cfab50897d9c181e8c2226268825aa6e42e930b05b99c5041a18c", + "sha256:dabffbf594a6be2fd6d054060846d1201569252efb10dfb749b504a7591f8af0" ], "markers": "python_version >= '3.6'", - "version": "==0.4.2" + "version": "==0.4.3" }, "gspread": { "hashes": [ - "sha256:273da28275eb8dc664b1ca944e59255949d75ac3cac62d65797003dbb419a2cd", - "sha256:e04f1a6267b3929fc1600424c5ec83906d439672cafdd61a9d5b916a139f841c" + "sha256:056ceb9fb4f439c15ec39d84c91653c6435f775a1c8afc8fe7f909f8393821fb", + "sha256:4bda4ab8c5edb9e41cf4ae40d4d5fb30447522b4e43608e05c01351ab1b96912" ], "index": "pypi", - "version": "==3.6.0" + "version": "==3.7.0" }, "idna": { "hashes": [ @@ -181,11 +196,11 @@ }, "rsa": { "hashes": [ - "sha256:69805d6b69f56eb05b62daea3a7dbd7aa44324ad1306445e05da8060232d00f4", - "sha256:a8774e55b59fd9fc893b0d05e9bfc6f47081f46ff5b46f39ccf24631b7be356b" + "sha256:78f9a9bf4e7be0c5ded4583326e7461e3a3c5aae24073648b4bdfa797d78c9d2", + "sha256:9d689e6ca1b3038bc82bf8d23e944b6b6037bc02301a574935b2dd946e0353b9" ], "markers": "python_version >= '3.6'", - "version": "==4.7" + "version": "==4.7.2" }, "s3transfer": { "hashes": [ @@ -204,19 +219,19 @@ }, "urllib3": { "hashes": [ - "sha256:1b465e494e3e0d8939b50680403e3aedaa2bc434b7d5af64dfd3c958d7f5ae80", - "sha256:de3eedaad74a2683334e282005cd8d7f22f4d55fa690a2a1020a416cb0a47e73" + "sha256:2f4da4594db7e1e110a944bb1b551fdf4e6c136ad42e4234131391e21eb5b0df", + "sha256:e7b021f7241115872f92f43c6508082facffbd1c048e3c6e2bb9c2a157e28937" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", - "version": "==1.26.3" + "version": "==1.26.4" }, "youtube-dl": { "hashes": [ - "sha256:831a29b2d34493ef8181ff288f403135bb4b00df1cd201eb8cbe80b5b5425760", - "sha256:b337f20563094decc6b3c16e6fcad14ec3df9d99519344d6e95e40878b2c8075" + "sha256:c287ad8dd33471aabaabab5ab1dd825bebc70eb8b83ebfa93fd71022e01a1d08", + "sha256:d414166efe52447877db06803816277f52f405faeee2bdf5ef816b30e352b3b8" ], "index": "pypi", - "version": "==2021.2.4.1" + "version": "==2021.3.14" } }, "develop": {} diff --git a/auto-archive.py b/auto-archive.py index c75328f..afb80b1 100644 --- a/auto-archive.py +++ b/auto-archive.py @@ -9,10 +9,10 @@ from dotenv import load_dotenv from botocore.errorfactory import ClientError import argparse import math +import ffmpeg load_dotenv() - def col_to_index(col): col = list(col) ndigits = len(col) @@ -27,10 +27,9 @@ def col_to_index(col): return v - 1 - def index_to_col(index): alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' - + if index > 25: t = index dig = 0 @@ -41,10 +40,61 @@ def index_to_col(index): else: return alphabet[index] +def get_thumbnails(filename, s3_client): + if not os.path.exists(filename.split('.')[0]): + os.mkdir(filename.split('.')[0]) + + stream = ffmpeg.input(filename) + stream = ffmpeg.filter(stream, 'fps', fps=0.5).filter('scale', 512, -1) + stream.output(filename.split('.')[0] + '/out%d.jpg').run() + + thumbnails = os.listdir(filename.split('.')[0] + '/') + cdn_urls = [] + + for fname in thumbnails: + thumbnail_filename = filename.split('.')[0] + '/' + fname + key = filename.split('/')[1].split('.')[0] + '/' + fname + + cdn_url = 'https://{}.{}.cdn.digitaloceanspaces.com/{}'.format( + os.getenv('DO_BUCKET'), os.getenv('DO_SPACES_REGION'), key) + + with open(thumbnail_filename, 'rb') as f: + s3_client.upload_fileobj(f, Bucket=os.getenv( + 'DO_BUCKET'), Key=key, ExtraArgs={'ACL': 'public-read'}) + + cdn_urls.append(cdn_url) + os.remove(thumbnail_filename) + + key_thumb = cdn_urls[int(len(cdn_urls)*0.25)] + + index_page = f'''{filename} + ''' + + for t in cdn_urls: + index_page += f'' + + index_page += f"" + index_fname = filename.split('.')[0] + '/index.html' + + with open(index_fname, 'w') as f: + f.write(index_page) + + thumb_index = filename.split('/')[1].split('.')[0] + '/index.html' + + s3_client.upload_fileobj(open(index_fname, 'rb'), Bucket=os.getenv( + 'DO_BUCKET'), Key=thumb_index, ExtraArgs={'ACL': 'public-read', 'ContentType': 'text/html'}) + + thumb_index_cdn_url = 'https://{}.{}.cdn.digitaloceanspaces.com/{}'.format( + os.getenv('DO_BUCKET'), os.getenv('DO_SPACES_REGION'), thumb_index) + + return (key_thumb, thumb_index_cdn_url) + def download_vid(url, s3_client, check_if_exists=False): ydl_opts = {'outtmpl': 'tmp/%(id)s.%(ext)s', 'quiet': False} ydl = youtube_dl.YoutubeDL(ydl_opts) + cdn_url = None + status = 'success' if check_if_exists: info = ydl.extract_info(url, download=False) @@ -59,14 +109,15 @@ def download_vid(url, s3_client, check_if_exists=False): filename = ydl.prepare_filename(info) key = filename.split('/')[1] - cdn_url = 'https://{}.{}.cdn.digitaloceanspaces.com/{}'.format( - os.getenv('DO_BUCKET'), os.getenv('DO_SPACES_REGION'), key) try: s3_client.head_object(Bucket=os.getenv('DO_BUCKET'), Key=key) # file exists - return (cdn_url, 'already archived') + cdn_url = 'https://{}.{}.cdn.digitaloceanspaces.com/{}'.format( + os.getenv('DO_BUCKET'), os.getenv('DO_SPACES_REGION'), key) + + status = 'already archived' except ClientError: pass @@ -86,41 +137,82 @@ def download_vid(url, s3_client, check_if_exists=False): if not os.path.exists(filename): filename = filename.split('.')[0] + '.mkv' - key = filename.split('/')[1] - cdn_url = 'https://{}.{}.cdn.digitaloceanspaces.com/{}'.format( - os.getenv('DO_BUCKET'), os.getenv('DO_SPACES_REGION'), key) + if status != 'already archived': + key = filename.split('/')[1] + cdn_url = 'https://{}.{}.cdn.digitaloceanspaces.com/{}'.format( + os.getenv('DO_BUCKET'), os.getenv('DO_SPACES_REGION'), key) - with open(filename, 'rb') as f: - s3_client.upload_fileobj(f, Bucket=os.getenv( - 'DO_BUCKET'), Key=key, ExtraArgs={'ACL': 'public-read'}) + with open(filename, 'rb') as f: + s3_client.upload_fileobj(f, Bucket=os.getenv( + 'DO_BUCKET'), Key=key, ExtraArgs={'ACL': 'public-read'}) + key_thumb, thumb_index = get_thumbnails(filename, s3_client) os.remove(filename) - return (cdn_url, 'success') + video_data = { + 'cdn_url': cdn_url, + 'thumbnail': key_thumb, + 'thumbnail_index': thumb_index, + 'duration': info['duration'] if 'duration' in info else None, + 'title': info['title'] if 'title' in info else None, + 'timestamp': info['timestamp'] if 'timestamp' in info else datetime.datetime.strptime(info['upload_date'], '%Y%m%d').timestamp() if 'upload_date' in info else None, + } + + return (video_data, status) -def update_sheet(wks, row, status, url, columns): +def update_sheet(wks, row, status, video_data, columns, v): update = [] - if url is not None and columns['archive'] is not None: - update += [{ - 'range': columns['archive'] + str(row), - 'values': [[url]] - }] - if columns['status'] is not None: update += [{ 'range': columns['status'] + str(row), 'values': [[status]] }] - if columns['date'] is not None: + if 'cdn_url' in video_data and video_data['cdn_url'] is not None and columns['archive'] is not None and v[col_to_index(columns['archive'])] == '': + update += [{ + 'range': columns['archive'] + str(row), + 'values': [[video_data['cdn_url']]] + }] + + if 'date' in video_data and columns['date'] is not None and v[col_to_index(columns['date'])] == '': update += [{ 'range': columns['date'] + str(row), 'values': [[datetime.datetime.now().isoformat()]] }] - wks.batch_update(update) + if 'thumbnail' in video_data and columns['thumbnail'] is not None and v[col_to_index(columns['thumbnail'])] == '': + update += [{ + 'range': columns['thumbnail'] + str(row), + 'values': [['=IMAGE("' + video_data['thumbnail'] + '")']] + }] + + if 'thumbnail_index' in video_data and columns['thumbnail_index'] is not None and v[col_to_index(columns['thumbnail_index'])] == '': + update += [{ + 'range': columns['thumbnail_index'] + str(row), + 'values': [[video_data['thumbnail_index']]] + }] + + if 'timestamp' in video_data and columns['timestamp'] is not None and video_data['timestamp'] is not None and v[col_to_index(columns['timestamp'])] == '': + update += [{ + 'range': columns['timestamp'] + str(row), + 'values': [[datetime.datetime.fromtimestamp(video_data['timestamp']).isoformat()]] + }] + + if 'title' in video_data and columns['title'] is not None and video_data['title'] is not None and v[col_to_index(columns['title'])] == '': + update += [{ + 'range': columns['title'] + str(row), + 'values': [[video_data['title']]] + }] + + if 'duration' in video_data and columns['duration'] is not None and video_data['duration'] is not None and v[col_to_index(columns['duration'])] == '': + update += [{ + 'range': columns['duration'] + str(row), + 'values': [[str(video_data['duration'])]] + }] + + wks.batch_update(update, value_input_option='USER_ENTERED') def main(): @@ -164,6 +256,16 @@ def main(): 'Archive date')) if 'Archive date' in headers else None columns['status'] = index_to_col(headers.index( 'Archive status')) if 'Archive status' in headers else None + columns['thumbnail'] = index_to_col(headers.index( + 'Thumbnail')) if 'Thumbnail' in headers else None + columns['thumbnail_index'] = index_to_col(headers.index( + 'Thumbnail index')) if 'Thumbnail index' in headers else None + columns['timestamp'] = index_to_col(headers.index( + 'Upload timestamp')) if 'Upload timestamp' in headers else None + columns['title'] = index_to_col(headers.index( + 'Upload title')) if 'Upload title' in headers else None + columns['duration'] = index_to_col(headers.index( + 'Duration')) if 'Duration' in headers else None if columns['url'] is None: print("No 'Media URL' column found, skipping") @@ -182,18 +284,18 @@ def main(): if args.streaming and 'is_live' in info and info['is_live']: wks.update(columns['status'] + str(i), 'Recording stream') - cdn_url, status = download_vid(v[url_index], s3_client) - update_sheet(wks, i, status, cdn_url, columns) + video_data, status = download_vid(v[url_index], s3_client) + update_sheet(wks, i, status, video_data, columns, v) sys.exit() elif not args.streaming and ('is_live' not in info or not info['is_live']): - cdn_url, status = download_vid( + video_data, status = download_vid( v[url_index], s3_client, check_if_exists=True) - update_sheet(wks, i, status, cdn_url, columns) + update_sheet(wks, i, status, video_data, columns, v) except: # if any unexpected errors occured, log these into the Google Sheet t, value, traceback = sys.exc_info() - update_sheet(wks, i, str(value), None) + update_sheet(wks, i, str(value), {}, columns, v) if __name__ == "__main__":