diff --git a/auto_archive.py b/auto_archive.py index 01390c2..ed5c8cd 100644 --- a/auto_archive.py +++ b/auto_archive.py @@ -139,7 +139,7 @@ def download_telegram_video(url, s3_client, check_if_exists=False): if status != 'already archived': cdn_url = 'https://{}.{}.cdn.digitaloceanspaces.com/{}'.format( - os.getenv('DO_BUCKET'), os.getenv('DO_SPACES_REGION'), key) + os.getenv('DO_BUCKET'), os.getenv('DO_SPACES_REGION'), key) with open(filename, 'rb') as f: s3_client.upload_fileobj(f, Bucket=os.getenv( @@ -225,7 +225,7 @@ def download_vid(url, s3_client, check_if_exists=False): 'thumbnail_index': thumb_index, 'duration': info['duration'] if 'duration' in info else None, 'title': info['title'] if 'title' in info else None, - 'timestamp': info['timestamp'] if 'timestamp' in info else datetime.datetime.strptime(info['upload_date'], '%Y%m%d').timestamp() if 'upload_date' in info else None, + 'timestamp': info['timestamp'] if 'timestamp' in info else datetime.datetime.strptime(info['upload_date'], '%Y%m%d').timestamp() if 'upload_date' in info else None, } return (video_data, status) @@ -284,10 +284,12 @@ def update_sheet(wks, row, status, video_data, columns, v): wks.batch_update(update, value_input_option='USER_ENTERED') + def record_stream(url, s3_client, wks, i, columns, v): video_data, status = download_vid(url, s3_client) update_sheet(wks, i, status, video_data, columns, v) + def process_sheet(sheet): gc = gspread.service_account() sh = gc.open(sheet) @@ -345,35 +347,55 @@ def process_sheet(sheet): v = values[i-1] if v[url_index] != "" and v[col_to_index(columns['status'])] == "": - try: - if 'http://t.me/' in v[url_index] or 'https://t.me/' in v[url_index]: - video_data, status = download_telegram_video(v[url_index], s3_client, check_if_exists=True) - update_sheet(wks, i, status, video_data, columns, v) - else: + + if 'http://t.me/' in v[url_index] or 'https://t.me/' in v[url_index]: + video_data, status = download_telegram_video( + v[url_index], s3_client, check_if_exists=True) + update_sheet(wks, i, status, video_data, columns, v) + else: + try: ydl_opts = { 'outtmpl': 'tmp/%(id)s.%(ext)s', 'quiet': False} ydl = youtube_dl.YoutubeDL(ydl_opts) info = ydl.extract_info(v[url_index], download=False) + print(info) if 'is_live' in info and info['is_live']: - wks.update(columns['status'] + str(i), 'Recording stream') - t = threading.Thread(target=record_stream, args=(v[url_index], s3_client, wks, i, columns, v)) + wks.update(columns['status'] + + str(i), 'Recording stream') + t = threading.Thread(target=record_stream, args=( + v[url_index], s3_client, wks, i, columns, v)) t.start() continue elif 'is_live' not in info or not info['is_live']: - latest_val = wks.acell(columns['status'] + str(i)).value + latest_val = wks.acell( + columns['status'] + str(i)).value # check so we don't step on each others' toes if latest_val == '' or latest_val is None: - wks.update(columns['status'] + str(i), 'Archive in progress') + wks.update( + columns['status'] + str(i), 'Archive in progress') video_data, status = download_vid( v[url_index], s3_client, check_if_exists=True) - update_sheet(wks, i, status, video_data, columns, v) + update_sheet(wks, i, status, + video_data, columns, v) + except: + # i'm sure there's a better way to handle this than nested try/catch blocks + try: + r = requests.get( + 'https://web.archive.org/save/' + v[url_index]) + + parsed = BeautifulSoup(r.content, 'html.parser') + title = parsed.find_all('title')[0].text + + update_sheet(wks, i, str('Internet Archive fallback'), { + 'cdn_url': r.url, 'title': title}, columns, v) + except: + # if any unexpected errors occured, log these into the Google Sheet + t, value, traceback = sys.exc_info() + + update_sheet(wks, i, str(value), {}, columns, v) - except: - # if any unexpected errors occured, log these into the Google Sheet - t, value, traceback = sys.exc_info() - update_sheet(wks, i, str(value), {}, columns, v) def main(): parser = argparse.ArgumentParser( @@ -385,5 +407,6 @@ def main(): process_sheet(args.sheet) + if __name__ == "__main__": main() diff --git a/docs/after.png b/docs/after.png deleted file mode 100644 index 1c52b11..0000000 Binary files a/docs/after.png and /dev/null differ diff --git a/docs/before.png b/docs/before.png deleted file mode 100644 index 5fe4f09..0000000 Binary files a/docs/before.png and /dev/null differ