From 39ec190e56733b6951c66f795906fab820fcbe66 Mon Sep 17 00:00:00 2001 From: msramalho <19508417+msramalho@users.noreply.github.com> Date: Wed, 9 Mar 2022 11:44:05 +0100 Subject: [PATCH 01/16] adds README instructions for geckodriver --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 7910e30..86991f2 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,8 @@ If you are using `pipenv` (recommended), `pipenv install` is sufficient to insta [ffmpeg](https://www.ffmpeg.org/) must also be installed locally for this tool to work. +[firefox](https://www.mozilla.org/en-US/firefox/new/) and [geckodriver](https://github.com/mozilla/geckodriver/releases) on a path folder like `/usr/local/bin`. + A `.env` file is required for saving content to a Digital Ocean space, and for archiving pages to the Internet Archive. This file should also be in the script directory, and should contain the following variables: ``` From 59027ac4772c518c0c4ddcb65f093607cee44c34 Mon Sep 17 00:00:00 2001 From: msramalho <19508417+msramalho@users.noreply.github.com> Date: Wed, 9 Mar 2022 11:44:19 +0100 Subject: [PATCH 02/16] simplification --- storages/s3_storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storages/s3_storage.py b/storages/s3_storage.py index 188db7e..5d8bf18 100644 --- a/storages/s3_storage.py +++ b/storages/s3_storage.py @@ -45,5 +45,5 @@ class S3Storage(Storage): return False def uploadf(self, file, key, **kwargs): - extra_args = kwargs["extra_args"] if "extra_args" in kwargs else {'ACL': 'public-read'} + extra_args = kwargs.get("extra_args", {'ACL': 'public-read'}) self.s3.upload_fileobj(file, Bucket=self.bucket, Key=self._get_path(key), ExtraArgs=extra_args) From 544e7578a63f80d952da2a4e13927fa2fc8e96a0 Mon Sep 17 00:00:00 2001 From: msramalho <19508417+msramalho@users.noreply.github.com> Date: Wed, 9 Mar 2022 11:46:14 +0100 Subject: [PATCH 03/16] removes duplicate code --- auto_archive.py | 54 +++++++++++++++++++++++-------------------------- 1 file changed, 25 insertions(+), 29 deletions(-) diff --git a/auto_archive.py b/auto_archive.py index d54d2f1..b5dae88 100644 --- a/auto_archive.py +++ b/auto_archive.py @@ -111,40 +111,36 @@ def process_sheet(sheet, header=1): url = gw.get_cell(row_values, 'url') status = gw.get_cell(row_values, 'status') if url != '' and status in ['', None]: - url = gw.get_cell(row, 'url') - status = gw.get_cell(status, 'status') + gw.set_cell(row, 'status', 'Archive in progress') - if url != '' and status in ['', None]: - gw.set_cell(row, 'status', 'Archive in progress') + url = expand_url(url) - url = expand_url(url) + for archiver in active_archivers: + logger.debug(f'Trying {archiver} on row {row}') - for archiver in active_archivers: - logger.debug(f'Trying {archiver} on row {row}') - - try: - result = archiver.download(url, check_if_exists=True) - except Exception as e: - result = False - logger.error( - f'Got unexpected error in row {row} with archiver {archiver} for url {url}: {e}') - - if result: - if result.status in ['success', 'already archived']: - result.status = archiver.name + \ - ": " + str(result.status) - logger.success( - f'{archiver} succeeded on row {row}') - break - logger.warning( - f'{archiver} did not succeed on row {row}, final status: {result.status}') - result.status = archiver.name + \ - ": " + str(result.status) + try: + result = archiver.download(url, check_if_exists=True) + except Exception as e: + result = False + logger.error( + f'Got unexpected error in row {row} with archiver {archiver} for url {url}: {e}') if result: - update_sheet(gw, row, result) - else: - gw.set_cell(row, 'status', 'failed: no archiver') + if result.status in ['success', 'already archived']: + result.status = archiver.name + \ + ": " + str(result.status) + logger.success( + f'{archiver} succeeded on row {row}') + break + logger.warning( + f'{archiver} did not succeed on row {row}, final status: {result.status}') + result.status = archiver.name + \ + ": " + str(result.status) + + if result: + update_sheet(gw, row, result) + else: + gw.set_cell(row, 'status', 'failed: no archiver') driver.quit() From ff874fe0d3f16bcbcd8fe4f62b79958882cdce04 Mon Sep 17 00:00:00 2001 From: msramalho <19508417+msramalho@users.noreply.github.com> Date: Wed, 9 Mar 2022 12:17:51 +0100 Subject: [PATCH 04/16] simplifies access to google sheets, single get_values --- auto_archive.py | 10 ++++------ utils/gworksheet.py | 9 +++++---- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/auto_archive.py b/auto_archive.py index b5dae88..2891ea0 100644 --- a/auto_archive.py +++ b/auto_archive.py @@ -7,6 +7,7 @@ import gspread from loguru import logger from dotenv import load_dotenv from selenium import webdriver +import traceback import archivers from storages import S3Storage, S3Config @@ -104,12 +105,10 @@ def process_sheet(sheet, header=1): archivers.WaybackArchiver(s3_client, driver) ] - values = gw.get_values() # loop through rows in worksheet for row in range(1 + header, gw.count_rows() + 1): - row_values = values[row-1] - url = gw.get_cell(row_values, 'url') - status = gw.get_cell(row_values, 'status') + url = gw.get_cell(row, 'url') + status = gw.get_cell(row, 'status') if url != '' and status in ['', None]: gw.set_cell(row, 'status', 'Archive in progress') @@ -122,8 +121,7 @@ def process_sheet(sheet, header=1): result = archiver.download(url, check_if_exists=True) except Exception as e: result = False - logger.error( - f'Got unexpected error in row {row} with archiver {archiver} for url {url}: {e}') + logger.error(f'Got unexpected error in row {row} with archiver {archiver} for url {url}: {e}\n{traceback.format_exc()}') if result: if result.status in ['success', 'already archived']: diff --git a/utils/gworksheet.py b/utils/gworksheet.py index f7f0549..e10df10 100644 --- a/utils/gworksheet.py +++ b/utils/gworksheet.py @@ -18,7 +18,8 @@ class GWorksheet: def __init__(self, worksheet, columns=COLUMN_NAMES, header_row=1): self.wks = worksheet - self.headers = [v.lower() for v in self.wks.row_values(header_row)] + self.values = self.wks.get_values() + self.headers = [v.lower() for v in self.values[header_row - 1]] self.columns = columns def _check_col_exists(self, col: str): @@ -34,14 +35,14 @@ class GWorksheet: return self.columns[col] in self.headers def count_rows(self): - return len(self.wks.get_values()) + return len(self.values) def get_row(self, row: int): # row is 1-based - return self.wks.row_values(row) + return self.values[row - 1] def get_values(self): - return self.wks.get_values() + return self.values def get_cell(self, row, col: str): """ From 077c71f941a64d40f47bf0dfc773d4d2c2f493bb Mon Sep 17 00:00:00 2001 From: msramalho <19508417+msramalho@users.noreply.github.com> Date: Wed, 9 Mar 2022 12:18:06 +0100 Subject: [PATCH 05/16] fixes index out fo range bug --- archivers/telegram_archiver.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/archivers/telegram_archiver.py b/archivers/telegram_archiver.py index e5b017f..9e735bb 100644 --- a/archivers/telegram_archiver.py +++ b/archivers/telegram_archiver.py @@ -43,8 +43,10 @@ class TelegramArchiver(Archiver): images += urls page_cdn, page_hash, thumbnail = self.generate_media_page(images, url, html.escape(str(t.content))) + time_elements = s.find_all('time') + timestamp = time_elements[0].get('datetime') if len(time_elements) else None - return ArchiveResult(status="success", cdn_url=page_cdn, screenshot=screenshot, hash=page_hash, thumbnail=thumbnail, timestamp=s.find_all('time')[0].get('datetime')) + return ArchiveResult(status="success", cdn_url=page_cdn, screenshot=screenshot, hash=page_hash, thumbnail=thumbnail, timestamp=timestamp) video_url = video.get('src') video_id = video_url.split('/')[-1].split('?')[0] From 52333874c9e07735cf0d38aa42de92fb13098940 Mon Sep 17 00:00:00 2001 From: msramalho <19508417+msramalho@users.noreply.github.com> Date: Wed, 9 Mar 2022 12:38:04 +0100 Subject: [PATCH 06/16] making column names configurable through the command line --- auto_archive.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/auto_archive.py b/auto_archive.py index 2891ea0..0c0a033 100644 --- a/auto_archive.py +++ b/auto_archive.py @@ -61,7 +61,7 @@ def expand_url(url): return url -def process_sheet(sheet, header=1): +def process_sheet(sheet, header=1, columns=GWorksheet.COLUMN_NAMES): gc = gspread.service_account(filename='service_account.json') sh = gc.open(sheet) @@ -80,16 +80,16 @@ def process_sheet(sheet, header=1): # loop through worksheets to check for ii, wks in enumerate(sh.worksheets()): logger.info(f'Opening worksheet {ii}: "{wks.title}"') - gw = GWorksheet(wks, header_row=header) + gw = GWorksheet(wks, header_row=header, columns=columns) if not gw.col_exists('url'): logger.warning( - f'No "Media URL" column found, skipping worksheet {wks.title}') + f'No "{columns["url"]}" column found, skipping worksheet {wks.title}') continue if not gw.col_exists('status'): logger.warning( - f'No "Archive status" column found, skipping worksheet {wks.title}') + f'No "{columns["status"]}" column found, skipping worksheet {wks.title}') continue # archives will be in a folder 'doc_name/worksheet_name' @@ -139,7 +139,7 @@ def process_sheet(sheet, header=1): update_sheet(gw, row, result) else: gw.set_cell(row, 'status', 'failed: no archiver') - + logger.success(f'Finshed worksheet {wks.title}') driver.quit() @@ -147,13 +147,17 @@ def main(): parser = argparse.ArgumentParser( description='Automatically archive social media videos from a Google Sheets document') parser.add_argument('--sheet', action='store', dest='sheet') - parser.add_argument('--header', action='store', dest='header', default=1, type=int) + parser.add_argument('--header', action='store', dest='header', default=1, type=int, help='1-based index for the header row') + for k, v in GWorksheet.COLUMN_NAMES.items(): + parser.add_argument(f'--col-{k}', action='store', dest=k, default=v, help=f'the name of the column to fill with {k} (defaults={v})') + args = parser.parse_args() + config_columns = {k: getattr(args, k) for k in GWorksheet.COLUMN_NAMES.keys()} logger.info(f'Opening document {args.sheet}') mkdir_if_not_exists('tmp') - process_sheet(args.sheet, header=args.header) + process_sheet(args.sheet, header=args.header, columns=config_columns) shutil.rmtree('tmp') From 6c5d6f521e44c77c7ecb399ab0022e3c897d84ff Mon Sep 17 00:00:00 2001 From: msramalho <19508417+msramalho@users.noreply.github.com> Date: Thu, 10 Mar 2022 19:00:02 +0100 Subject: [PATCH 07/16] implements fresh status retrieval if needed --- auto_archive.py | 5 +++-- utils/gworksheet.py | 8 ++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/auto_archive.py b/auto_archive.py index 0c0a033..e0638ae 100644 --- a/auto_archive.py +++ b/auto_archive.py @@ -108,7 +108,8 @@ def process_sheet(sheet, header=1, columns=GWorksheet.COLUMN_NAMES): # loop through rows in worksheet for row in range(1 + header, gw.count_rows() + 1): url = gw.get_cell(row, 'url') - status = gw.get_cell(row, 'status') + original_status = gw.get_cell(row, 'status') + status = gw.get_cell(row, 'status', fresh=original_status in ['', None]) if url != '' and status in ['', None]: gw.set_cell(row, 'status', 'Archive in progress') @@ -146,7 +147,7 @@ def process_sheet(sheet, header=1, columns=GWorksheet.COLUMN_NAMES): def main(): parser = argparse.ArgumentParser( description='Automatically archive social media videos from a Google Sheets document') - parser.add_argument('--sheet', action='store', dest='sheet') + parser.add_argument('--sheet', action='store', dest='sheet', help='the name of the google sheets document', required=True) parser.add_argument('--header', action='store', dest='header', default=1, type=int, help='1-based index for the header row') for k, v in GWorksheet.COLUMN_NAMES.items(): parser.add_argument(f'--col-{k}', action='store', dest=k, default=v, help=f'the name of the column to fill with {k} (defaults={v})') diff --git a/utils/gworksheet.py b/utils/gworksheet.py index e10df10..cf6535c 100644 --- a/utils/gworksheet.py +++ b/utils/gworksheet.py @@ -44,16 +44,20 @@ class GWorksheet: def get_values(self): return self.values - def get_cell(self, row, col: str): + def get_cell(self, row, col: str, fresh=False): """ returns the cell value from (row, col), where row can be an index (1-based) OR list of values as received from self.get_row(row) + if fresh=True, the sheet is queried again for this cell """ + col_index = self._col_index(col) + + if fresh: + return self.wks.cell(row, col_index + 1).value if type(row) == int: row = self.get_row(row) - col_index = self._col_index(col) if col_index >= len(row): return '' return row[col_index] From 486c3295b5ae9a99903682eb5a9704d0b5ab6c32 Mon Sep 17 00:00:00 2001 From: msramalho <19508417+msramalho@users.noreply.github.com> Date: Sat, 12 Mar 2022 19:54:10 +0100 Subject: [PATCH 08/16] log --- auto_archive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auto_archive.py b/auto_archive.py index e0638ae..5a3969c 100644 --- a/auto_archive.py +++ b/auto_archive.py @@ -155,7 +155,7 @@ def main(): args = parser.parse_args() config_columns = {k: getattr(args, k) for k in GWorksheet.COLUMN_NAMES.keys()} - logger.info(f'Opening document {args.sheet}') + logger.info(f'Opening document {args.sheet} for header {args.header}') mkdir_if_not_exists('tmp') process_sheet(args.sheet, header=args.header, columns=config_columns) From 6e5e7212c273b0266e1f0717bdac80c689853090 Mon Sep 17 00:00:00 2001 From: msramalho <19508417+msramalho@users.noreply.github.com> Date: Sat, 12 Mar 2022 19:56:00 +0100 Subject: [PATCH 09/16] fixes header offset --- utils/gworksheet.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/gworksheet.py b/utils/gworksheet.py index cf6535c..4a2a82f 100644 --- a/utils/gworksheet.py +++ b/utils/gworksheet.py @@ -18,8 +18,8 @@ class GWorksheet: def __init__(self, worksheet, columns=COLUMN_NAMES, header_row=1): self.wks = worksheet - self.values = self.wks.get_values() - self.headers = [v.lower() for v in self.values[header_row - 1]] + self.values = self.wks.get_values()[header_row-1:] + self.headers = [v.lower() for v in self.values[0]] self.columns = columns def _check_col_exists(self, col: str): From 69483d432c13452aeea30f15f013fb84bfe15d07 Mon Sep 17 00:00:00 2001 From: msramalho <19508417+msramalho@users.noreply.github.com> Date: Sat, 12 Mar 2022 20:04:08 +0100 Subject: [PATCH 10/16] adds logs --- auto_archive.py | 2 +- utils/gworksheet.py | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/auto_archive.py b/auto_archive.py index 5a3969c..b800c3e 100644 --- a/auto_archive.py +++ b/auto_archive.py @@ -79,7 +79,7 @@ def process_sheet(sheet, header=1, columns=GWorksheet.COLUMN_NAMES): # loop through worksheets to check for ii, wks in enumerate(sh.worksheets()): - logger.info(f'Opening worksheet {ii}: "{wks.title}"') + logger.info(f'Opening worksheet {ii}: "{wks.title}" header={header}') gw = GWorksheet(wks, header_row=header, columns=columns) if not gw.col_exists('url'): diff --git a/utils/gworksheet.py b/utils/gworksheet.py index 4a2a82f..f095962 100644 --- a/utils/gworksheet.py +++ b/utils/gworksheet.py @@ -18,8 +18,10 @@ class GWorksheet: def __init__(self, worksheet, columns=COLUMN_NAMES, header_row=1): self.wks = worksheet - self.values = self.wks.get_values()[header_row-1:] + self.values = self.wks.get_values()[header_row - 1:] self.headers = [v.lower() for v in self.values[0]] + print(self.headers) + self.row_offset = header_row - 1 self.columns = columns def _check_col_exists(self, col: str): @@ -54,7 +56,7 @@ class GWorksheet: col_index = self._col_index(col) if fresh: - return self.wks.cell(row, col_index + 1).value + return self.wks.cell(row + self.row_offset, col_index + 1).value if type(row) == int: row = self.get_row(row) @@ -65,7 +67,7 @@ class GWorksheet: def set_cell(self, row: int, col: str, val): # row is 1-based col_index = self._col_index(col) + 1 - self.wks.update_cell(row, col_index, val) + self.wks.update_cell(row + self.row_offset, col_index, val) def batch_set_cell(self, cell_updates): """ @@ -82,4 +84,4 @@ class GWorksheet: def to_a1(self, row: int, col: str): # row is 1-based - return utils.rowcol_to_a1(row, self._col_index(col) + 1) + return utils.rowcol_to_a1(row + self.row_offset, self._col_index(col) + 1) From ec4ae844873fc840e20afa851c919b841996525a Mon Sep 17 00:00:00 2001 From: msramalho <19508417+msramalho@users.noreply.github.com> Date: Sat, 12 Mar 2022 20:06:31 +0100 Subject: [PATCH 11/16] case-insensitive is a bad idea --- utils/gworksheet.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/utils/gworksheet.py b/utils/gworksheet.py index f095962..c536f37 100644 --- a/utils/gworksheet.py +++ b/utils/gworksheet.py @@ -19,8 +19,7 @@ class GWorksheet: def __init__(self, worksheet, columns=COLUMN_NAMES, header_row=1): self.wks = worksheet self.values = self.wks.get_values()[header_row - 1:] - self.headers = [v.lower() for v in self.values[0]] - print(self.headers) + self.headers = [v for v in self.values[0]] self.row_offset = header_row - 1 self.columns = columns From 67b16064bb78ab315f3b1ea86a634a3e0b33d0ec Mon Sep 17 00:00:00 2001 From: msramalho <19508417+msramalho@users.noreply.github.com> Date: Sat, 12 Mar 2022 20:11:38 +0100 Subject: [PATCH 12/16] offby1 --- utils/gworksheet.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/gworksheet.py b/utils/gworksheet.py index c536f37..7e0c82a 100644 --- a/utils/gworksheet.py +++ b/utils/gworksheet.py @@ -66,7 +66,7 @@ class GWorksheet: def set_cell(self, row: int, col: str, val): # row is 1-based col_index = self._col_index(col) + 1 - self.wks.update_cell(row + self.row_offset, col_index, val) + self.wks.update_cell(row, col_index, val) def batch_set_cell(self, cell_updates): """ @@ -83,4 +83,4 @@ class GWorksheet: def to_a1(self, row: int, col: str): # row is 1-based - return utils.rowcol_to_a1(row + self.row_offset, self._col_index(col) + 1) + return utils.rowcol_to_a1(row, self._col_index(col) + 1) From f121c9dab703795b2607fbb60a41c2c51193e186 Mon Sep 17 00:00:00 2001 From: msramalho <19508417+msramalho@users.noreply.github.com> Date: Sat, 12 Mar 2022 20:14:16 +0100 Subject: [PATCH 13/16] enable tolower --- auto_archive.py | 2 +- utils/gworksheet.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/auto_archive.py b/auto_archive.py index b800c3e..484b56f 100644 --- a/auto_archive.py +++ b/auto_archive.py @@ -153,7 +153,7 @@ def main(): parser.add_argument(f'--col-{k}', action='store', dest=k, default=v, help=f'the name of the column to fill with {k} (defaults={v})') args = parser.parse_args() - config_columns = {k: getattr(args, k) for k in GWorksheet.COLUMN_NAMES.keys()} + config_columns = {k: getattr(args, k).lower() for k in GWorksheet.COLUMN_NAMES.keys()} logger.info(f'Opening document {args.sheet} for header {args.header}') diff --git a/utils/gworksheet.py b/utils/gworksheet.py index 7e0c82a..355f074 100644 --- a/utils/gworksheet.py +++ b/utils/gworksheet.py @@ -19,7 +19,7 @@ class GWorksheet: def __init__(self, worksheet, columns=COLUMN_NAMES, header_row=1): self.wks = worksheet self.values = self.wks.get_values()[header_row - 1:] - self.headers = [v for v in self.values[0]] + self.headers = [v.lower() for v in self.values[0]] self.row_offset = header_row - 1 self.columns = columns From d8d9cf17dc997f9e21197fcde36f4a5fdf033b5a Mon Sep 17 00:00:00 2001 From: msramalho <19508417+msramalho@users.noreply.github.com> Date: Sat, 12 Mar 2022 20:25:52 +0100 Subject: [PATCH 14/16] fix offset --- utils/gworksheet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/gworksheet.py b/utils/gworksheet.py index 355f074..f9b0ddd 100644 --- a/utils/gworksheet.py +++ b/utils/gworksheet.py @@ -55,7 +55,7 @@ class GWorksheet: col_index = self._col_index(col) if fresh: - return self.wks.cell(row + self.row_offset, col_index + 1).value + return self.wks.cell(row, col_index + 1).value if type(row) == int: row = self.get_row(row) From 4c5492654872b95ec5d047eb69bdc09ac02ea049 Mon Sep 17 00:00:00 2001 From: msramalho <19508417+msramalho@users.noreply.github.com> Date: Sat, 12 Mar 2022 20:29:43 +0100 Subject: [PATCH 15/16] offset fix --- utils/gworksheet.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/utils/gworksheet.py b/utils/gworksheet.py index f9b0ddd..cf6535c 100644 --- a/utils/gworksheet.py +++ b/utils/gworksheet.py @@ -18,9 +18,8 @@ class GWorksheet: def __init__(self, worksheet, columns=COLUMN_NAMES, header_row=1): self.wks = worksheet - self.values = self.wks.get_values()[header_row - 1:] - self.headers = [v.lower() for v in self.values[0]] - self.row_offset = header_row - 1 + self.values = self.wks.get_values() + self.headers = [v.lower() for v in self.values[header_row - 1]] self.columns = columns def _check_col_exists(self, col: str): From 07bbf443cab35c66ae5042e48f6083947b4ce113 Mon Sep 17 00:00:00 2001 From: msramalho <19508417+msramalho@users.noreply.github.com> Date: Sun, 13 Mar 2022 12:05:09 +0100 Subject: [PATCH 16/16] improves documentation --- README.md | 1 + utils/gworksheet.py | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/README.md b/README.md index 86991f2..c4d4eb8 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,7 @@ graph TD A -->|parent of| C(TikTokArchiver) A -->|parent of| D(YoutubeDLArchiver) A -->|parent of| E(WaybackArchiver) + A -->|parent of| F(TwitterArchiver) ``` ### Current Storages ```mermaid diff --git a/utils/gworksheet.py b/utils/gworksheet.py index cf6535c..6dec9b2 100644 --- a/utils/gworksheet.py +++ b/utils/gworksheet.py @@ -2,6 +2,12 @@ from gspread import utils class GWorksheet: + """ + This class makes read/write operations to the a worksheet easier. + It can read the headers from a custom row number, but the row references + should always include the offset of the header. + eg: if header=4, row 5 will be the first with data. + """ COLUMN_NAMES = { 'url': 'link', 'archive': 'archive location',