From b3599dee718f157b39ef03308e78ad3598d33c13 Mon Sep 17 00:00:00 2001
From: Dave Mateer <davemateer@gmail.com>
Date: Wed, 11 May 2022 14:01:22 +0100
Subject: [PATCH 1/9] working

---
 Pipfile                | 5 +++++
 auto_archive.py        | 1 +
 storages/gd_storage.py | 0
 3 files changed, 6 insertions(+)
 create mode 100644 storages/gd_storage.py

diff --git a/Pipfile b/Pipfile
index 0f20e24..48f1429 100644
--- a/Pipfile
+++ b/Pipfile
@@ -17,8 +17,13 @@ selenium = "*"
 snscrape = "*"
 yt-dlp = "*"
 telethon = "*"
+google-api-python-client = "*"
+google-auth-httplib2 = "*"
+google-auth-oauthlib = "*"
+oauth2client = "*"
 
 [dev-packages]
+autopep8 = "*"
 
 [requires]
 python_version = "3.9"
diff --git a/auto_archive.py b/auto_archive.py
index ed262e9..c0ae085 100644
--- a/auto_archive.py
+++ b/auto_archive.py
@@ -11,6 +11,7 @@ import traceback
 
 import archivers
 from storages import S3Storage, S3Config
+from storages.gd_storage import GDConfig, GDStorage
 from utils import GWorksheet, mkdir_if_not_exists
 import sys
 
diff --git a/storages/gd_storage.py b/storages/gd_storage.py
new file mode 100644
index 0000000..e69de29

From dbac5accbd584749682c91da79c2fcf5ab935a14 Mon Sep 17 00:00:00 2001
From: Dave Mateer <davemateer@gmail.com>
Date: Wed, 11 May 2022 15:39:44 +0100
Subject: [PATCH 2/9] Save to folders for S3 and GD. Google Drive (GD) storage

---
 .example.env                    |  10 +-
 .gitignore                      |   3 +-
 .vscode/launch.json             |  27 +++++
 README.md                       |  36 +++++-
 archivers/base_archiver.py      |  60 ++++++++--
 archivers/telegram_archiver.py  |   9 +-
 archivers/telethon_archiver.py  |  23 +++-
 archivers/tiktok_archiver.py    |   6 +-
 archivers/twitter_archiver.py   |  15 ++-
 archivers/wayback_archiver.py   |   8 +-
 archivers/youtubedl_archiver.py |  25 +++-
 auto_archive.py                 |  89 +++++++++++---
 storages/base_storage.py        |  11 +-
 storages/gd_storage.py          | 202 ++++++++++++++++++++++++++++++++
 utils/gworksheet.py             |   1 +
 15 files changed, 469 insertions(+), 56 deletions(-)
 create mode 100644 .vscode/launch.json

diff --git a/.example.env b/.example.env
index 4a200cf..94d25af 100644
--- a/.example.env
+++ b/.example.env
@@ -7,4 +7,12 @@ INTERNET_ARCHIVE_S3_SECRET=
 TELEGRAM_API_ID=
 TELEGRAM_API_HASH=
 
-FACEBOOK_COOKIE=cookie: datr= xxxx
\ No newline at end of file
+FACEBOOK_COOKIE=cookie: datr= xxxx
+
+# Google Drive, Right click on folder, Get link, eg  
+# https://drive.google.com/drive/folders/1ljwzoAdKdJMJzRW9gPHDC8fkRykVH83X?usp=sharing
+# we want:  1ljwzoAdKdJMJzRW9gPHDC8fkRykVH83X
+GD_ROOT_FOLDER_ID=
+
+# Remeber to share the folder with the service eg
+# autoarchiverservice@auto-archiver-333333.iam.gserviceaccount.com
diff --git a/.gitignore b/.gitignore
index 9d83858..f76014b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,4 +8,5 @@ __pycache__/
 anu.html
 *.log
 .pytest_cach
-anon*
\ No newline at end of file
+
+anon*
diff --git a/.vscode/launch.json b/.vscode/launch.json
new file mode 100644
index 0000000..1a82e0c
--- /dev/null
+++ b/.vscode/launch.json
@@ -0,0 +1,27 @@
+{
+	// Use IntelliSense to learn about possible attributes.
+	// Hover to view descriptions of existing attributes.
+	// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+	"version": "0.2.0",
+	"configurations": [
+		{
+			"name": "Test Hashing",
+			"type": "python",
+			"request": "launch",
+			"program": "auto_archive.py",
+			"console": "integratedTerminal",
+			"justMyCode": true,
+			// "args": ["--sheet","Test Hashing"]
+			// "args": ["--sheet","Test Hashing","--use-filenumber-as-directory"]
+			"args": ["--sheet","Test Hashing","--use-filenumber-as-directory", "--storage=gd"]
+		},
+		{
+			"name": "Python: Current File",
+			"type": "python",
+			"request": "launch",
+			"program": "${file}",
+			"console": "integratedTerminal",
+			"justMyCode": true
+		}
+	]
+}
\ No newline at end of file
diff --git a/README.md b/README.md
index 5854f68..0f1b2e1 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
 # auto-archiver
 
-This Python script will look for links to Youtube, Twitter, etc,. in a specified column of a Google Sheet, uses YoutubeDL to download the media, stores the result in a Digital Ocean space, and updates the Google Sheet with the archive location, status, and date. It can be run manually or on an automated basis.
+This Python script will look for links to Youtube, Twitter, etc,. in a specified column of a Google Sheet, uses YoutubeDL to download the media, stores the result in a Digital Ocean space or Google Drive, and updates the Google Sheet with the archive location, status, and date. It can be run manually or on an automated basis.
 
 ## Setup
 
@@ -14,7 +14,7 @@ If you are using `pipenv` (recommended), `pipenv install` is sufficient to insta
 
 [fonts-noto](https://fonts.google.com/noto) to deal with multiple unicode characters during selenium/geckodriver's screenshots: `sudo apt install fonts-noto -y`. 
 
-A `.env` file is required for saving content to a Digital Ocean space, and for archiving pages to the Internet Archive. This file should also be in the script directory, and should contain the following variables:
+A `.env` file is required for saving content to a Digital Ocean space and Google Drive, and for archiving pages to the Internet Archive. This file should also be in the script directory, and should contain the following variables:
 
 ```
 DO_SPACES_REGION=
@@ -23,8 +23,14 @@ DO_SPACES_KEY=
 DO_SPACES_SECRET=
 INTERNET_ARCHIVE_S3_KEY=
 INTERNET_ARCHIVE_S3_SECRET=
+TELEGRAM_API_ID=
+TELEGRAM_API_HASH=
+FACEBOOK_COOKIE=
+GD_ROOT_FOLDER_ID=
 ```
 
+`.example.env` is an example of this file
+
 Internet Archive credentials can be retrieved from https://archive.org/account/s3.php.
 
 ## Running
@@ -93,3 +99,29 @@ graph TD
 graph TD
     A(BaseStorage) -->|parent of| B(S3Storage)
 ```
+
+## Saving into Folders
+
+To use a column from the spreadsheet called `File Number` eg SM001234 as a directory on the cloud storage, you need to pass in
+
+```bash
+python auto_archive.py --sheet 'Sheet Name' --use-filenumber-as-directory
+```
+
+## Google Drive
+
+To use Google Drive storage you need the id of the shared folder in the `.env` file which must be shared with the service account eg `autoarchiverservice@auto-archiver-111111.iam.gserviceaccount.com`
+
+```bash
+python auto_archive.py --sheet 'Sheet Name' --use-filenumber-as-directory --storage='gd'
+```
+
+Note the you must use filenumber for Google Drive Storage.
+
+## Telethon (Telegrams API Library)
+
+Put your `anon.session` in the root, so that it doesn't stall and ask for authentication
+
+
+
+
diff --git a/archivers/base_archiver.py b/archivers/base_archiver.py
index 7ab5a9c..367b483 100644
--- a/archivers/base_archiver.py
+++ b/archivers/base_archiver.py
@@ -14,6 +14,9 @@ from selenium.common.exceptions import TimeoutException
 from storages import Storage
 from utils import mkdir_if_not_exists
 
+from selenium.webdriver.common.by import By
+from loguru import logger
+from selenium.common.exceptions import TimeoutException
 
 @dataclass
 class ArchiveResult:
@@ -39,7 +42,7 @@ class Archiver(ABC):
         return self.__class__.__name__
 
     @abstractmethod
-    def download(self, url, check_if_exists=False): pass
+    def download(self, url, check_if_exists=False, filenumber=None): pass
 
     def get_netloc(self, url):
         return urlparse(url).netloc
@@ -47,7 +50,8 @@ class Archiver(ABC):
     def get_html_key(self, url):
         return self.get_key(urlparse(url).path.replace("/", "_") + ".html")
 
-    def generate_media_page_html(self, url, urls_info: dict, object, thumbnail=None):
+    # generate the html page eg SM3013/twitter__minmyatnaing13_status_1499415562937503751.html
+    def generate_media_page_html(self, url, urls_info: dict, object, thumbnail=None, filenumber=None):
         page = f'''<html><head><title>{url}</title><meta charset="UTF-8"></head>
             <body>
             <h2>Archived media from {self.name}</h2>
@@ -61,18 +65,24 @@ class Archiver(ABC):
 
         page_key = self.get_key(urlparse(url).path.replace("/", "_") + ".html")
         page_filename = 'tmp/' + page_key
-        page_cdn = self.storage.get_cdn_url(page_key)
 
         with open(page_filename, "w") as f:
             f.write(page)
 
         page_hash = self.get_hash(page_filename)
 
+        if filenumber != None:
+            logger.trace(f'filenumber for directory is {filenumber}')
+            page_key = filenumber + "/" + page_key
+
         self.storage.upload(page_filename, page_key, extra_args={
             'ACL': 'public-read', 'ContentType': 'text/html'})
+
+        page_cdn = self.storage.get_cdn_url(page_key)
         return (page_cdn, page_hash, thumbnail)
 
-    def generate_media_page(self, urls, url, object):
+    # eg images in a tweet save to cloud storage
+    def generate_media_page(self, urls, url, object, filenumber=None):
         headers = {
             'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36'
         }
@@ -87,19 +97,30 @@ class Archiver(ABC):
 
             filename = 'tmp/' + key
 
+            # eg media_url: https://pbs.twimg.com/media/FM7-ggCUYAQHKWW?format=jpg&name=orig
             d = requests.get(media_url, headers=headers)
             with open(filename, 'wb') as f:
                 f.write(d.content)
 
+            if filenumber is not None:
+                logger.debug(f'filenumber for directory is {filenumber}')
+                key = filenumber + "/" + key
+
+            # eg filename: 'tmp/twitter__media_FM7-ggCUYAQHKWW.jpg'
+            # eg key: 'twitter__media_FM7-ggCUYAQHKWW.jpg'
+            # or if using filename key: 'SM3013/twitter__media_FM7-ggCUYAQHKWW.jpg'
             self.storage.upload(filename, key)
+
             hash = self.get_hash(filename)
+
+            # eg 'https://testhashing.fra1.cdn.digitaloceanspaces.com/Test_Hashing/Sheet1/twitter__media_FM7-ggCUYAQHKWW.jpg'
             cdn_url = self.storage.get_cdn_url(key)
 
             if thumbnail is None:
                 thumbnail = cdn_url
             uploaded_media.append({'cdn_url': cdn_url, 'key': key, 'hash': hash})
 
-        return self.generate_media_page_html(url, uploaded_media, object, thumbnail=thumbnail)
+        return self.generate_media_page_html(url, uploaded_media, object, thumbnail=thumbnail, filenumber=filenumber)
 
     def get_key(self, filename):
         """
@@ -119,15 +140,33 @@ class Archiver(ABC):
     def get_hash(self, filename):
         f = open(filename, "rb")
         bytes = f.read()  # read entire file as bytes
+        
         hash = hashlib.sha256(bytes)
+        # option to use SHA3_512 instead
+        # hash = hashlib.sha3_512(bytes)
         f.close()
         return hash.hexdigest()
 
-    def get_screenshot(self, url):
+    # eg SA3013/twitter__minmyatnaing13_status_14994155629375037512022-04-27T13:51:43.701962.png
+    # def get_screenshot(self, url, filenumber, storage="GD"):
+    def get_screenshot(self, url, filenumber):
         key = self.get_key(urlparse(url).path.replace(
             "/", "_") + datetime.datetime.utcnow().isoformat().replace(" ", "_") + ".png")
         filename = 'tmp/' + key
 
+        # Accept cookies popup dismiss for ytdlp video
+        if 'facebook.com' in url:
+            try:
+                logger.debug(f'Trying fb click accept cookie popup for {url}')
+                self.driver.get("http://www.facebook.com") 
+                foo = self.driver.find_element(By.XPATH,"//button[@data-cookiebanner='accept_only_essential_button']")
+                foo.click()
+                logger.debug(f'fb click worked')
+                # linux server needs a sleep otherwise facebook cookie wont have worked and we'll get a popup on next page
+                time.sleep(2)
+            except:
+                logger.warning(f'Failed on fb accept cookies for url {url}')
+
         try:
             self.driver.get(url)
             time.sleep(6)
@@ -135,8 +174,14 @@ class Archiver(ABC):
             logger.info("TimeoutException loading page for screenshot")
 
         self.driver.save_screenshot(filename)
+
+        if filenumber is not None:
+            logger.debug(f'filenumber for directory is {filenumber}')
+            key = filenumber + "/" + key
+
         self.storage.upload(filename, key, extra_args={
                             'ACL': 'public-read', 'ContentType': 'image/png'})
+
         return self.storage.get_cdn_url(key)
 
     def get_thumbnails(self, filename, key, duration=None):
@@ -167,10 +212,9 @@ class Archiver(ABC):
                 thumbnail_filename = thumbnails_folder + fname
                 key = key_folder + fname
 
-                cdn_url = self.storage.get_cdn_url(key)
-
                 self.storage.upload(thumbnail_filename, key)
 
+                cdn_url = self.storage.get_cdn_url(key)
                 cdn_urls.append(cdn_url)
 
         if len(cdn_urls) == 0:
diff --git a/archivers/telegram_archiver.py b/archivers/telegram_archiver.py
index 5a7f63c..b19ab8f 100644
--- a/archivers/telegram_archiver.py
+++ b/archivers/telegram_archiver.py
@@ -11,7 +11,7 @@ from .base_archiver import Archiver, ArchiveResult
 class TelegramArchiver(Archiver):
     name = "telegram"
 
-    def download(self, url, check_if_exists=False):
+    def download(self, url, check_if_exists=False, filenumber=None):
         # detect URLs that we definitely cannot handle
         if 't.me' != self.get_netloc(url):
             return False
@@ -27,7 +27,7 @@ class TelegramArchiver(Archiver):
         if url[-8:] != "?embed=1":
             url += "?embed=1"
 
-        screenshot = self.get_screenshot(url)
+        screenshot = self.get_screenshot(url, filenumber=filenumber)
 
         t = requests.get(url, headers=headers)
         s = BeautifulSoup(t.content, 'html.parser')
@@ -42,7 +42,7 @@ class TelegramArchiver(Archiver):
                 urls = [u.replace("'", "") for u in re.findall(r'url\((.*?)\)', im['style'])]
                 images += urls
 
-            page_cdn, page_hash, thumbnail = self.generate_media_page(images, url, html.escape(str(t.content)))
+            page_cdn, page_hash, thumbnail = self.generate_media_page(images, url, html.escape(str(t.content)),filenumber=filenumber)
             time_elements = s.find_all('time')
             timestamp = time_elements[0].get('datetime') if len(time_elements) else None
 
@@ -52,6 +52,9 @@ class TelegramArchiver(Archiver):
         video_id = video_url.split('/')[-1].split('?')[0]
         key = self.get_key(video_id)
 
+        if filenumber is not None:
+            key = filenumber + "/" + key
+
         filename = 'tmp/' + key
         cdn_url = self.storage.get_cdn_url(key)
 
diff --git a/archivers/telethon_archiver.py b/archivers/telethon_archiver.py
index 88bec58..5cee791 100644
--- a/archivers/telethon_archiver.py
+++ b/archivers/telethon_archiver.py
@@ -41,20 +41,22 @@ class TelethonArchiver(Archiver):
                 media.append(post)
         return media
 
-    def download(self, url, check_if_exists=False):
+    def download(self, url, check_if_exists=False, filenumber=None):
         # detect URLs that we definitely cannot handle
         matches = self.link_pattern.findall(url)
         if not len(matches):
             return False
 
         status = "success"
-        screenshot = self.get_screenshot(url)
+        screenshot = self.get_screenshot(url, filenumber)
 
+        # app will ask (stall for user input!) for phone number and auth code if anon.session not found
         with self.client.start():
             matches = list(matches[0])
             chat, post_id = matches[1], matches[2]
 
             post_id = int(post_id)
+
             try:
                 post = self.client.get_messages(chat, ids=post_id)
             except ValueError as e:
@@ -65,9 +67,13 @@ class TelethonArchiver(Archiver):
 
             if len(media_posts) > 1:
                 key = self.get_html_key(url)
-                cdn_url = self.storage.get_cdn_url(key)
+
+                if filenumber is not None:
+                    key = filenumber + "/" + key          
 
                 if check_if_exists and self.storage.exists(key):
+                     # only s3 storage supports storage.exists as not implemented on gd
+                    cdn_url = self.storage.get_cdn_url(key)
                     status = 'already archived'
                     return ArchiveResult(status='already archived', cdn_url=cdn_url, title=post.message, timestamp=post.date, screenshot=screenshot)
 
@@ -78,19 +84,26 @@ class TelethonArchiver(Archiver):
                     if len(mp.message) > len(message): message = mp.message
                     filename = self.client.download_media(mp.media, f'tmp/{chat}_{group_id}/{mp.id}')
                     key = filename.split('tmp/')[1]
+
+                    if filenumber is not None:
+                        key = filenumber + "/" + key
                     self.storage.upload(filename, key)
                     hash = self.get_hash(filename)
                     cdn_url = self.storage.get_cdn_url(key)
                     uploaded_media.append({'cdn_url': cdn_url, 'key': key, 'hash': hash})
                     os.remove(filename)
 
-                page_cdn, page_hash, _ = self.generate_media_page_html(url, uploaded_media, html.escape(str(post)))
+                page_cdn, page_hash, _ = self.generate_media_page_html(url, uploaded_media, html.escape(str(post)), filenumber=filenumber)
 
                 return ArchiveResult(status=status, cdn_url=page_cdn, title=post.message, timestamp=post.date, hash=page_hash, screenshot=screenshot)
             elif len(media_posts) == 1:
                 key = self.get_key(f'{chat}_{post_id}')
                 filename = self.client.download_media(post.media, f'tmp/{key}')
                 key = filename.split('tmp/')[1].replace(" ", "")
+
+                if filenumber is not None:
+                    key = filenumber + "/" + key
+
                 self.storage.upload(filename, key)
                 hash = self.get_hash(filename)
                 cdn_url = self.storage.get_cdn_url(key)
@@ -99,5 +112,5 @@ class TelethonArchiver(Archiver):
 
                 return ArchiveResult(status=status, cdn_url=cdn_url, title=post.message, thumbnail=key_thumb, thumbnail_index=thumb_index, timestamp=post.date, hash=hash, screenshot=screenshot)
 
-            page_cdn, page_hash, _ = self.generate_media_page_html(url, [], html.escape(str(post)))
+            page_cdn, page_hash, _ = self.generate_media_page_html(url, [], html.escape(str(post)), filenumber=filenumber)
             return ArchiveResult(status=status, cdn_url=page_cdn, title=post.message, timestamp=post.date, hash=page_hash, screenshot=screenshot)
diff --git a/archivers/tiktok_archiver.py b/archivers/tiktok_archiver.py
index 6b5116f..9b90efa 100644
--- a/archivers/tiktok_archiver.py
+++ b/archivers/tiktok_archiver.py
@@ -8,7 +8,7 @@ from .base_archiver import Archiver, ArchiveResult
 class TiktokArchiver(Archiver):
     name = "tiktok"
 
-    def download(self, url, check_if_exists=False):
+    def download(self, url, check_if_exists=False, filenumber=None):
         if 'tiktok.com' not in url:
             return False
 
@@ -54,11 +54,13 @@ class TiktokArchiver(Archiver):
                                  thumbnail_index=thumb_index, duration=info.duration, title=info.caption, timestamp=info.create.isoformat(),
                                  hash=hash, screenshot=screenshot)
 
-        except tiktok_downloader.Except.InvalidUrl:
+        except tiktok_downloader.Except.InvalidUrl as e:
             status = 'Invalid URL'
+            logger.warning(f'Invalid URL on {url}  {e}\n{traceback.format_exc()}')
             return ArchiveResult(status=status)
 
         except:
             error = traceback.format_exc()
             status = 'Other Tiktok error: ' + str(error)
+            logger.warning(f'Other Tiktok error' + str(error))
             return ArchiveResult(status=status)
diff --git a/archivers/twitter_archiver.py b/archivers/twitter_archiver.py
index 099d279..05e7ec0 100644
--- a/archivers/twitter_archiver.py
+++ b/archivers/twitter_archiver.py
@@ -1,6 +1,5 @@
 from snscrape.modules.twitter import TwitterTweetScraper, Video, Gif, Photo
 from loguru import logger
-import requests
 from urllib.parse import urlparse
 
 from .base_archiver import Archiver, ArchiveResult
@@ -9,7 +8,8 @@ from .base_archiver import Archiver, ArchiveResult
 class TwitterArchiver(Archiver):
     name = "twitter"
 
-    def download(self, url, check_if_exists=False):
+    def download(self, url, check_if_exists=False, filenumber=None):
+        
         if 'twitter.com' != self.get_netloc(url):
             return False
 
@@ -24,11 +24,14 @@ class TwitterArchiver(Archiver):
 
         try:
             tweet = next(scr.get_items())
-        except:
-            logger.warning('wah wah')
+        except Exception as ex:
+            template = "TwitterArchiver cant get tweet and threw, which can happen if a media sensitive tweet. \n type: {0} occurred. \n arguments:{1!r}"
+            message = template.format(type(ex).__name__, ex.args)
+            logger.warning(message)
             return False
 
         if tweet.media is None:
+            logger.trace(f'No media found')
             return False
 
         urls = []
@@ -45,8 +48,8 @@ class TwitterArchiver(Archiver):
             else:
                 logger.warning(f"Could not get media URL of {media}")
 
-        page_cdn, page_hash, thumbnail = self.generate_media_page(urls, url, tweet.json())
+        page_cdn, page_hash, thumbnail = self.generate_media_page(urls, url, tweet.json(), filenumber)
 
-        screenshot = self.get_screenshot(url)
+        screenshot = self.get_screenshot(url, filenumber)
 
         return ArchiveResult(status="success", cdn_url=page_cdn, screenshot=screenshot, hash=page_hash, thumbnail=thumbnail, timestamp=tweet.date)
diff --git a/archivers/wayback_archiver.py b/archivers/wayback_archiver.py
index 1fa98aa..652798a 100644
--- a/archivers/wayback_archiver.py
+++ b/archivers/wayback_archiver.py
@@ -4,6 +4,8 @@ from bs4 import BeautifulSoup
 from storages import Storage
 from .base_archiver import Archiver, ArchiveResult
 
+from loguru import logger
+
 
 class WaybackArchiver(Archiver):
     name = "wayback"
@@ -12,7 +14,7 @@ class WaybackArchiver(Archiver):
         super(WaybackArchiver, self).__init__(storage, driver)
         self.seen_urls = {}
 
-    def download(self, url, check_if_exists=False):
+    def download(self, url, check_if_exists=False, filenumber=None):
         if check_if_exists and url in self.seen_urls:
             return self.seen_urls[url]
 
@@ -25,9 +27,11 @@ class WaybackArchiver(Archiver):
             'https://web.archive.org/save/', headers=ia_headers, data={'url': url})
 
         if r.status_code != 200:
+            logger.warning(f"Internet archive failed with status of {r.status_code}")
             return ArchiveResult(status="Internet archive failed")
 
         if 'job_id' not in r.json() and 'message' in r.json():
+            logger.warning(f"Internet archive failed json \n {r.json()}")
             return ArchiveResult(status=f"Internet archive failed: {r.json()['message']}")
 
         job_id = r.json()['job_id']
@@ -71,7 +75,7 @@ class WaybackArchiver(Archiver):
         except:
             title = "Could not get title"
 
-        screenshot = self.get_screenshot(url)
+        screenshot = self.get_screenshot(url, filenumber)
         result = ArchiveResult(status='Internet Archive fallback', cdn_url=archive_url, title=title, screenshot=screenshot)
         self.seen_urls[url] = result
         return result
diff --git a/archivers/youtubedl_archiver.py b/archivers/youtubedl_archiver.py
index ad8756b..9983950 100644
--- a/archivers/youtubedl_archiver.py
+++ b/archivers/youtubedl_archiver.py
@@ -15,7 +15,7 @@ class YoutubeDLArchiver(Archiver):
         super().__init__(storage, driver)
         self.fb_cookie = fb_cookie
 
-    def download(self, url, check_if_exists=False):
+    def download(self, url, check_if_exists=False, filenumber=None):
         netloc = self.get_netloc(url)
         if netloc in ['facebook.com', 'www.facebook.com']:
             logger.debug('Using Facebook cookie')
@@ -27,13 +27,17 @@ class YoutubeDLArchiver(Archiver):
 
         try:
             info = ydl.extract_info(url, download=False)
-        except yt_dlp.utils.DownloadError:
-            # no video here
+        except yt_dlp.utils.DownloadError as e:
+            logger.debug(f'No video - Youtube normal control flow: {e}')
+            return False
+        except Exception as e:
+            logger.debug(f'ytdlp exception which is normal for example a facebook page with images only will cause a IndexError: list index out of range. Exception here is: \n  {e}')
             return False
 
         if info.get('is_live', False):
             logger.warning("Live streaming media, not archiving now")
             return ArchiveResult(status="Streaming media")
+
         if 'twitter.com' in netloc:
             if 'https://twitter.com/' in info['webpage_url']:
                 logger.info('Found https://twitter.com/ in the download url from Twitter')
@@ -41,7 +45,6 @@ class YoutubeDLArchiver(Archiver):
                 logger.info('Found a linked video probably in a link in a tweet - not getting that video as there may be images in the tweet')
                 return False
 
-
         if check_if_exists:
             if 'entries' in info:
                 if len(info['entries']) > 1:
@@ -58,6 +61,9 @@ class YoutubeDLArchiver(Archiver):
 
             key = self.get_key(filename)
 
+            if filenumber is not None:
+                key = filenumber + "/" + key
+
             if self.storage.exists(key):
                 status = 'already archived'
                 cdn_url = self.storage.get_cdn_url(key)
@@ -81,12 +87,19 @@ class YoutubeDLArchiver(Archiver):
 
         if status != 'already archived':
             key = self.get_key(filename)
-            cdn_url = self.storage.get_cdn_url(key)
+            
+            if filenumber is not None:
+                key = filenumber + "/" + key
 
             self.storage.upload(filename, key)
 
+            # filename ='tmp/sDE-qZdi8p8.webm'
+            # key ='SM0022/youtube_dl_sDE-qZdi8p8.webm'
+            cdn_url = self.storage.get_cdn_url(key)
+
         hash = self.get_hash(filename)
-        screenshot = self.get_screenshot(url)
+        screenshot = self.get_screenshot(url, filenumber)
+
 
         # get duration
         duration = info.get('duration')
diff --git a/auto_archive.py b/auto_archive.py
index c0ae085..a3c17d1 100644
--- a/auto_archive.py
+++ b/auto_archive.py
@@ -68,7 +68,7 @@ def expand_url(url):
     return url
 
 
-def process_sheet(sheet, header=1, columns=GWorksheet.COLUMN_NAMES):
+def process_sheet(sheet, usefilenumber=False, storage="s3", header=1, columns=GWorksheet.COLUMN_NAMES):
     gc = gspread.service_account(filename='service_account.json')
     sh = gc.open(sheet)
 
@@ -78,6 +78,9 @@ def process_sheet(sheet, header=1, columns=GWorksheet.COLUMN_NAMES):
         key=os.getenv('DO_SPACES_KEY'),
         secret=os.getenv('DO_SPACES_SECRET')
     )
+    gd_config = GDConfig(
+        root_folder_id=os.getenv('GD_ROOT_FOLDER_ID'),
+    )
     telegram_config = archivers.TelegramConfig(
         api_id=os.getenv('TELEGRAM_API_ID'),
         api_hash=os.getenv('TELEGRAM_API_HASH')
@@ -91,12 +94,12 @@ def process_sheet(sheet, header=1, columns=GWorksheet.COLUMN_NAMES):
         gw = GWorksheet(wks, header_row=header, columns=columns)
 
         if not gw.col_exists('url'):
-            logger.warning(
+            logger.info(
                 f'No "{columns["url"]}" column found, skipping worksheet {wks.title}')
             continue
 
         if not gw.col_exists('status'):
-            logger.warning(
+            logger.info(
                 f'No "{columns["status"]}" column found, skipping worksheet {wks.title}')
             continue
 
@@ -104,26 +107,30 @@ def process_sheet(sheet, header=1, columns=GWorksheet.COLUMN_NAMES):
         s3_config.folder = f'{sheet.replace(" ", "_")}/{wks.title.replace(" ", "_")}/'
         s3_client = S3Storage(s3_config)
 
-        # order matters, first to succeed excludes remaining
-        active_archivers = [
-            archivers.TelethonArchiver(s3_client, driver, telegram_config),
-            archivers.TelegramArchiver(s3_client, driver),
-            archivers.TiktokArchiver(s3_client, driver),
-            archivers.YoutubeDLArchiver(s3_client, driver, os.getenv('FACEBOOK_COOKIE')),
-            archivers.TwitterArchiver(s3_client, driver),
-            archivers.WaybackArchiver(s3_client, driver)
-        ]
+        gd_config.folder = f'{sheet.replace(" ", "_")}/{wks.title.replace(" ", "_")}/'
+        gd_client = GDStorage(gd_config)
 
         # loop through rows in worksheet
         for row in range(1 + header, gw.count_rows() + 1):
             url = gw.get_cell(row, 'url')
             original_status = gw.get_cell(row, 'status')
             status = gw.get_cell(row, 'status', fresh=original_status in ['', None] and url != '')
+
             if url != '' and status in ['', None]:
                 gw.set_cell(row, 'status', 'Archive in progress')
 
                 url = expand_url(url)
                 
+                if usefilenumber:
+                    filenumber = gw.get_cell(row, 'filenumber')
+                    logger.debug(f'filenumber is {filenumber}')
+                    if filenumber == "":
+                        logger.warning(f'Logic error on row {row} with url {url} - the feature flag for usefilenumber is True, yet cant find a corresponding filenumber')
+                        gw.set_cell(row, 'status', 'Missing filenumber')
+                        continue
+                else:
+                    # We will use this through the app to differentiate between where to save
+                    filenumber = None
 
                 # make a new driver so each spreadsheet row is idempotent
                 options = webdriver.FirefoxOptions()
@@ -134,24 +141,58 @@ def process_sheet(sheet, header=1, columns=GWorksheet.COLUMN_NAMES):
                 driver.set_window_size(1400, 2000)
                 # in seconds, telegram screenshots catch which don't come back
                 driver.set_page_load_timeout(120)
+
+                 # client
+                storage_client = None
+                if storage == "s3":
+                    storage_client = s3_client
+                elif storage == "gd":
+                    storage_client = gd_client
+                else:
+                    raise ValueError(f'Cant get storage_client {storage_client}')
+
+                # order matters, first to succeed excludes remaining
+                active_archivers = [
+                    archivers.TelethonArchiver(storage_client, driver, telegram_config),
+                    archivers.TelegramArchiver(storage_client, driver),
+                    archivers.TiktokArchiver(storage_client, driver),
+                    archivers.YoutubeDLArchiver(storage_client, driver, os.getenv('FACEBOOK_COOKIE')),
+                    archivers.TwitterArchiver(storage_client, driver),
+                    archivers.WaybackArchiver(storage_client, driver)
+                ]
                 for archiver in active_archivers:
                     logger.debug(f'Trying {archiver} on row {row}')
 
                     try:
-                        result = archiver.download(url, check_if_exists=True)
+                        if usefilenumber:
+                            # using filenumber to store in folders so not checking for existence of that url
+                            result = archiver.download(url, check_if_exists=False, filenumber=filenumber)
+                        else:
+                            result = archiver.download(url, check_if_exists=True)
+
                     except Exception as e:
                         result = False
                         logger.error(f'Got unexpected error in row {row} with archiver {archiver} for url {url}: {e}\n{traceback.format_exc()}')
 
                     if result:
-                        if result.status in ['success', 'already archived']:
+                        # IA is a Success I believe - or do we want to display a logger warning for it?
+                        if result.status in ['success', 'already archived', 'Internet Archive fallback']:
                             result.status = archiver.name + \
                                 ": " + str(result.status)
                             logger.success(
-                                f'{archiver} succeeded on row {row}')
+                                 f'{archiver} succeeded on row {row}, url {url}')
                             break
+                        
+                        # wayback has seen this url before so keep existing status
+                        if "wayback: Internet Archive fallback" in result.status:
+                            logger.success(
+                                f'wayback has seen this url before so keep existing status on row {row}')
+                            result.status = result.status.replace(' (duplicate)', '')
+                            result.status = str(result.status) + " (duplicate)"
+                            break
+
                         logger.warning(
-                            f'{archiver} did not succeed on row {row}, final status: {result.status}')
+                            f'{archiver} did not succeed on {row=}, final status: {result.status}')
                         result.status = archiver.name + \
                             ": " + str(result.status)
                 # get rid of driver so can reload on next row
@@ -165,22 +206,34 @@ def process_sheet(sheet, header=1, columns=GWorksheet.COLUMN_NAMES):
 @logger.catch
 def main():
     logger.debug(f'Passed args:{sys.argv}')
+
     parser = argparse.ArgumentParser(
         description='Automatically archive social media videos from a Google Sheets document')
     parser.add_argument('--sheet', action='store', dest='sheet', help='the name of the google sheets document', required=True)
     parser.add_argument('--header', action='store', dest='header', default=1, type=int, help='1-based index for the header row')
     parser.add_argument('--private', action='store_true', help='Store content without public access permission')
 
+    parser.add_argument('--use-filenumber-as-directory', action=argparse.BooleanOptionalAction, dest='usefilenumber',  \
+         help='Will save files into a subfolder on cloud storage which has the File Number eg SM3012')
+    parser.add_argument('--storage', action='store', dest='storage', default='s3', \
+         help='s3 or gd storage. Default is s3. NOTE GD storage supports only using filenumber')
+
     for k, v in GWorksheet.COLUMN_NAMES.items():
         parser.add_argument(f'--col-{k}', action='store', dest=k, default=v, help=f'the name of the column to fill with {k} (defaults={v})')
 
     args = parser.parse_args()
     config_columns = {k: getattr(args, k).lower() for k in GWorksheet.COLUMN_NAMES.keys()}
 
-    logger.info(f'Opening document {args.sheet} for header {args.header}')
+    logger.info(f'Opening document {args.sheet} for header {args.header} using filenumber: {args.usefilenumber} and storage {args.storage}')
+
+    # https://stackoverflow.com/questions/15008758/parsing-boolean-values-with-argparse
+    # args.filenumber is True (of type bool) when set or None when argument is not there
+    usefilenumber = False
+    if args.usefilenumber:
+        usefilenumber = True
 
     mkdir_if_not_exists('tmp')
-    process_sheet(args.sheet, header=args.header, columns=config_columns)
+    process_sheet(args.sheet, usefilenumber, args.storage, args.header, config_columns)
     shutil.rmtree('tmp')
 
 
diff --git a/storages/base_storage.py b/storages/base_storage.py
index e1bf9c7..79a555b 100644
--- a/storages/base_storage.py
+++ b/storages/base_storage.py
@@ -17,5 +17,12 @@ class Storage(ABC):
 
     def upload(self, filename: str, key: str, **kwargs):
         logger.debug(f'[{self.__class__.__name__}] uploading file {filename} with key {key}')
-        with open(filename, 'rb') as f:
-            self.uploadf(f, key, **kwargs)
+        # S3 requires an open file, GD only the filename
+        storage = type(self).__name__
+        if storage == "GDStorage":
+            self.uploadf(filename, key, **kwargs)
+        elif storage == "S3Storage":
+            with open(filename, 'rb') as f:
+                self.uploadf(f, key, **kwargs)
+        else:
+            raise ValueError('Cant get storage thrown from base_storage.py')
\ No newline at end of file
diff --git a/storages/gd_storage.py b/storages/gd_storage.py
index e69de29..4dab7d0 100644
--- a/storages/gd_storage.py
+++ b/storages/gd_storage.py
@@ -0,0 +1,202 @@
+from loguru import logger
+from .base_storage import Storage
+from dataclasses import dataclass
+
+from googleapiclient.discovery import build
+from googleapiclient.http import MediaFileUpload
+from google.oauth2 import service_account
+
+import time
+
+@dataclass
+class GDConfig:
+    root_folder_id: str
+
+class GDStorage(Storage):
+
+    def __init__(self, config: GDConfig):
+        self.root_folder_id = config.root_folder_id
+        SCOPES = ['https://www.googleapis.com/auth/drive']
+        creds = service_account.Credentials.from_service_account_file('service_account.json', scopes=SCOPES)
+        self.service = build('drive', 'v3', credentials=creds)
+
+    def _get_path(self, key):
+        return self.folder + key
+
+    def get_cdn_url(self, key):
+        # only support files saved in a folders for GD
+        # S3 supports folder and all stored in the root
+
+        # key will be SM0002/twitter__media_ExeUSW2UcAE6RbN.jpg
+        foldername = key.split('/', 1)[0]
+        # eg twitter__media_asdf.jpg
+        filename = key.split('/', 1)[1]
+
+        logger.debug(f'Looking for {foldername} and filename: {filename} on GD')
+
+        # retry policy on Google Drive
+        try_again = True
+        counter = 1
+        folder_id = None
+        while try_again:
+            # need to lookup the id of folder eg SM0002 which should be there already as this is get_cdn_url
+            results = self.service.files().list(q=f"'{self.root_folder_id}' in parents \
+                                                and name = '{foldername}' ",
+                                            spaces='drive', # ie not appDataFolder or photos
+                                            fields='files(id, name)'
+                                            ).execute()
+            items = results.get('files', [])
+
+            for item in items:
+                logger.debug(f"found folder of {item['name']}")
+                folder_id= item['id']
+                try_again = False
+
+            if folder_id is None:
+                logger.debug(f'Cant find {foldername=} waiting and trying again {counter=}')
+                counter += 1
+                time.sleep(10)
+                if counter > 18:
+                    raise ValueError(f'Cant find  {foldername} and retried 18 times pausing 10seconds at a time which is 3 minutes')
+
+        # check for sub folder in file eg youtube_dl_sDE-qZdi8p8/index.html'
+        # happens doing thumbnails
+        a, _, b = filename.partition('/')
+
+        if b != '':
+            # a: 'youtube_dl_sDE-qZdi8p8'
+            # b: 'index.html'
+            logger.debug(f'get_cdn_url: Found a subfolder so need to split on a: {a} and {b}')
+
+            # get id of the sub folder
+            results = self.service.files().list(q=f"'{folder_id}' in parents \
+                                                and mimeType='application/vnd.google-apps.folder' \
+                                                and name = '{a}' ",
+                                            spaces='drive', # ie not appDataFolder or photos
+                                            fields='files(id, name)'
+                                            ).execute()
+            items = results.get('files', [])
+
+            filename = None
+            for item in items:
+                folder_id = item['id']
+                filename = b
+            if filename is None:
+                raise ValueError(f'Problem finding sub folder {a}')
+
+        # get id of file inside folder (or sub folder)
+        results = self.service.files().list(q=f"'{folder_id}' in parents \
+                                            and name = '{filename}' ",
+                                        spaces='drive', 
+                                        fields='files(id, name)'
+                                        ).execute()
+        items = results.get('files', [])
+        
+        file_id = None
+        for item in items:
+            logger.debug(f"found file of {item['name']}")
+            file_id= item['id']
+
+        if file_id is None:
+            raise ValueError(f'Problem finding file {filename} in folder_id {folder_id}')
+            
+        foo = "https://drive.google.com/file/d/" + file_id + "/view?usp=sharing"
+        return foo
+
+    def exists(self, key):
+        # Not implemented yet
+        # Google drive will accept duplicate named filenames as it is stored as a different fileid 
+
+        # try:
+        #     self.s3.head_object(Bucket=self.bucket, Key=self._get_path(key))
+        #     return True
+        # except ClientError:
+        #     return False
+        return False
+
+    def uploadf(self, file, key, **kwargs):
+        # split on first occurance of /
+        # eg SM0005
+        foldername = key.split('/', 1)[0]
+        # eg twitter__media_asdf.jpg
+        filename = key.split('/', 1)[1]
+
+         # does folder eg SM0005 exist already inside parent of Files auto-archiver
+        results = self.service.files().list(q=f"'{self.root_folder_id}' in parents \
+                                            and mimeType='application/vnd.google-apps.folder' \
+                                            and name = '{foldername}' ",
+                                        spaces='drive', 
+                                        fields='files(id, name)'
+                                        ).execute()
+        items = results.get('files', [])
+        folder_id_to_upload_to = None
+        if len(items) > 1:
+            logger.error(f'Duplicate folder name of {foldername} which should never happen, but continuing anyway')
+
+        for item in items:
+            logger.debug(f"Found existing folder of {item['name']}")
+            folder_id_to_upload_to = item['id']
+
+        if folder_id_to_upload_to is None:
+            logger.debug(f'Creating new folder {foldername}')
+            file_metadata = {
+                'name': [foldername],
+                'mimeType': 'application/vnd.google-apps.folder',
+                'parents': [self.root_folder_id]
+            }
+            gd_file = self.service.files().create(body=file_metadata, fields='id').execute()
+            folder_id_to_upload_to = gd_file.get('id')
+        
+        # check for subfolder nema in file eg youtube_dl_sDE-qZdi8p8/out1.jpg'
+        # happens doing thumbnails
+
+        # will always return a and a blank b even if there is nothing to split
+        # https://stackoverflow.com/a/38149500/26086
+        a, _, b = filename.partition('/')
+
+        if b != '':
+            # a: 'youtube_dl_sDE-qZdi8p8'
+            # b: 'out1.jpg'
+            logger.debug(f'uploadf: Found a subfolder so need to split on a: {a} and {b}')
+
+            # does the 'a' folder exist already in folder_id_to_upload_to eg SM0005
+            results = self.service.files().list(q=f"'{folder_id_to_upload_to}' in parents \
+                                                and mimeType='application/vnd.google-apps.folder' \
+                                                and name = '{a}' ",
+                                            spaces='drive', # ie not appDataFolder or photos
+                                            fields='files(id, name)'
+                                            ).execute()
+            items = results.get('files', [])
+            sub_folder_id_to_upload_to = None
+            if len(items) > 1:
+                logger.error(f'Duplicate folder name of {a} which should never happen')
+
+            for item in items:
+                logger.debug(f"Found existing folder of {item['name']}")
+                sub_folder_id_to_upload_to = item['id']
+
+            if sub_folder_id_to_upload_to is None:
+                # create new folder
+                file_metadata = {
+                    'name': [a],
+                    'mimeType': 'application/vnd.google-apps.folder',
+                    'parents': [folder_id_to_upload_to]
+                }
+                gd_file = self.service.files().create(body=file_metadata, fields='id').execute()
+                sub_folder_id_to_upload_to = gd_file.get('id')
+
+            filename = b
+            folder_id_to_upload_to = sub_folder_id_to_upload_to
+            # back to normal control flow
+            
+        # else:
+        # upload file to gd
+        file_metadata = {
+            # 'name': 'twitter__media_FMQg7yeXwAAwNEi.jpg',
+            'name': [filename],
+            'parents': [folder_id_to_upload_to]
+        }
+        media = MediaFileUpload(file, resumable=True)
+        gd_file = self.service.files().create(body=file_metadata,
+                                            media_body=media,
+                                            fields='id').execute()
diff --git a/utils/gworksheet.py b/utils/gworksheet.py
index 6dec9b2..42afe04 100644
--- a/utils/gworksheet.py
+++ b/utils/gworksheet.py
@@ -9,6 +9,7 @@ class GWorksheet:
     eg: if header=4, row 5 will be the first with data. 
     """
     COLUMN_NAMES = {
+        'filenumber': 'file number',
         'url': 'link',
         'archive': 'archive location',
         'date': 'archive date',

From c802a1516062f3f7428dfde167b3a90c616d946e Mon Sep 17 00:00:00 2001
From: msramalho <19508417+msramalho@users.noreply.github.com>
Date: Wed, 25 May 2022 12:19:04 +0200
Subject: [PATCH 3/9] ignoring new files

---
 .gitignore | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index f76014b..0cb7788 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,5 @@
 tmp/
+temp/
 .env*
 .DS_Store
 expmt/
@@ -8,5 +9,5 @@ __pycache__/
 anu.html
 *.log
 .pytest_cach
-
 anon*
+config*.json

From 0c1cb6d6af00f70417bbe7f1b615fed24ee2d6f4 Mon Sep 17 00:00:00 2001
From: msramalho <19508417+msramalho@users.noreply.github.com>
Date: Wed, 25 May 2022 12:19:18 +0200
Subject: [PATCH 4/9] improve comments

---
 .example.env | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.example.env b/.example.env
index 94d25af..46d7d7a 100644
--- a/.example.env
+++ b/.example.env
@@ -9,10 +9,10 @@ TELEGRAM_API_HASH=
 
 FACEBOOK_COOKIE=cookie: datr= xxxx
 
-# Google Drive, Right click on folder, Get link, eg  
-# https://drive.google.com/drive/folders/1ljwzoAdKdJMJzRW9gPHDC8fkRykVH83X?usp=sharing
-# we want:  1ljwzoAdKdJMJzRW9gPHDC8fkRykVH83X
+# Google Drive, Right click on folder, Get link: 
+# https://drive.google.com/drive/folders/123456789987654321abcdefghijk?usp=sharing
+# we want:  123456789987654321abcdefghijk
+# Remember to share the folder with the service email
+# autoarchiverservice@auto-archiver-333333.iam.gserviceaccount.com
 GD_ROOT_FOLDER_ID=
 
-# Remeber to share the folder with the service eg
-# autoarchiverservice@auto-archiver-333333.iam.gserviceaccount.com

From b58cbd2e858486cc21f693470648d24bec28b5b7 Mon Sep 17 00:00:00 2001
From: msramalho <19508417+msramalho@users.noreply.github.com>
Date: Wed, 25 May 2022 12:19:29 +0200
Subject: [PATCH 5/9] package management

---
 Pipfile      |   3 -
 Pipfile.lock | 389 ++++++++++++++++++++++++++++++++-------------------
 2 files changed, 246 insertions(+), 146 deletions(-)

diff --git a/Pipfile b/Pipfile
index 48f1429..7e5cbd7 100644
--- a/Pipfile
+++ b/Pipfile
@@ -22,8 +22,5 @@ google-auth-httplib2 = "*"
 google-auth-oauthlib = "*"
 oauth2client = "*"
 
-[dev-packages]
-autopep8 = "*"
-
 [requires]
 python_version = "3.9"
diff --git a/Pipfile.lock b/Pipfile.lock
index c48a8ca..9b5cbe7 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "e27ea0a6fdf6e588c14fbb90af45f784b9e55a9b986a3b50770490648ba96720"
+            "sha256": "25b858227d74cc232bba525d34dcf30f15d18d535a6e9c59555e85a0a2bd8c61"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -42,27 +42,27 @@
         },
         "beautifulsoup4": {
             "hashes": [
-                "sha256:9a315ce70049920ea4572a4055bc4bd700c940521d36fc858205ad4fcde149bf",
-                "sha256:c23ad23c521d818955a4151a67d81580319d4bf548d3d49f4223ae041ff98891"
+                "sha256:58d5c3d29f5a36ffeb94f02f0d786cd53014cf9b3b3951d42e0080d8a9498d30",
+                "sha256:ad9aa55b65ef2808eb405f46cf74df7fcb7044d5cbc26487f96eb2ef2e436693"
             ],
             "index": "pypi",
-            "version": "==4.10.0"
+            "version": "==4.11.1"
         },
         "boto3": {
             "hashes": [
-                "sha256:76d5b90400c54b25278150768e946edf166acce2c1597c0ecfbebb1dbe9acf2c",
-                "sha256:7bb2e6506a6ad44d111dd20a5d510374b6958fe989b4ef887109c79d812f926f"
+                "sha256:3fb956d097105a0fb98c29a622ff233fa8de68519aabd7088d7ffd36dfc33214",
+                "sha256:b59a210fa6a87f0c755b40403ffc66b9b285680bbc5ad5245cf167e2def33620"
             ],
             "index": "pypi",
-            "version": "==1.21.19"
+            "version": "==1.23.7"
         },
         "botocore": {
             "hashes": [
-                "sha256:5ed2be0e413961134f4c17eab16396d41a5b4b73a637588260c04d20806d52ea",
-                "sha256:d0d77bce152ca51f3c2cd0f9bf05cb3b623e719406ad58b4c20444e237fe82eb"
+                "sha256:0f4a467188644382856e96e85bff0b453442d5cf0c0f554154571a6e2468a005",
+                "sha256:9f8d5e8d65b24d97fcb7804b84831e5627fceb52707167d2f496477675c98ded"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==1.24.19"
+            "version": "==1.26.7"
         },
         "brotli": {
             "hashes": [
@@ -141,18 +141,19 @@
         },
         "cachetools": {
             "hashes": [
-                "sha256:486471dfa8799eb7ec503a8059e263db000cdda20075ce5e48903087f79d5fd6",
-                "sha256:8fecd4203a38af17928be7b90689d8083603073622229ca7077b72d8e5a976e4"
+                "sha256:4ebbd38701cdfd3603d1f751d851ed248ab4570929f2d8a7ce69e30c420b141c",
+                "sha256:8b3b8fa53f564762e5b221e9896798951e7f915513abf2ba072ce0f07f3f5a98"
             ],
             "markers": "python_version ~= '3.7'",
-            "version": "==5.0.0"
+            "version": "==5.1.0"
         },
         "certifi": {
             "hashes": [
-                "sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872",
-                "sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569"
+                "sha256:9c5705e395cd70084351dd8ad5c41e65655e08ce46f2ec9cf6c2c08390f71eb7",
+                "sha256:f1d53542ee8cbedbe2118b5686372fb33c297fcd6379b050cca0ef13a597382a"
             ],
-            "version": "==2021.10.8"
+            "markers": "python_version >= '3.6'",
+            "version": "==2022.5.18.1"
         },
         "cffi": {
             "hashes": [
@@ -219,11 +220,11 @@
         },
         "click": {
             "hashes": [
-                "sha256:6a7a62563bbfabfda3a38f3023a1db4a35978c0abd76f6c9605ecd6554d6d9b1",
-                "sha256:8458d7b1287c5fb128c90e23381cf99dcde74beaf6c7ff6384ce84d6fe090adb"
+                "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e",
+                "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==8.0.4"
+            "markers": "python_version >= '3.7'",
+            "version": "==8.1.3"
         },
         "cloudscraper": {
             "hashes": [
@@ -234,36 +235,38 @@
         },
         "cryptography": {
             "hashes": [
-                "sha256:0a817b961b46894c5ca8a66b599c745b9a3d9f822725221f0e0fe49dc043a3a3",
-                "sha256:2d87cdcb378d3cfed944dac30596da1968f88fb96d7fc34fdae30a99054b2e31",
-                "sha256:30ee1eb3ebe1644d1c3f183d115a8c04e4e603ed6ce8e394ed39eea4a98469ac",
-                "sha256:391432971a66cfaf94b21c24ab465a4cc3e8bf4a939c1ca5c3e3a6e0abebdbcf",
-                "sha256:39bdf8e70eee6b1c7b289ec6e5d84d49a6bfa11f8b8646b5b3dfe41219153316",
-                "sha256:4caa4b893d8fad33cf1964d3e51842cd78ba87401ab1d2e44556826df849a8ca",
-                "sha256:53e5c1dc3d7a953de055d77bef2ff607ceef7a2aac0353b5d630ab67f7423638",
-                "sha256:596f3cd67e1b950bc372c33f1a28a0692080625592ea6392987dba7f09f17a94",
-                "sha256:5d59a9d55027a8b88fd9fd2826c4392bd487d74bf628bb9d39beecc62a644c12",
-                "sha256:6c0c021f35b421ebf5976abf2daacc47e235f8b6082d3396a2fe3ccd537ab173",
-                "sha256:73bc2d3f2444bcfeac67dd130ff2ea598ea5f20b40e36d19821b4df8c9c5037b",
-                "sha256:74d6c7e80609c0f4c2434b97b80c7f8fdfaa072ca4baab7e239a15d6d70ed73a",
-                "sha256:7be0eec337359c155df191d6ae00a5e8bbb63933883f4f5dffc439dac5348c3f",
-                "sha256:94ae132f0e40fe48f310bba63f477f14a43116f05ddb69d6fa31e93f05848ae2",
-                "sha256:bb5829d027ff82aa872d76158919045a7c1e91fbf241aec32cb07956e9ebd3c9",
-                "sha256:ca238ceb7ba0bdf6ce88c1b74a87bffcee5afbfa1e41e173b1ceb095b39add46",
-                "sha256:ca28641954f767f9822c24e927ad894d45d5a1e501767599647259cbf030b903",
-                "sha256:e0344c14c9cb89e76eb6a060e67980c9e35b3f36691e15e1b7a9e58a0a6c6dc3",
-                "sha256:ebc15b1c22e55c4d5566e3ca4db8689470a0ca2babef8e3a9ee057a8b82ce4b1",
-                "sha256:ec63da4e7e4a5f924b90af42eddf20b698a70e58d86a72d943857c4c6045b3ee"
+                "sha256:093cb351031656d3ee2f4fa1be579a8c69c754cf874206be1d4cf3b542042804",
+                "sha256:0cc20f655157d4cfc7bada909dc5cc228211b075ba8407c46467f63597c78178",
+                "sha256:1b9362d34363f2c71b7853f6251219298124aa4cc2075ae2932e64c91a3e2717",
+                "sha256:1f3bfbd611db5cb58ca82f3deb35e83af34bb8cf06043fa61500157d50a70982",
+                "sha256:2bd1096476aaac820426239ab534b636c77d71af66c547b9ddcd76eb9c79e004",
+                "sha256:31fe38d14d2e5f787e0aecef831457da6cec68e0bb09a35835b0b44ae8b988fe",
+                "sha256:3b8398b3d0efc420e777c40c16764d6870bcef2eb383df9c6dbb9ffe12c64452",
+                "sha256:3c81599befb4d4f3d7648ed3217e00d21a9341a9a688ecdd615ff72ffbed7336",
+                "sha256:419c57d7b63f5ec38b1199a9521d77d7d1754eb97827bbb773162073ccd8c8d4",
+                "sha256:46f4c544f6557a2fefa7ac8ac7d1b17bf9b647bd20b16decc8fbcab7117fbc15",
+                "sha256:471e0d70201c069f74c837983189949aa0d24bb2d751b57e26e3761f2f782b8d",
+                "sha256:59b281eab51e1b6b6afa525af2bd93c16d49358404f814fe2c2410058623928c",
+                "sha256:731c8abd27693323b348518ed0e0705713a36d79fdbd969ad968fbef0979a7e0",
+                "sha256:95e590dd70642eb2079d280420a888190aa040ad20f19ec8c6e097e38aa29e06",
+                "sha256:a68254dd88021f24a68b613d8c51d5c5e74d735878b9e32cc0adf19d1f10aaf9",
+                "sha256:a7d5137e556cc0ea418dca6186deabe9129cee318618eb1ffecbd35bee55ddc1",
+                "sha256:aeaba7b5e756ea52c8861c133c596afe93dd716cbcacae23b80bc238202dc023",
+                "sha256:dc26bb134452081859aa21d4990474ddb7e863aa39e60d1592800a8865a702de",
+                "sha256:e53258e69874a306fcecb88b7534d61820db8a98655662a3dd2ec7f1afd9132f",
+                "sha256:ef15c2df7656763b4ff20a9bc4381d8352e6640cfeb95c2972c38ef508e75181",
+                "sha256:f224ad253cc9cea7568f49077007d2263efa57396a2f2f78114066fd54b5c68e",
+                "sha256:f8ec91983e638a9bcd75b39f1396e5c0dc2330cbd9ce4accefe68717e6779e0a"
             ],
-            "version": "==36.0.1"
+            "version": "==37.0.2"
         },
         "faker": {
             "hashes": [
-                "sha256:66db859b6abe376d02e805ad81eb8dcfce38f0945f17ee7cdf74ed349985ea52",
-                "sha256:fe969607836ce7100e38b88dcb598aacb733d895e6e9401894dd603e35623000"
+                "sha256:c6ff91847d7c820afc0a74d95e824b48aab71ddfd9003f300641e42d58ae886f",
+                "sha256:cad1f69d72a68878cd67855140b6fe3e44c11628971cd838595d289c98bc45de"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==13.3.2"
+            "version": "==13.11.1"
         },
         "ffmpeg-python": {
             "hashes": [
@@ -275,19 +278,19 @@
         },
         "filelock": {
             "hashes": [
-                "sha256:9cd540a9352e432c7246a48fe4e8712b10acb1df2ad1f30e8c070b82ae1fed85",
-                "sha256:f8314284bfffbdcfa0ff3d7992b023d4c628ced6feb957351d4c48d059f56bc0"
+                "sha256:b795f1b42a61bbf8ec7113c341dad679d772567b936fbd1bf43c9a238e673e20",
+                "sha256:c7b5fdb219b398a5b28c8e4c1893ef5f98ece6a38c6ab2c22e26ec161556fed6"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==3.6.0"
+            "version": "==3.7.0"
         },
         "flask": {
             "hashes": [
-                "sha256:59da8a3170004800a2837844bfa84d49b022550616070f7cb1a659682b2e7c9f",
-                "sha256:e1120c228ca2f553b470df4a5fa927ab66258467526069981b3eb0a91902687d"
+                "sha256:315ded2ddf8a6281567edb27393010fe3406188bafbfe65a3339d5787d89e477",
+                "sha256:fad5b446feb0d6db6aec0c3184d16a8c1f6c3e464b511649c8918a9be100b4fe"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==2.0.3"
+            "markers": "python_version >= '3.7'",
+            "version": "==2.1.2"
         },
         "future": {
             "hashes": [
@@ -296,29 +299,61 @@
             "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==0.18.2"
         },
+        "google-api-core": {
+            "hashes": [
+                "sha256:065bb8e11c605fd232707ae50963dc1c8af5b3c95b4568887515985e6c1156b3",
+                "sha256:1b9f59236ce1bae9a687c1d4f22957e79a2669e53d032893f6bf0fca54f6931d"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==2.8.0"
+        },
+        "google-api-python-client": {
+            "hashes": [
+                "sha256:4527f7b8518a795624ab68da412d55628f83b98c67dd6a5d6edf725454f8b30b",
+                "sha256:600c43d7eac6e3536fdcad1d14ba9ee503edf4c7db0bd827e791bbf03b9f1330"
+            ],
+            "index": "pypi",
+            "version": "==2.48.0"
+        },
         "google-auth": {
             "hashes": [
-                "sha256:218ca03d7744ca0c8b6697b6083334be7df49b7bf76a69d555962fd1a7657b5f",
-                "sha256:ad160fc1ea8f19e331a16a14a79f3d643d813a69534ba9611d2c80dc10439dad"
+                "sha256:1ba4938e032b73deb51e59c4656a00e0939cf0b1112575099f136babb4563312",
+                "sha256:349ac49b18b01019453cc99c11c92ed772739778c92f184002b7ab3a5b7ac77d"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
-            "version": "==2.6.0"
+            "version": "==2.6.6"
+        },
+        "google-auth-httplib2": {
+            "hashes": [
+                "sha256:31e49c36c6b5643b57e82617cb3e021e3e1d2df9da63af67252c02fa9c1f4a10",
+                "sha256:a07c39fd632becacd3f07718dfd6021bf396978f03ad3ce4321d060015cc30ac"
+            ],
+            "index": "pypi",
+            "version": "==0.1.0"
         },
         "google-auth-oauthlib": {
             "hashes": [
                 "sha256:24f67735513c4c7134dbde2f1dee5a1deb6acc8dfcb577d7bff30d213a28e7b0",
                 "sha256:30596b824fc6808fdaca2f048e4998cc40fb4b3599eaea66d28dc7085b36c5b8"
             ],
-            "markers": "python_version >= '3.6'",
+            "index": "pypi",
             "version": "==0.5.1"
         },
+        "googleapis-common-protos": {
+            "hashes": [
+                "sha256:6b5ee59dc646eb61a8eb65ee1db186d3df6687c8804830024f32573298bca19b",
+                "sha256:ddcd955b5bb6589368f659fa475373faa1ed7d09cde5ba25e88513d87007e174"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==1.56.1"
+        },
         "gspread": {
             "hashes": [
-                "sha256:05297b49587b5e89c2a0aa39967f43e5b7f170b62c11ddd43214baa1085131a8",
-                "sha256:25173ac081469cf9d621514c6576c6cf46f39c825f178b8cb9e78374a637b0bf"
+                "sha256:319766d90db05056293f7ee0ad2b35503a1a40683a75897a2922398cd2016283",
+                "sha256:c719e1c024a2a6f3b7d818fbe07c3886b26fd6504b64d1b1359cf242968213cd"
             ],
             "index": "pypi",
-            "version": "==5.2.0"
+            "version": "==5.3.2"
         },
         "h11": {
             "hashes": [
@@ -328,6 +363,14 @@
             "markers": "python_version >= '3.6'",
             "version": "==0.13.0"
         },
+        "httplib2": {
+            "hashes": [
+                "sha256:58a98e45b4b1a48273073f905d2961666ecf0fbac4250ea5b47aef259eb5c585",
+                "sha256:8b6a905cb1c79eefd03f8669fd993c36dc341f7c558f056cb5a33b5c2f458543"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==0.20.4"
+        },
         "idna": {
             "hashes": [
                 "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff",
@@ -336,29 +379,37 @@
             "markers": "python_version >= '3'",
             "version": "==3.3"
         },
+        "importlib-metadata": {
+            "hashes": [
+                "sha256:5d26852efe48c0a32b0509ffbc583fda1a2266545a78d104a6f4aff3db17d700",
+                "sha256:c58c8eb8a762858f49e18436ff552e83914778e50e9d2f1660535ffb364552ec"
+            ],
+            "markers": "python_version < '3.10'",
+            "version": "==4.11.4"
+        },
         "itsdangerous": {
             "hashes": [
-                "sha256:7b7d3023cd35d9cb0c1fd91392f8c95c6fa02c59bf8ad64b8849be3401b95afb",
-                "sha256:935642cd4b987cdbee7210080004033af76306757ff8b4c0a506a4b6e06f02cf"
+                "sha256:2c2349112351b88699d8d4b6b075022c0808887cb7ad10069318a8b0bc88db44",
+                "sha256:5dbbc68b317e5e42f327f9021763545dc3fc3bfe22e6deb96aaf1fc38874156a"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.1.1"
+            "version": "==2.1.2"
         },
         "jinja2": {
             "hashes": [
-                "sha256:077ce6014f7b40d03b47d1f1ca4b0fc8328a692bd284016f806ed0eaca390ad8",
-                "sha256:611bb273cd68f3b993fabdc4064fc858c5b47a973cb5aa7999ec1ba405c87cd7"
+                "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852",
+                "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==3.0.3"
+            "markers": "python_version >= '3.7'",
+            "version": "==3.1.2"
         },
         "jmespath": {
             "hashes": [
-                "sha256:b85d0567b8666149a93172712e68920734333c0ce7e89b78b3e987f71e5ed4f9",
-                "sha256:cdf6525904cc597730141d61b36f2e4b8ecc257c420fa2f4549bac2c2d0cb72f"
+                "sha256:a490e280edd1f57d6de88636992d05b71e97d69a26a19f058ecf7d304474bf5e",
+                "sha256:e8dcd576ed616f14ec02eed0005c85973b5890083313860136657e24784e4c04"
             ],
-            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==0.10.0"
+            "markers": "python_version >= '3.7'",
+            "version": "==1.0.0"
         },
         "loguru": {
             "hashes": [
@@ -489,6 +540,14 @@
             "markers": "python_version >= '3.5' and python_version < '4'",
             "version": "==1.45.1"
         },
+        "oauth2client": {
+            "hashes": [
+                "sha256:b8a81cc5d60e2d364f0b1b98f958dbd472887acaf1a5b05e21c28c31a2d6d3ac",
+                "sha256:d486741e451287f69568a4d26d70d9acd73a2bbfa275746c535b4209891cccc6"
+            ],
+            "index": "pypi",
+            "version": "==4.1.3"
+        },
         "oauthlib": {
             "hashes": [
                 "sha256:23a8208d75b902797ea29fd31fa80a15ed9dc2c6c16fe73f5d346f83f6fa27a2",
@@ -505,6 +564,36 @@
             "markers": "python_version >= '3.6'",
             "version": "==1.1.0"
         },
+        "protobuf": {
+            "hashes": [
+                "sha256:06059eb6953ff01e56a25cd02cca1a9649a75a7e65397b5b9b4e929ed71d10cf",
+                "sha256:097c5d8a9808302fb0da7e20edf0b8d4703274d140fd25c5edabddcde43e081f",
+                "sha256:284f86a6207c897542d7e956eb243a36bb8f9564c1742b253462386e96c6b78f",
+                "sha256:32ca378605b41fd180dfe4e14d3226386d8d1b002ab31c969c366549e66a2bb7",
+                "sha256:3cc797c9d15d7689ed507b165cd05913acb992d78b379f6014e013f9ecb20996",
+                "sha256:62f1b5c4cd6c5402b4e2d63804ba49a327e0c386c99b1675c8a0fefda23b2067",
+                "sha256:69ccfdf3657ba59569c64295b7d51325f91af586f8d5793b734260dfe2e94e2c",
+                "sha256:6f50601512a3d23625d8a85b1638d914a0970f17920ff39cec63aaef80a93fb7",
+                "sha256:7403941f6d0992d40161aa8bb23e12575637008a5a02283a930addc0508982f9",
+                "sha256:755f3aee41354ae395e104d62119cb223339a8f3276a0cd009ffabfcdd46bb0c",
+                "sha256:77053d28427a29987ca9caf7b72ccafee011257561259faba8dd308fda9a8739",
+                "sha256:7e371f10abe57cee5021797126c93479f59fccc9693dafd6bd5633ab67808a91",
+                "sha256:9016d01c91e8e625141d24ec1b20fed584703e527d28512aa8c8707f105a683c",
+                "sha256:9be73ad47579abc26c12024239d3540e6b765182a91dbc88e23658ab71767153",
+                "sha256:adc31566d027f45efe3f44eeb5b1f329da43891634d61c75a5944e9be6dd42c9",
+                "sha256:adfc6cf69c7f8c50fd24c793964eef18f0ac321315439d94945820612849c388",
+                "sha256:af0ebadc74e281a517141daad9d0f2c5d93ab78e9d455113719a45a49da9db4e",
+                "sha256:cb29edb9eab15742d791e1025dd7b6a8f6fcb53802ad2f6e3adcb102051063ab",
+                "sha256:cd68be2559e2a3b84f517fb029ee611546f7812b1fdd0aa2ecc9bc6ec0e4fdde",
+                "sha256:cdee09140e1cd184ba9324ec1df410e7147242b94b5f8b0c64fc89e38a8ba531",
+                "sha256:db977c4ca738dd9ce508557d4fce0f5aebd105e158c725beec86feb1f6bc20d8",
+                "sha256:dd5789b2948ca702c17027c84c2accb552fc30f4622a98ab5c51fcfe8c50d3e7",
+                "sha256:e250a42f15bf9d5b09fe1b293bdba2801cd520a9f5ea2d7fb7536d4441811d20",
+                "sha256:ff8d8fa42675249bb456f5db06c00de6c2f4c27a065955917b28c4f15978b9c3"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==3.20.1"
+        },
         "pyaes": {
             "hashes": [
                 "sha256:02c1b1405c38d3c370b085fb952dd8bea3fadcee6411ad99f312cc129c536d8f"
@@ -596,11 +685,11 @@
         },
         "pyparsing": {
             "hashes": [
-                "sha256:18ee9022775d270c55187733956460083db60b37d0d0fb357445f3094eed3eea",
-                "sha256:a6c06a88f252e6c322f65faf8f418b16213b51bdfaece0524c1c1bc30c63c484"
+                "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb",
+                "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==3.0.7"
+            "markers": "python_full_version >= '3.6.8'",
+            "version": "==3.0.9"
         },
         "pysocks": {
             "hashes": [
@@ -620,14 +709,13 @@
         },
         "python-dotenv": {
             "hashes": [
-                "sha256:32b2bdc1873fd3a3c346da1c6db83d0053c3c62f28f1f38516070c4c8971b1d3",
-                "sha256:a5de49a31e953b45ff2d2fd434bbc2670e8db5273606c1e737cc6b93eff3655f"
+                "sha256:b7e3b04a59693c42c36f9ab1cc2acc46fa5df8c78e178fc33a8d4cd05c8d498f",
+                "sha256:d92a187be61fe482e4fd675b6d52200e7be63a12b724abbf931a40ce4fa92938"
             ],
             "index": "pypi",
-            "version": "==0.19.2"
+            "version": "==0.20.0"
         },
         "requests": {
-            "extras": [],
             "hashes": [
                 "sha256:68d7c56fd5a8999887728ef304a6d12edc7be74f1cfa47714fc8b414525c9a61",
                 "sha256:f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d"
@@ -668,10 +756,10 @@
         },
         "selenium": {
             "hashes": [
-                "sha256:14d28a628c831c105d38305c881c9c7847199bfd728ec84240c5e86fa1c9bd5a"
+                "sha256:866b6dd6c459210662bff922ee7c33162d21920fbf6811e8e5a52be3866a687f"
             ],
             "index": "pypi",
-            "version": "==4.1.3"
+            "version": "==4.1.5"
         },
         "six": {
             "hashes": [
@@ -706,11 +794,11 @@
         },
         "soupsieve": {
             "hashes": [
-                "sha256:1a3cca2617c6b38c0343ed661b1fa5de5637f257d4fe22bd9f1338010a1efefb",
-                "sha256:b8d49b1cd4f037c7082a9683dfa1801aa2597fb11c3a1155b7a5b94829b4f1f9"
+                "sha256:3b2503d3c7084a42b1ebd08116e5f81aadfaea95863628c80a3b774a11b7c759",
+                "sha256:fc53893b3da2c33de295667a0e19f078c14bf86544af307354de5fcf12a3f30d"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==2.3.1"
+            "version": "==2.3.2.post1"
         },
         "telethon": {
             "hashes": [
@@ -740,76 +828,83 @@
             "markers": "python_version >= '3.5'",
             "version": "==0.9.2"
         },
-        "urllib3": {
-            "extras": [],
+        "uritemplate": {
             "hashes": [
-                "sha256:000ca7f471a233c2251c6c7023ee85305721bfdf18621ebff4fd17a8653427ed",
-                "sha256:0e7c33d9a63e7ddfcb86780aac87befc2fbddf46c58dbb487e0855f7ceec283c"
+                "sha256:4346edfc5c3b79f694bccd6d6099a322bbeb628dbf2cd86eea55a456ce5124f0",
+                "sha256:830c08b8d99bdd312ea4ead05994a38e8936266f84b9a7878232db50b044e02e"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==4.1.1"
+        },
+        "urllib3": {
+            "hashes": [
+                "sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14",
+                "sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e"
             ],
             "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
-            "version": "==1.26.8"
+            "version": "==1.26.9"
         },
         "websockets": {
             "hashes": [
-                "sha256:038afef2a05893578d10dadbdbb5f112bd115c46347e1efe99f6a356ff062138",
-                "sha256:05f6e9757017270e7a92a2975e2ae88a9a582ffc4629086fd6039aa80e99cd86",
-                "sha256:0b66421f9f13d4df60cd48ab977ed2c2b6c9147ae1a33caf5a9f46294422fda1",
-                "sha256:0cd02f36d37e503aca88ab23cc0a1a0e92a263d37acf6331521eb38040dcf77b",
-                "sha256:0f73cb2526d6da268e86977b2c4b58f2195994e53070fe567d5487c6436047e6",
-                "sha256:117383d0a17a0dda349f7a8790763dde75c1508ff8e4d6e8328b898b7df48397",
-                "sha256:1c1f3b18c8162e3b09761d0c6a0305fd642934202541cc511ef972cb9463261e",
-                "sha256:1c9031e90ebfc486e9cdad532b94004ade3aa39a31d3c46c105bb0b579cd2490",
-                "sha256:2349fa81b6b959484bb2bda556ccb9eb70ba68987646a0f8a537a1a18319fb03",
-                "sha256:24b879ba7db12bb525d4e58089fcbe6a3df3ce4666523183654170e86d372cbe",
-                "sha256:2aa9b91347ecd0412683f28aabe27f6bad502d89bd363b76e0a3508b1596402e",
-                "sha256:56d48eebe9e39ce0d68701bce3b21df923aa05dcc00f9fd8300de1df31a7c07c",
-                "sha256:5a38a0175ae82e4a8c4bac29fc01b9ee26d7d5a614e5ee11e7813c68a7d938ce",
-                "sha256:5b04270b5613f245ec84bb2c6a482a9d009aefad37c0575f6cda8499125d5d5c",
-                "sha256:6193bbc1ee63aadeb9a4d81de0e19477401d150d506aee772d8380943f118186",
-                "sha256:669e54228a4d9457abafed27cbf0e2b9f401445c4dfefc12bf8e4db9751703b8",
-                "sha256:6a009eb551c46fd79737791c0c833fc0e5b56bcd1c3057498b262d660b92e9cd",
-                "sha256:71a4491cfe7a9f18ee57d41163cb6a8a3fa591e0f0564ca8b0ed86b2a30cced4",
-                "sha256:7b38a5c9112e3dbbe45540f7b60c5204f49b3cb501b40950d6ab34cd202ab1d0",
-                "sha256:7bb9d8a6beca478c7e9bdde0159bd810cc1006ad6a7cb460533bae39da692ca2",
-                "sha256:82bc33db6d8309dc27a3bee11f7da2288ad925fcbabc2a4bb78f7e9c56249baf",
-                "sha256:8351c3c86b08156337b0e4ece0e3c5ec3e01fcd14e8950996832a23c99416098",
-                "sha256:8beac786a388bb99a66c3be4ab0fb38273c0e3bc17f612a4e0a47c4fc8b9c045",
-                "sha256:97950c7c844ec6f8d292440953ae18b99e3a6a09885e09d20d5e7ecd9b914cf8",
-                "sha256:98f57b3120f8331cd7440dbe0e776474f5e3632fdaa474af1f6b754955a47d71",
-                "sha256:9ca2ca05a4c29179f06cf6727b45dba5d228da62623ec9df4184413d8aae6cb9",
-                "sha256:a03a25d95cc7400bd4d61a63460b5d85a7761c12075ee2f51de1ffe73aa593d3",
-                "sha256:a10c0c1ee02164246f90053273a42d72a3b2452a7e7486fdae781138cf7fbe2d",
-                "sha256:a72b92f96e5e540d5dda99ee3346e199ade8df63152fa3c737260da1730c411f",
-                "sha256:ac081aa0307f263d63c5ff0727935c736c8dad51ddf2dc9f5d0c4759842aefaa",
-                "sha256:b22bdc795e62e71118b63e14a08bacfa4f262fd2877de7e5b950f5ac16b0348f",
-                "sha256:b4059e2ccbe6587b6dc9a01db5fc49ead9a884faa4076eea96c5ec62cb32f42a",
-                "sha256:b7fe45ae43ac814beb8ca09d6995b56800676f2cfa8e23f42839dc69bba34a42",
-                "sha256:bef03a51f9657fb03d8da6ccd233fe96e04101a852f0ffd35f5b725b28221ff3",
-                "sha256:bffc65442dd35c473ca9790a3fa3ba06396102a950794f536783f4b8060af8dd",
-                "sha256:c21a67ab9a94bd53e10bba21912556027fea944648a09e6508415ad14e37c325",
-                "sha256:c67d9cacb3f6537ca21e9b224d4fd08481538e43bcac08b3d93181b0816def39",
-                "sha256:c6e56606842bb24e16e36ae7eb308d866b4249cf0be8f63b212f287eeb76b124",
-                "sha256:cb316b87cbe3c0791c2ad92a5a36bf6adc87c457654335810b25048c1daa6fd5",
-                "sha256:cef40a1b183dcf39d23b392e9dd1d9b07ab9c46aadf294fff1350fb79146e72b",
-                "sha256:cf931c33db9c87c53d009856045dd524e4a378445693382a920fa1e0eb77c36c",
-                "sha256:d4d110a84b63c5cfdd22485acc97b8b919aefeecd6300c0c9d551e055b9a88ea",
-                "sha256:d5396710f86a306cf52f87fd8ea594a0e894ba0cc5a36059eaca3a477dc332aa",
-                "sha256:f09f46b1ff6d09b01c7816c50bd1903cf7d02ebbdb63726132717c2fcda835d5",
-                "sha256:f14bd10e170abc01682a9f8b28b16e6f20acf6175945ef38db6ffe31b0c72c3f",
-                "sha256:f5c335dc0e7dc271ef36df3f439868b3c790775f345338c2f61a562f1074187b",
-                "sha256:f8296b8408ec6853b26771599990721a26403e62b9de7e50ac0a056772ac0b5e",
-                "sha256:fa35c5d1830d0fb7b810324e9eeab9aa92e8f273f11fdbdc0741dcded6d72b9f"
+                "sha256:07cdc0a5b2549bcfbadb585ad8471ebdc7bdf91e32e34ae3889001c1c106a6af",
+                "sha256:210aad7fdd381c52e58777560860c7e6110b6174488ef1d4b681c08b68bf7f8c",
+                "sha256:28dd20b938a57c3124028680dc1600c197294da5db4292c76a0b48efb3ed7f76",
+                "sha256:2f94fa3ae454a63ea3a19f73b95deeebc9f02ba2d5617ca16f0bbdae375cda47",
+                "sha256:31564a67c3e4005f27815634343df688b25705cccb22bc1db621c781ddc64c69",
+                "sha256:347974105bbd4ea068106ec65e8e8ebd86f28c19e529d115d89bd8cc5cda3079",
+                "sha256:379e03422178436af4f3abe0aa8f401aa77ae2487843738542a75faf44a31f0c",
+                "sha256:3eda1cb7e9da1b22588cefff09f0951771d6ee9fa8dbe66f5ae04cc5f26b2b55",
+                "sha256:51695d3b199cd03098ae5b42833006a0f43dc5418d3102972addc593a783bc02",
+                "sha256:54c000abeaff6d8771a4e2cef40900919908ea7b6b6a30eae72752607c6db559",
+                "sha256:5b936bf552e4f6357f5727579072ff1e1324717902127ffe60c92d29b67b7be3",
+                "sha256:6075fd24df23133c1b078e08a9b04a3bc40b31a8def4ee0b9f2c8865acce913e",
+                "sha256:661f641b44ed315556a2fa630239adfd77bd1b11cb0b9d96ed8ad90b0b1e4978",
+                "sha256:6ea6b300a6bdd782e49922d690e11c3669828fe36fc2471408c58b93b5535a98",
+                "sha256:6ed1d6f791eabfd9808afea1e068f5e59418e55721db8b7f3bfc39dc831c42ae",
+                "sha256:7934e055fd5cd9dee60f11d16c8d79c4567315824bacb1246d0208a47eca9755",
+                "sha256:7ab36e17af592eec5747c68ef2722a74c1a4a70f3772bc661079baf4ae30e40d",
+                "sha256:7f6d96fdb0975044fdd7953b35d003b03f9e2bcf85f2d2cf86285ece53e9f991",
+                "sha256:83e5ca0d5b743cde3d29fda74ccab37bdd0911f25bd4cdf09ff8b51b7b4f2fa1",
+                "sha256:85506b3328a9e083cc0a0fb3ba27e33c8db78341b3eb12eb72e8afd166c36680",
+                "sha256:8af75085b4bc0b5c40c4a3c0e113fa95e84c60f4ed6786cbb675aeb1ee128247",
+                "sha256:8b1359aba0ff810d5830d5ab8e2c4a02bebf98a60aa0124fb29aa78cfdb8031f",
+                "sha256:8fbd7d77f8aba46d43245e86dd91a8970eac4fb74c473f8e30e9c07581f852b2",
+                "sha256:907e8247480f287aa9bbc9391bd6de23c906d48af54c8c421df84655eef66af7",
+                "sha256:93d5ea0b5da8d66d868b32c614d2b52d14304444e39e13a59566d4acb8d6e2e4",
+                "sha256:97bc9d41e69a7521a358f9b8e44871f6cdeb42af31815c17aed36372d4eec667",
+                "sha256:994cdb1942a7a4c2e10098d9162948c9e7b235df755de91ca33f6e0481366fdb",
+                "sha256:a141de3d5a92188234afa61653ed0bbd2dde46ad47b15c3042ffb89548e77094",
+                "sha256:a1e15b230c3613e8ea82c9fc6941b2093e8eb939dd794c02754d33980ba81e36",
+                "sha256:aad5e300ab32036eb3fdc350ad30877210e2f51bceaca83fb7fef4d2b6c72b79",
+                "sha256:b529fdfa881b69fe563dbd98acce84f3e5a67df13de415e143ef053ff006d500",
+                "sha256:b9c77f0d1436ea4b4dc089ed8335fa141e6a251a92f75f675056dac4ab47a71e",
+                "sha256:bb621ec2dbbbe8df78a27dbd9dd7919f9b7d32a73fafcb4d9252fc4637343582",
+                "sha256:c7250848ce69559756ad0086a37b82c986cd33c2d344ab87fea596c5ac6d9442",
+                "sha256:c8d1d14aa0f600b5be363077b621b1b4d1eb3fbf90af83f9281cda668e6ff7fd",
+                "sha256:d1655a6fc7aecd333b079d00fb3c8132d18988e47f19740c69303bf02e9883c6",
+                "sha256:d6353ba89cfc657a3f5beabb3b69be226adbb5c6c7a66398e17809b0ce3c4731",
+                "sha256:da4377904a3379f0c1b75a965fff23b28315bcd516d27f99a803720dfebd94d4",
+                "sha256:e49ea4c1a9543d2bd8a747ff24411509c29e4bdcde05b5b0895e2120cb1a761d",
+                "sha256:e4e08305bfd76ba8edab08dcc6496f40674f44eb9d5e23153efa0a35750337e8",
+                "sha256:e6fa05a680e35d0fcc1470cb070b10e6fe247af54768f488ed93542e71339d6f",
+                "sha256:e7e6f2d6fd48422071cc8a6f8542016f350b79cc782752de531577d35e9bd677",
+                "sha256:e904c0381c014b914136c492c8fa711ca4cced4e9b3d110e5e7d436d0fc289e8",
+                "sha256:ec2b0ab7edc8cd4b0eb428b38ed89079bdc20c6bdb5f889d353011038caac2f9",
+                "sha256:ef5ce841e102278c1c2e98f043db99d6755b1c58bde475516aef3a008ed7f28e",
+                "sha256:f351c7d7d92f67c0609329ab2735eee0426a03022771b00102816a72715bb00b",
+                "sha256:fab7c640815812ed5f10fbee7abbf58788d602046b7bb3af9b1ac753a6d5e916",
+                "sha256:fc06cc8073c8e87072138ba1e431300e2d408f054b27047d047b549455066ff4"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==10.2"
+            "version": "==10.3"
         },
         "werkzeug": {
             "hashes": [
-                "sha256:1421ebfc7648a39a5c58c601b154165d05cf47a3cd0ccb70857cbdacf6c8f2b8",
-                "sha256:b863f8ff057c522164b6067c9e28b041161b4be5ba4d0daceeaa50a163822d3c"
+                "sha256:1ce08e8093ed67d638d63879fd1ba3735817f7a80de3674d293f5984f25fb6e6",
+                "sha256:72a4b735692dd3135217911cbeaa1be5fa3f62bffb8745c5215420a03dc55255"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==2.0.3"
+            "markers": "python_version >= '3.7'",
+            "version": "==2.1.2"
         },
         "wsproto": {
             "hashes": [
@@ -821,11 +916,19 @@
         },
         "yt-dlp": {
             "hashes": [
-                "sha256:05179f0f2c34f06910003bb9f80af68ff798b072ca0f826c0e6704a3fbd5b306",
-                "sha256:68546578c18e6ce87450b53769d5d5b7f5a23e5209784976db6c7ccbf7954b21"
+                "sha256:3a7b59d2fb4b39ce8ba8e0b9c5a37fe20e5624f46a2346b4ae66ab1320e35134",
+                "sha256:deec1009442312c1e2ee5298966842194d0e950b433f0d4fc844ef464b9c32a7"
             ],
             "index": "pypi",
-            "version": "==2022.3.8.2"
+            "version": "==2022.5.18"
+        },
+        "zipp": {
+            "hashes": [
+                "sha256:56bf8aadb83c24db6c4b577e13de374ccfb67da2078beba1d037c17980bf43ad",
+                "sha256:c4f6e5bbf48e74f7a38e7cc5b0480ff42b0ae5178957d564d18932525d5cf099"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==3.8.0"
         }
     },
     "develop": {}

From 93cf3a89378ac57657a1553820d796d0dbb0b401 Mon Sep 17 00:00:00 2001
From: msramalho <19508417+msramalho@users.noreply.github.com>
Date: Wed, 25 May 2022 12:19:37 +0200
Subject: [PATCH 6/9] remove vscode files for now

---
 .vscode/launch.json | 27 ---------------------------
 1 file changed, 27 deletions(-)
 delete mode 100644 .vscode/launch.json

diff --git a/.vscode/launch.json b/.vscode/launch.json
deleted file mode 100644
index 1a82e0c..0000000
--- a/.vscode/launch.json
+++ /dev/null
@@ -1,27 +0,0 @@
-{
-	// Use IntelliSense to learn about possible attributes.
-	// Hover to view descriptions of existing attributes.
-	// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
-	"version": "0.2.0",
-	"configurations": [
-		{
-			"name": "Test Hashing",
-			"type": "python",
-			"request": "launch",
-			"program": "auto_archive.py",
-			"console": "integratedTerminal",
-			"justMyCode": true,
-			// "args": ["--sheet","Test Hashing"]
-			// "args": ["--sheet","Test Hashing","--use-filenumber-as-directory"]
-			"args": ["--sheet","Test Hashing","--use-filenumber-as-directory", "--storage=gd"]
-		},
-		{
-			"name": "Python: Current File",
-			"type": "python",
-			"request": "launch",
-			"program": "${file}",
-			"console": "integratedTerminal",
-			"justMyCode": true
-		}
-	]
-}
\ No newline at end of file

From b895def4323918106b43f1a98d15636c94214d14 Mon Sep 17 00:00:00 2001
From: msramalho <19508417+msramalho@users.noreply.github.com>
Date: Wed, 25 May 2022 12:23:52 +0200
Subject: [PATCH 7/9] method customization to children

---
 storages/base_storage.py | 11 ++-----
 storages/gd_storage.py   | 68 ++++++++++++++++++++--------------------
 2 files changed, 36 insertions(+), 43 deletions(-)

diff --git a/storages/base_storage.py b/storages/base_storage.py
index 79a555b..e1bf9c7 100644
--- a/storages/base_storage.py
+++ b/storages/base_storage.py
@@ -17,12 +17,5 @@ class Storage(ABC):
 
     def upload(self, filename: str, key: str, **kwargs):
         logger.debug(f'[{self.__class__.__name__}] uploading file {filename} with key {key}')
-        # S3 requires an open file, GD only the filename
-        storage = type(self).__name__
-        if storage == "GDStorage":
-            self.uploadf(filename, key, **kwargs)
-        elif storage == "S3Storage":
-            with open(filename, 'rb') as f:
-                self.uploadf(f, key, **kwargs)
-        else:
-            raise ValueError('Cant get storage thrown from base_storage.py')
\ No newline at end of file
+        with open(filename, 'rb') as f:
+            self.uploadf(f, key, **kwargs)
diff --git a/storages/gd_storage.py b/storages/gd_storage.py
index 4dab7d0..0e21dfa 100644
--- a/storages/gd_storage.py
+++ b/storages/gd_storage.py
@@ -8,10 +8,12 @@ from google.oauth2 import service_account
 
 import time
 
+
 @dataclass
 class GDConfig:
     root_folder_id: str
 
+
 class GDStorage(Storage):
 
     def __init__(self, config: GDConfig):
@@ -42,14 +44,14 @@ class GDStorage(Storage):
             # need to lookup the id of folder eg SM0002 which should be there already as this is get_cdn_url
             results = self.service.files().list(q=f"'{self.root_folder_id}' in parents \
                                                 and name = '{foldername}' ",
-                                            spaces='drive', # ie not appDataFolder or photos
-                                            fields='files(id, name)'
-                                            ).execute()
+                                                spaces='drive',  # ie not appDataFolder or photos
+                                                fields='files(id, name)'
+                                                ).execute()
             items = results.get('files', [])
 
             for item in items:
                 logger.debug(f"found folder of {item['name']}")
-                folder_id= item['id']
+                folder_id = item['id']
                 try_again = False
 
             if folder_id is None:
@@ -72,9 +74,9 @@ class GDStorage(Storage):
             results = self.service.files().list(q=f"'{folder_id}' in parents \
                                                 and mimeType='application/vnd.google-apps.folder' \
                                                 and name = '{a}' ",
-                                            spaces='drive', # ie not appDataFolder or photos
-                                            fields='files(id, name)'
-                                            ).execute()
+                                                spaces='drive',  # ie not appDataFolder or photos
+                                                fields='files(id, name)'
+                                                ).execute()
             items = results.get('files', [])
 
             filename = None
@@ -87,47 +89,40 @@ class GDStorage(Storage):
         # get id of file inside folder (or sub folder)
         results = self.service.files().list(q=f"'{folder_id}' in parents \
                                             and name = '{filename}' ",
-                                        spaces='drive', 
-                                        fields='files(id, name)'
-                                        ).execute()
+                                            spaces='drive',
+                                            fields='files(id, name)'
+                                            ).execute()
         items = results.get('files', [])
-        
+
         file_id = None
         for item in items:
             logger.debug(f"found file of {item['name']}")
-            file_id= item['id']
+            file_id = item['id']
 
         if file_id is None:
             raise ValueError(f'Problem finding file {filename} in folder_id {folder_id}')
-            
+
         foo = "https://drive.google.com/file/d/" + file_id + "/view?usp=sharing"
         return foo
 
-    def exists(self, key):
-        # Not implemented yet
-        # Google drive will accept duplicate named filenames as it is stored as a different fileid 
-
-        # try:
-        #     self.s3.head_object(Bucket=self.bucket, Key=self._get_path(key))
-        #     return True
-        # except ClientError:
-        #     return False
+    def exists(self, _key):
+        # TODO: How to check for google drive, as it accepts different names
         return False
 
-    def uploadf(self, file, key, **kwargs):
+    def uploadf(self, file, key, **_kwargs):
         # split on first occurance of /
         # eg SM0005
         foldername = key.split('/', 1)[0]
         # eg twitter__media_asdf.jpg
         filename = key.split('/', 1)[1]
 
-         # does folder eg SM0005 exist already inside parent of Files auto-archiver
+        # does folder eg SM0005 exist already inside parent of Files auto-archiver
         results = self.service.files().list(q=f"'{self.root_folder_id}' in parents \
                                             and mimeType='application/vnd.google-apps.folder' \
                                             and name = '{foldername}' ",
-                                        spaces='drive', 
-                                        fields='files(id, name)'
-                                        ).execute()
+                                            spaces='drive',
+                                            fields='files(id, name)'
+                                            ).execute()
         items = results.get('files', [])
         folder_id_to_upload_to = None
         if len(items) > 1:
@@ -146,7 +141,7 @@ class GDStorage(Storage):
             }
             gd_file = self.service.files().create(body=file_metadata, fields='id').execute()
             folder_id_to_upload_to = gd_file.get('id')
-        
+
         # check for subfolder nema in file eg youtube_dl_sDE-qZdi8p8/out1.jpg'
         # happens doing thumbnails
 
@@ -163,9 +158,9 @@ class GDStorage(Storage):
             results = self.service.files().list(q=f"'{folder_id_to_upload_to}' in parents \
                                                 and mimeType='application/vnd.google-apps.folder' \
                                                 and name = '{a}' ",
-                                            spaces='drive', # ie not appDataFolder or photos
-                                            fields='files(id, name)'
-                                            ).execute()
+                                                spaces='drive',  # ie not appDataFolder or photos
+                                                fields='files(id, name)'
+                                                ).execute()
             items = results.get('files', [])
             sub_folder_id_to_upload_to = None
             if len(items) > 1:
@@ -188,7 +183,7 @@ class GDStorage(Storage):
             filename = b
             folder_id_to_upload_to = sub_folder_id_to_upload_to
             # back to normal control flow
-            
+
         # else:
         # upload file to gd
         file_metadata = {
@@ -198,5 +193,10 @@ class GDStorage(Storage):
         }
         media = MediaFileUpload(file, resumable=True)
         gd_file = self.service.files().create(body=file_metadata,
-                                            media_body=media,
-                                            fields='id').execute()
+                                              media_body=media,
+                                              fields='id').execute()
+
+    def upload(self, filename: str, key: str, **kwargs):
+        # GD only requires the filename not a file reader
+        logger.debug(f'[{self.__class__.__name__}] uploading file {filename} with key {key}')
+        self.uploadf(filename, key, **kwargs)

From 03aa02e88b8a61801b7e813bf48fe7074a8c36ec Mon Sep 17 00:00:00 2001
From: msramalho <19508417+msramalho@users.noreply.github.com>
Date: Wed, 25 May 2022 12:23:59 +0200
Subject: [PATCH 8/9] diagram

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 0f1b2e1..e95949e 100644
--- a/README.md
+++ b/README.md
@@ -98,6 +98,7 @@ graph TD
 ```mermaid
 graph TD
     A(BaseStorage) -->|parent of| B(S3Storage)
+    A(BaseStorage) -->|parent of| C(GoogleDriveStorage)
 ```
 
 ## Saving into Folders

From 159adf9afed369cb52ea65bbcffe66ac4c39b32c Mon Sep 17 00:00:00 2001
From: msramalho <19508417+msramalho@users.noreply.github.com>
Date: Thu, 26 May 2022 19:18:29 +0200
Subject: [PATCH 9/9] refactoring filenumber into subfolder

---
 README.md                       |  12 +---
 archivers/base_archiver.py      |  33 ++++------
 archivers/telegram_archiver.py  |   9 +--
 archivers/telethon_archiver.py  |  27 ++++-----
 archivers/tiktok_archiver.py    |   2 +-
 archivers/twitter_archiver.py   |  14 ++---
 archivers/wayback_archiver.py   |   4 +-
 archivers/youtubedl_archiver.py |  17 ++----
 auto_archive.py                 |  58 +++++++-----------
 storages/base_storage.py        |  23 +++++++
 storages/gd_storage.py          | 103 ++++++++++++++------------------
 storages/s3_storage.py          |  10 ++--
 utils/gworksheet.py             |  11 +++-
 13 files changed, 144 insertions(+), 179 deletions(-)

diff --git a/README.md b/README.md
index e95949e..c52b077 100644
--- a/README.md
+++ b/README.md
@@ -101,24 +101,18 @@ graph TD
     A(BaseStorage) -->|parent of| C(GoogleDriveStorage)
 ```
 
-## Saving into Folders
+## Saving into Subfolders
 
-To use a column from the spreadsheet called `File Number` eg SM001234 as a directory on the cloud storage, you need to pass in
-
-```bash
-python auto_archive.py --sheet 'Sheet Name' --use-filenumber-as-directory
-```
+You can have a column in the spreadsheet for the argument `--col-subfolder` that is passed to the storage and can specify a subfolder to put the archived link into.
 
 ## Google Drive
 
 To use Google Drive storage you need the id of the shared folder in the `.env` file which must be shared with the service account eg `autoarchiverservice@auto-archiver-111111.iam.gserviceaccount.com`
 
 ```bash
-python auto_archive.py --sheet 'Sheet Name' --use-filenumber-as-directory --storage='gd'
+python auto_archive.py --sheet 'Sheet Name' --storage='gd'
 ```
 
-Note the you must use filenumber for Google Drive Storage.
-
 ## Telethon (Telegrams API Library)
 
 Put your `anon.session` in the root, so that it doesn't stall and ask for authentication
diff --git a/archivers/base_archiver.py b/archivers/base_archiver.py
index 367b483..6e11957 100644
--- a/archivers/base_archiver.py
+++ b/archivers/base_archiver.py
@@ -18,6 +18,7 @@ from selenium.webdriver.common.by import By
 from loguru import logger
 from selenium.common.exceptions import TimeoutException
 
+
 @dataclass
 class ArchiveResult:
     status: str
@@ -42,7 +43,7 @@ class Archiver(ABC):
         return self.__class__.__name__
 
     @abstractmethod
-    def download(self, url, check_if_exists=False, filenumber=None): pass
+    def download(self, url, check_if_exists=False): pass
 
     def get_netloc(self, url):
         return urlparse(url).netloc
@@ -51,7 +52,7 @@ class Archiver(ABC):
         return self.get_key(urlparse(url).path.replace("/", "_") + ".html")
 
     # generate the html page eg SM3013/twitter__minmyatnaing13_status_1499415562937503751.html
-    def generate_media_page_html(self, url, urls_info: dict, object, thumbnail=None, filenumber=None):
+    def generate_media_page_html(self, url, urls_info: dict, object, thumbnail=None):
         page = f'''<html><head><title>{url}</title><meta charset="UTF-8"></head>
             <body>
             <h2>Archived media from {self.name}</h2>
@@ -71,10 +72,6 @@ class Archiver(ABC):
 
         page_hash = self.get_hash(page_filename)
 
-        if filenumber != None:
-            logger.trace(f'filenumber for directory is {filenumber}')
-            page_key = filenumber + "/" + page_key
-
         self.storage.upload(page_filename, page_key, extra_args={
             'ACL': 'public-read', 'ContentType': 'text/html'})
 
@@ -82,7 +79,7 @@ class Archiver(ABC):
         return (page_cdn, page_hash, thumbnail)
 
     # eg images in a tweet save to cloud storage
-    def generate_media_page(self, urls, url, object, filenumber=None):
+    def generate_media_page(self, urls, url, object):
         headers = {
             'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36'
         }
@@ -102,10 +99,6 @@ class Archiver(ABC):
             with open(filename, 'wb') as f:
                 f.write(d.content)
 
-            if filenumber is not None:
-                logger.debug(f'filenumber for directory is {filenumber}')
-                key = filenumber + "/" + key
-
             # eg filename: 'tmp/twitter__media_FM7-ggCUYAQHKWW.jpg'
             # eg key: 'twitter__media_FM7-ggCUYAQHKWW.jpg'
             # or if using filename key: 'SM3013/twitter__media_FM7-ggCUYAQHKWW.jpg'
@@ -120,7 +113,7 @@ class Archiver(ABC):
                 thumbnail = cdn_url
             uploaded_media.append({'cdn_url': cdn_url, 'key': key, 'hash': hash})
 
-        return self.generate_media_page_html(url, uploaded_media, object, thumbnail=thumbnail, filenumber=filenumber)
+        return self.generate_media_page_html(url, uploaded_media, object, thumbnail=thumbnail)
 
     def get_key(self, filename):
         """
@@ -140,16 +133,15 @@ class Archiver(ABC):
     def get_hash(self, filename):
         f = open(filename, "rb")
         bytes = f.read()  # read entire file as bytes
-        
+
+        # TODO: customizable hash
         hash = hashlib.sha256(bytes)
         # option to use SHA3_512 instead
         # hash = hashlib.sha3_512(bytes)
         f.close()
         return hash.hexdigest()
 
-    # eg SA3013/twitter__minmyatnaing13_status_14994155629375037512022-04-27T13:51:43.701962.png
-    # def get_screenshot(self, url, filenumber, storage="GD"):
-    def get_screenshot(self, url, filenumber):
+    def get_screenshot(self, url):
         key = self.get_key(urlparse(url).path.replace(
             "/", "_") + datetime.datetime.utcnow().isoformat().replace(" ", "_") + ".png")
         filename = 'tmp/' + key
@@ -158,8 +150,8 @@ class Archiver(ABC):
         if 'facebook.com' in url:
             try:
                 logger.debug(f'Trying fb click accept cookie popup for {url}')
-                self.driver.get("http://www.facebook.com") 
-                foo = self.driver.find_element(By.XPATH,"//button[@data-cookiebanner='accept_only_essential_button']")
+                self.driver.get("http://www.facebook.com")
+                foo = self.driver.find_element(By.XPATH, "//button[@data-cookiebanner='accept_only_essential_button']")
                 foo.click()
                 logger.debug(f'fb click worked')
                 # linux server needs a sleep otherwise facebook cookie wont have worked and we'll get a popup on next page
@@ -174,11 +166,6 @@ class Archiver(ABC):
             logger.info("TimeoutException loading page for screenshot")
 
         self.driver.save_screenshot(filename)
-
-        if filenumber is not None:
-            logger.debug(f'filenumber for directory is {filenumber}')
-            key = filenumber + "/" + key
-
         self.storage.upload(filename, key, extra_args={
                             'ACL': 'public-read', 'ContentType': 'image/png'})
 
diff --git a/archivers/telegram_archiver.py b/archivers/telegram_archiver.py
index b19ab8f..5a7f63c 100644
--- a/archivers/telegram_archiver.py
+++ b/archivers/telegram_archiver.py
@@ -11,7 +11,7 @@ from .base_archiver import Archiver, ArchiveResult
 class TelegramArchiver(Archiver):
     name = "telegram"
 
-    def download(self, url, check_if_exists=False, filenumber=None):
+    def download(self, url, check_if_exists=False):
         # detect URLs that we definitely cannot handle
         if 't.me' != self.get_netloc(url):
             return False
@@ -27,7 +27,7 @@ class TelegramArchiver(Archiver):
         if url[-8:] != "?embed=1":
             url += "?embed=1"
 
-        screenshot = self.get_screenshot(url, filenumber=filenumber)
+        screenshot = self.get_screenshot(url)
 
         t = requests.get(url, headers=headers)
         s = BeautifulSoup(t.content, 'html.parser')
@@ -42,7 +42,7 @@ class TelegramArchiver(Archiver):
                 urls = [u.replace("'", "") for u in re.findall(r'url\((.*?)\)', im['style'])]
                 images += urls
 
-            page_cdn, page_hash, thumbnail = self.generate_media_page(images, url, html.escape(str(t.content)),filenumber=filenumber)
+            page_cdn, page_hash, thumbnail = self.generate_media_page(images, url, html.escape(str(t.content)))
             time_elements = s.find_all('time')
             timestamp = time_elements[0].get('datetime') if len(time_elements) else None
 
@@ -52,9 +52,6 @@ class TelegramArchiver(Archiver):
         video_id = video_url.split('/')[-1].split('?')[0]
         key = self.get_key(video_id)
 
-        if filenumber is not None:
-            key = filenumber + "/" + key
-
         filename = 'tmp/' + key
         cdn_url = self.storage.get_cdn_url(key)
 
diff --git a/archivers/telethon_archiver.py b/archivers/telethon_archiver.py
index 5cee791..9e92383 100644
--- a/archivers/telethon_archiver.py
+++ b/archivers/telethon_archiver.py
@@ -7,6 +7,7 @@ from loguru import logger
 from storages import Storage
 from .base_archiver import Archiver, ArchiveResult
 from telethon.sync import TelegramClient
+from telethon.errors import ChannelInvalidError
 
 
 @dataclass
@@ -41,14 +42,14 @@ class TelethonArchiver(Archiver):
                 media.append(post)
         return media
 
-    def download(self, url, check_if_exists=False, filenumber=None):
+    def download(self, url, check_if_exists=False):
         # detect URLs that we definitely cannot handle
         matches = self.link_pattern.findall(url)
         if not len(matches):
             return False
 
         status = "success"
-        screenshot = self.get_screenshot(url, filenumber)
+        screenshot = self.get_screenshot(url)
 
         # app will ask (stall for user input!) for phone number and auth code if anon.session not found
         with self.client.start():
@@ -60,7 +61,11 @@ class TelethonArchiver(Archiver):
             try:
                 post = self.client.get_messages(chat, ids=post_id)
             except ValueError as e:
-                logger.warning(f'Could not fetch telegram {url} possibly it\'s private: {e}')
+                logger.error(f'Could not fetch telegram {url} possibly it\'s private: {e}')
+                return False
+            except ChannelInvalidError as e:
+                # TODO: check followup here: https://github.com/LonamiWebs/Telethon/issues/3819
+                logger.error(f'Could not fetch telegram {url} possibly it\'s private or not displayable in : {e}')
                 return False
 
             media_posts = self._get_media_posts_in_group(chat, post)
@@ -68,11 +73,8 @@ class TelethonArchiver(Archiver):
             if len(media_posts) > 1:
                 key = self.get_html_key(url)
 
-                if filenumber is not None:
-                    key = filenumber + "/" + key          
-
                 if check_if_exists and self.storage.exists(key):
-                     # only s3 storage supports storage.exists as not implemented on gd
+                    # only s3 storage supports storage.exists as not implemented on gd
                     cdn_url = self.storage.get_cdn_url(key)
                     status = 'already archived'
                     return ArchiveResult(status='already archived', cdn_url=cdn_url, title=post.message, timestamp=post.date, screenshot=screenshot)
@@ -84,26 +86,19 @@ class TelethonArchiver(Archiver):
                     if len(mp.message) > len(message): message = mp.message
                     filename = self.client.download_media(mp.media, f'tmp/{chat}_{group_id}/{mp.id}')
                     key = filename.split('tmp/')[1]
-
-                    if filenumber is not None:
-                        key = filenumber + "/" + key
                     self.storage.upload(filename, key)
                     hash = self.get_hash(filename)
                     cdn_url = self.storage.get_cdn_url(key)
                     uploaded_media.append({'cdn_url': cdn_url, 'key': key, 'hash': hash})
                     os.remove(filename)
 
-                page_cdn, page_hash, _ = self.generate_media_page_html(url, uploaded_media, html.escape(str(post)), filenumber=filenumber)
+                page_cdn, page_hash, _ = self.generate_media_page_html(url, uploaded_media, html.escape(str(post)))
 
                 return ArchiveResult(status=status, cdn_url=page_cdn, title=post.message, timestamp=post.date, hash=page_hash, screenshot=screenshot)
             elif len(media_posts) == 1:
                 key = self.get_key(f'{chat}_{post_id}')
                 filename = self.client.download_media(post.media, f'tmp/{key}')
                 key = filename.split('tmp/')[1].replace(" ", "")
-
-                if filenumber is not None:
-                    key = filenumber + "/" + key
-
                 self.storage.upload(filename, key)
                 hash = self.get_hash(filename)
                 cdn_url = self.storage.get_cdn_url(key)
@@ -112,5 +107,5 @@ class TelethonArchiver(Archiver):
 
                 return ArchiveResult(status=status, cdn_url=cdn_url, title=post.message, thumbnail=key_thumb, thumbnail_index=thumb_index, timestamp=post.date, hash=hash, screenshot=screenshot)
 
-            page_cdn, page_hash, _ = self.generate_media_page_html(url, [], html.escape(str(post)), filenumber=filenumber)
+            page_cdn, page_hash, _ = self.generate_media_page_html(url, [], html.escape(str(post)))
             return ArchiveResult(status=status, cdn_url=page_cdn, title=post.message, timestamp=post.date, hash=page_hash, screenshot=screenshot)
diff --git a/archivers/tiktok_archiver.py b/archivers/tiktok_archiver.py
index 9b90efa..47b8374 100644
--- a/archivers/tiktok_archiver.py
+++ b/archivers/tiktok_archiver.py
@@ -8,7 +8,7 @@ from .base_archiver import Archiver, ArchiveResult
 class TiktokArchiver(Archiver):
     name = "tiktok"
 
-    def download(self, url, check_if_exists=False, filenumber=None):
+    def download(self, url, check_if_exists=False):
         if 'tiktok.com' not in url:
             return False
 
diff --git a/archivers/twitter_archiver.py b/archivers/twitter_archiver.py
index 05e7ec0..04ed578 100644
--- a/archivers/twitter_archiver.py
+++ b/archivers/twitter_archiver.py
@@ -8,15 +8,15 @@ from .base_archiver import Archiver, ArchiveResult
 class TwitterArchiver(Archiver):
     name = "twitter"
 
-    def download(self, url, check_if_exists=False, filenumber=None):
-        
+    def download(self, url, check_if_exists=False):
+
         if 'twitter.com' != self.get_netloc(url):
             return False
 
         tweet_id = urlparse(url).path.split('/')
         if 'status' in tweet_id:
             i = tweet_id.index('status')
-            tweet_id = tweet_id[i+1]
+            tweet_id = tweet_id[i + 1]
         else:
             return False
 
@@ -25,9 +25,7 @@ class TwitterArchiver(Archiver):
         try:
             tweet = next(scr.get_items())
         except Exception as ex:
-            template = "TwitterArchiver cant get tweet and threw, which can happen if a media sensitive tweet. \n type: {0} occurred. \n arguments:{1!r}"
-            message = template.format(type(ex).__name__, ex.args)
-            logger.warning(message)
+            logger.warning(f"can't get tweet: {type(ex).__name__} occurred. args: {ex.args}")
             return False
 
         if tweet.media is None:
@@ -48,8 +46,8 @@ class TwitterArchiver(Archiver):
             else:
                 logger.warning(f"Could not get media URL of {media}")
 
-        page_cdn, page_hash, thumbnail = self.generate_media_page(urls, url, tweet.json(), filenumber)
+        page_cdn, page_hash, thumbnail = self.generate_media_page(urls, url, tweet.json())
 
-        screenshot = self.get_screenshot(url, filenumber)
+        screenshot = self.get_screenshot(url)
 
         return ArchiveResult(status="success", cdn_url=page_cdn, screenshot=screenshot, hash=page_hash, thumbnail=thumbnail, timestamp=tweet.date)
diff --git a/archivers/wayback_archiver.py b/archivers/wayback_archiver.py
index 652798a..d8479f1 100644
--- a/archivers/wayback_archiver.py
+++ b/archivers/wayback_archiver.py
@@ -14,7 +14,7 @@ class WaybackArchiver(Archiver):
         super(WaybackArchiver, self).__init__(storage, driver)
         self.seen_urls = {}
 
-    def download(self, url, check_if_exists=False, filenumber=None):
+    def download(self, url, check_if_exists=False):
         if check_if_exists and url in self.seen_urls:
             return self.seen_urls[url]
 
@@ -75,7 +75,7 @@ class WaybackArchiver(Archiver):
         except:
             title = "Could not get title"
 
-        screenshot = self.get_screenshot(url, filenumber)
+        screenshot = self.get_screenshot(url)
         result = ArchiveResult(status='Internet Archive fallback', cdn_url=archive_url, title=title, screenshot=screenshot)
         self.seen_urls[url] = result
         return result
diff --git a/archivers/youtubedl_archiver.py b/archivers/youtubedl_archiver.py
index 9983950..a6ea615 100644
--- a/archivers/youtubedl_archiver.py
+++ b/archivers/youtubedl_archiver.py
@@ -7,6 +7,7 @@ from loguru import logger
 from .base_archiver import Archiver, ArchiveResult
 from storages import Storage
 
+
 class YoutubeDLArchiver(Archiver):
     name = "youtube_dl"
     ydl_opts = {'outtmpl': 'tmp/%(id)s.%(ext)s', 'quiet': False}
@@ -15,7 +16,7 @@ class YoutubeDLArchiver(Archiver):
         super().__init__(storage, driver)
         self.fb_cookie = fb_cookie
 
-    def download(self, url, check_if_exists=False, filenumber=None):
+    def download(self, url, check_if_exists=False):
         netloc = self.get_netloc(url)
         if netloc in ['facebook.com', 'www.facebook.com']:
             logger.debug('Using Facebook cookie')
@@ -61,9 +62,6 @@ class YoutubeDLArchiver(Archiver):
 
             key = self.get_key(filename)
 
-            if filenumber is not None:
-                key = filenumber + "/" + key
-
             if self.storage.exists(key):
                 status = 'already archived'
                 cdn_url = self.storage.get_cdn_url(key)
@@ -87,10 +85,6 @@ class YoutubeDLArchiver(Archiver):
 
         if status != 'already archived':
             key = self.get_key(filename)
-            
-            if filenumber is not None:
-                key = filenumber + "/" + key
-
             self.storage.upload(filename, key)
 
             # filename ='tmp/sDE-qZdi8p8.webm'
@@ -98,8 +92,7 @@ class YoutubeDLArchiver(Archiver):
             cdn_url = self.storage.get_cdn_url(key)
 
         hash = self.get_hash(filename)
-        screenshot = self.get_screenshot(url, filenumber)
-
+        screenshot = self.get_screenshot(url)
 
         # get duration
         duration = info.get('duration')
@@ -115,9 +108,9 @@ class YoutubeDLArchiver(Archiver):
 
         timestamp = datetime.datetime.utcfromtimestamp(info['timestamp']).replace(tzinfo=datetime.timezone.utc).isoformat() \
             if 'timestamp' in info else \
-                datetime.datetime.strptime(info['upload_date'], '%Y%m%d').replace(tzinfo=datetime.timezone.utc) \
+            datetime.datetime.strptime(info['upload_date'], '%Y%m%d').replace(tzinfo=datetime.timezone.utc) \
             if 'upload_date' in info and info['upload_date'] is not None else \
-                None
+            None
 
         return ArchiveResult(status=status, cdn_url=cdn_url, thumbnail=key_thumb, thumbnail_index=thumb_index, duration=duration,
                              title=info['title'] if 'title' in info else None, timestamp=timestamp, hash=hash, screenshot=screenshot)
diff --git a/auto_archive.py b/auto_archive.py
index a3c17d1..8044e06 100644
--- a/auto_archive.py
+++ b/auto_archive.py
@@ -23,6 +23,7 @@ logger.add("logs/5error.log", level="ERROR")
 
 load_dotenv()
 
+
 def update_sheet(gw, row, result: archivers.ArchiveResult):
     cell_updates = []
     row_values = gw.get_row(row)
@@ -68,7 +69,7 @@ def expand_url(url):
     return url
 
 
-def process_sheet(sheet, usefilenumber=False, storage="s3", header=1, columns=GWorksheet.COLUMN_NAMES):
+def process_sheet(sheet, storage="s3", header=1, columns=GWorksheet.COLUMN_NAMES):
     gc = gspread.service_account(filename='service_account.json')
     sh = gc.open(sheet)
 
@@ -86,8 +87,6 @@ def process_sheet(sheet, usefilenumber=False, storage="s3", header=1, columns=GW
         api_hash=os.getenv('TELEGRAM_API_HASH')
     )
 
-    
-
     # loop through worksheets to check
     for ii, wks in enumerate(sh.worksheets()):
         logger.info(f'Opening worksheet {ii=}: {wks.title=} {header=}')
@@ -120,17 +119,8 @@ def process_sheet(sheet, usefilenumber=False, storage="s3", header=1, columns=GW
                 gw.set_cell(row, 'status', 'Archive in progress')
 
                 url = expand_url(url)
-                
-                if usefilenumber:
-                    filenumber = gw.get_cell(row, 'filenumber')
-                    logger.debug(f'filenumber is {filenumber}')
-                    if filenumber == "":
-                        logger.warning(f'Logic error on row {row} with url {url} - the feature flag for usefilenumber is True, yet cant find a corresponding filenumber')
-                        gw.set_cell(row, 'status', 'Missing filenumber')
-                        continue
-                else:
-                    # We will use this through the app to differentiate between where to save
-                    filenumber = None
+
+                subfolder = gw.get_cell_or_default(row, 'subfolder')
 
                 # make a new driver so each spreadsheet row is idempotent
                 options = webdriver.FirefoxOptions()
@@ -142,7 +132,7 @@ def process_sheet(sheet, usefilenumber=False, storage="s3", header=1, columns=GW
                 # in seconds, telegram screenshots catch which don't come back
                 driver.set_page_load_timeout(120)
 
-                 # client
+                # client
                 storage_client = None
                 if storage == "s3":
                     storage_client = s3_client
@@ -150,6 +140,7 @@ def process_sheet(sheet, usefilenumber=False, storage="s3", header=1, columns=GW
                     storage_client = gd_client
                 else:
                     raise ValueError(f'Cant get storage_client {storage_client}')
+                storage_client.update_properties(subfolder=subfolder)
 
                 # order matters, first to succeed excludes remaining
                 active_archivers = [
@@ -164,12 +155,12 @@ def process_sheet(sheet, usefilenumber=False, storage="s3", header=1, columns=GW
                     logger.debug(f'Trying {archiver} on row {row}')
 
                     try:
-                        if usefilenumber:
-                            # using filenumber to store in folders so not checking for existence of that url
-                            result = archiver.download(url, check_if_exists=False, filenumber=filenumber)
-                        else:
-                            result = archiver.download(url, check_if_exists=True)
-
+                        result = archiver.download(url, check_if_exists=True)
+                    except KeyboardInterrupt:
+                        logger.warning("caught interrupt")
+                        gw.set_cell(row, 'status', '')
+                        driver.quit()
+                        exit()
                     except Exception as e:
                         result = False
                         logger.error(f'Got unexpected error in row {row} with archiver {archiver} for url {url}: {e}\n{traceback.format_exc()}')
@@ -180,9 +171,9 @@ def process_sheet(sheet, usefilenumber=False, storage="s3", header=1, columns=GW
                             result.status = archiver.name + \
                                 ": " + str(result.status)
                             logger.success(
-                                 f'{archiver} succeeded on row {row}, url {url}')
+                                f'{archiver} succeeded on row {row}, url {url}')
                             break
-                        
+
                         # wayback has seen this url before so keep existing status
                         if "wayback: Internet Archive fallback" in result.status:
                             logger.success(
@@ -203,6 +194,7 @@ def process_sheet(sheet, usefilenumber=False, storage="s3", header=1, columns=GW
                     gw.set_cell(row, 'status', 'failed: no archiver')
         logger.success(f'Finshed worksheet {wks.title}')
 
+
 @logger.catch
 def main():
     logger.debug(f'Passed args:{sys.argv}')
@@ -213,27 +205,21 @@ def main():
     parser.add_argument('--header', action='store', dest='header', default=1, type=int, help='1-based index for the header row')
     parser.add_argument('--private', action='store_true', help='Store content without public access permission')
 
-    parser.add_argument('--use-filenumber-as-directory', action=argparse.BooleanOptionalAction, dest='usefilenumber',  \
-         help='Will save files into a subfolder on cloud storage which has the File Number eg SM3012')
-    parser.add_argument('--storage', action='store', dest='storage', default='s3', \
-         help='s3 or gd storage. Default is s3. NOTE GD storage supports only using filenumber')
+    parser.add_argument('--storage', action='store', dest='storage', default='s3', help='which storage to use.', choices={"s3", "gd"})
 
     for k, v in GWorksheet.COLUMN_NAMES.items():
-        parser.add_argument(f'--col-{k}', action='store', dest=k, default=v, help=f'the name of the column to fill with {k} (defaults={v})')
+        help = f"the name of the column to fill with {k} (defaults={v})"
+        if k == "subfolder":
+            help = f"the name of the column to read the {k} from (defaults={v})"
+        parser.add_argument(f'--col-{k}', action='store', dest=k, default=v, help=help)
 
     args = parser.parse_args()
     config_columns = {k: getattr(args, k).lower() for k in GWorksheet.COLUMN_NAMES.keys()}
 
-    logger.info(f'Opening document {args.sheet} for header {args.header} using filenumber: {args.usefilenumber} and storage {args.storage}')
-
-    # https://stackoverflow.com/questions/15008758/parsing-boolean-values-with-argparse
-    # args.filenumber is True (of type bool) when set or None when argument is not there
-    usefilenumber = False
-    if args.usefilenumber:
-        usefilenumber = True
+    logger.info(f'Opening document {args.sheet} for header {args.header} and storage {args.storage}')
 
     mkdir_if_not_exists('tmp')
-    process_sheet(args.sheet, usefilenumber, args.storage, args.header, config_columns)
+    process_sheet(args.sheet, args.storage, args.header, config_columns)
     shutil.rmtree('tmp')
 
 
diff --git a/storages/base_storage.py b/storages/base_storage.py
index e1bf9c7..108e05f 100644
--- a/storages/base_storage.py
+++ b/storages/base_storage.py
@@ -1,5 +1,6 @@
 from loguru import logger
 from abc import ABC, abstractmethod
+from pathlib import Path
 
 
 class Storage(ABC):
@@ -19,3 +20,25 @@ class Storage(ABC):
         logger.debug(f'[{self.__class__.__name__}] uploading file {filename} with key {key}')
         with open(filename, 'rb') as f:
             self.uploadf(f, key, **kwargs)
+
+    def update_properties(self, **kwargs):
+        """
+        method used to update general properties that some children may use 
+        and others not, but that all can call
+        """
+        for k, v in kwargs.items():
+            if k in self.get_allowed_properties():
+                setattr(self, k, v)
+            else:
+                logger.warning(f'[{self.__class__.__name__}] does not accept dynamic property "{k}"')
+
+    def get_allowed_properties(self):
+        """
+        child classes should specify which properties they allow to be set
+        """
+        return set(["subfolder"])
+
+    def clean_path(self, folder, default="", add_forward_slash=True):
+        if folder is None or type(folder) != str or len(folder.strip()) == 0:
+            return default
+        return str(Path(folder)) + ("/" if add_forward_slash else "")
diff --git a/storages/gd_storage.py b/storages/gd_storage.py
index 0e21dfa..3d65519 100644
--- a/storages/gd_storage.py
+++ b/storages/gd_storage.py
@@ -15,6 +15,7 @@ class GDConfig:
 
 
 class GDStorage(Storage):
+    DEFAULT_UPLOAD_FOLDER_NAME = "default"
 
     def __init__(self, config: GDConfig):
         self.root_folder_id = config.root_folder_id
@@ -22,19 +23,14 @@ class GDStorage(Storage):
         creds = service_account.Credentials.from_service_account_file('service_account.json', scopes=SCOPES)
         self.service = build('drive', 'v3', credentials=creds)
 
-    def _get_path(self, key):
-        return self.folder + key
-
     def get_cdn_url(self, key):
-        # only support files saved in a folders for GD
-        # S3 supports folder and all stored in the root
-
-        # key will be SM0002/twitter__media_ExeUSW2UcAE6RbN.jpg
-        foldername = key.split('/', 1)[0]
-        # eg twitter__media_asdf.jpg
-        filename = key.split('/', 1)[1]
-
-        logger.debug(f'Looking for {foldername} and filename: {filename} on GD')
+        """
+        only support files saved in a folder for GD
+        S3 supports folder and all stored in the root
+        """
+        self.subfolder = self.clean_path(self.subfolder, GDStorage.DEFAULT_UPLOAD_FOLDER_NAME, False)
+        filename = key
+        logger.debug(f'Looking for {self.subfolder} and filename: {filename} on GD')
 
         # retry policy on Google Drive
         try_again = True
@@ -42,11 +38,11 @@ class GDStorage(Storage):
         folder_id = None
         while try_again:
             # need to lookup the id of folder eg SM0002 which should be there already as this is get_cdn_url
-            results = self.service.files().list(q=f"'{self.root_folder_id}' in parents \
-                                                and name = '{foldername}' ",
-                                                spaces='drive',  # ie not appDataFolder or photos
-                                                fields='files(id, name)'
-                                                ).execute()
+            results = self.service.files().list(
+                q=f"'{self.root_folder_id}' in parents and name = '{self.subfolder}' ",
+                spaces='drive',  # ie not appDataFolder or photos
+                fields='files(id, name)'
+            ).execute()
             items = results.get('files', [])
 
             for item in items:
@@ -55,11 +51,11 @@ class GDStorage(Storage):
                 try_again = False
 
             if folder_id is None:
-                logger.debug(f'Cant find {foldername=} waiting and trying again {counter=}')
+                logger.debug(f'Cannot find {self.subfolder=} waiting and trying again {counter=}')
                 counter += 1
                 time.sleep(10)
                 if counter > 18:
-                    raise ValueError(f'Cant find  {foldername} and retried 18 times pausing 10seconds at a time which is 3 minutes')
+                    raise ValueError(f'Cannot find  {self.subfolder} and retried 18 times pausing 10s at a time which is 3 minutes')
 
         # check for sub folder in file eg youtube_dl_sDE-qZdi8p8/index.html'
         # happens doing thumbnails
@@ -71,12 +67,11 @@ class GDStorage(Storage):
             logger.debug(f'get_cdn_url: Found a subfolder so need to split on a: {a} and {b}')
 
             # get id of the sub folder
-            results = self.service.files().list(q=f"'{folder_id}' in parents \
-                                                and mimeType='application/vnd.google-apps.folder' \
-                                                and name = '{a}' ",
-                                                spaces='drive',  # ie not appDataFolder or photos
-                                                fields='files(id, name)'
-                                                ).execute()
+            results = self.service.files().list(
+                q=f"'{folder_id}' in parents and mimeType='application/vnd.google-apps.folder' and name = '{a}' ",
+                spaces='drive',  # ie not appDataFolder or photos
+                fields='files(id, name)'
+            ).execute()
             items = results.get('files', [])
 
             filename = None
@@ -87,11 +82,11 @@ class GDStorage(Storage):
                 raise ValueError(f'Problem finding sub folder {a}')
 
         # get id of file inside folder (or sub folder)
-        results = self.service.files().list(q=f"'{folder_id}' in parents \
-                                            and name = '{filename}' ",
-                                            spaces='drive',
-                                            fields='files(id, name)'
-                                            ).execute()
+        results = self.service.files().list(
+            q=f"'{folder_id}' in parents and name = '{filename}' ",
+            spaces='drive',
+            fields='files(id, name)'
+        ).execute()
         items = results.get('files', [])
 
         file_id = None
@@ -110,41 +105,36 @@ class GDStorage(Storage):
         return False
 
     def uploadf(self, file, key, **_kwargs):
-        # split on first occurance of /
-        # eg SM0005
-        foldername = key.split('/', 1)[0]
-        # eg twitter__media_asdf.jpg
-        filename = key.split('/', 1)[1]
-
+        logger.debug(f"before {self.subfolder=}")
+        self.subfolder = self.clean_path(self.subfolder, GDStorage.DEFAULT_UPLOAD_FOLDER_NAME, False)
+        filename = key
+        logger.debug(f"after {self.subfolder=}")
         # does folder eg SM0005 exist already inside parent of Files auto-archiver
-        results = self.service.files().list(q=f"'{self.root_folder_id}' in parents \
-                                            and mimeType='application/vnd.google-apps.folder' \
-                                            and name = '{foldername}' ",
-                                            spaces='drive',
-                                            fields='files(id, name)'
-                                            ).execute()
+        results = self.service.files().list(
+            q=f"'{self.root_folder_id}' in parents and mimeType='application/vnd.google-apps.folder' and name = '{self.subfolder}' ",
+            spaces='drive',
+            fields='files(id, name)'
+        ).execute()
         items = results.get('files', [])
         folder_id_to_upload_to = None
         if len(items) > 1:
-            logger.error(f'Duplicate folder name of {foldername} which should never happen, but continuing anyway')
+            logger.error(f'Duplicate folder name of {self.subfolder} which should never happen, but continuing anyway')
 
         for item in items:
             logger.debug(f"Found existing folder of {item['name']}")
             folder_id_to_upload_to = item['id']
 
         if folder_id_to_upload_to is None:
-            logger.debug(f'Creating new folder {foldername}')
+            logger.debug(f'Creating new folder {self.subfolder}')
             file_metadata = {
-                'name': [foldername],
+                'name': [self.subfolder],
                 'mimeType': 'application/vnd.google-apps.folder',
                 'parents': [self.root_folder_id]
             }
             gd_file = self.service.files().create(body=file_metadata, fields='id').execute()
             folder_id_to_upload_to = gd_file.get('id')
 
-        # check for subfolder nema in file eg youtube_dl_sDE-qZdi8p8/out1.jpg'
-        # happens doing thumbnails
-
+        # check for subfolder name in file eg youtube_dl_sDE-qZdi8p8/out1.jpg', eg: thumbnails
         # will always return a and a blank b even if there is nothing to split
         # https://stackoverflow.com/a/38149500/26086
         a, _, b = filename.partition('/')
@@ -155,12 +145,11 @@ class GDStorage(Storage):
             logger.debug(f'uploadf: Found a subfolder so need to split on a: {a} and {b}')
 
             # does the 'a' folder exist already in folder_id_to_upload_to eg SM0005
-            results = self.service.files().list(q=f"'{folder_id_to_upload_to}' in parents \
-                                                and mimeType='application/vnd.google-apps.folder' \
-                                                and name = '{a}' ",
-                                                spaces='drive',  # ie not appDataFolder or photos
-                                                fields='files(id, name)'
-                                                ).execute()
+            results = self.service.files().list(
+                q=f"'{folder_id_to_upload_to}' in parents and mimeType='application/vnd.google-apps.folder' and name = '{a}' ",
+                spaces='drive',  # ie not appDataFolder or photos
+                fields='files(id, name)'
+            ).execute()
             items = results.get('files', [])
             sub_folder_id_to_upload_to = None
             if len(items) > 1:
@@ -184,17 +173,13 @@ class GDStorage(Storage):
             folder_id_to_upload_to = sub_folder_id_to_upload_to
             # back to normal control flow
 
-        # else:
         # upload file to gd
         file_metadata = {
-            # 'name': 'twitter__media_FMQg7yeXwAAwNEi.jpg',
             'name': [filename],
             'parents': [folder_id_to_upload_to]
         }
         media = MediaFileUpload(file, resumable=True)
-        gd_file = self.service.files().create(body=file_metadata,
-                                              media_body=media,
-                                              fields='id').execute()
+        gd_file = self.service.files().create(body=file_metadata, media_body=media, fields='id').execute()
 
     def upload(self, filename: str, key: str, **kwargs):
         # GD only requires the filename not a file reader
diff --git a/storages/s3_storage.py b/storages/s3_storage.py
index d7c9644..fd127e2 100644
--- a/storages/s3_storage.py
+++ b/storages/s3_storage.py
@@ -2,6 +2,7 @@ import boto3
 from botocore.errorfactory import ClientError
 from .base_storage import Storage
 from dataclasses import dataclass
+from loguru import logger
 
 
 @dataclass
@@ -19,12 +20,9 @@ class S3Storage(Storage):
     def __init__(self, config: S3Config):
         self.bucket = config.bucket
         self.region = config.region
-        self.folder = config.folder
+        self.folder = self.clean_path(config.folder)
         self.private = config.private
 
-        if len(self.folder) and self.folder[-1] != '/':
-            self.folder += '/'
-
         self.s3 = boto3.client(
             's3',
             region_name=self.region,
@@ -34,7 +32,7 @@ class S3Storage(Storage):
         )
 
     def _get_path(self, key):
-        return self.folder + key
+        return self.folder + self.clean_path(self.subfolder) + key
 
     def get_cdn_url(self, key):
         return f'https://{self.bucket}.{self.region}.cdn.digitaloceanspaces.com/{self._get_path(key)}'
@@ -47,9 +45,9 @@ class S3Storage(Storage):
             return False
 
     def uploadf(self, file, key, **kwargs):
+        logger.debug(f'[S3 storage] uploading {file=}, {key=}')
         if self.private:
             extra_args = kwargs.get("extra_args", {})
         else:
             extra_args = kwargs.get("extra_args", {'ACL': 'public-read'})
-
         self.s3.upload_fileobj(file, Bucket=self.bucket, Key=self._get_path(key), ExtraArgs=extra_args)
diff --git a/utils/gworksheet.py b/utils/gworksheet.py
index 42afe04..403c453 100644
--- a/utils/gworksheet.py
+++ b/utils/gworksheet.py
@@ -9,8 +9,8 @@ class GWorksheet:
     eg: if header=4, row 5 will be the first with data. 
     """
     COLUMN_NAMES = {
-        'filenumber': 'file number',
         'url': 'link',
+        'subfolder': 'sub folder',
         'archive': 'archive location',
         'date': 'archive date',
         'status': 'archive status',
@@ -69,6 +69,15 @@ class GWorksheet:
             return ''
         return row[col_index]
 
+    def get_cell_or_default(self, row, col: str, default: str = None, fresh=False):
+        """
+        return self.get_cell or default value on error (eg: column is missing)
+        """
+        try:
+            return self.get_cell(row, col, fresh)
+        except:
+            return default
+
     def set_cell(self, row: int, col: str, val):
         # row is 1-based
         col_index = self._col_index(col) + 1