Refactoring of storage code:

1. Fix some bugs in local_storage
2. Refactor unit tests to not set Media.key explicitly (unless it's well-known beforehand, which it isn't)
3. Limit length of URL for 'url' type path_generator
4. Throw an error if 'save_to' of local storage is too long
5. A few other tidyups

src/auto_archiver/modules/cli_feeder/cli_feeder.py

@@ -15,7 +15,6 @@ class CLIFeeder(Feeder):
         for url in urls:
             logger.debug(f"Processing {url}")
             m = Metadata().set_url(url)
-            m.set_context("folder", "cli")
             yield m
 
         logger.success(f"Processed {len(urls)} URL(s)")

src/auto_archiver/modules/local_storage/local_storage.py

@@ -6,37 +6,34 @@ from loguru import logger
 
 from auto_archiver.core import Media
 from auto_archiver.core import Storage
-
+from auto_archiver.core.consts import SetupError
 
 class LocalStorage(Storage):
 
-    MAX_FILE_LENGTH = 255
+
+    def setup(self) -> None:
+        if len(self.save_to) > 200:
+            raise SetupError(f"Your save_to path is  long, this will cause issues saving files on your computer. Please use a shorter path")
 
     def get_cdn_url(self, media: Media) -> str:
-        # TODO: is this viable with Storage.configs on path/filename?
-        dest = os.path.join(self.save_to, media.key)
+        dest = media.key
+
         if self.save_absolute:
             dest = os.path.abspath(dest)
         return dest
 
+    def set_key(self, media, url, metadata):
+        # clarify we want to save the file to the save_to folder
+
+        old_folder = metadata.get('folder', '')
+        metadata.set_context('folder', os.path.join(self.save_to, metadata.get('folder', '')))
+        super().set_key(media, url, metadata)
+        # don't impact other storages that might want a different 'folder' set
+        metadata.set_context('folder', old_folder)
+
     def upload(self, media: Media, **kwargs) -> bool:
         # override parent so that we can use shutil.copy2 and keep metadata
-        dest = os.path.join(self.save_to, media.key)
-
-        if len(dest) > self.max_file_length():
-            old_dest_length = len(dest)
-            filename, ext = os.path.splitext(media.key)
-            dir, filename = os.path.split(filename)
-            # see whether we should truncate filename or dir
-            if len(dir) > len(filename):
-                dir = dir[:self.MAX_FILE_LENGTH - len(self.save_to) - len(ext) - len(filename) - 1]
-            else:
-                filename = filename[:self.MAX_FILE_LENGTH - len(self.save_to) - len(ext) - len(filename) - 1]
-
-            # override media.key
-            media.key = os.path.join(dir, f"{filename}{ext}")
-            dest = os.path.join(self.save_to, dir, f"{filename}{ext}")
-            logger.warning(f'Filename too long ({old_dest_length} characters), truncating to {len(dest)} characters')
+        dest = media.key
 
         os.makedirs(os.path.dirname(dest), exist_ok=True)
         logger.debug(f'[{self.__class__.__name__}] storing file {media.filename} with key {media.key} to {dest}')
@@ -46,7 +43,5 @@ class LocalStorage(Storage):
         return True
 
     # must be implemented even if unused
-    def uploadf(self, file: IO[bytes], key: str, **kwargs: dict) -> bool: pass
-
-    def max_file_length(self):
-        return self.MAX_FILE_LENGTH
+    def uploadf(self, file: IO[bytes], key: str, **kwargs: dict) -> bool:
+        pass

src/auto_archiver/modules/s3_storage/s3_storage.py

@@ -42,7 +42,7 @@ class S3Storage(Storage):
                 logger.warning(f"Unable to get mimetype for {media.key=}, error: {e}")
         self.s3.upload_fileobj(file, Bucket=self.bucket, Key=media.key, ExtraArgs=extra_args)
         return True
-    
+
     def is_upload_needed(self, media: Media) -> bool:
         if self.random_no_duplicate:
             # checks if a folder with the hash already exists, if so it skips the upload
@@ -50,13 +50,13 @@ class S3Storage(Storage):
             path = os.path.join(NO_DUPLICATES_FOLDER, hd[:24])
 
             if existing_key:=self.file_in_folder(path):
-                media.key = existing_key
+                media._key = existing_key
                 media.set("previously archived", True)
                 logger.debug(f"skipping upload of {media.filename} because it already exists in {media.key}")
                 return False
             
             _, ext = os.path.splitext(media.key)
-            media.key = os.path.join(path, f"{random_str(24)}{ext}")
+            media._key = os.path.join(path, f"{random_str(24)}{ext}")
         return True
 
     def file_in_folder(self, path:str) -> str:
@@ -66,9 +66,4 @@ class S3Storage(Storage):
         resp = self.s3.list_objects(Bucket=self.bucket, Prefix=path, Delimiter='/', MaxKeys=1)
         if 'Contents' in resp:
             return resp['Contents'][0]['Key']
-        return False
-    
-    def max_file_length(self):
-        # Amazon AWS max file length is 1024, but we will use 1000 to be safe
-        return 1000
-
+        return False