Refactoring of storage code:

1. Fix some bugs in local_storage
2. Refactor unit tests to not set Media.key explicitly (unless it's well-known beforehand, which it isn't)
3. Limit length of URL for 'url' type path_generator
4. Throw an error if 'save_to' of local storage is too long
5. A few other tidyups

src/auto_archiver/core/config.py

@@ -14,7 +14,6 @@ from loguru import logger
 from copy import deepcopy
 from auto_archiver.core.consts import MODULE_TYPES
 
-from typing import Any, List, Type, Tuple
 
 _yaml: YAML = YAML()
 

src/auto_archiver/core/consts.py

@@ -1,3 +1,5 @@
+class SetupError(ValueError):
+    pass
 
 MODULE_TYPES = [
     'feeder',

src/auto_archiver/core/extractor.py

@@ -81,7 +81,8 @@ class Extractor(BaseModule):
             if len(to_filename) > 64:
                 to_filename = to_filename[-64:]
         to_filename = os.path.join(self.tmp_dir, to_filename)
-        if verbose: logger.debug(f"downloading {url[0:50]=} {to_filename=}")
+        if verbose:
+            logger.debug(f"downloading {url[0:50]=} {to_filename=}")
         headers = {
             'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36'
         }

src/auto_archiver/core/media.py

@@ -21,14 +21,13 @@ class Media:
     Represents a media file with associated properties and storage details.
 
     Attributes:
-    - filename: The file path of the media.
-    - key: An optional identifier for the media.
+    - filename: The file path of the media as saved locally (temporarily, before uploading to the storage).
     - urls: A list of URLs where the media is stored or accessible.
     - properties: Additional metadata or transformations for the media.
     - _mimetype: The media's mimetype (e.g., image/jpeg, video/mp4).
     """
     filename: str
-    key: str = None
+    _key: str = None
     urls: List[str] = field(default_factory=list)
     properties: dict = field(default_factory=dict)
     _mimetype: str = None  # eg: image/jpeg
@@ -67,6 +66,10 @@ class Media:
         # checks if the media is already stored in the given storage
         return len(self.urls) > 0 and len(self.urls) == len(in_storage.config["steps"]["storages"])
 
+    @property
+    def key(self) -> str:
+        return self._key
+
     def set(self, key: str, value: Any) -> Media:
         self.properties[key] = value
         return self

src/auto_archiver/core/orchestrator.py

@@ -23,15 +23,13 @@ from .config import read_yaml, store_yaml, to_dot_notation, merge_dicts, is_vali
     DefaultValidatingParser, UniqueAppendAction, AuthenticationJsonParseAction, DEFAULT_CONFIG_FILE
 from .module import ModuleFactory, LazyBaseModule
 from . import validators, Feeder, Extractor, Database, Storage, Formatter, Enricher
-from .consts import MODULE_TYPES
+from .consts import MODULE_TYPES, SetupError
 from auto_archiver.utils.url import check_url_or_raise
 
 if TYPE_CHECKING:
     from .base_module import BaseModule
     from .module import LazyBaseModule
 
-class SetupError(ValueError):
-    pass
 class ArchivingOrchestrator:
 
     # instance variables

src/auto_archiver/core/storage.py

@@ -6,6 +6,7 @@ from __future__ import annotations
 from abc import abstractmethod
 from typing import IO
 import os
+import platform
 
 from loguru import logger
 from slugify import slugify
@@ -43,17 +44,30 @@ class Storage(BaseModule):
     def uploadf(self, file: IO[bytes], key: str, **kwargs: dict) -> bool:
         """
         Uploads (or saves) a file to the storage service/location.
+
+        This method should not be called directly, but instead through the 'store' method,
+        which sets up the media for storage.
         """
         pass
 
     def upload(self, media: Media, **kwargs) -> bool:
+        """
+        Uploads a media object to the storage service.
+
+        This method should not be called directly, but instead be called through the 'store' method,
+        which sets up the media for storage.
+        """
         logger.debug(f'[{self.__class__.__name__}] storing file {media.filename} with key {media.key}')
         with open(media.filename, 'rb') as f:
             return self.uploadf(f, media, **kwargs)
 
     def set_key(self, media: Media, url: str, metadata: Metadata) -> None:
         """takes the media and optionally item info and generates a key"""
-        if media.key is not None and len(media.key) > 0: return
+        
+        if media.key is not None and len(media.key) > 0:
+            # media key is already set
+            return
+
         folder = metadata.get_context('folder', '')
         filename, ext = os.path.splitext(media.filename)
 
@@ -61,10 +75,8 @@ class Storage(BaseModule):
         path_generator = self.path_generator
         if path_generator == "flat":
             path = ""
-            # TODO: this is never used
-            filename = slugify(filename)  # Ensure filename is slugified
         elif path_generator == "url":
-            path = slugify(url)
+            path = slugify(url)[:70]
         elif path_generator == "random":
             path = random_str(24)
         else:
@@ -81,25 +93,7 @@ class Storage(BaseModule):
             filename = hd[:24]
         else:
             raise ValueError(f"Invalid filename_generator: {filename_generator}")
-
+        
         key = os.path.join(folder, path, f"{filename}{ext}")
-        if len(key) > self.max_file_length():
-            # truncate the path
-            max_path_length = self.max_file_length() - len(filename) - len(ext) - len(folder) - 1
-            path = path[:max_path_length]
-            logger.warning(f'Filename too long ({len(key)} characters), truncating to {self.max_file_length()} characters')
-            key = os.path.join(folder, path, f"{filename}{ext}")
-
-
-        media.key = key
-
-
-    def max_file_length(self) -> int:
-        """
-        Returns the maximum length of a file name that can be stored in the storage service.
-
-        Files are truncated if they exceed this length.
-        Override this method in subclasses if the storage service has a different maximum file length.
-        """
-        return 255 # safe max file length for most filesystems (macOS, Windows, Linux)
 
+        media._key = key