fix datetime parsing

Co-authored-by: msramalho <19508417+msramalho@users.noreply.github.com>

Dockerfile

@@ -4,12 +4,11 @@ ENV RUNNING_IN_DOCKER=1
 
 WORKDIR /app
 
-# TODO: use custom ffmpeg builds instead of apt-get install
 RUN pip install --upgrade pip && \
 	pip install pipenv && \
 	add-apt-repository ppa:mozillateam/ppa && \
 	apt-get update && \
-	apt-get install -y gcc ffmpeg fonts-noto && \
+	apt-get install -y gcc ffmpeg fonts-noto exiftool && \
 	apt-get install -y --no-install-recommends firefox-esr && \
 	ln -s /usr/bin/firefox-esr /usr/bin/firefox && \
 	wget https://github.com/mozilla/geckodriver/releases/download/v0.33.0/geckodriver-v0.33.0-linux64.tar.gz && \
@@ -18,19 +17,15 @@ RUN pip install --upgrade pip && \
 	rm geckodriver-v*
 
 
-# TODO: avoid copying unnecessary files, including .git
 COPY Pipfile* ./
-RUN pipenv install
-
+# install from pipenv, with browsertrix-only requirements
+RUN pipenv install && \
+	pipenv install pywb uwsgi
+	
 # doing this at the end helps during development, builds are quick
 COPY ./src/ . 
 
-# TODO: figure out how to make volumes not be root, does it depend on host or dockerfile?
-# RUN useradd --system --groups sudo --shell /bin/bash archiver && chown -R archiver:sudo .
-# USER archiver
-
-
 ENTRYPOINT ["pipenv", "run", "python3", "-m", "auto_archiver"]
 
 # should be executed with 2 volumes (3 if local_storage is used)
-# docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive aa pipenv run python3 -m auto_archiver --config secrets/orchestration.yaml
+# docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive aa pipenv run python3 -m auto_archiver --config secrets/orchestration.yaml

Pipfile

@@ -32,14 +32,14 @@ cryptography = "*"
 dataclasses-json = "*"
 yt-dlp = "*"
 vk-url-scraper = "*"
-uwsgi = "*"
 requests = {extras = ["socks"], version = "*"}
-# wacz = "==0.4.8"
 numpy = "*"
-
-[requires]
-python_version = "3.10"
+warcio = "*"
+jsonlines = "*"
 
 [dev-packages]
 autopep8 = "*"
 setuptools-pipfile = "*"
+
+[requires]
+python_version = "3.10"

example.orchestration.yaml

@@ -12,12 +12,14 @@ steps:
     # - tiktok_archiver
     - youtubedl_archiver
     # - wayback_archiver_enricher
+    # - wacz_archiver_enricher
   enrichers:
     - hash_enricher
+    # - metadata_enricher
     # - screenshot_enricher
     # - thumbnail_enricher
     # - wayback_archiver_enricher
-    # - wacz_enricher
+    # - wacz_archiver_enricher
     # - pdq_hash_enricher # if you want to calculate hashes for thumbnails, include this after thumbnail_enricher
   formatter: html_formatter # defaults to mute_formatter
   storages:
@@ -95,7 +97,7 @@ configurations:
     secret: "wayback secret"
   hash_enricher:
     algorithm: "SHA3-512" # can also be SHA-256
-  wacz_enricher:
+  wacz_archiver_enricher:
     profile: secrets/profile.tar.gz
   local_storage:
     save_to: "./local_archive"

scripts/release.sh

@@ -8,8 +8,8 @@ TAG=$(python -c 'from src.auto_archiver.version import __version__; print("v" +
 read -p "Creating new release for $TAG. Do you want to continue? [Y/n] " prompt
 
 if [[ $prompt == "y" || $prompt == "Y" || $prompt == "yes" || $prompt == "Yes" ]]; then
-    git add -A
-    git commit -m "Bump version to $TAG for release" || true && git push
+    # git add -A
+    # git commit -m "Bump version to $TAG for release" || true && git push
     echo "Creating new git tag $TAG"
     git tag "$TAG" -m "$TAG"
     git push --tags

src/auto_archiver/archivers/archiver.py

@@ -27,11 +27,6 @@ class Archiver(Step):
         # used to clean unnecessary URL parameters OR unfurl redirect links
         return url
 
-    def is_rearchivable(self, url: str) -> bool:
-        # archivers can signal if it does not make sense to rearchive a piece of content
-        # default is rearchiving
-        return True
-
     def _guess_file_type(self, path: str) -> str:
         """
         Receives a URL or filename and returns global mimetype like 'image' or 'video'
@@ -56,6 +51,7 @@ class Archiver(Step):
             'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36'
         }
         d = requests.get(url, headers=headers)
+        assert d.status_code == 200, f"got response code {d.status_code} for {url=}"
         with open(to_filename, 'wb') as f:
             f.write(d.content)
         return to_filename

src/auto_archiver/archivers/telegram_archiver.py

@@ -19,10 +19,6 @@ class TelegramArchiver(Archiver):
     def configs() -> dict:
         return {}
 
-    def is_rearchivable(self, url: str) -> bool:
-        # telegram posts are static
-        return False
-
     def download(self, item: Metadata) -> Metadata:
         url = item.get_url()
         # detect URLs that we definitely cannot handle
@@ -57,10 +53,10 @@ class TelegramArchiver(Archiver):
 
             if not len(image_urls): return False
             for img_url in image_urls:
-                result.add_media(Media(self.download_from_url(img_url)))
+                result.add_media(Media(self.download_from_url(img_url, item=item)))
         else:
             video_url = video.get('src')
-            m_video = Media(self.download_from_url(video_url))
+            m_video = Media(self.download_from_url(video_url, item=item))
             # extract duration from HTML
             try:
                 duration = s.find_all('time')[0].contents[0]

src/auto_archiver/archivers/telethon_archiver.py

@@ -38,10 +38,6 @@ class TelethonArchiver(Archiver):
             }
         }
 
-    def is_rearchivable(self, url: str) -> bool:
-        # telegram posts are static
-        return False
-
     def setup(self) -> None:
         """
         1. trigger login process for telegram or proceed if already saved in a session file

src/auto_archiver/archivers/tiktok_archiver.py

@@ -1,9 +1,11 @@
-import json, os, traceback, uuid
+import json, os, traceback
 import tiktok_downloader
 from loguru import logger
 
+
 from . import Archiver
 from ..core import Metadata, Media, ArchivingContext
+from ..utils.misc import random_str
 
 
 class TiktokArchiver(Archiver):
@@ -16,10 +18,6 @@ class TiktokArchiver(Archiver):
     def configs() -> dict:
         return {}
 
-    def is_rearchivable(self, url: str) -> bool:
-        # TikTok posts are static
-        return False
-
     def download(self, item: Metadata) -> Metadata:
         url = item.get_url()
         if 'tiktok.com' not in url:
@@ -41,7 +39,7 @@ class TiktokArchiver(Archiver):
             logger.warning(f'Other Tiktok error {error}')
 
         try:
-            filename = os.path.join(ArchivingContext.get_tmp_dir(), f'{str(uuid.uuid4())[0:8]}.mp4')
+            filename = os.path.join(ArchivingContext.get_tmp_dir(), f'{random_str(8)}.mp4')
             tiktok_media = tiktok_downloader.snaptik(url).get_media()
 
             if len(tiktok_media) <= 0:

src/auto_archiver/archivers/twitter_archiver.py

@@ -6,7 +6,7 @@ from slugify import slugify
 
 from . import Archiver
 from ..core import Metadata, Media
-from ..utils.misc import remove_get_parameters
+from ..utils import UrlUtil
 
 
 class TwitterArchiver(Archiver):
@@ -15,8 +15,8 @@ class TwitterArchiver(Archiver):
     """
 
     name = "twitter_archiver"
-    link_pattern = re.compile(r"twitter.com\/(?:\#!\/)?(\w+)\/status(?:es)?\/(\d+)")
-    link_clean_pattern = re.compile(r"(.+twitter\.com\/.+\/\d+)(\?)*.*")
+    link_pattern = re.compile(r"(?:twitter|x).com\/(?:\#!\/)?(\w+)\/status(?:es)?\/(\d+)")
+    link_clean_pattern = re.compile(r"(.+(?:twitter|x)\.com\/.+\/\d+)(\?)*.*")
 
     def __init__(self, config: dict) -> None:
         super().__init__(config)
@@ -37,10 +37,6 @@ class TwitterArchiver(Archiver):
         # https://twitter.com/MeCookieMonster/status/1617921633456640001?s=20&t=3d0g4ZQis7dCbSDg-mE7-w
         return self.link_clean_pattern.sub("\\1", url)
 
-    def is_rearchivable(self, url: str) -> bool:
-        # Twitter posts are static (for now)
-        return False
-
     def download(self, item: Metadata) -> Metadata:
         """
         if this url is archivable will download post info and look for other posts from the same group with media.
@@ -78,7 +74,7 @@ class TwitterArchiver(Archiver):
                 media.set("src", variant.url)
                 mimetype = variant.contentType
             elif type(tweet_media) == Photo:
-                media.set("src", tweet_media.fullUrl.replace('name=large', 'name=orig'))
+                media.set("src", UrlUtil.twitter_best_quality_url(tweet_media.fullUrl))
                 mimetype = "image/jpeg"
             else:
                 logger.warning(f"Could not get media URL of {tweet_media}")
@@ -96,21 +92,7 @@ class TwitterArchiver(Archiver):
         https://github.com/JustAnotherArchivist/snscrape/issues/996#issuecomment-1615937362
         next to test: https://cdn.embedly.com/widgets/media.html?&schema=twitter&url=https://twitter.com/bellingcat/status/1674700676612386816
         """
-        headers = {
-            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/114.0",
-            "Accept": "*/*",
-            "Accept-Language": "en-US,en;q=0.5",
-            "Accept-Encoding": "gzip, deflate, br",
-            "Origin": "https://platform.twitter.com",
-            "Connection": "keep-alive",
-            "Referer": "https://platform.twitter.com/",
-            "Sec-Fetch-Dest": "empty",
-            "Sec-Fetch-Mode": "cors",
-            "Sec-Fetch-Site": "cross-site",
-            "Pragma": "no-cache",
-            "Cache-Control": "no-cache",
-            "TE": "trailers"
-        }
+
         logger.debug(f"Trying twitter hack for {url=}")
         result = Metadata()
 
@@ -132,9 +114,10 @@ class TwitterArchiver(Archiver):
 
         for i, u in enumerate(urls):
             media = Media(filename="")
+            u = UrlUtil.twitter_best_quality_url(u)
             media.set("src", u)
             ext = ""
-            if (mtype := mimetypes.guess_type(remove_get_parameters(u))[0]):
+            if (mtype := mimetypes.guess_type(UrlUtil.remove_get_parameters(u))[0]):
                 ext = mimetypes.guess_extension(mtype)
 
             media.filename = self.download_from_url(u, f'{slugify(url)}_{i}{ext}', item)

src/auto_archiver/archivers/vk_archiver.py

@@ -27,10 +27,6 @@ class VkArchiver(Archiver):
             "session_file": {"default": "secrets/vk_config.v2.json", "help": "valid VKontakte password"},
         }
 
-    def is_rearchivable(self, url: str) -> bool:
-        # VK content is static
-        return False
-
     def download(self, item: Metadata) -> Metadata:
         url = item.get_url()
 

src/auto_archiver/archivers/youtubedl_archiver.py

@@ -25,7 +25,7 @@ class YoutubeDLArchiver(Archiver):
             logger.debug('Using Facebook cookie')
             yt_dlp.utils.std_headers['cookie'] = self.facebook_cookie
 
-        ydl = yt_dlp.YoutubeDL({'outtmpl': os.path.join(ArchivingContext.get_tmp_dir(), f'%(id)s.%(ext)s'), 'quiet': False})
+        ydl = yt_dlp.YoutubeDL({'outtmpl': os.path.join(ArchivingContext.get_tmp_dir(), f'%(id)s.%(ext)s'), 'quiet': False, 'noplaylist': True})
 
         try:
             # don'd download since it can be a live stream

src/auto_archiver/core/media.py

@@ -1,10 +1,15 @@
 
 from __future__ import annotations
+import os
+import traceback
 from typing import Any, List
 from dataclasses import dataclass, field
 from dataclasses_json import dataclass_json, config
 import mimetypes
 
+import ffmpeg
+from ffmpeg._run import Error
+
 from .context import ArchivingContext
 
 from loguru import logger
@@ -60,7 +65,9 @@ class Media:
 
     @property  # getter .mimetype
     def mimetype(self) -> str:
-        assert self.filename is not None and len(self.filename) > 0, "cannot get mimetype from media without filename"
+        if not self.filename or len(self.filename) == 0:
+            logger.warning(f"cannot get mimetype from media without filename: {self}")
+            return ""
         if not self._mimetype:
             self._mimetype = mimetypes.guess_type(self.filename)[0]
         return self._mimetype or ""
@@ -74,6 +81,23 @@ class Media:
 
     def is_audio(self) -> bool:
         return self.mimetype.startswith("audio")
-    
+
     def is_image(self) -> bool:
         return self.mimetype.startswith("image")
+
+    def is_valid_video(self) -> bool:
+        # checks for video streams with ffmpeg, or min file size for a video
+        # self.is_video() should be used together with this method
+        try:
+            streams = ffmpeg.probe(self.filename, select_streams='v')['streams']
+            logger.warning(f"STREAMS FOR {self.filename} {streams}")
+            return any(s.get("duration_ts", 0) > 0 for s in streams)
+        except Error: return False # ffmpeg errors when reading bad files
+        except Exception as e:
+            logger.error(e)
+            logger.error(traceback.format_exc())
+            try:
+                fsize = os.path.getsize(self.filename)
+                return fsize > 20_000
+            except: pass
+        return True

src/auto_archiver/core/metadata.py

@@ -1,11 +1,14 @@
 
 from __future__ import annotations
+import hashlib
 from typing import Any, List, Union, Dict
 from dataclasses import dataclass, field
 from dataclasses_json import dataclass_json, config
 import datetime
 from urllib.parse import urlparse
 from dateutil.parser import parse as parse_dt
+from loguru import logger
+
 from .media import Media
 from .context import ArchivingContext
 
@@ -16,7 +19,6 @@ class Metadata:
     status: str = "no archiver"
     metadata: Dict[str, Any] = field(default_factory=dict)
     media: List[Media] = field(default_factory=list)
-    rearchivable: bool = True  # defaults to true, archivers can overwrite
 
     def __post_init__(self):
         self.set("_processed_at", datetime.datetime.utcnow())
@@ -29,7 +31,6 @@ class Metadata:
         if overwrite_left:
             if right.status and len(right.status):
                 self.status = right.status
-            self.rearchivable |= right.rearchivable
             for k, v in right.metadata.items():
                 assert k not in self.metadata or type(v) == type(self.get(k))
                 if type(v) not in [dict, list, set] or k not in self.metadata:
@@ -44,6 +45,7 @@ class Metadata:
 
     def store(self: Metadata, override_storages: List = None):
         # calls .store for all contained media. storages [Storage]
+        self.remove_duplicate_media_by_hash()
         storages = override_storages or ArchivingContext.get("storages")
         for media in self.media:
             media.store(override_storages=storages, url=self.get_url())
@@ -105,10 +107,16 @@ class Metadata:
 
     def get_timestamp(self, utc=True, iso=True) -> datetime.datetime:
         ts = self.get("timestamp")
-        if not ts: return ts
-        if utc: ts = ts.replace(tzinfo=datetime.timezone.utc)
-        if iso: return ts.isoformat()
-        return ts
+        if not ts: return 
+        try:
+            if type(ts) == str: ts = datetime.datetime.fromisoformat(ts)
+            if type(ts) == float: ts = datetime.datetime.fromtimestamp(ts)
+            if utc: ts = ts.replace(tzinfo=datetime.timezone.utc)
+            if iso: return ts.isoformat()
+            return ts
+        except Exception as e:
+            logger.error(f"Unable to parse timestamp {ts}: {e}")
+            return
 
     def add_media(self, media: Media, id: str = None) -> Metadata:
         # adds a new media, optionally including an id
@@ -124,6 +132,27 @@ class Metadata:
             if m.get("id") == id: return m
         return default
 
+    def remove_duplicate_media_by_hash(self) -> None:
+        # iterates all media, calculates a hash if it's missing and deletes duplicates
+        def calculate_hash_in_chunks(hash_algo, chunksize, filename) -> str:
+            # taken from hash_enricher, cannot be isolated to misc due to circular imports
+            with open(filename, "rb") as f:
+                while True:
+                    buf = f.read(chunksize)
+                    if not buf: break
+                    hash_algo.update(buf)
+            return hash_algo.hexdigest()
+
+        media_hashes = set()
+        new_media = []
+        for m in self.media:
+            h = m.get("hash")
+            if not h: h = calculate_hash_in_chunks(hashlib.sha256(), int(1.6e7), m.filename)
+            if len(h) and h in media_hashes: continue
+            media_hashes.add(h)
+            new_media.append(m)
+        self.media = new_media
+
     def get_first_image(self, default=None) -> Media:
         for m in self.media:
             if "image" in m.mimetype: return m
@@ -136,10 +165,23 @@ class Metadata:
     def get_final_media(self) -> Media:
         _default = self.media[0] if len(self.media) else None
         return self.get_media_by_id("_final_media", _default)
-    
+
     def get_all_media(self) -> List[Media]:
         # returns a list with all the media and inner media
         return [inner for m in self.media for inner in m.all_inner_media(True)]
 
     def __str__(self) -> str:
         return self.__repr__()
+
+
+    @staticmethod
+    def choose_most_complete(results: List[Metadata]) -> Metadata:
+        # returns the most complete result from a list of results
+        # prioritizes results with more media, then more metadata
+        if len(results) == 0: return None
+        if len(results) == 1: return results[0]
+        most_complete = results[0]
+        for r in results[1:]:
+            if len(r.media) > len(most_complete.media): most_complete = r
+            elif len(r.media) == len(most_complete.media) and len(r.metadata) > len(most_complete.metadata): most_complete = r
+        return most_complete

src/auto_archiver/core/orchestrator.py

@@ -62,11 +62,7 @@ class ArchivingOrchestrator:
         result.set_url(url)
         if original_url != url: result.set("original_url", original_url)
 
-        # 2 - rearchiving logic + notify start to DB
-        # archivers can signal whether the content is rearchivable: eg: tweet vs webpage
-        for a in self.archivers: result.rearchivable |= a.is_rearchivable(url)
-        logger.debug(f"{result.rearchivable=} for {url=}")
-
+        # 2 - notify start to DB
         # signal to DB that archiving has started
         # and propagate already archived if it exists
         cached_result = None
@@ -78,10 +74,10 @@ class ArchivingOrchestrator:
             d.started(result)
             if (local_result := d.fetch(result)):
                 cached_result = (cached_result or Metadata()).merge(local_result)
-        if cached_result and not cached_result.rearchivable:
+        if cached_result:
             logger.debug("Found previously archived entry")
             for d in self.databases:
-                d.done(cached_result)
+                d.done(cached_result, cached=True)
             return cached_result
 
         # 3 - call archivers until one succeeds
@@ -109,6 +105,7 @@ class ArchivingOrchestrator:
         # looks for Media in result.media and also result.media[x].properties (as list or dict values)
         result.store()
 
+
         # 6 - format and store formatted if needed
         # enrichers typically need access to already stored URLs etc
         if (final_media := self.formatter.format(result)):

src/auto_archiver/core/step.py

@@ -1,9 +1,8 @@
 from __future__ import annotations
-from dataclasses import dataclass, field
+from dataclasses import dataclass
 from inspect import ClassFoundException
 from typing import Type
 from abc import ABC
-# from collections.abc import Iterable
 
 
 @dataclass

src/auto_archiver/databases/api_db.py

@@ -1,3 +1,4 @@
+from typing import Union
 import requests, os
 from loguru import logger
 
@@ -14,23 +15,48 @@ class AAApiDb(Database):
     def __init__(self, config: dict) -> None:
         # without this STEP.__init__ is not called
         super().__init__(config)
+        self.allow_rearchive = bool(self.allow_rearchive)
+        self.store_results = bool(self.store_results)
         self.assert_valid_string("api_endpoint")
-        self.assert_valid_string("api_secret")
 
     @staticmethod
     def configs() -> dict:
         return {
             "api_endpoint": {"default": None, "help": "API endpoint where calls are made to"},
-            "api_secret": {"default": None, "help": "API authentication secret"},
+            "api_secret": {"default": None, "help": "API Basic authentication secret [deprecating soon]"},
+            "api_token": {"default": None, "help": "API Bearer token, to be preferred over secret (Basic auth) going forward"},
             "public": {"default": False, "help": "whether the URL should be publicly available via the API"},
             "author_id": {"default": None, "help": "which email to assign as author"},
             "group_id": {"default": None, "help": "which group of users have access to the archive in case public=false as author"},
+            "allow_rearchive": {"default": True, "help": "if False then the API database will be queried prior to any archiving operations and stop if the link has already been archived"},
+            "store_results": {"default": True, "help": "when set, will send the results to the API database."},
             "tags": {"default": [], "help": "what tags to add to the archived URL", "cli_set": lambda cli_val, cur_val: set(cli_val.split(","))},
         }
+    def fetch(self, item: Metadata) -> Union[Metadata, bool]:
+        """ query the database for the existence of this item"""
+        if not self.allow_rearchive: return
+        
+        params = {"url": item.get_url(), "limit": 15}
+        headers = {"Authorization": f"Bearer {self.api_token}", "accept": "application/json"}
+        response = requests.get(os.path.join(self.api_endpoint, "tasks/search-url"), params=params, headers=headers)
 
-    def done(self, item: Metadata) -> None:
+        if response.status_code == 200:
+            if len(response.json()):
+                logger.success(f"API returned {len(response.json())} previously archived instance(s)")
+                fetched_metadata = [Metadata.from_dict(r["result"]) for r in response.json()]
+                return Metadata.choose_most_complete(fetched_metadata)
+        else:
+            logger.error(f"AA API FAIL ({response.status_code}): {response.json()}")
+        return False
+
+
+    def done(self, item: Metadata, cached: bool=False) -> None:
         """archival result ready - should be saved to DB"""
-        logger.info(f"saving archive of {item.get_url()} to the AA API.")
+        if not self.store_results: return
+        if cached: 
+            logger.debug(f"skipping saving archive of {item.get_url()} to the AA API because it was cached")
+            return
+        logger.debug(f"saving archive of {item.get_url()} to the AA API.")
 
         payload = {'result': item.to_json(), 'public': self.public, 'author_id': self.author_id, 'group_id': self.group_id, 'tags': list(self.tags)}
         response = requests.post(os.path.join(self.api_endpoint, "submit-archive"), json=payload, auth=("abc", self.api_secret))
@@ -39,3 +65,5 @@ class AAApiDb(Database):
             logger.success(f"AA API: {response.json()}")
         else:
             logger.error(f"AA API FAIL ({response.status_code}): {response.json()}")
+
+    

src/auto_archiver/databases/console_db.py

@@ -27,6 +27,6 @@ class ConsoleDb(Database):
     def aborted(self, item: Metadata) -> None:
         logger.warning(f"ABORTED {item}")
 
-    def done(self, item: Metadata) -> None:
+    def done(self, item: Metadata, cached: bool=False) -> None:
         """archival result ready - should be saved to DB"""
         logger.success(f"DONE {item}")

src/auto_archiver/databases/csv_db.py

@@ -24,7 +24,7 @@ class CSVDb(Database):
             "csv_file": {"default": "db.csv", "help": "CSV file name"}
         }
 
-    def done(self, item: Metadata) -> None:
+    def done(self, item: Metadata, cached: bool=False) -> None:
         """archival result ready - should be saved to DB"""
         logger.success(f"DONE {item}")
         is_empty = not os.path.isfile(self.csv_file) or os.path.getsize(self.csv_file) == 0

src/auto_archiver/databases/database.py

@@ -36,6 +36,6 @@ class Database(Step, ABC):
         return False
 
     @abstractmethod
-    def done(self, item: Metadata) -> None:
+    def done(self, item: Metadata, cached: bool=False) -> None:
         """archival result ready - should be saved to DB"""
         pass

src/auto_archiver/databases/gsheet_db.py

@@ -41,7 +41,7 @@ class GsheetsDb(Database):
         """check if the given item has been archived already"""
         return False
 
-    def done(self, item: Metadata) -> None:
+    def done(self, item: Metadata, cached: bool=False) -> None:
         """archival result ready - should be saved to DB"""
         logger.success(f"DONE {item.get_url()}")
         gw, row = self._retrieve_gsheet(item)
@@ -57,8 +57,10 @@ class GsheetsDb(Database):
                     cell_updates.append((row, col, final_value))
             except Exception as e:
                 logger.error(f"Unable to batch {col}={final_value} due to {e}")
-
-        cell_updates.append((row, 'status', item.status))
+        status_message = item.status
+        if cached:
+            status_message = f"[cached] {status_message}"
+        cell_updates.append((row, 'status', status_message))
 
         media: Media = item.get_final_media()
         if hasattr(media, "urls"):

src/auto_archiver/enrichers/__init__.py

@@ -3,6 +3,7 @@ from .screenshot_enricher import ScreenshotEnricher
 from .wayback_enricher import WaybackArchiverEnricher
 from .hash_enricher import HashEnricher
 from .thumbnail_enricher import ThumbnailEnricher
-from .wacz_enricher import WaczEnricher
+from .wacz_enricher import WaczArchiverEnricher
 from .whisper_enricher import WhisperEnricher
-from .pdq_hash_enricher import PdqHashEnricher
+from .pdq_hash_enricher import PdqHashEnricher
+from .metadata_enricher import MetadataEnricher

src/auto_archiver/enrichers/hash_enricher.py

@@ -23,7 +23,7 @@ class HashEnricher(Enricher):
     def configs() -> dict:
         return {
             "algorithm": {"default": "SHA-256", "help": "hash algorithm to use", "choices": ["SHA-256", "SHA3-512"]},
-            "chunksize": {"default": 1.6e7, "help": "number of bytes to use when reading files in chunks (if this value is too large you will run out of RAM), default is 16MB"},
+            "chunksize": {"default": int(1.6e7), "help": "number of bytes to use when reading files in chunks (if this value is too large you will run out of RAM), default is 16MB"},
         }
 
     def enrich(self, to_enrich: Metadata) -> None:
@@ -34,7 +34,7 @@ class HashEnricher(Enricher):
             if len(hd := self.calculate_hash(m.filename)):
                 to_enrich.media[i].set("hash", f"{self.algorithm}:{hd}")
 
-    def calculate_hash(self, filename):
+    def calculate_hash(self, filename) -> str:
         hash = None
         if self.algorithm == "SHA-256":
             hash = hashlib.sha256()

src/auto_archiver/enrichers/metadata_enricher.py

@@ -0,0 +1,47 @@
+import subprocess
+import traceback
+from loguru import logger
+
+from . import Enricher
+from ..core import Metadata
+
+
+class MetadataEnricher(Enricher):
+    """
+    Extracts metadata information from files using exiftool.
+    """
+    name = "metadata_enricher"
+
+    def __init__(self, config: dict) -> None:
+        # without this STEP.__init__ is not called
+        super().__init__(config)
+
+    @staticmethod
+    def configs() -> dict:
+        return {}
+
+    def enrich(self, to_enrich: Metadata) -> None:
+        url = to_enrich.get_url()
+        logger.debug(f"extracting EXIF metadata for {url=}")
+
+        for i, m in enumerate(to_enrich.media):
+            if len(md := self.get_metadata(m.filename)):
+                to_enrich.media[i].set("metadata", md)
+
+    def get_metadata(self, filename: str) -> dict:
+        try:
+            # Run ExifTool command to extract metadata from the file
+            cmd = ['exiftool', filename]
+            result = subprocess.run(cmd, capture_output=True, text=True)
+
+            # Process the output to extract individual metadata fields
+            metadata = {}
+            for line in result.stdout.splitlines():
+                field, value = line.strip().split(':', 1)
+                metadata[field.strip()] = value.strip()
+            return metadata
+        except FileNotFoundError:
+            logger.error("[exif_enricher] ExifTool not found. Make sure ExifTool is installed and added to PATH.")
+        except Exception as e:
+            logger.error(f"Error occurred: {e}: {traceback.format_exc()}")
+        return {}

src/auto_archiver/enrichers/pdq_hash_enricher.py

@@ -1,6 +1,7 @@
+import traceback
 import pdqhash
 import numpy as np
-from PIL import Image
+from PIL import Image, UnidentifiedImageError
 from loguru import logger
 
 from . import Enricher
@@ -25,18 +26,27 @@ class PdqHashEnricher(Enricher):
     def enrich(self, to_enrich: Metadata) -> None:
         url = to_enrich.get_url()
         logger.debug(f"calculating perceptual hashes for {url=}")
+        media_with_hashes = []
 
         for m in to_enrich.media:
             for media in m.all_inner_media(True):
-                if media.is_image() and media.get("id") != "screenshot" and len(hd := self.calculate_pdq_hash(media.filename)):
-                        media.set("pdq_hash", hd)    
+                media_id = media.get("id", "")
+                if media.is_image() and "screenshot" not in media_id and "warc-file-" not in media_id and len(hd := self.calculate_pdq_hash(media.filename)):
+                    media.set("pdq_hash", hd)
+                    media_with_hashes.append(media.filename)
+
+        logger.debug(f"calculated '{len(media_with_hashes)}' perceptual hashes for {url=}: {media_with_hashes}")
 
     def calculate_pdq_hash(self, filename):
-        # returns a hexadecimal string with the perceptual hash for the given filename 
-        with Image.open(filename) as img:
-            # convert the image to RGB
-            image_rgb = np.array(img.convert("RGB"))
-            # compute the 256-bit PDQ hash (we do not store the quality score)
-            hash_array, _ = pdqhash.compute(image_rgb)
-            hash = "".join(str(b) for b in hash_array)
-            return hex(int(hash, 2))[2:]
+        # returns a hexadecimal string with the perceptual hash for the given filename
+        try:
+            with Image.open(filename) as img:
+                # convert the image to RGB
+                image_rgb = np.array(img.convert("RGB"))
+                # compute the 256-bit PDQ hash (we do not store the quality score)
+                hash_array, _ = pdqhash.compute(image_rgb)
+                hash = "".join(str(b) for b in hash_array)
+                return hex(int(hash, 2))[2:]
+        except UnidentifiedImageError as e:
+            logger.error(f"Image {filename=} is likely corrupted or in unsupported format {e}: {traceback.format_exc()}")
+        return ""

src/auto_archiver/enrichers/screenshot_enricher.py

@@ -1,9 +1,10 @@
 from loguru import logger
-import time, uuid, os
+import time, os
 from selenium.common.exceptions import TimeoutException
 
+
 from . import Enricher
-from ..utils import Webdriver, UrlUtil
+from ..utils import Webdriver, UrlUtil, random_str  
 from ..core import Media, Metadata, ArchivingContext
 
 class ScreenshotEnricher(Enricher):
@@ -29,7 +30,7 @@ class ScreenshotEnricher(Enricher):
             try:
                 driver.get(url)
                 time.sleep(int(self.sleep_before_screenshot))
-                screenshot_file = os.path.join(ArchivingContext.get_tmp_dir(), f"screenshot_{str(uuid.uuid4())[0:8]}.png")
+                screenshot_file = os.path.join(ArchivingContext.get_tmp_dir(), f"screenshot_{random_str(8)}.png")
                 driver.save_screenshot(screenshot_file)
                 to_enrich.add_media(Media(filename=screenshot_file), id="screenshot")
             except TimeoutException:

src/auto_archiver/enrichers/thumbnail_enricher.py

@@ -1,8 +1,9 @@
-import ffmpeg, os, uuid
+import ffmpeg, os
 from loguru import logger
 
 from . import Enricher
 from ..core import Media, Metadata, ArchivingContext
+from ..utils.misc import random_str
 
 
 class ThumbnailEnricher(Enricher):
@@ -23,7 +24,7 @@ class ThumbnailEnricher(Enricher):
         logger.debug(f"generating thumbnails")
         for i, m in enumerate(to_enrich.media[::]):
             if m.is_video():
-                folder = os.path.join(ArchivingContext.get_tmp_dir(), str(uuid.uuid4()))
+                folder = os.path.join(ArchivingContext.get_tmp_dir(), random_str(24))
                 os.makedirs(folder, exist_ok=True)
                 logger.debug(f"generating thumbnails for {m.filename}")
                 fps, duration = 0.5, m.get("duration")

src/auto_archiver/enrichers/wacz_enricher.py

@@ -1,16 +1,24 @@
-import os, shutil, subprocess, uuid
+import jsonlines
+import mimetypes
+import os, shutil, subprocess
+from zipfile import ZipFile
 from loguru import logger
+from warcio.archiveiterator import ArchiveIterator
 
 from ..core import Media, Metadata, ArchivingContext
 from . import Enricher
-from ..utils import UrlUtil
+from ..archivers import Archiver
+from ..utils import UrlUtil, random_str
 
 
-class WaczEnricher(Enricher):
+class WaczArchiverEnricher(Enricher, Archiver):
     """
-    Submits the current URL to the webarchive and returns a job_id or completed archive
+    Uses https://github.com/webrecorder/browsertrix-crawler to generate a .WACZ archive of the URL
+    If used with [profiles](https://github.com/webrecorder/browsertrix-crawler#creating-and-using-browser-profiles)
+    it can become quite powerful for archiving private content.
+    When used as an archiver it will extract the media from the .WACZ archive so it can be enriched.
     """
-    name = "wacz_enricher"
+    name = "wacz_archiver_enricher"
 
     def __init__(self, config: dict) -> None:
         # without this STEP.__init__ is not called
@@ -20,58 +28,66 @@ class WaczEnricher(Enricher):
     def configs() -> dict:
         return {
             "profile": {"default": None, "help": "browsertrix-profile (for profile generation see https://github.com/webrecorder/browsertrix-crawler#creating-and-using-browser-profiles)."},
-            "timeout": {"default": 90, "help": "timeout for WACZ generation in seconds"},
-            "ignore_auth_wall": {"default": True, "help": "skip URL if it is behind authentication wall, set to False if you have browsertrix profile configured for private content."},
+            "docker_commands": {"default": None, "help":"if a custom docker invocation is needed"},
+            "timeout": {"default": 120, "help": "timeout for WACZ generation in seconds"},
+            "extract_media": {"default": True, "help": "If enabled all the images/videos/audio present in the WACZ archive will be extracted into separate Media. The .wacz file will be kept untouched."}
         }
 
+    def download(self, item: Metadata) -> Metadata:
+        # this new Metadata object is required to avoid duplication
+        result = Metadata()
+        result.merge(item)
+        if self.enrich(result):
+            return result.success("wacz")
+
     def enrich(self, to_enrich: Metadata) -> bool:
+        if to_enrich.get_media_by_id("browsertrix"):
+            logger.info(f"WACZ enricher had already been executed: {to_enrich.get_media_by_id('browsertrix')}")
+            return True
+
         url = to_enrich.get_url()
-        
-        collection = str(uuid.uuid4())[0:8]
-        browsertrix_home = os.path.abspath(ArchivingContext.get_tmp_dir())
-        
-        if os.getenv('RUNNING_IN_DOCKER'):
-            logger.debug(f"generating WACZ without Docker for {url=}")
 
-            cmd = [
-                "crawl",
-                "--url", url,
-                "--scopeType", "page",
-                "--generateWACZ",
-                "--text",
-                "--collection", collection,
-                "--id", collection,
-                "--saveState", "never",
-                "--behaviors", "autoscroll,autoplay,autofetch,siteSpecific",
-                "--behaviorTimeout", str(self.timeout),
-                "--timeout", str(self.timeout)]
-            
-            if self.profile:
-                cmd.extend(["--profile", os.path.join("/app", str(self.profile))])
-        else:
+        collection = random_str(8)
+        browsertrix_home_host = os.environ.get('BROWSERTRIX_HOME_HOST') or os.path.abspath(ArchivingContext.get_tmp_dir())
+        browsertrix_home_container = os.environ.get('BROWSERTRIX_HOME_CONTAINER') or browsertrix_home_host
+
+        cmd = [
+            "crawl",
+            "--url", url,
+            "--scopeType", "page",
+            "--generateWACZ",
+            "--text",
+            "--screenshot", "fullPage",
+            "--collection", collection,
+            "--id", collection,
+            "--saveState", "never",
+            "--behaviors", "autoscroll,autoplay,autofetch,siteSpecific",
+            "--behaviorTimeout", str(self.timeout),
+            "--timeout", str(self.timeout)]
+
+        # call docker if explicitly enabled or we are running on the host (not in docker)
+        use_docker = os.environ.get('WACZ_ENABLE_DOCKER') or not os.environ.get('RUNNING_IN_DOCKER')
+
+        if use_docker:
             logger.debug(f"generating WACZ in Docker for {url=}")
-            
-            cmd = [
-                "docker", "run",
-                "--rm",  # delete container once it has completed running
-                "-v", f"{browsertrix_home}:/crawls/",
-                # "-it", # this leads to "the input device is not a TTY"
-                "webrecorder/browsertrix-crawler", "crawl",
-                "--url", url,
-                "--scopeType", "page",
-                "--generateWACZ",
-                "--text",
-                "--collection", collection,
-                "--behaviors", "autoscroll,autoplay,autofetch,siteSpecific",
-                "--behaviorTimeout", str(self.timeout),
-                "--timeout", str(self.timeout)
-            ]
+            logger.debug(f"{browsertrix_home_host=} {browsertrix_home_container=}")
+            if self.docker_commands:
+                cmd = self.docker_commands + cmd
+            else:
+                cmd = ["docker", "run", "--rm", "-v", f"{browsertrix_home_host}:/crawls/", "webrecorder/browsertrix-crawler"] + cmd
 
             if self.profile:
-                profile_fn = os.path.join(browsertrix_home, "profile.tar.gz")
+                profile_fn = os.path.join(browsertrix_home_container, "profile.tar.gz")
+                logger.debug(f"copying {self.profile} to {profile_fn}")
                 shutil.copyfile(self.profile, profile_fn)
                 cmd.extend(["--profile", os.path.join("/crawls", "profile.tar.gz")])
 
+        else:
+            logger.debug(f"generating WACZ without Docker for {url=}")
+
+            if self.profile:
+                cmd.extend(["--profile", os.path.join("/app", str(self.profile))])
+
         try:
             logger.info(f"Running browsertrix-crawler: {' '.join(cmd)}")
             subprocess.run(cmd, check=True)
@@ -79,15 +95,109 @@ class WaczEnricher(Enricher):
             logger.error(f"WACZ generation failed: {e}")
             return False
 
-        
-
-        if os.getenv('RUNNING_IN_DOCKER'):
-            filename = os.path.join("collections", collection, f"{collection}.wacz")
+        if use_docker:
+            wacz_fn = os.path.join(browsertrix_home_container, "collections", collection, f"{collection}.wacz")
         else:
-            filename = os.path.join(browsertrix_home, "collections", collection, f"{collection}.wacz")
-        
-        if not os.path.exists(filename):
-            logger.warning(f"Unable to locate and upload WACZ  {filename=}")
+            wacz_fn = os.path.join("collections", collection, f"{collection}.wacz")
+
+        if not os.path.exists(wacz_fn):
+            logger.warning(f"Unable to locate and upload WACZ  {wacz_fn=}")
             return False
 
-        to_enrich.add_media(Media(filename), "browsertrix")
+        to_enrich.add_media(Media(wacz_fn), "browsertrix")
+        if self.extract_media:
+            self.extract_media_from_wacz(to_enrich, wacz_fn)
+
+        if use_docker:
+            jsonl_fn = os.path.join(browsertrix_home_container, "collections", collection, "pages", "pages.jsonl")
+        else:
+            jsonl_fn = os.path.join("collections", collection, "pages", "pages.jsonl")
+
+        if not os.path.exists(jsonl_fn):
+            logger.warning(f"Unable to locate and pages.jsonl  {jsonl_fn=}")
+        else:
+            logger.info(f"Parsing pages.jsonl  {jsonl_fn=}")
+            with jsonlines.open(jsonl_fn) as reader:
+                for obj in reader:
+                    if 'title' in obj:
+                        to_enrich.set_title(obj['title'])
+                    if 'text' in obj:
+                        to_enrich.set_content(obj['text'])
+
+
+        return True
+
+    def extract_media_from_wacz(self, to_enrich: Metadata, wacz_filename: str) -> None:
+        """
+        Receives a .wacz archive, and extracts all relevant media from it, adding them to to_enrich.
+        """
+        logger.info(f"WACZ extract_media flag is set, extracting media from {wacz_filename=}")
+
+        # unzipping the .wacz
+        tmp_dir = ArchivingContext.get_tmp_dir()
+        unzipped_dir = os.path.join(tmp_dir, "unzipped")
+        with ZipFile(wacz_filename, 'r') as z_obj:
+            z_obj.extractall(path=unzipped_dir)
+
+        # if warc is split into multiple gzip chunks, merge those
+        warc_dir = os.path.join(unzipped_dir, "archive")
+        warc_filename = os.path.join(tmp_dir, "merged.warc")
+        with open(warc_filename, 'wb') as outfile:
+            for filename in sorted(os.listdir(warc_dir)):
+                if filename.endswith('.gz'):
+                    chunk_file = os.path.join(warc_dir, filename)
+                    with open(chunk_file, 'rb') as infile:
+                        shutil.copyfileobj(infile, outfile)
+
+        # get media out of .warc
+        counter = 0
+        seen_urls = set()
+        with open(warc_filename, 'rb') as warc_stream:
+            for record in ArchiveIterator(warc_stream):
+                # only include fetched resources
+                if record.rec_type == "resource":  # screenshots
+                    fn = os.path.join(tmp_dir, f"warc-file-{counter}.png")
+                    with open(fn, "wb") as outf: outf.write(record.raw_stream.read())
+                    m = Media(filename=fn)
+                    to_enrich.add_media(m, "browsertrix-screenshot")
+                    counter += 1
+
+                if record.rec_type != 'response': continue
+                record_url = record.rec_headers.get_header('WARC-Target-URI')
+                if not UrlUtil.is_relevant_url(record_url):
+                    logger.debug(f"Skipping irrelevant URL {record_url} but it's still present in the WACZ.")
+                    continue
+                if record_url in seen_urls:
+                    logger.debug(f"Skipping already seen URL {record_url}.")
+                    continue
+
+                # filter by media mimetypes
+                content_type = record.http_headers.get("Content-Type")
+                if not content_type: continue
+                if not any(x in content_type for x in ["video", "image", "audio"]): continue
+
+                # create local file and add media
+                ext = mimetypes.guess_extension(content_type)
+                warc_fn = f"warc-file-{counter}{ext}"
+                fn = os.path.join(tmp_dir, warc_fn)
+
+                record_url_best_qual = UrlUtil.twitter_best_quality_url(record_url)
+                with open(fn, "wb") as outf: outf.write(record.raw_stream.read())
+
+                m = Media(filename=fn)
+                m.set("src", record_url)
+                # if a link with better quality exists, try to download that
+                if record_url_best_qual != record_url:
+                    try:
+                        m.filename = self.download_from_url(record_url_best_qual, warc_fn, to_enrich)
+                        m.set("src", record_url_best_qual)
+                        m.set("src_alternative", record_url)
+                    except Exception as e: logger.warning(f"Unable to download best quality URL for {record_url=} got error {e}, using original in WARC.")
+
+                # remove bad videos
+                if m.is_video() and not m.is_valid_video(): continue
+                
+                to_enrich.add_media(m, warc_fn)
+                counter += 1
+                seen_urls.add(record_url)
+        logger.info(f"WACZ extract_media finished, found {counter} relevant media file(s)")

src/auto_archiver/enrichers/wayback_enricher.py

@@ -23,11 +23,13 @@ class WaybackArchiverEnricher(Enricher, Archiver):
     def configs() -> dict:
         return {
             "timeout": {"default": 15, "help": "seconds to wait for successful archive confirmation from wayback, if more than this passes the result contains the job_id so the status can later be checked manually."},
+            "if_not_archived_within": {"default": None, "help": "only tell wayback to archive if no archive is available before the number of seconds specified, use None to ignore this option. For more information: https://docs.google.com/document/d/1Nsv52MvSjbLb2PCpHlat0gkzw0EvtSgpKHu4mk0MnrA"},
             "key": {"default": None, "help": "wayback API key. to get credentials visit https://archive.org/account/s3.php"},
             "secret": {"default": None, "help": "wayback API secret. to get credentials visit https://archive.org/account/s3.php"}
         }
 
     def download(self, item: Metadata) -> Metadata:
+        # this new Metadata object is required to avoid duplication
         result = Metadata()
         result.merge(item)
         if self.enrich(result):
@@ -49,7 +51,11 @@ class WaybackArchiverEnricher(Enricher, Archiver):
             "Accept": "application/json",
             "Authorization": f"LOW {self.key}:{self.secret}"
         }
-        r = requests.post('https://web.archive.org/save/', headers=ia_headers, data={'url': url})
+        post_data = {'url': url}
+        if self.if_not_archived_within:
+            post_data["if_not_archived_within"] = self.if_not_archived_within
+        # see https://docs.google.com/document/d/1Nsv52MvSjbLb2PCpHlat0gkzw0EvtSgpKHu4mk0MnrA for more options
+        r = requests.post('https://web.archive.org/save/', headers=ia_headers, data=post_data)
 
         if r.status_code != 200:
             logger.error(em := f"Internet archive failed with status of {r.status_code}: {r.json()}")

src/auto_archiver/enrichers/whisper_enricher.py

@@ -18,17 +18,18 @@ class WhisperEnricher(Enricher):
     def __init__(self, config: dict) -> None:
         # without this STEP.__init__ is not called
         super().__init__(config)
+        assert type(self.api_endpoint) == str and len(self.api_endpoint) > 0, "please provide a value for the whisper_enricher api_endpoint"
         assert type(self.api_key) == str and len(self.api_key) > 0, "please provide a value for the whisper_enricher api_key"
         self.timeout = int(self.timeout)
 
     @staticmethod
     def configs() -> dict:
         return {
-            "api_endpoint": {"default": "https://whisper.spoettel.dev/api/v1", "help": "WhisperApi api endpoint"},
+            "api_endpoint": {"default": None, "help": "WhisperApi api endpoint, eg: https://whisperbox-api.com/api/v1, a deployment of https://github.com/bellingcat/whisperbox-transcribe."},
             "api_key": {"default": None, "help": "WhisperApi api key for authentication"},
             "include_srt": {"default": False, "help": "Whether to include a subtitle SRT (SubRip Subtitle file) for the video (can be used in video players)."},
             "timeout": {"default": 90, "help": "How many seconds to wait at most for a successful job completion."},
-            "action": {"default": "translation", "help": "which Whisper operation to execute", "choices": ["transcript", "translation", "language_detection"]},
+            "action": {"default": "translate", "help": "which Whisper operation to execute", "choices": ["transcribe", "translate", "language_detection"]},
 
         }
 
@@ -56,9 +57,12 @@ class WhisperEnricher(Enricher):
 
         for i, m in enumerate(to_enrich.media):
             if m.is_video() or m.is_audio():
-                job_id = to_enrich.media[i].get("whisper_model")["job_id"]
+                job_id = to_enrich.media[i].get("whisper_model", {}).get("job_id")
+                if not job_id: continue
                 to_enrich.media[i].set("whisper_model", {
                     "job_id": job_id,
+                    "job_status_check": f"{self.api_endpoint}/jobs/{job_id}",
+                    "job_artifacts_check": f"{self.api_endpoint}/jobs/{job_id}/artifacts",
                     **(job_results[job_id] if job_results[job_id] else {"result": "incomplete or failed job"})
                 })
                 # append the extracted text to the content of the post so it gets written to the DBs like gsheets text column
@@ -76,6 +80,7 @@ class WhisperEnricher(Enricher):
             "type": self.action,
             # "language": "string" # may be a config
         }
+        logger.debug(f"calling API with {payload=}")
         response = requests.post(f'{self.api_endpoint}/jobs', json=payload, headers={'Authorization': f'Bearer {self.api_key}'})
         assert response.status_code == 201, f"calling the whisper api {self.api_endpoint} returned a non-success code: {response.status_code}"
         logger.debug(response.json())

src/auto_archiver/formatters/html_formatter.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 from dataclasses import dataclass
-import mimetypes, uuid, os, pathlib
+import mimetypes, os, pathlib
 from jinja2 import Environment, FileSystemLoader
 from urllib.parse import quote
 from loguru import logger
@@ -9,6 +9,7 @@ from ..version import __version__
 from ..core import Metadata, Media, ArchivingContext
 from . import Formatter
 from ..enrichers import HashEnricher
+from ..utils.misc import random_str
 
 
 @dataclass
@@ -44,10 +45,10 @@ class HtmlFormatter(Formatter):
             metadata=item.metadata,
             version=__version__
         )
-        html_path = os.path.join(ArchivingContext.get_tmp_dir(), f"formatted{str(uuid.uuid4())}.html")
+        html_path = os.path.join(ArchivingContext.get_tmp_dir(), f"formatted{random_str(24)}.html")
         with open(html_path, mode="w", encoding="utf-8") as outf:
             outf.write(content)
-        final_media = Media(filename=html_path)
+        final_media = Media(filename=html_path, _mimetype="text/html")
 
         he = HashEnricher({"hash_enricher": {"algorithm": ArchivingContext.get("hash_enricher.algorithm"), "chunksize": 1.6e7}})
         if len(hd := he.calculate_hash(final_media.filename)):

src/auto_archiver/formatters/templates/html_template.html

@@ -65,11 +65,12 @@
         }
 
         /* Disable grayscale on hover */
-        img:hover,
+        /* img:hover,
         video:hover {
             -webkit-filter: grayscale(0);
             filter: none;
-        }
+        } */
+
 
         .collapsible {
             background-color: #777;
@@ -101,57 +102,68 @@
 <body>
     <div id="notification"></div>
     <h2>Archived media for <a href="{{ url }}">{{ url }}</a></h2>
+    {% if title | string | length > 0 %}
     <p><b>title:</b> '<span class="copy">{{ title }}</span>'</p>
+    {% endif %}
     <h2 class="center">content {{ media | length }} item(s)</h2>
+    <form class="center">
+        <label>
+            <input type="checkbox" id="safe-media-view" checked>
+            Safe Media View
+        </label>
+    </form>
     <table class="content">
         <tr>
             <th>about</th>
             <th>preview(s)</th>
         </tr>
-        {% for m in media %}
-        <tr>
-            <td>
-                <ul>
-                    <li><b>key:</b> <span class="copy">{{ m.key }}</span></li>
-                    <li><b>type:</b> <span class="copy">{{ m.mimetype }}</span></li>
+        <tbody>
+            {% for m in media %}
+            <tr>
+                <td>
+                    <ul>
+                        <li><b>key:</b> <span class="copy">{{ m.key }}</span></li>
+                        <li><b>type:</b> <span class="copy">{{ m.mimetype }}</span></li>
 
-                    {% for prop in m.properties %}
+                        {% for prop in m.properties %}
 
-                    {% if m.properties[prop] | is_list %}
-                    <p></p>
-                    <div>
-                        <b class="collapsible" title="expand">{{ prop }}:</b>
+                        {% if m.properties[prop] | is_list %}
                         <p></p>
-                        <div class="collapsible-content">
-                            {% for subprop in m.properties[prop] %}
-                            {% if subprop | is_media %}
-                            {{ macros.display_media(subprop, true, url) }}
+                        <div>
+                            <b class="collapsible" title="expand">{{ prop }}:</b>
+                            <p></p>
+                            <div class="collapsible-content">
+                                {% for subprop in m.properties[prop] %}
+                                {% if subprop | is_media %}
+                                {{ macros.display_media(subprop, true, url) }}
 
-                            <ul>
-                                {% for subprop_prop in subprop.properties %}
-                                <li><b>{{ subprop_prop }}:</b> {{ macros.copy_urlize(subprop.properties[subprop_prop]) }}</li>
+                                <ul>
+                                    {% for subprop_prop in subprop.properties %}
+                                    <li><b>{{ subprop_prop }}:</b>
+                                        {{ macros.copy_urlize(subprop.properties[subprop_prop]) }}</li>
+                                    {% endfor %}
+                                </ul>
+
+                                {% else %}
+                                {{ subprop }}
+                                {% endif %}
                                 {% endfor %}
-                            </ul>
-
-                            {% else %}
-                            {{ subprop }}
-                            {% endif %}
-                            {% endfor %}
+                            </div>
                         </div>
-                    </div>
-                    <p></p>
-                    {% elif m.properties[prop] | string | length > 1 %}
-                    <li><b>{{ prop }}:</b> {{ macros.copy_urlize(m.properties[prop]) }}</li>
-                    {% endif %}
+                        <p></p>
+                        {% elif m.properties[prop] | string | length > 1 %}
+                        <li><b>{{ prop }}:</b> {{ macros.copy_urlize(m.properties[prop]) }}</li>
+                        {% endif %}
 
-                    {% endfor %}
-                </ul>
-            </td>
-            <td>
-                {{ macros.display_media(m, true, url) }}
-            </td>
-        </tr>
-        {% endfor %}
+                        {% endfor %}
+                    </ul>
+                </td>
+                <td>
+                    {{ macros.display_media(m, true, url) }}
+                </td>
+            </tr>
+            {% endfor %}
+        </tbody>
     </table>
     <h2 class="center">metadata</h2>
     <table class="metadata">
@@ -220,6 +232,49 @@
             }
         });
     }
+
+    // logic for enabled/disabled greyscale
+    // Get references to the checkboxes and images/videos
+    const safeImageViewCheckbox = document.getElementById('safe-media-view');
+    const imagesVideos = document.querySelectorAll('img, video');
+
+    // Function to toggle grayscale effect
+    function toggleGrayscale() {
+        imagesVideos.forEach(element => {
+            if (safeImageViewCheckbox.checked) {
+                // Enable grayscale effect
+                element.style.filter = 'grayscale(1)';
+                element.style.webkitFilter = 'grayscale(1)';
+            } else {
+                // Disable grayscale effect
+                element.style.filter = 'none';
+                element.style.webkitFilter = 'none';
+            }
+        });
+    }
+
+    // Add event listener to the checkbox to trigger the toggleGrayscale function
+    safeImageViewCheckbox.addEventListener('change', toggleGrayscale);
+
+    // Handle the hover effect using JavaScript
+    imagesVideos.forEach(element => {
+        element.addEventListener('mouseenter', () => {
+            // Disable grayscale effect on hover
+            element.style.filter = 'none';
+            element.style.webkitFilter = 'none';
+        });
+
+        element.addEventListener('mouseleave', () => {
+            // Re-enable grayscale effect if checkbox is checked
+            if (safeImageViewCheckbox.checked) {
+                element.style.filter = 'grayscale(1)';
+                element.style.webkitFilter = 'grayscale(1)';
+            }
+        });
+    });
+
+    // Call the function on page load to apply the initial state
+    toggleGrayscale();
 </script>
 
 </html>

src/auto_archiver/formatters/templates/macros.html

@@ -16,10 +16,7 @@ No URL available for {{ m.key }}.
         <a href="https://lens.google.com/uploadbyurl?url={{ url | quote }}">Google Lens</a>,&nbsp;
         <a href="https://yandex.ru/images/touch/search?rpt=imageview&url={{ url | quote }}">Yandex</a>,&nbsp;
         <a href="https://www.bing.com/images/search?view=detailv2&iss=sbi&form=SBIVSP&sbisrc=UrlPaste&q=imgurl:{{ url | quote }}">Bing</a>,&nbsp;
-        <a href="https://www.tineye.com/search/?url={{ url | quote }}">Tineye</a>,&nbsp;
-        <a href="https://iqdb.org/?url={{ url | quote }}">IQDB</a>,&nbsp;
-        <a href="https://saucenao.com/search.php?db=999&url={{ url | quote }}">SauceNAO</a>,&nbsp;
-        <a href="https://imgops.com/{{ url | quote }}">IMGOPS</a>
+        <a href="https://www.tineye.com/search/?url={{ url | quote }}">Tineye</a>
     </div>
     <p></p>
 </div>

src/auto_archiver/storages/gd.py

@@ -119,7 +119,7 @@ class GDriveStorage(Storage):
             'parents': [upload_to]
         }
         media = MediaFileUpload(media.filename, resumable=True)
-        gd_file = self.service.files().create(body=file_metadata, media_body=media, fields='id').execute()
+        gd_file = self.service.files().create(supportsAllDrives=True, body=file_metadata, media_body=media, fields='id').execute()
         logger.debug(f'uploadf: uploaded file {gd_file["id"]} successfully in folder={upload_to}')
 
     # must be implemented even if unused
@@ -150,6 +150,9 @@ class GDriveStorage(Storage):
 
         for attempt in range(retries):
             results = self.service.files().list(
+                # both below for Google Shared Drives
+                supportsAllDrives=True,
+                includeItemsFromAllDrives=True,
                 q=query_string,
                 spaces='drive',  # ie not appDataFolder or photos
                 fields='files(id, name)'
@@ -182,7 +185,7 @@ class GDriveStorage(Storage):
             'mimeType': 'application/vnd.google-apps.folder',
             'parents': [parent_id]
         }
-        gd_folder = self.service.files().create(body=file_metadata, fields='id').execute()
+        gd_folder = self.service.files().create(supportsAllDrives=True, body=file_metadata, fields='id').execute()
         return gd_folder.get('id')
 
     # def exists(self, key):

src/auto_archiver/storages/s3.py

@@ -1,14 +1,14 @@
 
-from typing import IO, Any
-import boto3, uuid, os, mimetypes
-from botocore.errorfactory import ClientError
-from ..core import Metadata
+from typing import IO
+import boto3, os
+
+from ..utils.misc import random_str
 from ..core import Media
 from ..storages import Storage
+from ..enrichers import HashEnricher
 from loguru import logger
-from slugify import slugify
-
 
+NO_DUPLICATES_FOLDER = "no-dups/"
 class S3Storage(Storage):
     name = "s3_storage"
 
@@ -21,6 +21,9 @@ class S3Storage(Storage):
             aws_access_key_id=self.key,
             aws_secret_access_key=self.secret
         )
+        self.random_no_duplicate = bool(self.random_no_duplicate)
+        if self.random_no_duplicate:
+            logger.warning("random_no_duplicate is set to True, this will override `path_generator`, `filename_generator` and `folder`.")
 
     @staticmethod
     def configs() -> dict:
@@ -31,7 +34,7 @@ class S3Storage(Storage):
                 "region": {"default": None, "help": "S3 region name"},
                 "key": {"default": None, "help": "S3 API key"},
                 "secret": {"default": None, "help": "S3 API secret"},
-                # TODO: how to have sth like a custom folder? has to come from the feeders
+                "random_no_duplicate": {"default": False, "help": f"if set, it will override `path_generator`, `filename_generator` and `folder`. It will check if the file already exists and if so it will not upload it again. Creates a new root folder path `{NO_DUPLICATES_FOLDER}`"},
                 "endpoint_url": {
                     "default": 'https://{region}.digitaloceanspaces.com',
                     "help": "S3 bucket endpoint, {region} are inserted at runtime"
@@ -47,6 +50,22 @@ class S3Storage(Storage):
         return self.cdn_url.format(bucket=self.bucket, region=self.region, key=media.key)
 
     def uploadf(self, file: IO[bytes], media: Media, **kwargs: dict) -> None:
+        if not self.is_upload_needed(media): return True
+
+        if self.random_no_duplicate:
+            # checks if a folder with the hash already exists, if so it skips the upload
+            he = HashEnricher({"hash_enricher": {"algorithm": "SHA-256", "chunksize": 1.6e7}})
+            hd = he.calculate_hash(media.filename)
+            path = os.path.join(NO_DUPLICATES_FOLDER, hd[:24])
+
+            if existing_key:=self.file_in_folder(path):
+                media.key = existing_key
+                logger.debug(f"skipping upload of {media.filename} because it already exists in {media.key}")
+                return True
+            
+            _, ext = os.path.splitext(media.key)
+            media.key = os.path.join(path, f"{random_str(24)}{ext}")
+
         extra_args = kwargs.get("extra_args", {})
         if not self.private and 'ACL' not in extra_args:
             extra_args['ACL'] = 'public-read'
@@ -60,14 +79,30 @@ class S3Storage(Storage):
 
         self.s3.upload_fileobj(file, Bucket=self.bucket, Key=media.key, ExtraArgs=extra_args)
         return True
+    
+    def is_upload_needed(self, media: Media) -> bool:
+        if self.random_no_duplicate:
+            # checks if a folder with the hash already exists, if so it skips the upload
+            he = HashEnricher({"hash_enricher": {"algorithm": "SHA-256", "chunksize": 1.6e7}})
+            hd = he.calculate_hash(media.filename)
+            path = os.path.join(NO_DUPLICATES_FOLDER, hd[:24])
+
+            if existing_key:=self.file_in_folder(path):
+                media.key = existing_key
+                logger.debug(f"skipping upload of {media.filename} because it already exists in {media.key}")
+                return False
+            
+            _, ext = os.path.splitext(media.key)
+            media.key = os.path.join(path, f"{random_str(24)}{ext}")
+        return True
+    
+    
+    def file_in_folder(self, path:str) -> str:
+        # checks if path exists and is not an empty folder
+        if not path.endswith('/'):
+            path = path + '/' 
+        resp = self.s3.list_objects(Bucket=self.bucket, Prefix=path, Delimiter='/', MaxKeys=1)
+        if 'Contents' in resp:
+            return resp['Contents'][0]['Key']
+        return False
 
-    # def exists(self, key: str) -> bool:
-    #     """
-    #     Tests if a given file with key=key exists in the bucket
-    #     """
-    #     try:
-    #         self.s3.head_object(Bucket=self.bucket, Key=key)
-    #         return True
-    #     except ClientError as e:
-    #         logger.warning(f"got a ClientError when checking if {key=} exists in bucket={self.bucket}: {e}")
-    #     return False

src/auto_archiver/storages/storage.py

@@ -2,11 +2,13 @@ from __future__ import annotations
 from abc import abstractmethod
 from dataclasses import dataclass
 from typing import IO
+import os
+
+from ..utils.misc import random_str
 
 from ..core import Media, Step, ArchivingContext
 from ..enrichers import HashEnricher
 from loguru import logger
-import os, uuid
 from slugify import slugify
 
 
@@ -72,10 +74,10 @@ class Storage(Step):
             filename = slugify(filename)  # in case it comes with os.sep
         elif self.path_generator == "url": path = slugify(url)
         elif self.path_generator == "random":
-            path = ArchivingContext.get("random_path", str(uuid.uuid4())[:16], True)
+            path = ArchivingContext.get("random_path", random_str(24), True)
 
         # filename_generator logic
-        if self.filename_generator == "random": filename = str(uuid.uuid4())[:16]
+        if self.filename_generator == "random": filename = random_str(24)
         elif self.filename_generator == "static":
             he = HashEnricher({"hash_enricher": {"algorithm": ArchivingContext.get("hash_enricher.algorithm"), "chunksize": 1.6e7}})
             hd = he.calculate_hash(media.filename)

src/auto_archiver/utils/misc.py

@@ -1,8 +1,8 @@
 
 import os, json, requests
+import uuid
 from datetime import datetime
 from loguru import logger
-from urllib.parse import urlparse, urlunparse
 
 
 def mkdir_if_not_exists(folder):
@@ -21,13 +21,6 @@ def expand_url(url):
             logger.error(f'Failed to expand url {url}')
     return url
 
-def remove_get_parameters(url):
-    # http://example.com/file.mp4?t=1 -> http://example.com/file.mp4
-    # useful for mimetypes to work
-    parsed_url = urlparse(url)
-    new_url = urlunparse(parsed_url._replace(query=''))
-    return new_url
-
 
 def getattr_or(o: object, prop: str, default=None):
     try:
@@ -57,3 +50,7 @@ def update_nested_dict(dictionary, update_dict):
             update_nested_dict(dictionary[key], value)
         else:
             dictionary[key] = value
+
+def random_str(length: int = 32) -> str:
+    assert length <= 32, "length must be less than 32 as UUID4 is used"
+    return str(uuid.uuid4()).replace("-", "")[:length]

src/auto_archiver/utils/url.py

@@ -1,14 +1,15 @@
 import re
+from urllib.parse import urlparse, urlunparse
 
 class UrlUtil:
     telegram_private = re.compile(r"https:\/\/t\.me(\/c)\/(.+)\/(\d+)")
     is_istagram = re.compile(r"https:\/\/www\.instagram\.com")
 
     @staticmethod
-    def clean(url): return url
+    def clean(url: str) -> str: return url
 
     @staticmethod
-    def is_auth_wall(url):
+    def is_auth_wall(url: str) -> bool:
         """
         checks if URL is behind an authentication wall meaning steps like wayback, wacz, ... may not work
         """
@@ -17,3 +18,62 @@ class UrlUtil:
 
         return False
 
+    @staticmethod
+    def remove_get_parameters(url: str) -> str:
+        # http://example.com/file.mp4?t=1 -> http://example.com/file.mp4
+        # useful for mimetypes to work
+        parsed_url = urlparse(url)
+        new_url = urlunparse(parsed_url._replace(query=''))
+        return new_url
+
+    @staticmethod
+    def is_relevant_url(url: str) -> bool:
+        """
+        Detect if a detected media URL is recurring and therefore irrelevant to a specific archive. Useful, for example, for the enumeration of the media files in WARC files which include profile pictures, favicons, etc.
+        """
+        clean_url = UrlUtil.remove_get_parameters(url)
+
+        # favicons
+        if "favicon" in url: return False
+        # ifnore icons
+        if clean_url.endswith(".ico"): return False
+        # ignore SVGs
+        if UrlUtil.remove_get_parameters(url).endswith(".svg"): return False
+
+        # twitter profile pictures
+        if "twimg.com/profile_images" in url: return False
+        if "twimg.com" in url and "/default_profile_images" in url: return False
+
+        # instagram profile pictures
+        if "https://scontent.cdninstagram.com/" in url and "150x150" in url: return False
+        # instagram recurring images
+        if "https://static.cdninstagram.com/rsrc.php/" in url: return False
+
+        # telegram
+        if "https://telegram.org/img/emoji/" in url: return False
+
+        # youtube
+        if "https://www.youtube.com/s/gaming/emoji/" in url: return False
+        if "https://yt3.ggpht.com" in url and "default-user=" in url: return False
+        if "https://www.youtube.com/s/search/audio/" in url: return False
+
+        # ok
+        if " https://ok.ru/res/i/" in url: return False
+
+        # vk
+        if "https://vk.com/emoji/" in url: return False
+        if "vk.com/images/" in url: return False
+        if "vk.com/images/reaction/" in url: return False
+
+        # wikipedia
+        if "wikipedia.org/static" in url: return False
+
+        return True
+
+    @staticmethod
+    def twitter_best_quality_url(url: str) -> str:
+        """
+        some twitter image URLs point to a less-than best quality
+        this returns the URL pointing to the highest (original) quality
+        """
+        return re.sub(r"name=(\w+)", "name=orig", url, 1)

src/auto_archiver/utils/webdriver.py

@@ -15,7 +15,7 @@ class Webdriver:
 
     def __enter__(self) -> webdriver:
         options = webdriver.FirefoxOptions()
-        options.headless = True
+        options.add_argument("--headless")
         options.set_preference('network.protocol-handler.external.tg', False)
         try:
             self.driver = webdriver.Firefox(options=options)

src/auto_archiver/version.py

@@ -1,9 +1,9 @@
 
 _MAJOR = "0"
-_MINOR = "5"
+_MINOR = "7"
 # On main and in a nightly release the patch should be one ahead of the last
 # released build.
-_PATCH = "28"
+_PATCH = "10"
 # This is mainly for nightly builds which have the suffix ".dev$DATE". See
 # https://semver.org/#is-v123-a-semantic-version for the semantics.
 _SUFFIX = ""