Bump version to v0.6.12 for release

Co-authored-by: msramalho <19508417+msramalho@users.noreply.github.com>

.github/workflows/docker-publish.yaml

@@ -9,7 +9,7 @@ on:
   release:
     types: [published]
   push:
-    branches: [ "dockerize" ]
+    # branches: [ "main" ]
     tags: [ "v*.*.*" ]
 
 env:

.github/workflows/python-publish.yaml

@@ -12,7 +12,7 @@ on:
   release:
     types: [published]
   push:
-    branches: [ "dockerize" ]
+    # branches: [ "main" ]
     tags: [ "v*.*.*" ]
 
 permissions:

Dockerfile

@@ -4,12 +4,11 @@ ENV RUNNING_IN_DOCKER=1
 
 WORKDIR /app
 
-# TODO: use custom ffmpeg builds instead of apt-get install
 RUN pip install --upgrade pip && \
 	pip install pipenv && \
 	add-apt-repository ppa:mozillateam/ppa && \
 	apt-get update && \
-	apt-get install -y gcc ffmpeg fonts-noto && \
+	apt-get install -y gcc ffmpeg fonts-noto exiftool && \
 	apt-get install -y --no-install-recommends firefox-esr && \
 	ln -s /usr/bin/firefox-esr /usr/bin/firefox && \
 	wget https://github.com/mozilla/geckodriver/releases/download/v0.33.0/geckodriver-v0.33.0-linux64.tar.gz && \
@@ -18,19 +17,15 @@ RUN pip install --upgrade pip && \
 	rm geckodriver-v*
 
 
-# TODO: avoid copying unnecessary files, including .git
 COPY Pipfile* ./
-RUN pipenv install
-
+# install from pipenv, with browsertrix-only requirements
+RUN pipenv install && \
+	pipenv install pywb uwsgi
+	
 # doing this at the end helps during development, builds are quick
 COPY ./src/ . 
 
-# TODO: figure out how to make volumes not be root, does it depend on host or dockerfile?
-# RUN useradd --system --groups sudo --shell /bin/bash archiver && chown -R archiver:sudo .
-# USER archiver
-
-
 ENTRYPOINT ["pipenv", "run", "python3", "-m", "auto_archiver"]
 
 # should be executed with 2 volumes (3 if local_storage is used)
-# docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive aa pipenv run python3 -m auto_archiver --config secrets/orchestration.yaml
+# docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive aa pipenv run python3 -m auto_archiver --config secrets/orchestration.yaml

Pipfile

@@ -19,6 +19,8 @@ google-api-python-client = "*"
 google-auth-httplib2 = "*"
 google-auth-oauthlib = "*"
 oauth2client = "*"
+pdqhash = "*"
+pillow = "*"
 python-slugify = "*"
 pyyaml = "*"
 dateparser = "*"
@@ -26,18 +28,17 @@ python-twitter-v2 = "*"
 instaloader = "*"
 tqdm = "*"
 jinja2 = "*"
-cryptography = "==38.0.4"
+cryptography = "*"
 dataclasses-json = "*"
-yt-dlp = ">=2023.2.17"
+yt-dlp = "*"
 vk-url-scraper = "*"
-uwsgi = "*"
 requests = {extras = ["socks"], version = "*"}
-# wacz = "==0.4.8"
-pywb = ">=2.7.3"
-
-[requires]
-python_version = "3.10"
+numpy = "*"
+warcio = "*"
 
 [dev-packages]
 autopep8 = "*"
 setuptools-pipfile = "*"
+
+[requires]
+python_version = "3.10"

README.md

@@ -1,7 +1,7 @@
 <h1 align="center">Auto Archiver</h1>
 
 [![PyPI version](https://badge.fury.io/py/auto-archiver.svg)](https://badge.fury.io/py/auto-archiver)
-[![Docker Image Version (latest by date)](https://img.shields.io/docker/v/bellingcat/auto-archiver?label=version&logo=docker)](https://pypi.org/project/auto-archiver/)
+[![Docker Image Version (latest by date)](https://img.shields.io/docker/v/bellingcat/auto-archiver?label=version&logo=docker)](https://hub.docker.com/r/bellingcat/auto-archiver)
 <!-- ![Docker Pulls](https://img.shields.io/docker/pulls/bellingcat/auto-archiver) -->
 <!-- [![PyPI download month](https://img.shields.io/pypi/dm/auto-archiver.svg)](https://pypi.python.org/pypi/auto-archiver/) -->
 <!-- [![Documentation Status](https://readthedocs.org/projects/vk-url-scraper/badge/?version=latest)](https://vk-url-scraper.readthedocs.io/en/latest/?badge=latest) -->
@@ -20,12 +20,10 @@ There are 3 ways to use the auto-archiver:
 But **you always need a configuration/orchestration file**, which is where you'll configure where/what/how to archive. Make sure you read [orchestration](#orchestration).
 
 
-## How to run the auto-archiver
+## How to install and run the auto-archiver
 
 ### Option 1 - docker
 
-<details><summary><code>Docker instructions</code></summary>
-
 [![dockeri.co](https://dockerico.blankenship.io/image/bellingcat/auto-archiver)](https://hub.docker.com/r/bellingcat/auto-archiver)
 
 Docker works like a virtual machine running inside your computer, it isolates everything and makes installation simple. Since it is an isolated environment when you need to pass it your orchestration file or get downloaded media out of docker you will need to connect folders on your machine with folders inside docker with the `-v` volume flag.
@@ -45,8 +43,6 @@ Docker works like a virtual machine running inside your computer, it isolates ev
        2.  `$PWD/local_archive` is a folder `local_archive/` in case you want to archive locally and have the files accessible outside docker
        3.  `/app/local_archive` is a folder inside docker that you can reference in your orchestration.yml file 
 
-</details>
-
 ### Option 2 - python package
 
 <details><summary><code>Python package instructions</code></summary>
@@ -54,8 +50,9 @@ Docker works like a virtual machine running inside your computer, it isolates ev
 1. make sure you have python 3.8 or higher installed
 2. install the package `pip/pipenv/conda install auto-archiver`
 3. test it's installed with `auto-archiver --help`
-4. run it with your orchestration file and pass any flags you want in the command line `auto-archiver --config secrets/orchestration.yaml`
-   1. if your orchestration file is inside a `secrets/` which we advise
+4. run it with your orchestration file and pass any flags you want in the command line `auto-archiver --config secrets/orchestration.yaml` if your orchestration file is inside a `secrets/`, which we advise
+   
+You will also need [ffmpeg](https://www.ffmpeg.org/), [firefox](https://www.mozilla.org/en-US/firefox/new/) and [geckodriver](https://github.com/mozilla/geckodriver/releases), and optionally [fonts-noto](https://fonts.google.com/noto). Similar to the local installation. 
 
 </details>
 
@@ -69,7 +66,7 @@ This can also be used for development.
 Install the following locally:
 1. [ffmpeg](https://www.ffmpeg.org/) must also be installed locally for this tool to work. 
 2. [firefox](https://www.mozilla.org/en-US/firefox/new/) and [geckodriver](https://github.com/mozilla/geckodriver/releases) on a path folder like `/usr/local/bin`. 
-3. [fonts-noto](https://fonts.google.com/noto) to deal with multiple unicode characters during selenium/geckodriver's screenshots: `sudo apt install fonts-noto -y`. 
+3. (optional) [fonts-noto](https://fonts.google.com/noto) to deal with multiple unicode characters during selenium/geckodriver's screenshots: `sudo apt install fonts-noto -y`. 
 
 Clone and run:
 1. `git clone https://github.com/bellingcat/auto-archiver`
@@ -87,7 +84,7 @@ The archiver work is orchestrated by the following workflow (we call each a **st
 4. **Formatter** creates a report from all the archived content (HTML, PDF, ...)
 5. **Database** knows what's been archived and also stores the archive result (spreadsheet, CSV, or just the console)
 
-To setup an auto-archiver instance, instance, create an `orchestration.yaml` which contains the workflow you would like. We advise you put this file into a `secrets/` folder and do not share it with others because it will contain passwords and other secrets. 
+To setup an auto-archiver instance create an `orchestration.yaml` which contains the workflow you would like. We advise you put this file into a `secrets/` folder and do not share it with others because it will contain passwords and other secrets. 
 
 The structure of orchestration file is split into 2 parts: `steps` (what **steps** to use) and `configurations` (how those steps should behave), here's a simplification:
 ```yaml
@@ -147,19 +144,30 @@ Use this to make sure you help making sure you did all the required steps:
   * [ ] (optional for browsertrix) `profile.tar.gz` file
 
 #### Example invocations
-These assume you've installed with pipenv, see docker section above for how to run through docker
+The recommended way to run the auto-archiver is through Docker. The invocations below will run the auto-archiver Docker image using a configuration file that you have specified
+
+```bash
+# all the configurations come from ./secrets/orchestration.yaml
+docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver --config secrets/orchestration.yaml
+# uses the same configurations but for another google docs sheet 
+# with a header on row 2 and with some different column names
+# notice that columns is a dictionary so you need to pass it as JSON and it will override only the values provided
+docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver --config secrets/orchestration.yaml --gsheet_feeder.sheet="use it on another sheets doc" --gsheet_feeder.header=2 --gsheet_feeder.columns='{"url": "link"}'
+# all the configurations come from orchestration.yaml and specifies that s3 files should be private
+docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver --config secrets/orchestration.yaml --s3_storage.private=1
+```
+
+The auto-archiver can also be run locally, if pre-requisites are correctly configured. Equivalent invocations are below.
 
 ```bash
-# all the configurations come from ./orchestration.yaml
-auto-archiver
 # all the configurations come from ./secrets/orchestration.yaml
 auto-archiver --config secrets/orchestration.yaml
 # uses the same configurations but for another google docs sheet 
 # with a header on row 2 and with some different column names
 # notice that columns is a dictionary so you need to pass it as JSON and it will override only the values provided
-auto-archiver --config orchestration.yaml --gsheet_feeder.sheet="use it on another sheets doc" --gsheet_feeder.header=2 --gsheet_feeder.columns='{"url": "link"}'
+auto-archiver --config secrets/orchestration.yaml --gsheet_feeder.sheet="use it on another sheets doc" --gsheet_feeder.header=2 --gsheet_feeder.columns='{"url": "link"}'
 # all the configurations come from orchestration.yaml and specifies that s3 files should be private
-auto-archiver --s3_storage.private=1
+auto-archiver --config secrets/orchestration.yaml --s3_storage.private=1
 ```
 
 ### Extra notes on configuration
@@ -173,18 +181,46 @@ The first time you run, you will be prompted to do a authentication with the pho
 ## Running on Google Sheets Feeder (gsheet_feeder)
 The `--gseets_feeder.sheet` property is the name of the Google Sheet to check for URLs. 
 This sheet must have been shared with the Google Service account used by `gspread`. 
-This sheet must also have specific columns (case-insensitive) in the `header` row - see [Gsheet.configs](src/auto_archiver/utils/gsheet.py) for all their names.
+This sheet must also have specific columns (case-insensitive) in the `header` as specified in [Gsheet.configs](src/auto_archiver/utils/gsheet.py). The default names of these columns and their purpose is:
 
-For example, for use with this spreadsheet:
+Inputs:
 
-![A screenshot of a Google Spreadsheet with column headers defined as above, and several Youtube and Twitter URLs in the "Media URL" column](docs/demo-before.png)
+* **Link** *(required)*: the URL of the post to archive
+* **Destination folder**: custom folder for archived file (regardless of storage)
+
+Outputs:
+* **Archive status** *(required)*: Status of archive operation
+* **Archive location**: URL of archived post
+* **Archive date**: Date archived
+* **Thumbnail**: Embeds a thumbnail for the post in the spreadsheet
+* **Timestamp**: Timestamp of original post
+* **Title**: Post title
+* **Text**: Post text
+* **Screenshot**: Link to screenshot of post
+* **Hash**: Hash of archived HTML file (which contains hashes of post media) - for checksums/verification
+* **Perceptual Hash**: Perceptual hashes of found images - these can be used for de-duplication of content
+* **WACZ**: Link to a WACZ web archive of post
+* **ReplayWebpage**: Link to a ReplayWebpage viewer of the WACZ archive
+
+For example, this is a spreadsheet configured with all of the columns for the auto archiver and a few URLs to archive. (Note that the column names are not case sensitive.)
+
+![A screenshot of a Google Spreadsheet with column headers defined as above, and several Youtube and Twitter URLs in the "Link" column](docs/demo-before.png)
+
+Now the auto archiver can be invoked, with this command in this example: `docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver:dockerize --config secrets/orchestration-global.yaml --gsheet_feeder.sheet "Auto archive test 2023-2"`. Note that the sheet name has been overridden/specified in the command line invocation.
 
 When the auto archiver starts running, it updates the "Archive status" column.
-![A screenshot of a Google Spreadsheet with column headers defined as above, and several Youtube and Twitter URLs in the "Media URL" column. The auto archiver has added "archive in progress" to one of the status columns.](docs/demo-progress.png)
+
+![A screenshot of a Google Spreadsheet with column headers defined as above, and several Youtube and Twitter URLs in the "Link" column. The auto archiver has added "archive in progress" to one of the status columns.](docs/demo-progress.png)
+
 The links are downloaded and archived, and the spreadsheet is updated to the following:
+
 ![A screenshot of a Google Spreadsheet with videos archived and metadata added per the description of the columns above.](docs/demo-after.png)
+
 Note that the first row is skipped, as it is assumed to be a header row (`--gsheet_feeder.header=1` and you can change it if you use more rows above). Rows with an empty URL column, or a non-empty archive column are also skipped. All sheets in the document will be checked.
 
+The "archive location" link contains the path of the archived file, in local storage, S3, or in Google Drive.
+
+![The archive result for a link in the demo sheet.](docs/demo-archive.png)
 
 ---
 ## Development
@@ -193,7 +229,7 @@ Use `python -m src.auto_archiver --config secrets/orchestration.yaml` to run fro
 #### Docker development
 working with docker locally:
   * `docker build . -t auto-archiver` to build a local image
-  * `docker run --rm -v $PWD/secrets:/app/secrets aa pipenv run python3 -m auto_archiver --config secrets/orchestration.yaml`
+  * `docker run --rm -v $PWD/secrets:/app/secrets auto-archiver  --config secrets/orchestration.yaml`
     * to use local archive, also create a volume `-v` for it by adding `-v $PWD/local_archive:/app/local_archive`
 
 
@@ -205,4 +241,4 @@ release to docker hub
 * update version in [version.py](src/auto_archiver/version.py)
 * run `bash ./scripts/release.sh` and confirm
 * package is automatically updated in pypi
-* docker image is automatically pushed to dockerhup
+* docker image is automatically pushed to dockerhup

example.orchestration.yaml

@@ -11,14 +11,16 @@ steps:
     # - instagram_archiver
     # - tiktok_archiver
     - youtubedl_archiver
-    - wayback_archiver_enricher
+    # - wayback_archiver_enricher
+    # - wacz_archiver_enricher
   enrichers:
     - hash_enricher
+    # - metadata_enricher
     # - screenshot_enricher
     # - thumbnail_enricher
     # - wayback_archiver_enricher
-    # - wacz_enricher
-    
+    # - wacz_archiver_enricher
+    # - pdq_hash_enricher # if you want to calculate hashes for thumbnails, include this after thumbnail_enricher
   formatter: html_formatter # defaults to mute_formatter
   storages:
     - local_storage
@@ -50,6 +52,7 @@ configurations:
       text: textual content
       screenshot: screenshot
       hash: hash
+      pdq_hash: perceptual hashes
       wacz: wacz
       replaywebpage: replaywebpage
   instagram_tbot_archiver:
@@ -94,7 +97,7 @@ configurations:
     secret: "wayback secret"
   hash_enricher:
     algorithm: "SHA3-512" # can also be SHA-256
-  wacz_enricher:
+  wacz_archiver_enricher:
     profile: secrets/profile.tar.gz
   local_storage:
     save_to: "./local_archive"
@@ -112,10 +115,11 @@ configurations:
     private: false
     # with 'random' you can generate a random UUID for the URL instead of a predictable path, useful to still have public but unlisted files, alternative is 'default' or not omitted from config
     key_path: random
-
   gdrive_storage:
     path_generator: url
     filename_generator: random
     root_folder_id: folder_id_from_url
     oauth_token: secrets/gd-token.json # needs to be generated with scripts/create_update_gdrive_oauth_token.py
     service_account: "secrets/service_account.json"
+  csv_db:
+    csv_file: "./local_archive/db.csv"

src/auto_archiver/__main__.py

@@ -5,7 +5,7 @@ def main():
     config = Config()
     config.parse()
     orchestrator = ArchivingOrchestrator(config)
-    orchestrator.feed()
+    for r in orchestrator.feed(): pass
 
 
 if __name__ == "__main__":

src/auto_archiver/archivers/archiver.py

@@ -27,11 +27,6 @@ class Archiver(Step):
         # used to clean unnecessary URL parameters OR unfurl redirect links
         return url
 
-    def is_rearchivable(self, url: str) -> bool:
-        # archivers can signal if it does not make sense to rearchive a piece of content
-        # default is rearchiving
-        return True
-
     def _guess_file_type(self, path: str) -> str:
         """
         Receives a URL or filename and returns global mimetype like 'image' or 'video'
@@ -56,6 +51,7 @@ class Archiver(Step):
             'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36'
         }
         d = requests.get(url, headers=headers)
+        assert d.status_code == 200, f"got response code {d.status_code} for {url=}"
         with open(to_filename, 'wb') as f:
             f.write(d.content)
         return to_filename

src/auto_archiver/archivers/telegram_archiver.py

@@ -19,10 +19,6 @@ class TelegramArchiver(Archiver):
     def configs() -> dict:
         return {}
 
-    def is_rearchivable(self, url: str) -> bool:
-        # telegram posts are static
-        return False
-
     def download(self, item: Metadata) -> Metadata:
         url = item.get_url()
         # detect URLs that we definitely cannot handle
@@ -48,7 +44,7 @@ class TelegramArchiver(Archiver):
         video = s.find("video")
         if video is None:
             logger.warning("could not find video")
-            image_tags = s.find_all(class_="js-message_photo")
+            image_tags = s.find_all(class_="tgme_widget_message_photo_wrap")
 
             image_urls = []
             for im in image_tags:
@@ -57,10 +53,10 @@ class TelegramArchiver(Archiver):
 
             if not len(image_urls): return False
             for img_url in image_urls:
-                result.add_media(Media(self.download_from_url(img_url)))
+                result.add_media(Media(self.download_from_url(img_url, item=item)))
         else:
             video_url = video.get('src')
-            m_video = Media(self.download_from_url(video_url))
+            m_video = Media(self.download_from_url(video_url, item=item))
             # extract duration from HTML
             try:
                 duration = s.find_all('time')[0].contents[0]

src/auto_archiver/archivers/telethon_archiver.py

@@ -38,10 +38,6 @@ class TelethonArchiver(Archiver):
             }
         }
 
-    def is_rearchivable(self, url: str) -> bool:
-        # telegram posts are static
-        return False
-
     def setup(self) -> None:
         """
         1. trigger login process for telegram or proceed if already saved in a session file

src/auto_archiver/archivers/tiktok_archiver.py

@@ -16,10 +16,6 @@ class TiktokArchiver(Archiver):
     def configs() -> dict:
         return {}
 
-    def is_rearchivable(self, url: str) -> bool:
-        # TikTok posts are static
-        return False
-
     def download(self, item: Metadata) -> Metadata:
         url = item.get_url()
         if 'tiktok.com' not in url:

src/auto_archiver/archivers/twitter_archiver.py

@@ -6,6 +6,7 @@ from slugify import slugify
 
 from . import Archiver
 from ..core import Metadata, Media
+from ..utils import UrlUtil
 
 
 class TwitterArchiver(Archiver):
@@ -14,8 +15,8 @@ class TwitterArchiver(Archiver):
     """
 
     name = "twitter_archiver"
-    link_pattern = re.compile(r"twitter.com\/(?:\#!\/)?(\w+)\/status(?:es)?\/(\d+)")
-    link_clean_pattern = re.compile(r"(.+twitter\.com\/.+\/\d+)(\?)*.*")
+    link_pattern = re.compile(r"(?:twitter|x).com\/(?:\#!\/)?(\w+)\/status(?:es)?\/(\d+)")
+    link_clean_pattern = re.compile(r"(.+(?:twitter|x)\.com\/.+\/\d+)(\?)*.*")
 
     def __init__(self, config: dict) -> None:
         super().__init__(config)
@@ -36,10 +37,6 @@ class TwitterArchiver(Archiver):
         # https://twitter.com/MeCookieMonster/status/1617921633456640001?s=20&t=3d0g4ZQis7dCbSDg-mE7-w
         return self.link_clean_pattern.sub("\\1", url)
 
-    def is_rearchivable(self, url: str) -> bool:
-        # Twitter posts are static (for now)
-        return False
-
     def download(self, item: Metadata) -> Metadata:
         """
         if this url is archivable will download post info and look for other posts from the same group with media.
@@ -77,7 +74,7 @@ class TwitterArchiver(Archiver):
                 media.set("src", variant.url)
                 mimetype = variant.contentType
             elif type(tweet_media) == Photo:
-                media.set("src", tweet_media.fullUrl.replace('name=large', 'name=orig'))
+                media.set("src", UrlUtil.twitter_best_quality_url(tweet_media.fullUrl))
                 mimetype = "image/jpeg"
             else:
                 logger.warning(f"Could not get media URL of {tweet_media}")
@@ -90,20 +87,22 @@ class TwitterArchiver(Archiver):
 
     def download_alternative(self, item: Metadata, url: str, tweet_id: str) -> Metadata:
         """
-        CURRENTLY STOPPED WORKING
+        Hack alternative working again.
+        https://stackoverflow.com/a/71867055/6196010 (OUTDATED URL)
+        https://github.com/JustAnotherArchivist/snscrape/issues/996#issuecomment-1615937362
+        next to test: https://cdn.embedly.com/widgets/media.html?&schema=twitter&url=https://twitter.com/bellingcat/status/1674700676612386816
         """
-        return False
-        # https://stackoverflow.com/a/71867055/6196010
+
         logger.debug(f"Trying twitter hack for {url=}")
         result = Metadata()
 
-        hack_url = f"https://cdn.syndication.twimg.com/tweet?id={tweet_id}"
+        hack_url = f"https://cdn.syndication.twimg.com/tweet-result?id={tweet_id}"
         r = requests.get(hack_url)
         if r.status_code != 200: return False
         tweet = r.json()
 
         urls = []
-        for p in tweet["photos"]:
+        for p in tweet.get("photos", []):
             urls.append(p["url"])
 
         # 1 tweet has 1 video max
@@ -113,14 +112,19 @@ class TwitterArchiver(Archiver):
 
         logger.debug(f"Twitter hack got {urls=}")
 
-        for u in urls:
-            media = Media()
+        for i, u in enumerate(urls):
+            media = Media(filename="")
+            u = UrlUtil.twitter_best_quality_url(u)
             media.set("src", u)
-            media.filename = self.download_from_url(u, f'{slugify(url)}_{i}', item)
+            ext = ""
+            if (mtype := mimetypes.guess_type(UrlUtil.remove_get_parameters(u))[0]):
+                ext = mimetypes.guess_extension(mtype)
+
+            media.filename = self.download_from_url(u, f'{slugify(url)}_{i}{ext}', item)
             result.add_media(media)
 
-        result.set_content(json.dumps(tweet, ensure_ascii=False)).set_timestamp(datetime.strptime(tweet["created_at"], "%Y-%m-%dT%H:%M:%S.%fZ"))
-        return result
+        result.set_title(tweet.get("text")).set_content(json.dumps(tweet, ensure_ascii=False)).set_timestamp(datetime.strptime(tweet["created_at"], "%Y-%m-%dT%H:%M:%S.%fZ"))
+        return result.success("twitter-hack")
 
     def get_username_tweet_id(self, url):
         # detect URLs that we definitely cannot handle

src/auto_archiver/archivers/vk_archiver.py

@@ -27,10 +27,6 @@ class VkArchiver(Archiver):
             "session_file": {"default": "secrets/vk_config.v2.json", "help": "valid VKontakte password"},
         }
 
-    def is_rearchivable(self, url: str) -> bool:
-        # VK content is static
-        return False
-
     def download(self, item: Metadata) -> Metadata:
         url = item.get_url()
 

src/auto_archiver/archivers/youtubedl_archiver.py

@@ -25,7 +25,7 @@ class YoutubeDLArchiver(Archiver):
             logger.debug('Using Facebook cookie')
             yt_dlp.utils.std_headers['cookie'] = self.facebook_cookie
 
-        ydl = yt_dlp.YoutubeDL({'outtmpl': os.path.join(ArchivingContext.get_tmp_dir(), f'%(id)s.%(ext)s'), 'quiet': False})
+        ydl = yt_dlp.YoutubeDL({'outtmpl': os.path.join(ArchivingContext.get_tmp_dir(), f'%(id)s.%(ext)s'), 'quiet': False, 'noplaylist': True})
 
         try:
             # don'd download since it can be a live stream

src/auto_archiver/core/config.py

@@ -13,6 +13,7 @@ from ..formatters import Formatter
 from ..storages import Storage
 from ..enrichers import Enricher
 from . import Step
+from ..utils import update_nested_dict
 
 
 @dataclass
@@ -38,10 +39,11 @@ class Config:
         self.cli_ops = {}
         self.config = {}
 
-    def parse(self, use_cli=True, yaml_config_filename: str = None):
+    def parse(self, use_cli=True, yaml_config_filename: str = None, overwrite_configs: str = {}):
         """
         if yaml_config_filename is provided, the --config argument is ignored, 
         useful for library usage when the config values are preloaded
+        overwrite_configs is a dict that overwrites the yaml file contents
         """
         # 1. parse CLI values
         if use_cli:
@@ -80,6 +82,7 @@ class Config:
 
         # 2. read YAML config file (or use provided value)
         self.yaml_config = self.read_yaml(yaml_config_filename)
+        update_nested_dict(self.yaml_config, overwrite_configs)
 
         # 3. CONFIGS: decide value with priority: CLI >> config.yaml >> default
         self.config = defaultdict(dict)

src/auto_archiver/core/media.py

@@ -1,11 +1,15 @@
 
 from __future__ import annotations
-from ast import List
-from typing import Any
+import os
+import traceback
+from typing import Any, List
 from dataclasses import dataclass, field
 from dataclasses_json import dataclass_json, config
 import mimetypes
 
+import ffmpeg
+from ffmpeg._run import Error
+
 from .context import ArchivingContext
 
 from loguru import logger
@@ -31,15 +35,19 @@ class Media:
             return
 
         for s in storages:
-            s.store(self, url)
-            # Media can be inside media properties, examples include transformations on original media
-            for prop in self.properties.values():
-                if isinstance(prop, Media):
-                    s.store(prop, url)
-                if isinstance(prop, list):
-                    for prop_media in prop:
-                        if isinstance(prop_media, Media):
-                            s.store(prop_media, url)
+            for any_media in self.all_inner_media(include_self=True):
+                s.store(any_media, url)
+
+    def all_inner_media(self, include_self=False):
+        """ Media can be inside media properties, examples include transformations on original media.
+        This function returns a generator for all the inner media.        
+        """
+        if include_self: yield self
+        for prop in self.properties.values():
+            if isinstance(prop, Media): yield prop
+            if isinstance(prop, list):
+                for prop_media in prop:
+                    if isinstance(prop_media, Media): yield prop_media
 
     def is_stored(self) -> bool:
         return len(self.urls) > 0 and len(self.urls) == len(ArchivingContext.get("storages"))
@@ -71,3 +79,23 @@ class Media:
 
     def is_audio(self) -> bool:
         return self.mimetype.startswith("audio")
+
+    def is_image(self) -> bool:
+        return self.mimetype.startswith("image")
+
+    def is_valid_video(self) -> bool:
+        # checks for video streams with ffmpeg, or min file size for a video
+        # self.is_video() should be used together with this method
+        try:
+            streams = ffmpeg.probe(self.filename, select_streams='v')['streams']
+            logger.warning(f"STREAMS FOR {self.filename} {streams}")
+            return any(s.get("duration_ts", 0) > 0 for s in streams)
+        except Error: return False # ffmpeg errors when reading bad files
+        except Exception as e:
+            logger.error(e)
+            logger.error(traceback.format_exc())
+            try:
+                fsize = os.path.getsize(self.filename)
+                return fsize > 20_000
+            except: pass
+        return True

src/auto_archiver/core/metadata.py

@@ -1,7 +1,7 @@
 
 from __future__ import annotations
-from ast import List, Set
-from typing import Any, Union, Dict
+import hashlib
+from typing import Any, List, Union, Dict
 from dataclasses import dataclass, field
 from dataclasses_json import dataclass_json, config
 import datetime
@@ -17,7 +17,6 @@ class Metadata:
     status: str = "no archiver"
     metadata: Dict[str, Any] = field(default_factory=dict)
     media: List[Media] = field(default_factory=list)
-    rearchivable: bool = True  # defaults to true, archivers can overwrite
 
     def __post_init__(self):
         self.set("_processed_at", datetime.datetime.utcnow())
@@ -30,7 +29,6 @@ class Metadata:
         if overwrite_left:
             if right.status and len(right.status):
                 self.status = right.status
-            self.rearchivable |= right.rearchivable
             for k, v in right.metadata.items():
                 assert k not in self.metadata or type(v) == type(self.get(k))
                 if type(v) not in [dict, list, set] or k not in self.metadata:
@@ -45,6 +43,7 @@ class Metadata:
 
     def store(self: Metadata, override_storages: List = None):
         # calls .store for all contained media. storages [Storage]
+        self.remove_duplicate_media_by_hash()
         storages = override_storages or ArchivingContext.get("storages")
         for media in self.media:
             media.store(override_storages=storages, url=self.get_url())
@@ -125,6 +124,27 @@ class Metadata:
             if m.get("id") == id: return m
         return default
 
+    def remove_duplicate_media_by_hash(self) -> None:
+        # iterates all media, calculates a hash if it's missing and deletes duplicates
+        def calculate_hash_in_chunks(hash_algo, chunksize, filename) -> str:
+            # taken from hash_enricher, cannot be isolated to misc due to circular imports
+            with open(filename, "rb") as f:
+                while True:
+                    buf = f.read(chunksize)
+                    if not buf: break
+                    hash_algo.update(buf)
+            return hash_algo.hexdigest()
+
+        media_hashes = set()
+        new_media = []
+        for m in self.media:
+            h = m.get("hash")
+            if not h: h = calculate_hash_in_chunks(hashlib.sha256(), int(1.6e7), m.filename)
+            if len(h) and h in media_hashes: continue
+            media_hashes.add(h)
+            new_media.append(m)
+        self.media = new_media
+
     def get_first_image(self, default=None) -> Media:
         for m in self.media:
             if "image" in m.mimetype: return m
@@ -138,5 +158,9 @@ class Metadata:
         _default = self.media[0] if len(self.media) else None
         return self.get_media_by_id("_final_media", _default)
 
+    def get_all_media(self) -> List[Media]:
+        # returns a list with all the media and inner media
+        return [inner for m in self.media for inner in m.all_inner_media(True)]
+
     def __str__(self) -> str:
         return self.__repr__()

src/auto_archiver/core/orchestrator.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
-from ast import List
-from typing import Union
+from typing import Generator, Union, List
 
 from .context import ArchivingContext
 
@@ -10,7 +9,6 @@ from ..formatters import Formatter
 from ..storages import Storage
 from ..enrichers import Enricher
 from ..databases import Database
-from .media import Media
 from .metadata import Metadata
 
 import tempfile, traceback
@@ -29,9 +27,9 @@ class ArchivingOrchestrator:
 
         for a in self.archivers: a.setup()
 
-    def feed(self) -> None:
+    def feed(self) -> Generator[Metadata]:
         for item in self.feeder:
-            self.feed_item(item)
+            yield self.feed_item(item)
 
     def feed_item(self, item: Metadata) -> Metadata:
         try:
@@ -64,11 +62,7 @@ class ArchivingOrchestrator:
         result.set_url(url)
         if original_url != url: result.set("original_url", original_url)
 
-        # 2 - rearchiving logic + notify start to DB
-        # archivers can signal whether the content is rearchivable: eg: tweet vs webpage
-        for a in self.archivers: result.rearchivable |= a.is_rearchivable(url)
-        logger.debug(f"{result.rearchivable=} for {url=}")
-
+        # 2 - notify start to DB
         # signal to DB that archiving has started
         # and propagate already archived if it exists
         cached_result = None
@@ -80,7 +74,7 @@ class ArchivingOrchestrator:
             d.started(result)
             if (local_result := d.fetch(result)):
                 cached_result = (cached_result or Metadata()).merge(local_result)
-        if cached_result and not cached_result.rearchivable:
+        if cached_result:
             logger.debug("Found previously archived entry")
             for d in self.databases:
                 d.done(cached_result)
@@ -111,6 +105,7 @@ class ArchivingOrchestrator:
         # looks for Media in result.media and also result.media[x].properties (as list or dict values)
         result.store()
 
+
         # 6 - format and store formatted if needed
         # enrichers typically need access to already stored URLs etc
         if (final_media := self.formatter.format(result)):

src/auto_archiver/core/step.py

@@ -1,9 +1,8 @@
 from __future__ import annotations
-from dataclasses import dataclass, field
+from dataclasses import dataclass
 from inspect import ClassFoundException
 from typing import Type
 from abc import ABC
-# from collections.abc import Iterable
 
 
 @dataclass
@@ -21,7 +20,7 @@ class Step(ABC):
 
     def init(name: str, config: dict, child: Type[Step]) -> Step:
         """
-        looks into direct subclasses of child for name and returns such ab object
+        looks into direct subclasses of child for name and returns such an object
         TODO: cannot find subclasses of child.subclasses
         """
         for sub in child.__subclasses__():

src/auto_archiver/databases/__init__.py

@@ -1,4 +1,5 @@
 from .database import Database
 from .gsheet_db import GsheetsDb
 from .console_db import ConsoleDb
-from .csv_db import CSVDb
+from .csv_db import CSVDb
+from .api_db import AAApiDb

src/auto_archiver/databases/api_db.py

@@ -0,0 +1,41 @@
+import requests, os
+from loguru import logger
+
+from . import Database
+from ..core import Metadata
+
+
+class AAApiDb(Database):
+    """
+        Connects to auto-archiver-api instance
+    """
+    name = "auto_archiver_api_db"
+
+    def __init__(self, config: dict) -> None:
+        # without this STEP.__init__ is not called
+        super().__init__(config)
+        self.assert_valid_string("api_endpoint")
+        self.assert_valid_string("api_secret")
+
+    @staticmethod
+    def configs() -> dict:
+        return {
+            "api_endpoint": {"default": None, "help": "API endpoint where calls are made to"},
+            "api_secret": {"default": None, "help": "API authentication secret"},
+            "public": {"default": False, "help": "whether the URL should be publicly available via the API"},
+            "author_id": {"default": None, "help": "which email to assign as author"},
+            "group_id": {"default": None, "help": "which group of users have access to the archive in case public=false as author"},
+            "tags": {"default": [], "help": "what tags to add to the archived URL", "cli_set": lambda cli_val, cur_val: set(cli_val.split(","))},
+        }
+
+    def done(self, item: Metadata) -> None:
+        """archival result ready - should be saved to DB"""
+        logger.info(f"saving archive of {item.get_url()} to the AA API.")
+
+        payload = {'result': item.to_json(), 'public': self.public, 'author_id': self.author_id, 'group_id': self.group_id, 'tags': list(self.tags)}
+        response = requests.post(os.path.join(self.api_endpoint, "submit-archive"), json=payload, auth=("abc", self.api_secret))
+
+        if response.status_code == 200:
+            logger.success(f"AA API: {response.json()}")
+        else:
+            logger.error(f"AA API FAIL ({response.status_code}): {response.json()}")

src/auto_archiver/databases/gsheet_db.py

@@ -52,8 +52,11 @@ class GsheetsDb(Database):
 
         def batch_if_valid(col, val, final_value=None):
             final_value = final_value or val
-            if val and gw.col_exists(col) and gw.get_cell(row_values, col) == '':
-                cell_updates.append((row, col, final_value))
+            try:
+                if val and gw.col_exists(col) and gw.get_cell(row_values, col) == '':
+                    cell_updates.append((row, col, final_value))
+            except Exception as e:
+                logger.error(f"Unable to batch {col}={final_value} due to {e}")
 
         cell_updates.append((row, 'status', item.status))
 
@@ -64,7 +67,17 @@ class GsheetsDb(Database):
         batch_if_valid('title', item.get_title())
         batch_if_valid('text', item.get("content", ""))
         batch_if_valid('timestamp', item.get_timestamp())
-        batch_if_valid('hash', media.get("hash", "not-calculated"))
+        if media: batch_if_valid('hash', media.get("hash", "not-calculated"))
+
+        # merge all pdq hashes into a single string, if present
+        pdq_hashes = []
+        all_media = item.get_all_media()
+        for m in all_media:
+            if pdq := m.get("pdq_hash"):
+                pdq_hashes.append(pdq)
+        if len(pdq_hashes):
+            batch_if_valid('pdq_hash', ",".join(pdq_hashes))
+
         if (screenshot := item.get_media_by_id("screenshot")) and hasattr(screenshot, "urls"):
             batch_if_valid('screenshot', "\n".join(screenshot.urls))
 

src/auto_archiver/enrichers/__init__.py

@@ -3,5 +3,7 @@ from .screenshot_enricher import ScreenshotEnricher
 from .wayback_enricher import WaybackArchiverEnricher
 from .hash_enricher import HashEnricher
 from .thumbnail_enricher import ThumbnailEnricher
-from .wacz_enricher import WaczEnricher
-from .whisper_enricher import WhisperEnricher
+from .wacz_enricher import WaczArchiverEnricher
+from .whisper_enricher import WhisperEnricher
+from .pdq_hash_enricher import PdqHashEnricher
+from .metadata_enricher import MetadataEnricher

src/auto_archiver/enrichers/hash_enricher.py

@@ -23,7 +23,7 @@ class HashEnricher(Enricher):
     def configs() -> dict:
         return {
             "algorithm": {"default": "SHA-256", "help": "hash algorithm to use", "choices": ["SHA-256", "SHA3-512"]},
-            "chunksize": {"default": 1.6e7, "help": "number of bytes to use when reading files in chunks (if this value is too large you will run out of RAM), default is 16MB"},
+            "chunksize": {"default": int(1.6e7), "help": "number of bytes to use when reading files in chunks (if this value is too large you will run out of RAM), default is 16MB"},
         }
 
     def enrich(self, to_enrich: Metadata) -> None:
@@ -34,7 +34,7 @@ class HashEnricher(Enricher):
             if len(hd := self.calculate_hash(m.filename)):
                 to_enrich.media[i].set("hash", f"{self.algorithm}:{hd}")
 
-    def calculate_hash(self, filename):
+    def calculate_hash(self, filename) -> str:
         hash = None
         if self.algorithm == "SHA-256":
             hash = hashlib.sha256()

src/auto_archiver/enrichers/metadata_enricher.py

@@ -0,0 +1,47 @@
+import subprocess
+import traceback
+from loguru import logger
+
+from . import Enricher
+from ..core import Metadata
+
+
+class MetadataEnricher(Enricher):
+    """
+    Extracts metadata information from files using exiftool.
+    """
+    name = "metadata_enricher"
+
+    def __init__(self, config: dict) -> None:
+        # without this STEP.__init__ is not called
+        super().__init__(config)
+
+    @staticmethod
+    def configs() -> dict:
+        return {}
+
+    def enrich(self, to_enrich: Metadata) -> None:
+        url = to_enrich.get_url()
+        logger.debug(f"extracting EXIF metadata for {url=}")
+
+        for i, m in enumerate(to_enrich.media):
+            if len(md := self.get_metadata(m.filename)):
+                to_enrich.media[i].set("metadata", md)
+
+    def get_metadata(self, filename: str) -> dict:
+        try:
+            # Run ExifTool command to extract metadata from the file
+            cmd = ['exiftool', filename]
+            result = subprocess.run(cmd, capture_output=True, text=True)
+
+            # Process the output to extract individual metadata fields
+            metadata = {}
+            for line in result.stdout.splitlines():
+                field, value = line.strip().split(':', 1)
+                metadata[field.strip()] = value.strip()
+            return metadata
+        except FileNotFoundError:
+            logger.error("[exif_enricher] ExifTool not found. Make sure ExifTool is installed and added to PATH.")
+        except Exception as e:
+            logger.error(f"Error occurred: {e}: {traceback.format_exc()}")
+        return {}

src/auto_archiver/enrichers/pdq_hash_enricher.py

@@ -0,0 +1,52 @@
+import traceback
+import pdqhash
+import numpy as np
+from PIL import Image, UnidentifiedImageError
+from loguru import logger
+
+from . import Enricher
+from ..core import Metadata
+
+
+class PdqHashEnricher(Enricher):
+    """
+    Calculates perceptual hashes for Media instances using PDQ, allowing for (near-)duplicate detection.
+    Ideally this enrichment is orchestrated to run after the thumbnail_enricher.
+    """
+    name = "pdq_hash_enricher"
+
+    def __init__(self, config: dict) -> None:
+        # Without this STEP.__init__ is not called
+        super().__init__(config)
+
+    @staticmethod
+    def configs() -> dict:
+        return {}
+
+    def enrich(self, to_enrich: Metadata) -> None:
+        url = to_enrich.get_url()
+        logger.debug(f"calculating perceptual hashes for {url=}")
+        media_with_hashes = []
+
+        for m in to_enrich.media:
+            for media in m.all_inner_media(True):
+                media_id = media.get("id", "")
+                if media.is_image() and "screenshot" not in media_id and "warc-file-" not in media_id and len(hd := self.calculate_pdq_hash(media.filename)):
+                    media.set("pdq_hash", hd)
+                    media_with_hashes.append(media.filename)
+
+        logger.debug(f"calculated '{len(media_with_hashes)}' perceptual hashes for {url=}: {media_with_hashes}")
+
+    def calculate_pdq_hash(self, filename):
+        # returns a hexadecimal string with the perceptual hash for the given filename
+        try:
+            with Image.open(filename) as img:
+                # convert the image to RGB
+                image_rgb = np.array(img.convert("RGB"))
+                # compute the 256-bit PDQ hash (we do not store the quality score)
+                hash_array, _ = pdqhash.compute(image_rgb)
+                hash = "".join(str(b) for b in hash_array)
+                return hex(int(hash, 2))[2:]
+        except UnidentifiedImageError as e:
+            logger.error(f"Image {filename=} is likely corrupted or in unsupported format {e}: {traceback.format_exc()}")
+        return ""

src/auto_archiver/enrichers/wacz_enricher.py

@@ -1,16 +1,23 @@
+import mimetypes
 import os, shutil, subprocess, uuid
+from zipfile import ZipFile
 from loguru import logger
+from warcio.archiveiterator import ArchiveIterator
 
 from ..core import Media, Metadata, ArchivingContext
 from . import Enricher
+from ..archivers import Archiver
 from ..utils import UrlUtil
 
 
-class WaczEnricher(Enricher):
+class WaczArchiverEnricher(Enricher, Archiver):
     """
-    Submits the current URL to the webarchive and returns a job_id or completed archive
+    Uses https://github.com/webrecorder/browsertrix-crawler to generate a .WACZ archive of the URL
+    If used with [profiles](https://github.com/webrecorder/browsertrix-crawler#creating-and-using-browser-profiles)
+    it can become quite powerful for archiving private content.
+    When used as an archiver it will extract the media from the .WACZ archive so it can be enriched.
     """
-    name = "wacz_enricher"
+    name = "wacz_archiver_enricher"
 
     def __init__(self, config: dict) -> None:
         # without this STEP.__init__ is not called
@@ -20,64 +27,65 @@ class WaczEnricher(Enricher):
     def configs() -> dict:
         return {
             "profile": {"default": None, "help": "browsertrix-profile (for profile generation see https://github.com/webrecorder/browsertrix-crawler#creating-and-using-browser-profiles)."},
-            "timeout": {"default": 90, "help": "timeout for WACZ generation in seconds"},
-            "ignore_auth_wall": {"default": True, "help": "skip URL if it is behind authentication wall, set to False if you have browsertrix profile configured for private content."},
+            "docker_commands": {"default": None, "help":"if a custom docker invocation is needed"},
+            "timeout": {"default": 120, "help": "timeout for WACZ generation in seconds"},
+            "extract_media": {"default": True, "help": "If enabled all the images/videos/audio present in the WACZ archive will be extracted into separate Media. The .wacz file will be kept untouched."}
         }
 
+    def download(self, item: Metadata) -> Metadata:
+        # this new Metadata object is required to avoid duplication
+        result = Metadata()
+        result.merge(item)
+        if self.enrich(result):
+            return result.success("wacz")
+
     def enrich(self, to_enrich: Metadata) -> bool:
-        # TODO: figure out support for browsertrix in docker
+        if to_enrich.get_media_by_id("browsertrix"):
+            logger.info(f"WACZ enricher had already been executed: {to_enrich.get_media_by_id('browsertrix')}")
+            return True
 
         url = to_enrich.get_url()
 
-        if UrlUtil.is_auth_wall(url):
-            logger.debug(f"[SKIP] SCREENSHOT since url is behind AUTH WALL: {url=}")
-            return
-        
         collection = str(uuid.uuid4())[0:8]
-        browsertrix_home = os.path.abspath(ArchivingContext.get_tmp_dir())
-        
-        if os.getenv('RUNNING_IN_DOCKER'):
-            logger.debug(f"generating WACZ without Docker for {url=}")
+        browsertrix_home_host = os.environ.get('BROWSERTRIX_HOME_HOST') or os.path.abspath(ArchivingContext.get_tmp_dir())
+        browsertrix_home_container = os.environ.get('BROWSERTRIX_HOME_CONTAINER') or browsertrix_home_host
 
-            cmd = [
-                "crawl",
-                "--url", url,
-                "--scopeType", "page",
-                "--generateWACZ",
-                "--text",
-                "--collection", collection,
-                "--id", collection,
-                "--saveState", "never",
-                "--behaviors", "autoscroll,autoplay,autofetch,siteSpecific",
-                "--behaviorTimeout", str(self.timeout),
-                "--timeout", str(self.timeout),
-                "--profile", str(self.profile)
-            ]
-        else:
+        cmd = [
+            "crawl",
+            "--url", url,
+            "--scopeType", "page",
+            "--generateWACZ",
+            "--text",
+            "--screenshot", "fullPage",
+            "--collection", collection,
+            "--id", collection,
+            "--saveState", "never",
+            "--behaviors", "autoscroll,autoplay,autofetch,siteSpecific",
+            "--behaviorTimeout", str(self.timeout),
+            "--timeout", str(self.timeout)]
+
+        # call docker if explicitly enabled or we are running on the host (not in docker)
+        use_docker = os.environ.get('WACZ_ENABLE_DOCKER') or not os.environ.get('RUNNING_IN_DOCKER')
+
+        if use_docker:
             logger.debug(f"generating WACZ in Docker for {url=}")
-            
-            cmd = [
-                "docker", "run",
-                "--rm",  # delete container once it has completed running
-                "-v", f"{browsertrix_home}:/crawls/",
-                # "-it", # this leads to "the input device is not a TTY"
-                "webrecorder/browsertrix-crawler", "crawl",
-                "--url", url,
-                "--scopeType", "page",
-                "--generateWACZ",
-                "--text",
-                "--collection", collection,
-                "--behaviors", "autoscroll,autoplay,autofetch,siteSpecific",
-                "--behaviorTimeout", str(self.timeout),
-                "--timeout", str(self.timeout)
-            ]
+            logger.debug(f"{browsertrix_home_host=} {browsertrix_home_container=}")
+            if self.docker_commands:
+                cmd = self.docker_commands + cmd
+            else:
+                cmd = ["docker", "run", "--rm", "-v", f"{browsertrix_home_host}:/crawls/", "webrecorder/browsertrix-crawler"] + cmd
 
             if self.profile:
-                profile_fn = os.path.join(browsertrix_home, "profile.tar.gz")
+                profile_fn = os.path.join(browsertrix_home_container, "profile.tar.gz")
+                logger.debug(f"copying {self.profile} to {profile_fn}")
                 shutil.copyfile(self.profile, profile_fn)
-                # TODO: test which is right
-                cmd.extend(["--profile", profile_fn])
-                # cmd.extend(["--profile", "/crawls/profile.tar.gz"])
+                cmd.extend(["--profile", os.path.join("/crawls", "profile.tar.gz")])
+
+        else:
+            logger.debug(f"generating WACZ without Docker for {url=}")
+
+            if self.profile:
+                cmd.extend(["--profile", os.path.join("/app", str(self.profile))])
 
         try:
             logger.info(f"Running browsertrix-crawler: {' '.join(cmd)}")
@@ -86,15 +94,91 @@ class WaczEnricher(Enricher):
             logger.error(f"WACZ generation failed: {e}")
             return False
 
-        
-
-        if os.getenv('RUNNING_IN_DOCKER'):
-            filename = os.path.join("collections", collection, f"{collection}.wacz")
+        if use_docker:
+            wacz_fn = os.path.join(browsertrix_home_container, "collections", collection, f"{collection}.wacz")
         else:
-            filename = os.path.join(browsertrix_home, "collections", collection, f"{collection}.wacz")
-        
-        if not os.path.exists(filename):
-            logger.warning(f"Unable to locate and upload WACZ  {filename=}")
+            wacz_fn = os.path.join("collections", collection, f"{collection}.wacz")
+
+        if not os.path.exists(wacz_fn):
+            logger.warning(f"Unable to locate and upload WACZ  {wacz_fn=}")
             return False
 
-        to_enrich.add_media(Media(filename), "browsertrix")
+        to_enrich.add_media(Media(wacz_fn), "browsertrix")
+        if self.extract_media:
+            self.extract_media_from_wacz(to_enrich, wacz_fn)
+        return True
+
+    def extract_media_from_wacz(self, to_enrich: Metadata, wacz_filename: str) -> None:
+        """
+        Receives a .wacz archive, and extracts all relevant media from it, adding them to to_enrich.
+        """
+        logger.info(f"WACZ extract_media flag is set, extracting media from {wacz_filename=}")
+
+        # unzipping the .wacz
+        tmp_dir = ArchivingContext.get_tmp_dir()
+        unzipped_dir = os.path.join(tmp_dir, "unzipped")
+        with ZipFile(wacz_filename, 'r') as z_obj:
+            z_obj.extractall(path=unzipped_dir)
+
+        # if warc is split into multiple gzip chunks, merge those
+        warc_dir = os.path.join(unzipped_dir, "archive")
+        warc_filename = os.path.join(tmp_dir, "merged.warc")
+        with open(warc_filename, 'wb') as outfile:
+            for filename in sorted(os.listdir(warc_dir)):
+                if filename.endswith('.gz'):
+                    chunk_file = os.path.join(warc_dir, filename)
+                    with open(chunk_file, 'rb') as infile:
+                        shutil.copyfileobj(infile, outfile)
+
+        # get media out of .warc
+        counter = 0
+        seen_urls = set()
+        with open(warc_filename, 'rb') as warc_stream:
+            for record in ArchiveIterator(warc_stream):
+                # only include fetched resources
+                if record.rec_type == "resource":  # screenshots
+                    fn = os.path.join(tmp_dir, f"warc-file-{counter}.png")
+                    with open(fn, "wb") as outf: outf.write(record.raw_stream.read())
+                    m = Media(filename=fn)
+                    to_enrich.add_media(m, "browsertrix-screenshot")
+                    counter += 1
+
+                if record.rec_type != 'response': continue
+                record_url = record.rec_headers.get_header('WARC-Target-URI')
+                if not UrlUtil.is_relevant_url(record_url):
+                    logger.debug(f"Skipping irrelevant URL {record_url} but it's still present in the WACZ.")
+                    continue
+                if record_url in seen_urls:
+                    logger.debug(f"Skipping already seen URL {record_url}.")
+                    continue
+
+                # filter by media mimetypes
+                content_type = record.http_headers.get("Content-Type")
+                if not content_type: continue
+                if not any(x in content_type for x in ["video", "image", "audio"]): continue
+
+                # create local file and add media
+                ext = mimetypes.guess_extension(content_type)
+                warc_fn = f"warc-file-{counter}{ext}"
+                fn = os.path.join(tmp_dir, warc_fn)
+
+                record_url_best_qual = UrlUtil.twitter_best_quality_url(record_url)
+                with open(fn, "wb") as outf: outf.write(record.raw_stream.read())
+
+                m = Media(filename=fn)
+                m.set("src", record_url)
+                # if a link with better quality exists, try to download that
+                if record_url_best_qual != record_url:
+                    try:
+                        m.filename = self.download_from_url(record_url_best_qual, warc_fn, to_enrich)
+                        m.set("src", record_url_best_qual)
+                        m.set("src_alternative", record_url)
+                    except Exception as e: logger.warning(f"Unable to download best quality URL for {record_url=} got error {e}, using original in WARC.")
+
+                # remove bad videos
+                if m.is_video() and not m.is_valid_video(): continue
+                
+                to_enrich.add_media(m, warc_fn)
+                counter += 1
+                seen_urls.add(record_url)
+        logger.info(f"WACZ extract_media finished, found {counter} relevant media file(s)")

src/auto_archiver/enrichers/wayback_enricher.py

@@ -23,11 +23,13 @@ class WaybackArchiverEnricher(Enricher, Archiver):
     def configs() -> dict:
         return {
             "timeout": {"default": 15, "help": "seconds to wait for successful archive confirmation from wayback, if more than this passes the result contains the job_id so the status can later be checked manually."},
+            "if_not_archived_within": {"default": None, "help": "only tell wayback to archive if no archive is available before the number of seconds specified, use None to ignore this option. For more information: https://docs.google.com/document/d/1Nsv52MvSjbLb2PCpHlat0gkzw0EvtSgpKHu4mk0MnrA"},
             "key": {"default": None, "help": "wayback API key. to get credentials visit https://archive.org/account/s3.php"},
             "secret": {"default": None, "help": "wayback API secret. to get credentials visit https://archive.org/account/s3.php"}
         }
 
     def download(self, item: Metadata) -> Metadata:
+        # this new Metadata object is required to avoid duplication
         result = Metadata()
         result.merge(item)
         if self.enrich(result):
@@ -49,7 +51,11 @@ class WaybackArchiverEnricher(Enricher, Archiver):
             "Accept": "application/json",
             "Authorization": f"LOW {self.key}:{self.secret}"
         }
-        r = requests.post('https://web.archive.org/save/', headers=ia_headers, data={'url': url})
+        post_data = {'url': url}
+        if self.if_not_archived_within:
+            post_data["if_not_archived_within"] = self.if_not_archived_within
+        # see https://docs.google.com/document/d/1Nsv52MvSjbLb2PCpHlat0gkzw0EvtSgpKHu4mk0MnrA for more options
+        r = requests.post('https://web.archive.org/save/', headers=ia_headers, data=post_data)
 
         if r.status_code != 200:
             logger.error(em := f"Internet archive failed with status of {r.status_code}: {r.json()}")

src/auto_archiver/enrichers/whisper_enricher.py

@@ -18,17 +18,18 @@ class WhisperEnricher(Enricher):
     def __init__(self, config: dict) -> None:
         # without this STEP.__init__ is not called
         super().__init__(config)
+        assert type(self.api_endpoint) == str and len(self.api_endpoint) > 0, "please provide a value for the whisper_enricher api_endpoint"
         assert type(self.api_key) == str and len(self.api_key) > 0, "please provide a value for the whisper_enricher api_key"
         self.timeout = int(self.timeout)
 
     @staticmethod
     def configs() -> dict:
         return {
-            "api_endpoint": {"default": "https://whisper.spoettel.dev/api/v1", "help": "WhisperApi api endpoint"},
+            "api_endpoint": {"default": None, "help": "WhisperApi api endpoint, eg: https://whisperbox-api.com/api/v1, a deployment of https://github.com/bellingcat/whisperbox-transcribe."},
             "api_key": {"default": None, "help": "WhisperApi api key for authentication"},
             "include_srt": {"default": False, "help": "Whether to include a subtitle SRT (SubRip Subtitle file) for the video (can be used in video players)."},
             "timeout": {"default": 90, "help": "How many seconds to wait at most for a successful job completion."},
-            "action": {"default": "translation", "help": "which Whisper operation to execute", "choices": ["transcript", "translation", "language_detection"]},
+            "action": {"default": "translate", "help": "which Whisper operation to execute", "choices": ["transcribe", "translate", "language_detection"]},
 
         }
 
@@ -56,9 +57,12 @@ class WhisperEnricher(Enricher):
 
         for i, m in enumerate(to_enrich.media):
             if m.is_video() or m.is_audio():
-                job_id = to_enrich.media[i].get("whisper_model")["job_id"]
+                job_id = to_enrich.media[i].get("whisper_model", {}).get("job_id")
+                if not job_id: continue
                 to_enrich.media[i].set("whisper_model", {
                     "job_id": job_id,
+                    "job_status_check": f"{self.api_endpoint}/jobs/{job_id}",
+                    "job_artifacts_check": f"{self.api_endpoint}/jobs/{job_id}/artifacts",
                     **(job_results[job_id] if job_results[job_id] else {"result": "incomplete or failed job"})
                 })
                 # append the extracted text to the content of the post so it gets written to the DBs like gsheets text column
@@ -76,6 +80,7 @@ class WhisperEnricher(Enricher):
             "type": self.action,
             # "language": "string" # may be a config
         }
+        logger.debug(f"calling API with {payload=}")
         response = requests.post(f'{self.api_endpoint}/jobs', json=payload, headers={'Authorization': f'Bearer {self.api_key}'})
         assert response.status_code == 201, f"calling the whisper api {self.api_endpoint} returned a non-success code: {response.status_code}"
         logger.debug(response.json())

src/auto_archiver/feeders/gsheet_feeder.py

@@ -39,7 +39,7 @@ class GsheetsFeeder(Gsheets, Feeder):
             })
 
     def __iter__(self) -> Metadata:
-        sh = self.gsheets_client.open(self.sheet)
+        sh = self.open_sheet()
         for ii, wks in enumerate(sh.worksheets()):
             if not self.should_process_sheet(wks.title):
                 logger.debug(f"SKIPPED worksheet '{wks.title}' due to allow/block rules")
@@ -64,7 +64,10 @@ class GsheetsFeeder(Gsheets, Feeder):
                 # All checks done - archival process starts here
                 m = Metadata().set_url(url)
                 ArchivingContext.set("gsheet", {"row": row, "worksheet": gw}, keep_on_reset=True)
-                folder = slugify(gw.get_cell(row, 'folder').strip())
+                if gw.get_cell_or_default(row, 'folder', "") is None:
+                    folder = ''
+                else:
+                    folder = slugify(gw.get_cell_or_default(row, 'folder', "").strip())
                 if len(folder):
                     if self.use_sheet_names_in_stored_paths:
                         ArchivingContext.set("folder", os.path.join(folder, slugify(self.sheet), slugify(wks.title)), True)

src/auto_archiver/formatters/templates/html_template.html

@@ -65,11 +65,12 @@
         }
 
         /* Disable grayscale on hover */
-        img:hover,
+        /* img:hover,
         video:hover {
             -webkit-filter: grayscale(0);
             filter: none;
-        }
+        } */
+
 
         .collapsible {
             background-color: #777;
@@ -101,50 +102,68 @@
 <body>
     <div id="notification"></div>
     <h2>Archived media for <a href="{{ url }}">{{ url }}</a></h2>
+    {% if title | string | length > 0 %}
     <p><b>title:</b> '<span class="copy">{{ title }}</span>'</p>
+    {% endif %}
     <h2 class="center">content {{ media | length }} item(s)</h2>
+    <form class="center">
+        <label>
+            <input type="checkbox" id="safe-media-view" checked>
+            Safe Media View
+        </label>
+    </form>
     <table class="content">
         <tr>
             <th>about</th>
             <th>preview(s)</th>
         </tr>
-        {% for m in media %}
-        <tr>
-            <td>
-                <ul>
-                    <li><b>key:</b> <span class="copy">{{ m.key }}</span></li>
-                    <li><b>type:</b> <span class="copy">{{ m.mimetype }}</span></li>
+        <tbody>
+            {% for m in media %}
+            <tr>
+                <td>
+                    <ul>
+                        <li><b>key:</b> <span class="copy">{{ m.key }}</span></li>
+                        <li><b>type:</b> <span class="copy">{{ m.mimetype }}</span></li>
 
-                    {% for prop in m.properties %}
+                        {% for prop in m.properties %}
 
-                    {% if m.properties[prop] | is_list %}
-                    <p></p>
-                    <div>
-                        <b class="collapsible" title="expand">{{ prop }}:</b>
+                        {% if m.properties[prop] | is_list %}
                         <p></p>
-                        <div class="collapsible-content">
-                            {% for subprop in m.properties[prop] %}
-                            {% if subprop | is_media %}
-                            {{ macros.display_media(subprop, false, url) }}
-                            {% else %}
-                            {{ subprop }}
-                            {% endif %}
-                            {% endfor %}
-                        </div>
-                    </div>
-                    <p></p>
-                    {% elif m.properties[prop] | string | length > 1 %}
-                    <li><b>{{ prop }}:</b> {{ macros.copy_urlize(m.properties[prop]) }}</li>
-                    {% endif %}
+                        <div>
+                            <b class="collapsible" title="expand">{{ prop }}:</b>
+                            <p></p>
+                            <div class="collapsible-content">
+                                {% for subprop in m.properties[prop] %}
+                                {% if subprop | is_media %}
+                                {{ macros.display_media(subprop, true, url) }}
 
-                    {% endfor %}
-                </ul>
-            </td>
-            <td>
-                {{ macros.display_media(m, true, url) }}
-            </td>
-        </tr>
-        {% endfor %}
+                                <ul>
+                                    {% for subprop_prop in subprop.properties %}
+                                    <li><b>{{ subprop_prop }}:</b>
+                                        {{ macros.copy_urlize(subprop.properties[subprop_prop]) }}</li>
+                                    {% endfor %}
+                                </ul>
+
+                                {% else %}
+                                {{ subprop }}
+                                {% endif %}
+                                {% endfor %}
+                            </div>
+                        </div>
+                        <p></p>
+                        {% elif m.properties[prop] | string | length > 1 %}
+                        <li><b>{{ prop }}:</b> {{ macros.copy_urlize(m.properties[prop]) }}</li>
+                        {% endif %}
+
+                        {% endfor %}
+                    </ul>
+                </td>
+                <td>
+                    {{ macros.display_media(m, true, url) }}
+                </td>
+            </tr>
+            {% endfor %}
+        </tbody>
     </table>
     <h2 class="center">metadata</h2>
     <table class="metadata">
@@ -162,7 +181,8 @@
         {% endfor %}
     </table>
 
-    <p style="text-align:center;">Made with <a href="https://github.com/bellingcat/auto-archiver">bellingcat/auto-archiver</a> v{{ version }}</p>
+    <p style="text-align:center;">Made with <a
+            href="https://github.com/bellingcat/auto-archiver">bellingcat/auto-archiver</a> v{{ version }}</p>
 </body>
 <script defer>
     // notification logic
@@ -201,7 +221,7 @@
     let i;
 
     for (i = 0; i < coll.length; i++) {
-        coll[i].addEventListener("click", function() {
+        coll[i].addEventListener("click", function () {
             this.classList.toggle("active");
             // let content = this.nextElementSibling;
             let content = this.parentElement.querySelector(".collapsible-content");
@@ -212,6 +232,49 @@
             }
         });
     }
+
+    // logic for enabled/disabled greyscale
+    // Get references to the checkboxes and images/videos
+    const safeImageViewCheckbox = document.getElementById('safe-media-view');
+    const imagesVideos = document.querySelectorAll('img, video');
+
+    // Function to toggle grayscale effect
+    function toggleGrayscale() {
+        imagesVideos.forEach(element => {
+            if (safeImageViewCheckbox.checked) {
+                // Enable grayscale effect
+                element.style.filter = 'grayscale(1)';
+                element.style.webkitFilter = 'grayscale(1)';
+            } else {
+                // Disable grayscale effect
+                element.style.filter = 'none';
+                element.style.webkitFilter = 'none';
+            }
+        });
+    }
+
+    // Add event listener to the checkbox to trigger the toggleGrayscale function
+    safeImageViewCheckbox.addEventListener('change', toggleGrayscale);
+
+    // Handle the hover effect using JavaScript
+    imagesVideos.forEach(element => {
+        element.addEventListener('mouseenter', () => {
+            // Disable grayscale effect on hover
+            element.style.filter = 'none';
+            element.style.webkitFilter = 'none';
+        });
+
+        element.addEventListener('mouseleave', () => {
+            // Re-enable grayscale effect if checkbox is checked
+            if (safeImageViewCheckbox.checked) {
+                element.style.filter = 'grayscale(1)';
+                element.style.webkitFilter = 'grayscale(1)';
+            }
+        });
+    });
+
+    // Call the function on page load to apply the initial state
+    toggleGrayscale();
 </script>
 
 </html>

src/auto_archiver/formatters/templates/macros.html

@@ -16,10 +16,7 @@ No URL available for {{ m.key }}.
         <a href="https://lens.google.com/uploadbyurl?url={{ url | quote }}">Google Lens</a>,&nbsp;
         <a href="https://yandex.ru/images/touch/search?rpt=imageview&url={{ url | quote }}">Yandex</a>,&nbsp;
         <a href="https://www.bing.com/images/search?view=detailv2&iss=sbi&form=SBIVSP&sbisrc=UrlPaste&q=imgurl:{{ url | quote }}">Bing</a>,&nbsp;
-        <a href="https://www.tineye.com/search/?url={{ url | quote }}">Tineye</a>,&nbsp;
-        <a href="https://iqdb.org/?url={{ url | quote }}">IQDB</a>,&nbsp;
-        <a href="https://saucenao.com/search.php?db=999&url={{ url | quote }}">SauceNAO</a>,&nbsp;
-        <a href="https://imgops.com/{{ url | quote }}">IMGOPS</a>
+        <a href="https://www.tineye.com/search/?url={{ url | quote }}">Tineye</a>
     </div>
     <p></p>
 </div>

src/auto_archiver/utils/gsheet.py

@@ -10,16 +10,17 @@ class Gsheets(Step):
         # without this STEP.__init__ is not called
         super().__init__(config)
         self.gsheets_client = gspread.service_account(filename=self.service_account)
-        #TODO: config should be responsible for conversions
+        # TODO: config should be responsible for conversions
         try: self.header = int(self.header)
         except: pass
         assert type(self.header) == int, f"header ({self.header}) value must be an integer not {type(self.header)}"
-        assert self.sheet is not None, "You need to define a sheet name in your orchestration file when using gsheets."
+        assert self.sheet is not None or self.sheet_id is not None, "You need to define either a 'sheet' name or a 'sheet_id' in your orchestration file when using gsheets."
 
     @staticmethod
     def configs() -> dict:
         return {
             "sheet": {"default": None, "help": "name of the sheet to archive"},
+            "sheet_id": {"default": None, "help": "(alternative to sheet name) the id of the sheet to archive"},
             "header": {"default": 1, "help": "index of the header row (starts at 1)"},
             "service_account": {"default": "secrets/service_account.json", "help": "service account JSON file path"},
             "columns": {
@@ -35,10 +36,17 @@ class Gsheets(Step):
                     'text': 'text content',
                     'screenshot': 'screenshot',
                     'hash': 'hash',
+                    'pdq_hash': 'perceptual hashes',
                     'wacz': 'wacz',
                     'replaywebpage': 'replaywebpage',
                 },
                 "help": "names of columns in the google sheet (stringified JSON object)",
                 "cli_set": lambda cli_val, cur_val: dict(cur_val, **json.loads(cli_val))
             },
-        }
+        }
+
+    def open_sheet(self):
+        if self.sheet:
+            return self.gsheets_client.open(self.sheet)
+        else:  # self.sheet_id
+            return self.gsheets_client.open_by_key(self.sheet_id)

src/auto_archiver/utils/gworksheet.py

@@ -19,6 +19,7 @@ class GWorksheet:
         'title': 'upload title',
         'screenshot': 'screenshot',
         'hash': 'hash',
+        'pdq_hash': 'perceptual hashes',
         'wacz': 'wacz',
         'replaywebpage': 'replaywebpage',
     }

src/auto_archiver/utils/misc.py

@@ -40,3 +40,12 @@ class DateTimeEncoder(json.JSONEncoder):
 
 def dump_payload(p):
     return json.dumps(p, ensure_ascii=False, indent=4, cls=DateTimeEncoder)
+
+
+def update_nested_dict(dictionary, update_dict):
+    # takes 2 dicts and overwrites the first with the second only on the changed balues
+    for key, value in update_dict.items():
+        if key in dictionary and isinstance(value, dict) and isinstance(dictionary[key], dict):
+            update_nested_dict(dictionary[key], value)
+        else:
+            dictionary[key] = value

src/auto_archiver/utils/url.py

@@ -1,14 +1,15 @@
 import re
+from urllib.parse import urlparse, urlunparse
 
 class UrlUtil:
     telegram_private = re.compile(r"https:\/\/t\.me(\/c)\/(.+)\/(\d+)")
     is_istagram = re.compile(r"https:\/\/www\.instagram\.com")
 
     @staticmethod
-    def clean(url): return url
+    def clean(url: str) -> str: return url
 
     @staticmethod
-    def is_auth_wall(url):
+    def is_auth_wall(url: str) -> bool:
         """
         checks if URL is behind an authentication wall meaning steps like wayback, wacz, ... may not work
         """
@@ -17,3 +18,59 @@ class UrlUtil:
 
         return False
 
+    @staticmethod
+    def remove_get_parameters(url: str) -> str:
+        # http://example.com/file.mp4?t=1 -> http://example.com/file.mp4
+        # useful for mimetypes to work
+        parsed_url = urlparse(url)
+        new_url = urlunparse(parsed_url._replace(query=''))
+        return new_url
+
+    @staticmethod
+    def is_relevant_url(url: str) -> bool:
+        """
+        Detect if a detected media URL is recurring and therefore irrelevant to a specific archive. Useful, for example, for the enumeration of the media files in WARC files which include profile pictures, favicons, etc.
+        """
+        clean_url = UrlUtil.remove_get_parameters(url)
+
+        # favicons
+        if "favicon" in url: return False
+        # ifnore icons
+        if clean_url.endswith(".ico"): return False
+        # ignore SVGs
+        if UrlUtil.remove_get_parameters(url).endswith(".svg"): return False
+
+        # twitter profile pictures
+        if "twimg.com/profile_images" in url: return False
+        if "twimg.com" in url and "/default_profile_images" in url: return False
+
+        # instagram profile pictures
+        if "https://scontent.cdninstagram.com/" in url and "150x150" in url: return False
+        # instagram recurring images
+        if "https://static.cdninstagram.com/rsrc.php/" in url: return False
+
+        # telegram
+        if "https://telegram.org/img/emoji/" in url: return False
+
+        # youtube
+        if "https://www.youtube.com/s/gaming/emoji/" in url: return False
+        if "https://yt3.ggpht.com" in url and "default-user=" in url: return False
+        if "https://www.youtube.com/s/search/audio/" in url: return False
+
+        # ok
+        if " https://ok.ru/res/i/" in url: return False
+
+        # vk
+        if "https://vk.com/emoji/" in url: return False
+        if "vk.com/images/" in url: return False
+        if "vk.com/images/reaction/" in url: return False
+
+        return True
+
+    @staticmethod
+    def twitter_best_quality_url(url: str) -> str:
+        """
+        some twitter image URLs point to a less-than best quality
+        this returns the URL pointing to the highest (original) quality
+        """
+        return re.sub(r"name=(\w+)", "name=orig", url, 1)

src/auto_archiver/version.py

@@ -1,6 +1,6 @@
 
 _MAJOR = "0"
-_MINOR = "5"
+_MINOR = "6"
 # On main and in a nightly release the patch should be one ahead of the last
 # released build.
 _PATCH = "12"