Bump version to v0.5.12 for release

.github/workflows/docker-publish.yaml

@@ -26,6 +26,14 @@ jobs:
     steps:
       - name: Check out the repo
         uses: actions/checkout@v3
+        
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      # https://github.com/docker/setup-buildx-action
+      
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v1
       
       - name: Log in to Docker Hub
         uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
@@ -40,9 +48,10 @@ jobs:
           images: bellingcat/auto-archiver
       
       - name: Build and push Docker image
-        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
+        uses: docker/build-push-action@v2
         with:
           context: .
-          push: true
+          platforms: linux/amd64,linux/arm64
+          push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
+          labels: ${{ steps.meta.outputs.labels }}

Dockerfile

@@ -1,35 +1,36 @@
-# stage 1 - all dependencies
-From python:3.10
+FROM webrecorder/browsertrix-crawler:latest
+
+ENV RUNNING_IN_DOCKER=1
 
 WORKDIR /app
 
 # TODO: use custom ffmpeg builds instead of apt-get install
 RUN pip install --upgrade pip && \
 	pip install pipenv && \
+	add-apt-repository ppa:mozillateam/ppa && \
 	apt-get update && \
-	apt-get install -y gcc ffmpeg fonts-noto firefox-esr && \
-	wget https://github.com/mozilla/geckodriver/releases/download/v0.32.0/geckodriver-v0.32.0-linux64.tar.gz && \
+	apt-get install -y gcc ffmpeg fonts-noto && \
+	apt-get install -y --no-install-recommends firefox-esr && \
+	ln -s /usr/bin/firefox-esr /usr/bin/firefox && \
+	wget https://github.com/mozilla/geckodriver/releases/download/v0.33.0/geckodriver-v0.33.0-linux64.tar.gz && \
 	tar -xvzf geckodriver* -C /usr/local/bin && \
 	chmod +x /usr/local/bin/geckodriver && \
-	rm geckodriver-v* 
+	rm geckodriver-v*
 
 
-# install docker for WACZ
-# TODO: currently disabled see https://github.com/bellingcat/auto-archiver/issues/66
-# RUN curl -fsSL https://get.docker.com | sh
-
 # TODO: avoid copying unnecessary files, including .git
-COPY Pipfile Pipfile.lock ./
-RUN pipenv install --python=3.10 --system --deploy
-# ENV IS_DOCKER=1
+COPY Pipfile* ./
+RUN pipenv install
+
 # doing this at the end helps during development, builds are quick
 COPY ./src/ . 
 
 # TODO: figure out how to make volumes not be root, does it depend on host or dockerfile?
 # RUN useradd --system --groups sudo --shell /bin/bash archiver && chown -R archiver:sudo .
 # USER archiver
-ENTRYPOINT ["python"]
-# ENTRYPOINT ["docker-entrypoint.sh"]
 
-# should be executed with 2 volumes (3 if local_storage)
-# docker run -v /var/run/docker.sock:/var/run/docker.sock -v $PWD/secrets:/app/secrets  -v $PWD/local_archive:/app/local_archive aa --help
+
+ENTRYPOINT ["pipenv", "run", "python3", "-m", "auto_archiver"]
+
+# should be executed with 2 volumes (3 if local_storage is used)
+# docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive aa pipenv run python3 -m auto_archiver --config secrets/orchestration.yaml

Pipfile

@@ -30,9 +30,13 @@ cryptography = "==38.0.4"
 dataclasses-json = "*"
 yt-dlp = ">=2023.2.17"
 vk-url-scraper = "*"
+uwsgi = "*"
+requests = {extras = ["socks"], version = "*"}
+# wacz = "==0.4.8"
+pywb = ">=2.7.3"
 
 [requires]
-python_version = "3.9"
+python_version = "3.10"
 
 [dev-packages]
 autopep8 = "*"

README.md

@@ -33,7 +33,7 @@ Docker works like a virtual machine running inside your computer, it isolates ev
 
 1. install [docker](https://docs.docker.com/get-docker/)
 2. pull the auto-archiver docker [image](https://hub.docker.com/r/bellingcat/auto-archiver) with `docker pull bellingcat/auto-archiver`
-3. run the docker image locally in a container: `docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver -m auto_archiver  --config secrets/orchestration.yaml` breaking this command down:
+3. run the docker image locally in a container: `docker run --rm -v $PWD/secrets:/app/secrets -v $PWD/local_archive:/app/local_archive bellingcat/auto-archiver --config secrets/orchestration.yaml` breaking this command down:
    1. `docker run` tells docker to start a new container (an instance of the image)
    2. `--rm` makes sure this container is removed after execution (less garbage locally)
    3. `-v $PWD/secrets:/app/secrets` - your secrets folder
@@ -87,11 +87,9 @@ The archiver work is orchestrated by the following workflow (we call each a **st
 4. **Formatter** creates a report from all the archived content (HTML, PDF, ...)
 5. **Database** knows what's been archived and also stores the archive result (spreadsheet, CSV, or just the console)
 
-To check all available steps (which archivers, storages, databses, ...) exist check the [example.orchestration.yaml](example.orchestration.yaml).
+To setup an auto-archiver instance, instance, create an `orchestration.yaml` which contains the workflow you would like. We advise you put this file into a `secrets/` folder and do not share it with others because it will contain passwords and other secrets. 
 
-The great thing is you configure all the workflow in your `orchestration.yaml` file which we advise you put into a `secrets/` folder and don't share it with others because it will contain passwords and other secrets. 
-
-The structure of orchestration file is split into 2 parts: `steps` (what **steps** to use) and `configs` (how those steps should behave), here's a simplification:
+The structure of orchestration file is split into 2 parts: `steps` (what **steps** to use) and `configurations` (how those steps should behave), here's a simplification:
 ```yaml
 # orchestration.yaml content
 steps:
@@ -113,10 +111,12 @@ configurations:
   # ... configurations for the other steps here ...
 ```
 
+To see all available `steps` (which archivers, storages, databses, ...) exist check the [example.orchestration.yaml](example.orchestration.yaml).
+
 All the `configurations` in the `orchestration.yaml` file (you can name it differently but need to pass it in the `--config FILENAME` argument) can be seen in the console by using the `--help` flag. They can also be overwritten, for example if you are using the `cli_feeder` to archive from the command line and want to provide the URLs you should do:
 
 ```bash
-auto-archiver --config orchestration.yaml --cli_feeder.urls="url1,url2,url3"
+auto-archiver --config secrets/orchestration.yaml --cli_feeder.urls="url1,url2,url3"
 ```
 
 Here's the complete workflow that the auto-archiver goes through:
@@ -193,7 +193,7 @@ Use `python -m src.auto_archiver --config secrets/orchestration.yaml` to run fro
 #### Docker development
 working with docker locally:
   * `docker build . -t auto-archiver` to build a local image
-  * `docker run --rm -v $PWD/secrets:/app/secrets aa --config secrets/config.yaml`
+  * `docker run --rm -v $PWD/secrets:/app/secrets aa pipenv run python3 -m auto_archiver --config secrets/orchestration.yaml`
     * to use local archive, also create a volume `-v` for it by adding `-v $PWD/local_archive:/app/local_archive`
 
 

example.orchestration.yaml

@@ -45,11 +45,9 @@ configurations:
       archive: archive location
       date: archive date
       thumbnail: thumbnail
-      thumbnail_index: thumbnail index
       timestamp: upload timestamp
       title: upload title
       text: textual content
-      duration: duration
       screenshot: screenshot
       hash: hash
       wacz: wacz

scripts/create_update_gdrive_oauth_token.py

@@ -10,6 +10,7 @@ from googleapiclient.errors import HttpError
 # You can run this code to get a new token and verify it belongs to the correct user
 # This token will be refresh automatically by the auto-archiver
 # Code below from https://developers.google.com/drive/api/quickstart/python
+# Example invocation: py scripts/create_update_gdrive_oauth_token.py -c secrets/credentials.json -t secrets/gd-token.json
 
 SCOPES = ['https://www.googleapis.com/auth/drive']
 

src/auto_archiver/auto_auto_archive.py

@@ -1,34 +0,0 @@
-
-#TODO: refactor GDriveStorage before merging to main
-# is it possible to have something like this with the new pipeline?
-
-
-# # import tempfile
-# import auto_archive
-# from loguru import logger
-# from configs import Config
-# from storages import Storage
-
-
-# def main():
-#     c = Config()
-#     c.parse()
-#     logger.info(f'Opening document {c.sheet} to look for sheet names to archive')
-
-#     gc = c.gsheets_client
-#     sh = gc.open(c.sheet)
-
-#     wks = sh.get_worksheet(0)
-#     values = wks.get_all_values()
-
-#     with tempfile.TemporaryDirectory(dir="./") as tmpdir:
-#         Storage.TMP_FOLDER = tmpdir
-#         for i in range(11, len(values)):
-#             c.sheet = values[i][0]
-#             logger.info(f"Processing {c.sheet}")
-#             auto_archive.process_sheet(c)
-#         c.destroy_webdriver()
-
-
-# if __name__ == "__main__":
-#     main()

src/auto_archiver/core/context.py

@@ -27,7 +27,6 @@ class ArchivingContext:
 
     @staticmethod
     def set(key, value, keep_on_reset: bool = False):
-        logger.debug(f"SET [{key}]={value}")
         ac = ArchivingContext.get_instance()
         ac.configs[key] = value
         if keep_on_reset: ac.keep_on_reset.add(key)

src/auto_archiver/core/media.py

@@ -19,7 +19,7 @@ class Media:
     urls: List[str] = field(default_factory=list)
     properties: dict = field(default_factory=dict)
     _mimetype: str = None  # eg: image/jpeg
-    _stored: bool = field(default=False, repr=False, metadata=config(exclude=lambda _: True)) # always exclude
+    _stored: bool = field(default=False, repr=False, metadata=config(exclude=lambda _: True))  # always exclude
 
     def store(self: Media, override_storages: List = None, url: str = "url-not-available"):
         # stores the media into the provided/available storages [Storage]
@@ -42,7 +42,7 @@ class Media:
                             s.store(prop_media, url)
 
     def is_stored(self) -> bool:
-        return len(self.urls) > 0
+        return len(self.urls) > 0 and len(self.urls) == len(ArchivingContext.get("storages"))
 
     def set(self, key: str, value: Any) -> Media:
         self.properties[key] = value

src/auto_archiver/databases/gsheet_db.py

@@ -64,6 +64,7 @@ class GsheetsDb(Database):
         batch_if_valid('title', item.get_title())
         batch_if_valid('text', item.get("content", ""))
         batch_if_valid('timestamp', item.get_timestamp())
+        batch_if_valid('hash', media.get("hash", "not-calculated"))
         if (screenshot := item.get_media_by_id("screenshot")) and hasattr(screenshot, "urls"):
             batch_if_valid('screenshot', "\n".join(screenshot.urls))
 

src/auto_archiver/enrichers/hash_enricher.py

@@ -2,7 +2,7 @@ import hashlib
 from loguru import logger
 
 from . import Enricher
-from ..core import Metadata
+from ..core import Metadata, ArchivingContext
 
 
 class HashEnricher(Enricher):
@@ -17,6 +17,7 @@ class HashEnricher(Enricher):
         algo_choices = self.configs()["algorithm"]["choices"]
         assert self.algorithm in algo_choices, f"Invalid hash algorithm selected, must be one of {algo_choices} (you selected {self.algorithm})."
         self.chunksize = int(self.chunksize)
+        ArchivingContext.set("hash_enricher.algorithm", self.algorithm, keep_on_reset=True)
 
     @staticmethod
     def configs() -> dict:

src/auto_archiver/enrichers/wacz_enricher.py

@@ -26,36 +26,58 @@ class WaczEnricher(Enricher):
 
     def enrich(self, to_enrich: Metadata) -> bool:
         # TODO: figure out support for browsertrix in docker
+
         url = to_enrich.get_url()
 
         if UrlUtil.is_auth_wall(url):
             logger.debug(f"[SKIP] SCREENSHOT since url is behind AUTH WALL: {url=}")
             return
-
-        logger.debug(f"generating WACZ for {url=}")
+        
         collection = str(uuid.uuid4())[0:8]
         browsertrix_home = os.path.abspath(ArchivingContext.get_tmp_dir())
-        cmd = [
-            "docker", "run",
-            "--rm",  # delete container once it has completed running
-            "-v", f"{browsertrix_home}:/crawls/",
-            # "-it", # this leads to "the input device is not a TTY"
-            "webrecorder/browsertrix-crawler", "crawl",
-            "--url", url,
-            "--scopeType", "page",
-            "--generateWACZ",
-            "--text",
-            "--collection", collection,
-            "--behaviors", "autoscroll,autoplay,autofetch,siteSpecific",
-            "--behaviorTimeout", str(self.timeout),
-            "--timeout", str(self.timeout)
-        ]
-        if self.profile:
-            profile_fn = os.path.join(browsertrix_home, "profile.tar.gz")
-            shutil.copyfile(self.profile, profile_fn)
-            # TODO: test which is right
-            cmd.extend(["--profile", profile_fn])
-            # cmd.extend(["--profile", "/crawls/profile.tar.gz"])
+        
+        if os.getenv('RUNNING_IN_DOCKER'):
+            logger.debug(f"generating WACZ without Docker for {url=}")
+
+            cmd = [
+                "crawl",
+                "--url", url,
+                "--scopeType", "page",
+                "--generateWACZ",
+                "--text",
+                "--collection", collection,
+                "--id", collection,
+                "--saveState", "never",
+                "--behaviors", "autoscroll,autoplay,autofetch,siteSpecific",
+                "--behaviorTimeout", str(self.timeout),
+                "--timeout", str(self.timeout),
+                "--profile", str(self.profile)
+            ]
+        else:
+            logger.debug(f"generating WACZ in Docker for {url=}")
+            
+            cmd = [
+                "docker", "run",
+                "--rm",  # delete container once it has completed running
+                "-v", f"{browsertrix_home}:/crawls/",
+                # "-it", # this leads to "the input device is not a TTY"
+                "webrecorder/browsertrix-crawler", "crawl",
+                "--url", url,
+                "--scopeType", "page",
+                "--generateWACZ",
+                "--text",
+                "--collection", collection,
+                "--behaviors", "autoscroll,autoplay,autofetch,siteSpecific",
+                "--behaviorTimeout", str(self.timeout),
+                "--timeout", str(self.timeout)
+            ]
+
+            if self.profile:
+                profile_fn = os.path.join(browsertrix_home, "profile.tar.gz")
+                shutil.copyfile(self.profile, profile_fn)
+                # TODO: test which is right
+                cmd.extend(["--profile", profile_fn])
+                # cmd.extend(["--profile", "/crawls/profile.tar.gz"])
 
         try:
             logger.info(f"Running browsertrix-crawler: {' '.join(cmd)}")
@@ -64,7 +86,13 @@ class WaczEnricher(Enricher):
             logger.error(f"WACZ generation failed: {e}")
             return False
 
-        filename = os.path.join(browsertrix_home, "collections", collection, f"{collection}.wacz")
+        
+
+        if os.getenv('RUNNING_IN_DOCKER'):
+            filename = os.path.join("collections", collection, f"{collection}.wacz")
+        else:
+            filename = os.path.join(browsertrix_home, "collections", collection, f"{collection}.wacz")
+        
         if not os.path.exists(filename):
             logger.warning(f"Unable to locate and upload WACZ  {filename=}")
             return False

src/auto_archiver/enrichers/whisper_enricher.py

@@ -87,7 +87,7 @@ class WhisperEnricher(Enricher):
         while not all_completed and (time.time() - start_time) <= self.timeout:
             all_completed = True
             for job_id in job_results:
-                if job_results[job_id]: continue
+                if job_results[job_id] != False: continue
                 all_completed = False  # at least one not ready
                 try: job_results[job_id] = self.check_job(job_id)
                 except Exception as e:
@@ -116,6 +116,9 @@ class WhisperEnricher(Enricher):
                 if not len(subtitle): continue
                 if self.include_srt: result[f"artifact_{art_id}_subtitle"] = "\n".join(subtitle)
                 result[f"artifact_{art_id}_text"] = "\n".join(full_text)
+            # call /delete endpoint on timely success
+            r_del = requests.delete(f'{self.api_endpoint}/jobs/{job_id}', headers={'Authorization': f'Bearer {self.api_key}'})
+            logger.debug(f"DELETE whisper {job_id=} result: {r_del.status_code}")
             return result
         return False
 

src/auto_archiver/feeders/gsheet_feeder.py

@@ -64,8 +64,13 @@ class GsheetsFeeder(Gsheets, Feeder):
                 # All checks done - archival process starts here
                 m = Metadata().set_url(url)
                 ArchivingContext.set("gsheet", {"row": row, "worksheet": gw}, keep_on_reset=True)
-                if self.use_sheet_names_in_stored_paths:
-                    ArchivingContext.set("folder", os.path.join(slugify(self.sheet), slugify(wks.title)), True)
+                folder = slugify(gw.get_cell(row, 'folder').strip())
+                if len(folder):
+                    if self.use_sheet_names_in_stored_paths:
+                        ArchivingContext.set("folder", os.path.join(folder, slugify(self.sheet), slugify(wks.title)), True)
+                    else:
+                        ArchivingContext.set("folder", folder, True)
+
                 yield m
 
             logger.success(f'Finished worksheet {wks.title}')

src/auto_archiver/formatters/html_formatter.py

@@ -8,6 +8,7 @@ from loguru import logger
 from ..version import __version__
 from ..core import Metadata, Media, ArchivingContext
 from . import Formatter
+from ..enrichers import HashEnricher
 
 
 @dataclass
@@ -46,11 +47,16 @@ class HtmlFormatter(Formatter):
         html_path = os.path.join(ArchivingContext.get_tmp_dir(), f"formatted{str(uuid.uuid4())}.html")
         with open(html_path, mode="w", encoding="utf-8") as outf:
             outf.write(content)
-        return Media(filename=html_path)
+        final_media = Media(filename=html_path)
+
+        he = HashEnricher({"hash_enricher": {"algorithm": ArchivingContext.get("hash_enricher.algorithm"), "chunksize": 1.6e7}})
+        if len(hd := he.calculate_hash(final_media.filename)):
+            final_media.set("hash", f"{he.algorithm}:{hd}")
+
+        return final_media
 
 
 # JINJA helper filters
-
 class JinjaHelpers:
     @staticmethod
     def is_list(v) -> bool:

src/auto_archiver/storages/storage.py

@@ -43,7 +43,7 @@ class Storage(Step):
 
     def store(self, media: Media, url: str) -> None:
         if media.is_stored(): 
-            logger.debug(f"{self.key} already stored, skipping")
+            logger.debug(f"{media.key} already stored, skipping")
             return
         self.set_key(media, url)
         self.upload(media)
@@ -77,7 +77,7 @@ class Storage(Step):
         # filename_generator logic
         if self.filename_generator == "random": filename = str(uuid.uuid4())[:16]
         elif self.filename_generator == "static":
-            he = HashEnricher({"hash_enricher": {"algorithm": "SHA-256", "chunksize": 1.6e7}})
+            he = HashEnricher({"hash_enricher": {"algorithm": ArchivingContext.get("hash_enricher.algorithm"), "chunksize": 1.6e7}})
             hd = he.calculate_hash(media.filename)
             filename = hd[:24]
 

src/auto_archiver/utils/gsheet.py

@@ -30,11 +30,9 @@ class Gsheets(Step):
                     'archive': 'archive location',
                     'date': 'archive date',
                     'thumbnail': 'thumbnail',
-                    'thumbnail_index': 'thumbnail index',
                     'timestamp': 'upload timestamp',
                     'title': 'upload title',
                     'text': 'text content',
-                    'duration': 'duration',
                     'screenshot': 'screenshot',
                     'hash': 'hash',
                     'wacz': 'wacz',

src/auto_archiver/utils/gworksheet.py

@@ -15,10 +15,8 @@ class GWorksheet:
         'archive': 'archive location',
         'date': 'archive date',
         'thumbnail': 'thumbnail',
-        'thumbnail_index': 'thumbnail index',
         'timestamp': 'upload timestamp',
         'title': 'upload title',
-        'duration': 'duration',
         'screenshot': 'screenshot',
         'hash': 'hash',
         'wacz': 'wacz',
@@ -98,7 +96,7 @@ class GWorksheet:
         cell_updates = [
             {
                 'range': self.to_a1(row, col),
-                'values': [[val]]
+                'values': [[str(val)[0:49999]]]
             }
             for row, col, val in cell_updates
         ]

src/auto_archiver/version.py

@@ -3,7 +3,7 @@ _MAJOR = "0"
 _MINOR = "5"
 # On main and in a nightly release the patch should be one ahead of the last
 # released build.
-_PATCH = "10"
+_PATCH = "12"
 # This is mainly for nightly builds which have the suffix ".dev$DATE". See
 # https://semver.org/#is-v123-a-semantic-version for the semantics.
 _SUFFIX = ""