dind exception

src/auto_archiver/enrichers/wacz_enricher.py

@@ -38,20 +38,19 @@ class WaczArchiverEnricher(Enricher, Archiver):
     
     def setup(self) -> None:
         self.use_docker = os.environ.get('WACZ_ENABLE_DOCKER') or not os.environ.get('RUNNING_IN_DOCKER')
+        self.docker_in_docker = os.environ.get('WACZ_ENABLE_DOCKER') and os.environ.get('RUNNING_IN_DOCKER')
+
+        self.cwd_dind = f"/crawls/crawls{random_str(8)}"
         self.browsertrix_home_host = os.environ.get('BROWSERTRIX_HOME_HOST')
         self.browsertrix_home_container = os.environ.get('BROWSERTRIX_HOME_CONTAINER') or self.browsertrix_home_host
         # create crawls folder if not exists, so it can be safely removed in cleanup
-        if self.use_docker:
+        if self.docker_in_docker:
-            if self.browsertrix_home_container:
+            os.makedirs(self.cwd_dind, exist_ok=True)
-                os.makedirs(self.browsertrix_home_container, exist_ok=True)
 
     def cleanup(self) -> None:
-        if self.use_docker:
+        if self.docker_in_docker:
-            if self.browsertrix_home_container:
+            logger.debug(f"Removing {self.cwd_dind=}")
-                logger.debug(f"Removing {self.browsertrix_home_container=}")
+            shutil.rmtree(self.cwd_dind, ignore_errors=True)
-                shutil.rmtree(self.browsertrix_home_container, ignore_errors=True)
-
-
 
     def download(self, item: Metadata) -> Metadata:
         # this new Metadata object is required to avoid duplication
@@ -85,6 +84,9 @@ class WaczArchiverEnricher(Enricher, Archiver):
             "--behaviorTimeout", str(self.timeout),
             "--timeout", str(self.timeout)]
         
+        if self.docker_in_docker:
+            cmd.extend(["--cwd", self.cwd_dind])
+
         # call docker if explicitly enabled or we are running on the host (not in docker)
         if self.use_docker:
             logger.debug(f"generating WACZ in Docker for {url=}")
@@ -118,7 +120,10 @@ class WaczArchiverEnricher(Enricher, Archiver):
             logger.error(f"WACZ generation failed: {e}")
             return False
 
-        if self.use_docker:
+        
+        if self.docker_in_docker:
+            wacz_fn = os.path.join(self.cwd_dind, "collections", collection, f"{collection}.wacz")
+        elif self.use_docker:
             wacz_fn = os.path.join(browsertrix_home_container, "collections", collection, f"{collection}.wacz")
         else:
             wacz_fn = os.path.join("collections", collection, f"{collection}.wacz")
@@ -131,7 +136,9 @@ class WaczArchiverEnricher(Enricher, Archiver):
         if self.extract_media or self.extract_screenshot:
             self.extract_media_from_wacz(to_enrich, wacz_fn)
 
-        if self.use_docker:
+        if self.docker_in_docker:
+            jsonl_fn = os.path.join(self.cwd_dind, "collections", collection, "pages", "pages.jsonl")
+        elif self.use_docker:
             jsonl_fn = os.path.join(browsertrix_home_container, "collections", collection, "pages", "pages.jsonl")
         else:
             jsonl_fn = os.path.join("collections", collection, "pages", "pages.jsonl")

src/auto_archiver/version.py

@@ -3,7 +3,7 @@ _MAJOR = "0"
 _MINOR = "9"
 # On main and in a nightly release the patch should be one ahead of the last
 # released build.
-_PATCH = "4"
+_PATCH = "7"
 # This is mainly for nightly builds which have the suffix ".dev$DATE". See
 # https://semver.org/#is-v123-a-semantic-version for the semantics.
 _SUFFIX = ""