Compare commits

...

4 Commits

Author SHA1 Message Date
msramalho
7a2119e6e9 Bump version to v0.6.9 for release 2023-09-12 20:08:00 +01:00
Miguel Sozinho Ramalho
3ae25e51e7 adds flexibile setup for wacz in docker (#94) 2023-09-12 20:07:21 +01:00
msramalho
9584193d69 Bump version to v0.6.8 for release 2023-09-08 15:10:02 +01:00
msramalho
0dd45d90f1 fix: docker+wacz troubles 2023-09-08 15:09:50 +01:00
3 changed files with 11 additions and 11 deletions

View File

@@ -2,7 +2,7 @@ FROM webrecorder/browsertrix-crawler:latest
ENV RUNNING_IN_DOCKER=1
WORKDIR /app/auto-archiver
WORKDIR /app
RUN pip install --upgrade pip && \
pip install pipenv && \

View File

@@ -27,6 +27,8 @@ class WaczArchiverEnricher(Enricher, Archiver):
def configs() -> dict:
return {
"profile": {"default": None, "help": "browsertrix-profile (for profile generation see https://github.com/webrecorder/browsertrix-crawler#creating-and-using-browser-profiles)."},
"docker_commands": {"default": None, "help":"if a custom docker invocation is needed"},
"browsertrix_home": {"default": None, "help": "Path to use with the custom browsertrix file locations, useful together with docker_commands"},
"timeout": {"default": 120, "help": "timeout for WACZ generation in seconds"},
"extract_media": {"default": True, "help": "If enabled all the images/videos/audio present in the WACZ archive will be extracted into separate Media. The .wacz file will be kept untouched."}
}
@@ -46,9 +48,9 @@ class WaczArchiverEnricher(Enricher, Archiver):
url = to_enrich.get_url()
collection = str(uuid.uuid4())[0:8]
browsertrix_home = os.path.abspath(ArchivingContext.get_tmp_dir())
browsertrix_home = self.browsertrix_home or os.path.abspath(ArchivingContext.get_tmp_dir())
if os.getenv('RUNNING_IN_DOCKER'):
if os.environ.get('RUNNING_IN_DOCKER', 0) == '1':
logger.debug(f"generating WACZ without Docker for {url=}")
cmd = [
@@ -69,13 +71,10 @@ class WaczArchiverEnricher(Enricher, Archiver):
cmd.extend(["--profile", os.path.join("/app", str(self.profile))])
else:
logger.debug(f"generating WACZ in Docker for {url=}")
cmd = [
"docker", "run",
"--rm", # delete container once it has completed running
"-v", f"{browsertrix_home}:/crawls/",
# "-it", # this leads to "the input device is not a TTY"
"webrecorder/browsertrix-crawler", "crawl",
if not self.docker_commands:
self.docker_commands = ["docker", "run", "--rm", "-v", f"{browsertrix_home}:/crawls/", "webrecorder/browsertrix-crawler"]
cmd = self.docker_commands + [
"crawl",
"--url", url,
"--scopeType", "page",
"--generateWACZ",
@@ -89,6 +88,7 @@ class WaczArchiverEnricher(Enricher, Archiver):
if self.profile:
profile_fn = os.path.join(browsertrix_home, "profile.tar.gz")
logger.debug(f"copying {self.profile} to {profile_fn}")
shutil.copyfile(self.profile, profile_fn)
cmd.extend(["--profile", os.path.join("/crawls", "profile.tar.gz")])

View File

@@ -3,7 +3,7 @@ _MAJOR = "0"
_MINOR = "6"
# On main and in a nightly release the patch should be one ahead of the last
# released build.
_PATCH = "7"
_PATCH = "9"
# This is mainly for nightly builds which have the suffix ".dev$DATE". See
# https://semver.org/#is-v123-a-semantic-version for the semantics.
_SUFFIX = ""