Compare commits

..

5 Commits

Author SHA1 Message Date
msramalho
0feeb0bd24 Bump version to v0.6.12 for release 2023-09-20 10:18:44 +01:00
msramalho
ddb9dc87d7 unfortunately needed twitter->x 2023-09-20 10:17:31 +01:00
msramalho
e8935b9a80 Bump version to v0.6.11 for release 2023-09-15 19:53:07 +01:00
msramalho
b157f9a6b1 renaming variable 2023-09-15 19:52:47 +01:00
msramalho
ea38a604bb fixes #96 by not assigning to self.prop 2023-09-15 19:35:35 +01:00
3 changed files with 13 additions and 12 deletions

View File

@@ -15,8 +15,8 @@ class TwitterArchiver(Archiver):
"""
name = "twitter_archiver"
link_pattern = re.compile(r"twitter.com\/(?:\#!\/)?(\w+)\/status(?:es)?\/(\d+)")
link_clean_pattern = re.compile(r"(.+twitter\.com\/.+\/\d+)(\?)*.*")
link_pattern = re.compile(r"(?:twitter|x).com\/(?:\#!\/)?(\w+)\/status(?:es)?\/(\d+)")
link_clean_pattern = re.compile(r"(.+(?:twitter|x)\.com\/.+\/\d+)(\?)*.*")
def __init__(self, config: dict) -> None:
super().__init__(config)

View File

@@ -70,9 +70,10 @@ class WaczArchiverEnricher(Enricher, Archiver):
if use_docker:
logger.debug(f"generating WACZ in Docker for {url=}")
logger.debug(f"{browsertrix_home_host=} {browsertrix_home_container=}")
if not self.docker_commands:
self.docker_commands = ["docker", "run", "--rm", "-v", f"{browsertrix_home_host}:/crawls/", "webrecorder/browsertrix-crawler"]
cmd = self.docker_commands + cmd
if self.docker_commands:
cmd = self.docker_commands + cmd
else:
cmd = ["docker", "run", "--rm", "-v", f"{browsertrix_home_host}:/crawls/", "webrecorder/browsertrix-crawler"] + cmd
if self.profile:
profile_fn = os.path.join(browsertrix_home_container, "profile.tar.gz")
@@ -94,17 +95,17 @@ class WaczArchiverEnricher(Enricher, Archiver):
return False
if use_docker:
filename = os.path.join(browsertrix_home_container, "collections", collection, f"{collection}.wacz")
wacz_fn = os.path.join(browsertrix_home_container, "collections", collection, f"{collection}.wacz")
else:
filename = os.path.join("collections", collection, f"{collection}.wacz")
wacz_fn = os.path.join("collections", collection, f"{collection}.wacz")
if not os.path.exists(filename):
logger.warning(f"Unable to locate and upload WACZ {filename=}")
if not os.path.exists(wacz_fn):
logger.warning(f"Unable to locate and upload WACZ {wacz_fn=}")
return False
to_enrich.add_media(Media(filename), "browsertrix")
to_enrich.add_media(Media(wacz_fn), "browsertrix")
if self.extract_media:
self.extract_media_from_wacz(to_enrich, filename)
self.extract_media_from_wacz(to_enrich, wacz_fn)
return True
def extract_media_from_wacz(self, to_enrich: Metadata, wacz_filename: str) -> None:

View File

@@ -3,7 +3,7 @@ _MAJOR = "0"
_MINOR = "6"
# On main and in a nightly release the patch should be one ahead of the last
# released build.
_PATCH = "10"
_PATCH = "12"
# This is mainly for nightly builds which have the suffix ".dev$DATE". See
# https://semver.org/#is-v123-a-semantic-version for the semantics.
_SUFFIX = ""