mirror of
https://github.com/bellingcat/auto-archiver.git
synced 2026-06-12 13:18:28 +03:00
Compare commits
5 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0feeb0bd24 | ||
|
|
ddb9dc87d7 | ||
|
|
e8935b9a80 | ||
|
|
b157f9a6b1 | ||
|
|
ea38a604bb |
@@ -15,8 +15,8 @@ class TwitterArchiver(Archiver):
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
name = "twitter_archiver"
|
name = "twitter_archiver"
|
||||||
link_pattern = re.compile(r"twitter.com\/(?:\#!\/)?(\w+)\/status(?:es)?\/(\d+)")
|
link_pattern = re.compile(r"(?:twitter|x).com\/(?:\#!\/)?(\w+)\/status(?:es)?\/(\d+)")
|
||||||
link_clean_pattern = re.compile(r"(.+twitter\.com\/.+\/\d+)(\?)*.*")
|
link_clean_pattern = re.compile(r"(.+(?:twitter|x)\.com\/.+\/\d+)(\?)*.*")
|
||||||
|
|
||||||
def __init__(self, config: dict) -> None:
|
def __init__(self, config: dict) -> None:
|
||||||
super().__init__(config)
|
super().__init__(config)
|
||||||
|
|||||||
@@ -70,9 +70,10 @@ class WaczArchiverEnricher(Enricher, Archiver):
|
|||||||
if use_docker:
|
if use_docker:
|
||||||
logger.debug(f"generating WACZ in Docker for {url=}")
|
logger.debug(f"generating WACZ in Docker for {url=}")
|
||||||
logger.debug(f"{browsertrix_home_host=} {browsertrix_home_container=}")
|
logger.debug(f"{browsertrix_home_host=} {browsertrix_home_container=}")
|
||||||
if not self.docker_commands:
|
if self.docker_commands:
|
||||||
self.docker_commands = ["docker", "run", "--rm", "-v", f"{browsertrix_home_host}:/crawls/", "webrecorder/browsertrix-crawler"]
|
cmd = self.docker_commands + cmd
|
||||||
cmd = self.docker_commands + cmd
|
else:
|
||||||
|
cmd = ["docker", "run", "--rm", "-v", f"{browsertrix_home_host}:/crawls/", "webrecorder/browsertrix-crawler"] + cmd
|
||||||
|
|
||||||
if self.profile:
|
if self.profile:
|
||||||
profile_fn = os.path.join(browsertrix_home_container, "profile.tar.gz")
|
profile_fn = os.path.join(browsertrix_home_container, "profile.tar.gz")
|
||||||
@@ -94,17 +95,17 @@ class WaczArchiverEnricher(Enricher, Archiver):
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
if use_docker:
|
if use_docker:
|
||||||
filename = os.path.join(browsertrix_home_container, "collections", collection, f"{collection}.wacz")
|
wacz_fn = os.path.join(browsertrix_home_container, "collections", collection, f"{collection}.wacz")
|
||||||
else:
|
else:
|
||||||
filename = os.path.join("collections", collection, f"{collection}.wacz")
|
wacz_fn = os.path.join("collections", collection, f"{collection}.wacz")
|
||||||
|
|
||||||
if not os.path.exists(filename):
|
if not os.path.exists(wacz_fn):
|
||||||
logger.warning(f"Unable to locate and upload WACZ {filename=}")
|
logger.warning(f"Unable to locate and upload WACZ {wacz_fn=}")
|
||||||
return False
|
return False
|
||||||
|
|
||||||
to_enrich.add_media(Media(filename), "browsertrix")
|
to_enrich.add_media(Media(wacz_fn), "browsertrix")
|
||||||
if self.extract_media:
|
if self.extract_media:
|
||||||
self.extract_media_from_wacz(to_enrich, filename)
|
self.extract_media_from_wacz(to_enrich, wacz_fn)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
def extract_media_from_wacz(self, to_enrich: Metadata, wacz_filename: str) -> None:
|
def extract_media_from_wacz(self, to_enrich: Metadata, wacz_filename: str) -> None:
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ _MAJOR = "0"
|
|||||||
_MINOR = "6"
|
_MINOR = "6"
|
||||||
# On main and in a nightly release the patch should be one ahead of the last
|
# On main and in a nightly release the patch should be one ahead of the last
|
||||||
# released build.
|
# released build.
|
||||||
_PATCH = "10"
|
_PATCH = "12"
|
||||||
# This is mainly for nightly builds which have the suffix ".dev$DATE". See
|
# This is mainly for nightly builds which have the suffix ".dev$DATE". See
|
||||||
# https://semver.org/#is-v123-a-semantic-version for the semantics.
|
# https://semver.org/#is-v123-a-semantic-version for the semantics.
|
||||||
_SUFFIX = ""
|
_SUFFIX = ""
|
||||||
|
|||||||
Reference in New Issue
Block a user