mirror of
https://github.com/bellingcat/auto-archiver.git
synced 2026-06-12 05:08:28 +03:00
Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
31c07a02e1 | ||
|
|
bd231488ff | ||
|
|
fb197f1064 |
@@ -139,7 +139,7 @@ class Metadata:
|
|||||||
new_media = []
|
new_media = []
|
||||||
for m in self.media:
|
for m in self.media:
|
||||||
h = m.get("hash")
|
h = m.get("hash")
|
||||||
if not h: h = calculate_hash_in_chunks(hashlib.sha256(), 1.6e7, m.filename)
|
if not h: h = calculate_hash_in_chunks(hashlib.sha256(), int(1.6e7), m.filename)
|
||||||
if len(h) and h in media_hashes: continue
|
if len(h) and h in media_hashes: continue
|
||||||
media_hashes.add(h)
|
media_hashes.add(h)
|
||||||
new_media.append(m)
|
new_media.append(m)
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ class HashEnricher(Enricher):
|
|||||||
def configs() -> dict:
|
def configs() -> dict:
|
||||||
return {
|
return {
|
||||||
"algorithm": {"default": "SHA-256", "help": "hash algorithm to use", "choices": ["SHA-256", "SHA3-512"]},
|
"algorithm": {"default": "SHA-256", "help": "hash algorithm to use", "choices": ["SHA-256", "SHA3-512"]},
|
||||||
"chunksize": {"default": 1.6e7, "help": "number of bytes to use when reading files in chunks (if this value is too large you will run out of RAM), default is 16MB"},
|
"chunksize": {"default": int(1.6e7), "help": "number of bytes to use when reading files in chunks (if this value is too large you will run out of RAM), default is 16MB"},
|
||||||
}
|
}
|
||||||
|
|
||||||
def enrich(self, to_enrich: Metadata) -> None:
|
def enrich(self, to_enrich: Metadata) -> None:
|
||||||
|
|||||||
@@ -49,6 +49,9 @@ class UrlUtil:
|
|||||||
# instagram recurring images
|
# instagram recurring images
|
||||||
if "https://static.cdninstagram.com/rsrc.php/" in url: return False
|
if "https://static.cdninstagram.com/rsrc.php/" in url: return False
|
||||||
|
|
||||||
|
# telegram
|
||||||
|
if "https://telegram.org/img/emoji/" in url: return False
|
||||||
|
|
||||||
return True
|
return True
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ _MAJOR = "0"
|
|||||||
_MINOR = "6"
|
_MINOR = "6"
|
||||||
# On main and in a nightly release the patch should be one ahead of the last
|
# On main and in a nightly release the patch should be one ahead of the last
|
||||||
# released build.
|
# released build.
|
||||||
_PATCH = "1"
|
_PATCH = "2"
|
||||||
# This is mainly for nightly builds which have the suffix ".dev$DATE". See
|
# This is mainly for nightly builds which have the suffix ".dev$DATE". See
|
||||||
# https://semver.org/#is-v123-a-semantic-version for the semantics.
|
# https://semver.org/#is-v123-a-semantic-version for the semantics.
|
||||||
_SUFFIX = ""
|
_SUFFIX = ""
|
||||||
|
|||||||
Reference in New Issue
Block a user