From 294033f156707e7f60da1841edc04844cd826d79 Mon Sep 17 00:00:00 2001 From: Patrick Robertson Date: Tue, 11 Mar 2025 15:44:04 +0000 Subject: [PATCH] Fix bug ordering tsr that only have one cert + more unit tests --- .../timestamping_enricher/__manifest__.py | 32 +++-- .../timestamping_enricher.py | 53 ++++---- tests/conftest.py | 8 +- tests/data/timestamping/intermediate.crt | 44 ------- tests/data/timestamping/leaf.crt | 39 ------ tests/data/timestamping/root.crt | 31 ----- tests/data/timestamping/self_signed.tsr | Bin 0 -> 1600 bytes ...mestamp_token_http-timestamp-identrust-com | Bin 0 -> 4767 bytes ...stamp_response.tsr => valid_timestamp.tsr} | Bin tests/enrichers/test_timestamping_enricher.py | 113 ++++++++++++++++-- tests/storages/test_local_storage.py | 11 -- 11 files changed, 158 insertions(+), 173 deletions(-) delete mode 100644 tests/data/timestamping/intermediate.crt delete mode 100644 tests/data/timestamping/leaf.crt delete mode 100644 tests/data/timestamping/root.crt create mode 100644 tests/data/timestamping/self_signed.tsr create mode 100644 tests/data/timestamping/timestamp_token_http-timestamp-identrust-com rename tests/data/timestamping/{timestamp_response.tsr => valid_timestamp.tsr} (100%) diff --git a/src/auto_archiver/modules/timestamping_enricher/__manifest__.py b/src/auto_archiver/modules/timestamping_enricher/__manifest__.py index e945350..2c81dbb 100644 --- a/src/auto_archiver/modules/timestamping_enricher/__manifest__.py +++ b/src/auto_archiver/modules/timestamping_enricher/__manifest__.py @@ -15,25 +15,21 @@ "configs": { "tsa_urls": { "default": [ - # [Adobe Approved Trust List] and [Windows Cert Store] - "http://timestamp.digicert.com", + # See https://github.com/trailofbits/rfc3161-client/issues/46 for a list of valid TSAs + # Full list of TSAs: https://gist.github.com/Manouchehri/fd754e402d98430243455713efada710 "http://timestamp.identrust.com", - # "https://timestamp.entrust.net/TSS/RFC3161sha2TS", # not valid for timestamping - # "https://timestamp.sectigo.com", # wait 15 seconds between each request. - - # [Adobe: European Union Trusted Lists]. - # "https://timestamp.sectigo.com/qualified", # wait 15 seconds between each request. - - # [Windows Cert Store] - "http://timestamp.globalsign.com/tsa/r6advanced1", - # [Adobe: European Union Trusted Lists] and [Windows Cert Store] - # "http://ts.quovadisglobal.com/eu", # not valid for timestamping - # "http://tsa.belgium.be/connect", # self-signed certificate in certificate chain - # "https://timestamp.aped.gov.gr/qtss", # self-signed certificate in certificate chain - # "http://tsa.sep.bg", # self-signed certificate in certificate chain - # "http://tsa.izenpe.com", #unable to get local issuer certificate - # "http://kstamp.keynectis.com/KSign", # unable to get local issuer certificate - "http://tss.accv.es:8318/tsa", + "http://timestamp.ssl.trustwave.com", #timeouts + "http://zeitstempel.dfn.de", + "http://ts.ssl.com", + "http://tsa.izenpe.com", + "http://tsa.lex-persona.com/tsa", + "http://ca.signfiles.com/TSAServer.aspx", + "http://aloahacoin.chain-provider.com/tsa.aspx", + "http://tsa.sinpe.fi.cr/tsaHttp/", + "http://tsa.cra.ge/signserver/tsa?workerName=qtsa", + "http://tss.cnbs.gob.hn/TSS/HttpTspServer", + "http://dss.nowina.lu/pki-factory/tsa/good-tsa", + "https://freetsa.org/tsr", ], "help": "List of RFC3161 Time Stamp Authorities to use, separate with commas if passed via the command line.", }, diff --git a/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py b/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py index 9742587..6f9aa56 100644 --- a/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py +++ b/src/auto_archiver/modules/timestamping_enricher/timestamping_enricher.py @@ -1,10 +1,11 @@ import os -from loguru import logger from importlib.metadata import version import hashlib +from slugify import slugify import requests +from loguru import logger from rfc3161_client import ( TimestampRequestBuilder, @@ -23,7 +24,6 @@ from auto_archiver.core import Metadata, Media from auto_archiver.version import __version__ - class TimestampingEnricher(Enricher): """ Uses several RFC3161 Time Stamp Authorities to generate a timestamp token that will be preserved. This can be used to prove that a certain file existed at a certain time, useful for legal purposes, for example, to prove that a certain file was not tampered with after a certain date. @@ -33,6 +33,8 @@ class TimestampingEnricher(Enricher): See https://gist.github.com/Manouchehri/fd754e402d98430243455713efada710 for list of timestamp authorities. """ + session = None + def setup(self): self.session = requests.Session() self.session.headers.update( @@ -43,11 +45,12 @@ class TimestampingEnricher(Enricher): } ) - def __del__(self) -> None: + def cleaup(self) -> None: """ Terminates the underlying network session. """ - self.session.close() + if self.session: + self.session.close() def enrich(self, to_enrich: Metadata) -> None: url = to_enrich.get_url() @@ -68,40 +71,47 @@ class TimestampingEnricher(Enricher): hashes_media = Media(filename=hashes_fn) timestamp_tokens = [] - from slugify import slugify for tsa_url in self.tsa_urls: try: message = bytes(data_to_sign, encoding='utf8') + + print(tsa_url) + logger.debug(f"Timestamping {url=} with {tsa_url=}") signed: TimeStampResponse = self.sign_data(tsa_url, message) # fail if there's any issue with the certificates, uses certifi list of trusted CAs or the user-defined `cert_authorities` root_cert = self.verify_signed(signed, message) - if not root_cert: - raise ValueError(f"No valid root certificate found for {tsa_url=}. Are you sure it's a trusted TSA? Or define an alternative trusted root with `cert_authorities`.") + raise ValueError(f"No valid root certificate found for {tsa_url=}. Are you sure it's a trusted TSA? Or define an alternative trusted root with `cert_authorities`. (tried: {self.cert_authorities or certifi.where()})") # save the timestamping certificate cert_chain = self.save_certificate(signed, root_cert) - # continue with saving the timestamp token - tst_fn = os.path.join(self.tmp_dir, f"timestamp_token_{slugify(tsa_url)}") - with open(tst_fn, "wb") as f: - f.write(signed) - timestamp_tokens.append(Media(filename=tst_fn).set("tsa", tsa_url).set("cert_chain", cert_chain)) + timestamp_token_path = self.save_timestamp_token(signed.time_stamp_token(), tsa_url) + timestamp_tokens.append(Media(filename=timestamp_token_path).set("tsa", tsa_url).set("cert_chain", cert_chain)) except Exception as e: logger.warning(f"Error while timestamping {url=} with {tsa_url=}: {e}") if len(timestamp_tokens): hashes_media.set("timestamp_authority_files", timestamp_tokens) hashes_media.set("certifi v", version("certifi")) - hashes_media.set("tsp_client v", version("tsp_client")) - hashes_media.set("certvalidator v", version("certvalidator")) + hashes_media.set("rfc3161-client v", version("rfc3161_client")) + hashes_media.set("cryptography v", version("cryptography")) to_enrich.add_media(hashes_media, id="timestamped_hashes") to_enrich.set("timestamped", True) logger.success(f"{len(timestamp_tokens)} timestamp tokens created for {url=}") else: logger.warning(f"No successful timestamps for {url=}") + def save_timestamp_token(self, timestamp_token: bytes, tsa_url: str) -> str: + """ + Takes a timestamp token, and saves it to a file with the TSA URL as part of the filename. + """ + tst_path = os.path.join(self.tmp_dir, f"timestamp_token_{slugify(tsa_url)}") + with open(tst_path, "wb") as f: + f.write(timestamp_token) + return tst_path + def verify_signed(self, timestamp_response: TimeStampResponse, message: bytes) -> x509.Certificate: """ Verify a Signed Timestamp Response is trusted by a known Certificate Authority. @@ -127,10 +137,7 @@ class TimestampingEnricher(Enricher): raise ValueError(f"No trusted roots found in {trusted_root_path}.") timestamp_certs = self.tst_certs(timestamp_response) - intermediate_certs = [] - for i, cert in enumerate(timestamp_certs): # cannot use list comprehension, it's a set - intermediate_certs.append(cert) - + intermediate_certs = timestamp_certs[1:-1] message_hash = None hash_algorithm = timestamp_response.tst_info.message_imprint.hash_algorithm @@ -155,10 +162,9 @@ class TimestampingEnricher(Enricher): verifier.verify(timestamp_response, message_hash) return certificate except Rfc3161VerificationError as e: - logger.debug(f"Unable to verify Timestamp with CA {certificate.subject}: {e}") continue - raise ValueError(f"No valid root certificate found in {trusted_root_path}.") + return None def sign_data(self, tsa_url: str, bytes_data: bytes) -> TimeStampResponse: # see https://github.com/sigstore/sigstore-python/blob/99948d5b80525a5a104e904ffea58169dc6e0629/sigstore/_internal/timestamp.py#L84-L121 @@ -186,15 +192,18 @@ class TimestampingEnricher(Enricher): certs = [x509.load_der_x509_certificate(c) for c in signed_data.certificates] # reorder the certs to be in the correct order ordered_certs = [] + if len(certs) == 1: + return certs + while(len(ordered_certs) < len(certs)): if len(ordered_certs) == 0: for cert in certs: - if not [c for c in certs if c.subject == cert.issuer]: + if not [c for c in certs if cert.subject == c.issuer]: ordered_certs.append(cert) break else: for cert in certs: - if cert.issuer == ordered_certs[-1].subject: + if cert.subject == ordered_certs[-1].issuer: ordered_certs.append(cert) break return ordered_certs diff --git a/tests/conftest.py b/tests/conftest.py index a94abcd..bf338b9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,7 +9,7 @@ from typing import Dict, Tuple import hashlib import pytest -from auto_archiver.core.metadata import Metadata +from auto_archiver.core.metadata import Metadata, Media from auto_archiver.core.module import ModuleFactory # Test names inserted into this list will be run last. This is useful for expensive/costly tests @@ -139,6 +139,12 @@ def mock_binary_dependencies(mocker): mock_shutil_which.return_value = "/usr/bin/fake_binary" return mock_shutil_which +@pytest.fixture +def sample_media(tmp_path) -> Media: + """Fixture creating a Media object with temporary source file""" + src_file = tmp_path / "source.txt" + src_file.write_text("test content") + return Media(key="subdir/test.txt", filename=str(src_file)) @pytest.fixture def sample_datetime(): diff --git a/tests/data/timestamping/intermediate.crt b/tests/data/timestamping/intermediate.crt deleted file mode 100644 index a9242f9..0000000 --- a/tests/data/timestamping/intermediate.crt +++ /dev/null @@ -1,44 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIHuDCCBaCgAwIBAgIQQAF/lJAVu6kSuFeWPUTs7jANBgkqhkiG9w0BAQsFADBK -MQswCQYDVQQGEwJVUzESMBAGA1UEChMJSWRlblRydXN0MScwJQYDVQQDEx5JZGVu -VHJ1c3QgQ29tbWVyY2lhbCBSb290IENBIDEwHhcNMjIwMzE2MjEwOTA1WhcNMzMw -NjEyMjEwOTA1WjBFMQswCQYDVQQGEwJVUzESMBAGA1UEChMJSWRlblRydXN0MSIw -IAYDVQQDExlUcnVzdElEIFRpbWVzdGFtcGluZyBDQSAzMIICIjANBgkqhkiG9w0B -AQEFAAOCAg8AMIICCgKCAgEAqGL0RYFG7mL0RgSXLynLNWhEVrhsKhrVL4rSG+NA -p4v8TbAP2YXsqWB8yZgj9DQ55AECnmQ2Uo/BqQSsI/AOr9ctqZykItmca/nGjKez -l1kZS2YoNc4Zjj+7QO9iNunclA06fBhI+iQHAam7isQLK3CwXRDLzKkMs7TisMoG -QOSd0M8P6YY/QOGYv/+tCxmfvUz2GjWzQQemgiuLjvGhPwo+hrcNzays9j0G7QtA -LkJ0KfvJS+guvCvSuEfDzt3BaPIpD2q6GYK+MUiNis3uwwngauyL4r048wdvUSsf -92Kyr6T1pAfjjPyVDNazf/w/BjzA6ewNevFVLNfE0DhQkXMmsNVGBzY5Phhlp5fb -TwsrD19K0FPgbGO/l/Zp2dheeiCbe09bxbhdeahSBtTVPca4Vu3Ljz+PRZjFodq7 -+lziqqpqqCP/ikEnmK/QkxjCG7AkX384dxg7yb5jjtXOnP5Yv4SXuV4SNNVVUBJf -bLXyYAf3Q0Dal85ZxNQd0QNPQIsYWv9ttTMVc6sVErdfTBPw355St6bHz91LUoDD -0S/GjUif8LYhVlZGXlwjYmVZOb2Z7+DAamzjwVrSsrxGCJ66Coy1rKapJuHVsfGA -W44p2ioIEZT3s6nQJkCt7te4ab1iWzaydZGAYyBao0K7kfK3vSp4AXsE5t5y6dpH -Yo0CAwEAAaOCAp0wggKZMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQD -AgGGMIGJBggrBgEFBQcBAQR9MHswMAYIKwYBBQUHMAGGJGh0dHA6Ly9jb21tZXJj -aWFsLm9jc3AuaWRlbnRydXN0LmNvbTBHBggrBgEFBQcwAoY7aHR0cDovL3ZhbGlk -YXRpb24uaWRlbnRydXN0LmNvbS9yb290cy9jb21tZXJjaWFscm9vdGNhMS5wN2Mw -HwYDVR0jBBgwFoAU7UQZwNPwBovupHu+QucmVMiONnYwggFEBgNVHSAEggE7MIIB -NzAIBgZngQwBBAIwDQYLYIZIAYb5LwAGDQMwggEaBgtghkgBhvkvAAYNATCCAQkw -SgYIKwYBBQUHAgEWPmh0dHBzOi8vc2VjdXJlLmlkZW50cnVzdC5jb20vY2VydGlm -aWNhdGVzL3BvbGljeS90cy9pbmRleC5odG1sMIG6BggrBgEFBQcCAjCBrQyBqlRo -aXMgVHJ1c3RJRCBDZXJ0aWZpY2F0ZSBoYXMgYmVlbiBpc3N1ZWQgaW4gYWNjb3Jk -YW5jZSB3aXRoIElkZW5UcnVzdCdzIFRydXN0SUQgQ2VydGlmaWNhdGUgUG9saWN5 -IGZvdW5kIGF0IGh0dHBzOi8vc2VjdXJlLmlkZW50cnVzdC5jb20vY2VydGlmaWNh -dGVzL3BvbGljeS90cy9pbmRleC5odG1sMEoGA1UdHwRDMEEwP6A9oDuGOWh0dHA6 -Ly92YWxpZGF0aW9uLmlkZW50cnVzdC5jb20vY3JsL2NvbW1lcmNpYWxyb290Y2Ex -LmNybDAdBgNVHQ4EFgQUyjLwNnzHKtqRtXyHihG9uCJsvwkwEwYDVR0lBAwwCgYI -KwYBBQUHAwgwDQYJKoZIhvcNAQELBQADggIBACtnO4f6QB6v2yDFeld3Pa1H7Bmb -y2tSwzQ/5dB5WXmTZHlV433s7BDkpwGzK4fGBuTcx814uPUWWSwcL+f8bpfozBv6 -p855j/AlKul1EiPMKkMlLtwdiK6sWT2x8qaTm4fMGbcpgUbQnxOo4BnzVUmgs3ep -m9/qXf9GaWXRz4maSWl4z3apD3X/5oMrriGWIiW6ivCq8bmOBUdm0I9quhW9Snk2 -JAaqkVCjs06rqE3rRblyNdrSypGzo5eBT498aCfcvDPJX2/q2PMkLvkKoXtVJ1g4 -sEwDQxm2sg8QMEd2GKo+X7TqOeF8An7KOPDq9v0xtSsF4+ufFrl43vl6v4uMey68 -wOHv6VmaGpCtWk1e6lSq9jLQqRBBg8CMwpw6niVyvkrdh4Tvu+5HLrareZBp98PJ -2cQzrHk1SiPcyDxSlhbXRks/TgKXTicUm3pZsKZcTvCXHcqulN1eoQ3z9azMIuR7 -RtdECz2RJUI6Xg//6ZdMMgaDnktMALgAyo9GgGGVimx4E2/aM5r0cm+RCrk956BT -qahEdiGLWjgjq8dAJe4XfLtp6EmGvsw650uBbDA0Mg9nlSCFdGTMFBKlw65o2oJa -enMysAvE/VC39ZIEOBfk2IOj2GTYlOgHi0iIBIZf7SqdjhTAtoW7U9TTZj8SHRen -/EEe1WEcfTawcOhJ ------END CERTIFICATE----- diff --git a/tests/data/timestamping/leaf.crt b/tests/data/timestamping/leaf.crt deleted file mode 100644 index 539ae6e..0000000 --- a/tests/data/timestamping/leaf.crt +++ /dev/null @@ -1,39 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIG4zCCBMugAwIBAgIQQAGSoSzKR+4rZmhPudMJdjANBgkqhkiG9w0BAQsFADBF -MQswCQYDVQQGEwJVUzESMBAGA1UEChMJSWRlblRydXN0MSIwIAYDVQQDExlUcnVz -dElEIFRpbWVzdGFtcGluZyBDQSAzMB4XDTI0MTAxODE5NDg0MFoXDTI2MDExNzE5 -NDgzOVowRzELMAkGA1UEBhMCVVMxEjAQBgNVBAoTCUlkZW5UcnVzdDEkMCIGA1UE -AxMbVHJ1c3RJRCBUaW1lc3RhbXAgQXV0aG9yaXR5MIICIjANBgkqhkiG9w0BAQEF -AAOCAg8AMIICCgKCAgEA27BXl6MHeJwySCub5fd8rD4bWC2yOQE4hWDkw6YjMHMl -mKQVPIqD2+ezxhIK5GmwwNxDopo1ovsTMrF/KFsTExw0l4ZguJ+xs5W6qLrRVm7Z -TnkB6wMKaxR8+fdxtq9lnqRDnHtrGiGEQGOqVvUpAhnzioYa/5qWmqhRhdf7bRpI -1D71LC4UtWm1YG0PmJVSsE2ZqjB4rP7baj6hAlQxe0bvNwL/Bqy6D0QHNk2Xf2Vk -w7BLN7knh/SngAlzfumBg0cQff0C5DQgeyCQoM5/XBAyYZUQc//t2XVZPYiGUvOj -QNjLCbqOpRQf9JBaU/gh9Mf6Mbs6xh5qn4gKHpxOUMy+QfYAU+RiLhV4X0/brdqc -UOqRHte9gcv8yU4mqyMzbmHn5y09SkuhPOIhYE5Fd72XHeR8RnC9CI9V5NjVW370 -iNZuO2R2hw3dwtlWFxJPM+jortoxO3BPoxmppp/EdEH60p5Sk12eeyx86zieLb1E -JS5gEvu+jYWwKKjCf3visWC4OJNzbknGr3WRVOI6UvjyKrck3hpCuOjLXST+pEiB -XJzbiHCR5UJn8mXJnPAvKabgCGk7/trqrqB3Rkd2l5rce7maoyn265r/3IRLZisf -QdBsfzQv6RAP/zBFQkzcE2Yl12M25k2B6kFZMLsTT60jBTt1W6utXM2T7DbOwO8C -AwEAAaOCAcswggHHMAwGA1UdEwEB/wQCMAAwDgYDVR0PAQH/BAQDAgeAMBYGA1Ud -JQEB/wQMMAoGCCsGAQUFBwMIMIGFBggrBgEFBQcBAQR5MHcwMAYIKwYBBQUHMAGG -JGh0dHA6Ly9jb21tZXJjaWFsLm9jc3AuaWRlbnRydXN0LmNvbTBDBggrBgEFBQcw -AoY3aHR0cDovL3ZhbGlkYXRpb24uaWRlbnRydXN0LmNvbS9jZXJ0cy90aW1lc3Rh -bXBpbmczLnA3YzAfBgNVHSMEGDAWgBTKMvA2fMcq2pG1fIeKEb24Imy/CTB/BgNV -HSAEeDB2MAgGBmeBDAEEAjANBgtghkgBhvkvAAYNATBbBgtghkgBhvkvAAYNAzBM -MEoGCCsGAQUFBwIBFj5odHRwczovL3NlY3VyZS5pZGVudHJ1c3QuY29tL2NlcnRp -ZmljYXRlcy9wb2xpY3kvdHMvaW5kZXguaHRtbDBGBgNVHR8EPzA9MDugOaA3hjVo -dHRwOi8vdmFsaWRhdGlvbi5pZGVudHJ1c3QuY29tL2NybC90aW1lc3RhbXBpbmcz -LmNybDAdBgNVHQ4EFgQUIYBfL7FSq7kLBlpOTWDaRGHtFgEwDQYJKoZIhvcNAQEL -BQADggIBAFIczKS3kFMFeX2WIC1uUj4Nvt+W2/kPNhuKukF2pOC+VcxrbbsTugTx -oO+X8J5JVX5sNP74p7YrkSY+dVPFQ+8rfJJeshSvJEQnt9DgsXPTRcU982OPmJCQ -QZY2Ux7xYfoEuWytuigyhMMoh+g5IzipQ56UCDY/sHY40SdLpSXg69RMy6y/L2Zm -9f4YPFrXPy7q3hGTemQHo+jmshg/hU/zIEjjfWx7uG223r0M5Ez9ks4y/EmtdT5l -KkF9RGALpKEBWIQaL+yi8X8NHRM1Qfs91GvEZe5wPri/5R9YvhiHWjizqxvDlqVW -ka6Mu13zbovM2vMppHJWYlvBTZ4z8vZBdNzN7fiTAdcd3lAl0A3wmpBIqrvFChhY -DX0Su1/3kA3X2PAJrbaZ7RNQ+Zjuz31T8QK0d1PyzHOZ/jK76f7Nb5Ic8fbALKzs -S+Yg3O4fmLqC0kERtL96xA/y+Y9oAJYMOOvrpBvpe1TdGZ5s3nRpLhWT4NEddkGg -AWodbBA3W0x5iKRWXAo+sATHTNXB6RGTIuF1/PC3R4fjbTMvpHXyW+bP+k8m+uIu -XLDMJ+XW5TZ01g6UGfg62ti4ohS0PyEHbnuEQDQogcqtqF0d8oihfcbDXOm/YgId -vhdRXJC/UZ0Q6Q6jtuj5uV76fvNU4j4FYKnXZqutzlCxeKn1odHN ------END CERTIFICATE----- diff --git a/tests/data/timestamping/root.crt b/tests/data/timestamping/root.crt deleted file mode 100644 index de6f07d..0000000 --- a/tests/data/timestamping/root.crt +++ /dev/null @@ -1,31 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIFYDCCA0igAwIBAgIQCgFCgAAAAUUjyES1AAAAAjANBgkqhkiG9w0BAQsFADBK -MQswCQYDVQQGEwJVUzESMBAGA1UEChMJSWRlblRydXN0MScwJQYDVQQDEx5JZGVu -VHJ1c3QgQ29tbWVyY2lhbCBSb290IENBIDEwHhcNMTQwMTE2MTgxMjIzWhcNMzQw -MTE2MTgxMjIzWjBKMQswCQYDVQQGEwJVUzESMBAGA1UEChMJSWRlblRydXN0MScw -JQYDVQQDEx5JZGVuVHJ1c3QgQ29tbWVyY2lhbCBSb290IENBIDEwggIiMA0GCSqG -SIb3DQEBAQUAA4ICDwAwggIKAoICAQCnUBneP5k91DNG8W9RYYKyqU+PZ4ldhNlT -3Qwo2dfw/66VQ3KZ+bVdfIrBQuExUHTRgQ18zZshq0PirK1ehm7zCYofWjK9ouuU -+ehcCuz/mNKvcbO0U59Oh++SvL3sTzIwiEsXXlfEU8L2ApeN2WIrvyQfYo3fw7gp -S0l4PJNgiCL8mdo2yMKi1CxUAGc1bnO/AljwpN3lsKImesrgNqUZFvX9t++uP0D1 -bVoE/c40yiTcdCMbXTMTEl3EASX2MN0CXZ/g1Ue9tOsbobtJSdifWwLziuQkkORi -T0/Br4sOdBeo0XKIanoBScy0RnnGF7HamB4HWfp1IYVl3ZBWzvurpWCdxJ35UrCL -vYf5jysjCiN2O/cz4ckA82n5S6LgTrx+kzmEB/dEcH7+B1rlsazRGMzyNeVJSQjK -Vsk9+w8YfYs7wRPCTY/JTw436R+hDmrfYi7LNQZReSzIJTj0+kuniVyc0uMNOYZK -dHzVWYfCP04MXFL0PfdSgvHqo6z9STQaKPNBiDoT7uje/5kdX7rL6B7yuVBgwDHT -c+XvvqDtMwt0viAgxGds8AgDelWAf0ZOlqf0Hj7h9tgJ4TNkK2PXMl6f+cB7D3hv -l7yTmvmcEpB4eoCHFddydJxVdHixuuFucAS6T6C6aMN7/zHwcz09lCqxC0EOoP5N -iGVreTO01wIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB -/zAdBgNVHQ4EFgQU7UQZwNPwBovupHu+QucmVMiONnYwDQYJKoZIhvcNAQELBQAD -ggIBAA2ukDL2pkt8RHYZYR4nKM1eVO8lvOMIkPkp165oCOGUAFjvLi5+U1KMtlwH -6oi6mYtQlNeCgN9hCQCTrQ0U5s7B8jeUeLBfnLOic7iPBZM4zY0+sLj7wM+x8uwt -LRvM7Kqas6pgghstO8OEPVeKlh6cdbjTMM1gCIOQ045U8U1mwF10A0Cj7oV+wh93 -nAbowacYXVKV7cndJZ5t+qntozo00Fl72u1Q8zW/7esUTTHHYPTa8Yec4kjixsU3 -+wYQ+nVZZjFHKdp2mhzpgq7vmrlR94gjmmmVYjzlVYA211QC//G5Xc7UI2/YRYRK -W2XviQzdFKcgyxilJbQN+QHwotL0AMh0jqEqSI5l2xPE4iUXfeu+h1sXIFRRk0pT -AwvsXcoz7WL9RccvW9xYoIA55vrX/hMUpu09lEpCdNTDd1lzzY9GvlU47/rokTLq -l1gEIt44w8y8bckzOmoKaT+gyOpyj4xjhiO9bTyWnpXgSUyqorkqG5w2gXjtw+hG -4iZZRHUe2XWJUc0QhJ1hYMtd+ZciTY6Y5uN/9lu7rs3KSoFrXgvzUeF0K+l+J6fZ -mUlO+KWA2yUPHGNiiskzZ2s8EIPGrd6ozRaOjfAHN3Gf8qv8QfXBi+wAN10J5U6A -7/qxXDgGpRtK4dw4LTzcqx+QGtVKnO7RcGzM7vRX+Bi6hG6H ------END CERTIFICATE----- diff --git a/tests/data/timestamping/self_signed.tsr b/tests/data/timestamping/self_signed.tsr new file mode 100644 index 0000000000000000000000000000000000000000..e7ffd8344a4476af45816719931c4864d7bc8b1f GIT binary patch literal 1600 zcmbW1X;71A5XbYrZxRwn2*^1o5{VWGf=>wHQ0@k#BBIElI4BsRBnSwhq*$yZ#T3EH zA&RzxNjNluNI)4u@F=K+dlV?(2q;mmQVEL6q0-oKDt_n}KR(a?XJ_}n`&%HyIs*e3 zLIE9uw*Mh^LPR1bE|%;c9gz%#*i~3ORpcXjON2CFi4eO?{iO{<2#o}YSbVt12Vx^= z7{Z`{pa84&Nr~5i&=SF~Xh8-L`~pJ|7NUafOZ5l_g?3+vzkm|QNoO`zBy#bsET^HP zCf%_klA}XgFp0*#HA>O;~sl){|F$Y?yo1Qcd&tjKR+9F|{#GrQ_ z$T3Id+KX1CX%BaqG07I#G60=MquJTfY#l>kJnD_($!9t1YQrQ#!~+Ns7a)Y!NDvr; zVO><3@7v{&#oba;k*TbTv$O7FiJw}4wA3xx(@cOd770RY5_I`|z73CaAa+0JAe(2y zii@Ec1H+Fogh296V6(}AJ3Poc*a=DO1hCSGNOJ@XIsoP>1yVQX*VuPr?g*P^gdWWIoc+jM1b(#Fxfo(?J zITHcv9%XYn+{;ZBxCb(Rz2Zxkd$}PG?lCQ-0eX~Ig~wk#G4}|+ zUeiN4d^^)w>YVtn%9JQoQl%|AJigF(x9jQyY5T(F@6>K*s(LE5OXX#uIC9Vjx;I_r zMZlfkOY*Dmb5#s#Ql@K5B8wa=E~OS1dRevx`ri*#bWdHGi8=O8qp2bxkKf2-;Lmh& ze;kwTE^Sq0=)wquf{MHW6KJU?V@g7!5m+Pj-!Yl4fuKb+0cc`zR#*sw(Liv(^rJ`{ zZHU&d&fXQwis*0TraY|E3+~d~oSHtk)M@oHsGWvT&Q0m=CDuW~q2(<$devhduV`63 z-8pUpN@qhe;uKf7s_$O-7)yM|!In2WcH(()wYkF4PaL1_+w#I_F1M_3yHq!FwMIbk z9=n(|izFgO4NKm5_3~r0l$~qGeh^oh()$m|b$&WXwch3Nl2L=j z-p;+#JifW@G1hH(ADe&orOaxT zY{eXdkz!Z^XswRwi%UwPAFwzN^=hCWLj>01?$p1K+P*8DzfO{n+@q<1|Gj2sd$mKRNPbYjBCY#CRfY zxGpFuecera&e_^%!iN?M!5?7_TNZOWwe*#zCqrhTYh>oc5aak`$xApeJB^@dG}CXVB+olT361ye4qDbx|Q*~qcivUKVj)=Q2+n{ literal 0 HcmV?d00001 diff --git a/tests/data/timestamping/timestamp_token_http-timestamp-identrust-com b/tests/data/timestamping/timestamp_token_http-timestamp-identrust-com new file mode 100644 index 0000000000000000000000000000000000000000..1def8d13da9563906351e1de95e369cdcac3c485 GIT binary patch literal 4767 zcmb`Kby$?!_Q#oEXa@-iMTU-%;T<}pkuE_v5>mq;qeI9HND2%oC?N`nNC?s(ijopa zsDxr5?Lj~Z=>}2mj6d~q&h_{BJolbIzx&;>*1PvwpS1uI%MBQvc$8k$1`~(|oK0f6 z1dv!_z#s@34lu#!9HaC=SO^phqNW1E3SbQ1XVNi)pxI%m(6FmOSP~cnf-z7gMr}z_ z!I&Vx@x3b{)KpOQ&*$=~vOIHEmG2PG&~;)I{|w=N@rIM_a}=VJre&E?{PL(6xx}VGwP(Z&?D+FP+N?mja|@0ckWE z4ai8#D%gPOs8+mkfH|Fsm~7I%Tz~`{2T0J)YzP&5S`bP{xG)_SuFD+!1w0{x2t*`iKM%2EmU6nrOxT?p@Ud2ji85cwRqyYl ztBb$Q#=DwmW3kD%`eYkjuhf3{OjZp2;S3YAu11Y@DvJqc6uX~;=}_5VhS^Ng;l75j z&Yfo_!X*OI9!`^!5-J9UIY-C%98Hd$Z%E~t2-5LtpiQux=(c9X5f*aPYRh54cbB|N0!fxmm@1g3m=CB zj@@;vQAj3u=(peTNjfp6WWKp9UM={JTeD`W^OWG%JiRcxjG-8>qz{^>m$A<>7A3{< zM`>}&Uk7K(v(M}3`lhB22iB(NimiW2-yM!HbVc%N^x#8eB&V6-yTCC`qhVH8p+1b< zN8_*=4QrsD)wE22TG_|8q|B~6c~y^aNlezbGD75NH<)2B?62 z6a}X!CJsh20{yjMg9;V5Y!(y4*)+j0E!ZHBNDxoBqcGP z?(SG`49*FU^289lP&jhD5GnD3B7Xt2ey#vV$$wwr>x9R-I1zE49)F!IiNSgk36jJ; zK_!jylE(nN6#E3AoWOxlwhpO9xu7TFgGmp9qGR_r)bQh*=ztK)1O(I%@C9gLu+w3T zAhMrKdvg*9u-z*mfDvHu!!$7HfGWi_0@)h^7UScM{e@jPR~*KPh$TpRdE#*xe@U{T zI1d-BAIgpBjt6uo2Jk}F02M$vTQOTcO7^$DdgK4*DaIQQ@KE;I2R#5~;|q0=ylY-k z%K)=6F?Jl(cA7f?+8glXoJ9_J5S96%uDt3*3u^!16oiC_xhhlRpD9CIaJhrAbsD~T zqm7nbXWi>r>!3^73#p5l`j!{)GG8|fs*p*-sy-HNS_{aaEB5!;?g(m&RQHVDCA>b? zrm})bNJ~uANRhKRyyUbEt;LtsiAqJZh(=E-3Mdq5WnQI~Q>*Y*cqw9dTWIvt8>7zB zCP`PvODTTWRwzf-RQ7_s?4_cM-0Cm8zfq+U zD~&hX%w?Hz zbY49crZexnssl}x6zi0A)3&O?i*xr%#=bf@u3iwgV_X3rh@35Q40@I(uqvZC0Rm^b z$eAZRDuR{@EbhD0CsCA<$3Kv9cB}nTL1n5nm!Yeu>~pSowR-ghXSwO&t4vBkoO;`W zG@zpT*j5IlSH&sj&aNWH%KKv#9WeEYn>{b!(@|>b@6(!g%NV$_8jRMtWh*sk@=3_J z_@x{*dex|ErtZ?xbrskggE~r+DE9f8;S}m2@>Pv)%kz;(Zp&iuGj&{~Mzr4L*zWlj zy3sSUabpb%D>RJe5ioA0Z^H8|VK-SbHb#UN~QHS3d>s~g1qK!MPY@4Tl6b)7GNCmr}E zx%RmZuuwASH`37pFf!7}De|wp14RL07k{LFGS`BE!2g14x<7URz){M-%k|#mkRj{= zB}HSL(Eq`Y`MkhJ%}L=CPvhl$$@tc$xP*X`K+0qx|NjvVLjK6TmvJES z0Np=Ixv;t)UW36vSQ%qj@d-B^0r8_u)gt5jJ0=lsP6UKA7VCk)5ePn57X;1&;e^3> zdb>DzV6ceuIHDWkXLb?!^%iE|K#Oqo^zm>(I1v&5Bccr`e~}le1!w?j*(%w}QHsC2 zHZqWZacO_W^e@X1R?1OAe=DZ`+HZVgH}bS{^tSrpJ41-JfIrTwlGrExJ79g}kmTf!N9t79!R>+FGOW_x)i(M>Us*R`Cb?r$wQ^($5;4dc#D^4=Ytu!eqH2!p~)?0S) zRYy`~ZfclmLXext@I&cm4xTgpD}tyk`kX*Zk&_A)Mi4Eos(Wx|K-ZVESk>YFjNhx(ds3Zw#D%rz#r&EojVBwjDMpQLa9SJ(2PS;}b zeypB@(&?+-o=NnzDwEk3MTOeFd~r4k0wqt>h2}Ye>Ty&0QH@9F`WhmV z=+ec;a=WG6ZIEOWK#;0nX#HYVy>);*PI6J#gsX->A`Wa{s!w#|DslCW9H#hX8JXT$V%G=o68rj;{$h-iBbF7SIXU0FRG` z0FwXS3X0~3A|zC9WIg6*V85_#FsVoEYd7hQr%MW55si=B@2?h{D808>Avd&85}3*% zCNv5)WEfpehD*d;C}yB(F6#gph(Kcs)k;s~} zT2I$Rtsc?BRv;rg=1dXDui5mMoidV;DePr|18}{PvvGOqB{zllV-Nvmv*BXGUiKtD z?e*bj?rDWXRQj~Cy$|#e^y>S>TZ*q~-cne%soN4>cx|I>Nj+dM_%z!xMycrWzW|E4 BXl4Ka literal 0 HcmV?d00001 diff --git a/tests/data/timestamping/timestamp_response.tsr b/tests/data/timestamping/valid_timestamp.tsr similarity index 100% rename from tests/data/timestamping/timestamp_response.tsr rename to tests/data/timestamping/valid_timestamp.tsr diff --git a/tests/enrichers/test_timestamping_enricher.py b/tests/enrichers/test_timestamping_enricher.py index 1db503c..850096a 100644 --- a/tests/enrichers/test_timestamping_enricher.py +++ b/tests/enrichers/test_timestamping_enricher.py @@ -1,22 +1,103 @@ +from pathlib import Path import pytest -from auto_archiver.modules.timestamping_enricher.timestamping_enricher import TimestampingEnricher + from rfc3161_client import ( TimeStampResponse, decode_timestamp_response, ) +import requests -from cryptography import x509 +from auto_archiver.modules.timestamping_enricher.timestamping_enricher import TimestampingEnricher +from auto_archiver.core import Metadata, Media @pytest.fixture def timestamp_response() -> TimeStampResponse: - with open("tests/data/timestamping/timestamp_response.tsr", "rb") as f: + with open("tests/data/timestamping/valid_timestamp.tsr", "rb") as f: return decode_timestamp_response(f.read()) @pytest.fixture def wrong_order_timestamp_response() -> TimeStampResponse: with open("tests/data/timestamping/rfc3161-client-issue-104.tsr", "rb") as f: return decode_timestamp_response(f.read()) + +@pytest.fixture +def selfsigned_response() -> TimeStampResponse: + with open("tests/data/timestamping/self_signed.tsr", "rb") as f: + return decode_timestamp_response(f.read()) +@pytest.fixture +def filehash(): + return "4b7b4e39f12b8c725e6e603e6d4422500316df94211070682ef10260ff5759ef" + +def test_full_enriching(setup_module, sample_media, mocker, timestamp_response, filehash): + mock_post = mocker.patch("requests.sessions.Session.post") + mock_post.return_value.status_code = 200 + mock_decode_timestamp_response = mocker.patch("auto_archiver.modules.timestamping_enricher.timestamping_enricher.decode_timestamp_response") + mock_decode_timestamp_response.return_value = timestamp_response + + tsp: TimestampingEnricher = setup_module("timestamping_enricher", {"tsa_urls": ["http://timestamp.identrust.com"]}) + metadata = Metadata().set_url("https://example.com") + sample_media.set('hash', filehash) + metadata.add_media(sample_media) + tsp.enrich(metadata) + + assert metadata.get('timestamped') == True + assert len(metadata.media) == 2 # the original 'sample_media' and the new 'timestamp_media' + + timestamp_media = metadata.media[1] + assert timestamp_media.filename == f"{tsp.tmp_dir}/hashes.txt" + assert Path(timestamp_media.filename).read_text() == filehash + + # we only have one authority file because we only used one TSA + assert len(timestamp_media.get('timestamp_authority_files')) == 1 + timestamp_authority_file = timestamp_media.get('timestamp_authority_files')[0] + assert Path(timestamp_authority_file.filename).read_bytes() == timestamp_response.time_stamp_token() + + cert_chain = timestamp_authority_file.get('cert_chain') + assert len(cert_chain) == 3 + assert cert_chain[0].filename == f"{tsp.tmp_dir}/1 – 85078758028491331763.crt" + assert cert_chain[1].filename == f"{tsp.tmp_dir}/2 – 85078371663472981624.crt" + assert cert_chain[2].filename == f"{tsp.tmp_dir}/3 – 13298821034946342390.crt" + +def test_full_enriching_multiple_tsa(setup_module, sample_media, mocker, timestamp_response, filehash): + mock_post = mocker.patch("requests.sessions.Session.post") + mock_post.return_value.status_code = 200 + + mock_decode_timestamp_response = mocker.patch("auto_archiver.modules.timestamping_enricher.timestamping_enricher.decode_timestamp_response") + mock_decode_timestamp_response.return_value = timestamp_response + + tsp: TimestampingEnricher = setup_module("timestamping_enricher", {"tsa_urls": ["http://example.com/timestamp1", "http://example.com/timestamp2"]}) + metadata = Metadata().set_url("https://example.com") + sample_media.set('hash', filehash) + metadata.add_media(sample_media) + tsp.enrich(metadata) + + assert metadata.get('timestamped') == True + assert len(metadata.media) == 2 # the original 'sample_media' and the new 'timestamp_media' + + timestamp_media = metadata.media[1] + assert len(timestamp_media.get('timestamp_authority_files')) == 2 + for timestamp_token_media in timestamp_media.get('timestamp_authority_files'): + assert Path(timestamp_token_media.filename).read_bytes() == timestamp_response.time_stamp_token() + assert len(timestamp_token_media.get('cert_chain')) == 3 + + + + + + + + + + +def test_enriching(setup_module, sample_media): + tsp: TimestampingEnricher = setup_module("timestamping_enricher") + + # test the enrich method + metadata = Metadata().set_url("https://example.com") + sample_media.set('hash', "4b7b4e39f12b8c725e6e603e6d4422500316df94211070682ef10260ff5759ef") + metadata.add_media(sample_media) + tsp.enrich(metadata) @pytest.mark.download def test_fails_for_digicert(setup_module): @@ -57,8 +138,8 @@ def test_verify_save(setup_module, timestamp_response): cert_chain = tsp.save_certificate(timestamp_response, verified_root_cert) assert len(cert_chain) == 3 - assert cert_chain[0].filename == f"{tsp.tmp_dir}/1 – 85078371663472981624.crt" - assert cert_chain[1].filename == f"{tsp.tmp_dir}/2 – 85078758028491331763.crt" + assert cert_chain[0].filename == f"{tsp.tmp_dir}/1 – 85078758028491331763.crt" + assert cert_chain[1].filename == f"{tsp.tmp_dir}/2 – 85078371663472981624.crt" assert cert_chain[2].filename == f"{tsp.tmp_dir}/3 – 13298821034946342390.crt" @@ -70,7 +151,25 @@ def test_order_crt_correctly(setup_module, wrong_order_timestamp_response): ordered_certs = tsp.tst_certs(wrong_order_timestamp_response) assert len(ordered_certs) == 2 - assert ordered_certs[0].subject.rfc4514_string() == "CN=TrustID Timestamping CA 3,O=IdenTrust,C=US" - assert ordered_certs[1].subject.rfc4514_string() == "CN=TrustID Timestamp Authority,O=IdenTrust,C=US" + assert ordered_certs[0].subject.rfc4514_string() == "CN=TrustID Timestamp Authority,O=IdenTrust,C=US" + assert ordered_certs[1].subject.rfc4514_string() == "CN=TrustID Timestamping CA 3,O=IdenTrust,C=US" +def test_invalid_tsa_404(setup_module, mocker): + tsp = setup_module("timestamping_enricher") + post_mock = mocker.patch("requests.sessions.Session.post") + post_mock.side_effect = Exception("error") + with pytest.raises(Exception, match="error"): + tsp.sign_data("http://bellingcat.com/", b"my-message") + +@pytest.mark.download +def test_invalid_tsa_invalid_response(setup_module, mocker): + tsp = setup_module("timestamping_enricher") + + with pytest.raises(requests.exceptions.HTTPError, match="404 Client Error"): + tsp.sign_data("http://bellingcat.com/page-not-found/", b"my-message") + +def test_fail_on_selfsigned_cert(setup_module, selfsigned_response): + tsp = setup_module("timestamping_enricher") + root_cert = tsp.verify_signed(selfsigned_response, b"my-message") + assert root_cert is None \ No newline at end of file diff --git a/tests/storages/test_local_storage.py b/tests/storages/test_local_storage.py index 85f97c6..c2bf1f0 100644 --- a/tests/storages/test_local_storage.py +++ b/tests/storages/test_local_storage.py @@ -18,22 +18,11 @@ def local_storage(setup_module) -> LocalStorage: } return setup_module("local_storage", configs) - -@pytest.fixture -def sample_media(tmp_path) -> Media: - """Fixture creating a Media object with temporary source file""" - src_file = tmp_path / "source.txt" - src_file.write_text("test content") - return Media(key="subdir/test.txt", filename=str(src_file)) - - def test_get_cdn_url_relative(local_storage): media = Media(key="test.txt", filename="dummy.txt") expected = os.path.join(local_storage.save_to, media.key) assert local_storage.get_cdn_url(media) == expected - - def test_get_cdn_url_absolute(local_storage): media = Media(key="test.txt", filename="dummy.txt") local_storage.save_absolute = True