diff --git a/CHANGELOG.md b/CHANGELOG.md index 9d48c97..f784d3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ recorded here; their history is the git log. on 3.12, so 3.10 and 3.11 are no longer tested (NAPALM/driver-standard#6). - The floor on napalm is 5.1: older releases import `pkg_resources`, which a Python 3.12 venv no longer has. +- `requests` is a declared dependency (floor 2.25.0), for `tls_certificates`; it came + with napalm before. - The SNMP health profile shapes (`SnmpHealthProfile`, `SnmpValueSpec`, `SnmpMemorySpec`, `SnmpInterfacesSpec`) are defined in `napalm_device_types.snmp_health_profile`. The package and `models` still export @@ -33,6 +35,18 @@ recorded here; their history is the git log. and how many there are), and `ClusterNodeDict.votes`. All `NotRequired`, so drivers written before stay valid; None means could not be read, `[]` read and empty. +- `napalm_device_types.tls_certificates`: TLS certificates by trust on first use, the + counterpart of `ssh_host_keys` for a driver's HTTPS API. A driver takes the + fingerprint to expect from `optional_args["tls_fingerprints"]` (by port, the SHA-256 + of the DER certificate, `SHA256:AB:CD:…`) and refuses any other certificate with + `TlsCertificateMismatch` (a `ConnectionException` naming both fingerprints) before a + password is sent; no CA is consulted, so a self-signed certificate verifies. Without + a fingerprint the first certificate is accepted. Every certificate seen goes to + `optional_args["tls_certificate_observer"]`. `CertificateCheck.verify_server()` + checks the certificate in a handshake of its own, `pin(session)` makes every later + connection of a `requests` session present the same one, `refusing()` turns a refusal + there into the mismatch, `observe()` reports it once the login stands + (NAPALM/napalm-hpe-aruba-procurve#17). - `LldpAgentMixin.configure_lldp_agent(interfaces)` (`napalm_device_types.fixes`): run an LLDP agent on the interfaces the caller names. The template installs the agent (`ensure_package(LLDP_AGENT_PACKAGE)`), has the driver write the interface list diff --git a/README.md b/README.md index 9c59ae8..ea8ba97 100644 --- a/README.md +++ b/README.md @@ -332,6 +332,26 @@ caller's; `ssh_host_keys.HostKeyCheck` is how every SSH driver verifies against A caller that passes neither key gets first use on every connection, as before; a driver that ignores both still connects. Old and new versions on either side keep working. +### TLS certificates: trust on first use + +The same for a driver's HTTPS API. Devices ship self-signed certificates, so a CA check +fails on all of them and turning verification off trusts whatever answers; +`tls_certificates.CertificateCheck` pins the certificate by its fingerprint instead +(NAPALM/napalm-hpe-aruba-procurve#17). No CA is consulted. + +- **In:** `optional_args["tls_fingerprints"]` = `{"443": "SHA256:AB:CD:…"}`, by port: the + SHA-256 of the DER certificate as `openssl x509 -fingerprint -sha256` prints it (prefix, + colons and case optional). With one, any other certificate is refused *before* a request + carries credentials and the connection raises `TlsCertificateMismatch` (a + `ConnectionException` naming both fingerprints). Without one, the first certificate is + accepted. A driver does not fall back to another transport on a mismatch. +- **Out:** `optional_args["tls_certificate_observer"]`, a callable, gets one dict per + connection: `{"host", "port", "seen": {"fingerprint"}, "expected", "accepted"}`. +- **requests:** `check = CertificateCheck(optional_args, host, port)`; + `check.verify_server(timeout)` before the login, `check.pin(session)` so every later + connection of the session must present that certificate, the login inside + `with check.refusing():`, then `check.observe()`. + ## Design principle: generic vs. device-specific logic When adding behavior to a device-type base class, split it along one line: **would diff --git a/napalm_device_types/tls_certificates.py b/napalm_device_types/tls_certificates.py new file mode 100644 index 0000000..6169383 --- /dev/null +++ b/napalm_device_types/tls_certificates.py @@ -0,0 +1,285 @@ +"""TLS certificates: trust on first use by fingerprint, a changed one refused before any login. + +Network devices ship self-signed certificates no CA vouches for. A driver that +then turns verification off logs in with the device's credentials at whatever +answers on its address (NetOrk/netork#1167, NAPALM/napalm-hpe-aruba-procurve#17). +The same answer as for SSH host keys (:mod:`napalm_device_types.ssh_host_keys`): +the layer above keeps the store of the certificates it trusts, a driver verifies. + +* It receives the fingerprints to expect in ``optional_args["tls_fingerprints"]`` + (:data:`KNOWN_FINGERPRINTS_ARG`), by the port they were seen on: + ``{"443": "SHA256:AB:CD:…"}``, the SHA-256 of the DER certificate as + ``openssl x509 -fingerprint -sha256`` prints it (the ``SHA256:`` prefix, + colons and case are optional, :func:`normalise_fingerprint`). With a + fingerprint, any other certificate is refused before a password leaves the + caller; no CA is consulted. Without one, the first certificate is accepted: + trust on first use. +* It reports every certificate it sees through + ``optional_args["tls_certificate_observer"]`` (:data:`OBSERVER_ARG`), a + callable that receives one dict per connection:: + + {"host": str, "port": int, + "seen": {"fingerprint"}, + "expected": {"fingerprint"} | None, + "accepted": bool} + + ``accepted`` is False only for a refused certificate; the connection then + raises :class:`TlsCertificateMismatch`. A failing observer is logged and + ignored. + +A caller that passes neither keeps working as before (first use, every time), +and a driver that ignores both still connects for a caller that passes them. + +:class:`CertificateCheck` is the one implementation. :meth:`~CertificateCheck.verify_server` +takes the certificate from a handshake of its own, before any request; +:meth:`~CertificateCheck.pin` makes every later connection of a ``requests`` +session present that same certificate (urllib3's ``assert_fingerprint``), so +another host cannot step in between the check and the login; +:meth:`~CertificateCheck.refusing` turns such a refusal into +:class:`TlsCertificateMismatch`; :meth:`~CertificateCheck.observe` reports the +certificate once the login stands. +""" + +from __future__ import annotations + +import hashlib +import logging +import re +import socket +import ssl +from collections.abc import Callable, Iterator, Mapping +from contextlib import contextmanager +from dataclasses import dataclass +from typing import Any + +from napalm.base.exceptions import ConnectionException + +logger = logging.getLogger(__name__) + +#: The ``optional_args`` key that carries the fingerprints to expect, by port. +KNOWN_FINGERPRINTS_ARG = "tls_fingerprints" +#: The ``optional_args`` key that carries the callable every seen certificate goes to. +OBSERVER_ARG = "tls_certificate_observer" + +_PREFIX = "SHA256:" +_SHA256_HEX = re.compile(r"^[0-9A-F]{64}$") +#: What urllib3 says when a connection presents another certificate than the pinned one +#: (``got "…"``; 1.26 writes the digest as ``b'…'``, escaped once requests wraps it). +_URLLIB3_MISMATCH = re.compile(r"Fingerprints did not match.*?got \"(?:b\\?')?([0-9a-fA-F]{64})") + + +def fingerprint(der: bytes) -> str: + """``SHA256:AB:CD:…``: the SHA-256 of a DER certificate, as openssl prints it.""" + return _format(hashlib.sha256(der).hexdigest()) + + +def _format(hex_digest: str) -> str: + upper = hex_digest.upper() + return _PREFIX + ":".join(upper[i : i + 2] for i in range(0, len(upper), 2)) + + +def normalise_fingerprint(value: Any) -> str: + """*value* as :func:`fingerprint` writes it; ``SHA256:``, colons, spaces and case optional.""" + if not isinstance(value, str): + raise ValueError(f"not a SHA-256 certificate fingerprint: {value!r}") + text = value.strip() + if text[: len(_PREFIX)].upper() == _PREFIX: + text = text[len(_PREFIX) :] + digits = re.sub(r"[\s:]", "", text).upper() + if not _SHA256_HEX.match(digits): + raise ValueError(f"not a SHA-256 certificate fingerprint: {value!r}") + return _format(digits) + + +def _hex(fingerprint_: str) -> str: + """The bare lower-case hex urllib3's ``assert_fingerprint`` takes.""" + return fingerprint_.removeprefix(_PREFIX).replace(":", "").lower() + + +def fetch_certificate( + host: str, + port: int, + timeout: float, + *, + connect: Callable[..., socket.socket] = socket.create_connection, +) -> bytes: + """The DER certificate *host* presents on *port*, from a handshake that sends nothing else. + + No CA and no host name are checked: the certificate is judged by its fingerprint. + """ + context = ssl.create_default_context() + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + with connect((host, port), timeout=timeout) as sock: + with context.wrap_socket(sock, server_hostname=host) as tls: + der = tls.getpeercert(binary_form=True) + if not der: # pragma: no cover - a TLS server always presents one + raise ConnectionException(f"{host}:{port} presented no TLS certificate") + return der + + +@dataclass(frozen=True) +class CertificateObservation: + """The certificate one connection saw, against the one it was to expect.""" + + host: str + port: int + #: The fingerprint of the certificate presented. + seen: str + #: The fingerprint it was to present; None on first use. + expected: str | None + + @property + def accepted(self) -> bool: + """True on first use and for the expected certificate; False for a refused one.""" + return self.expected is None or self.expected == self.seen + + def as_dict(self) -> dict[str, Any]: + return { + "host": self.host, + "port": self.port, + "seen": {"fingerprint": self.seen}, + "expected": {"fingerprint": self.expected} if self.expected else None, + "accepted": self.accepted, + } + + +class TlsCertificateMismatch(ConnectionException): # type: ignore[misc] + """The device presented another TLS certificate than the one it is known by. + + Raised before any request that carries credentials: nothing was sent to it. + """ + + def __init__(self, observation: CertificateObservation) -> None: + self.observation = observation + super().__init__( + f"TLS certificate mismatch at {observation.host}:{observation.port}: it presented " + f"{observation.seen}, the trusted certificate is {observation.expected or 'none'}. " + "Nothing was sent to it." + ) + + +def _expected(known: Any, port: int) -> str | None: + """The fingerprint in *known* (``{port: fingerprint}``, port as text or number) for *port*.""" + if not known: + return None + if not isinstance(known, Mapping): + raise ValueError(f"TLS fingerprints must be given by port, not as {type(known).__name__}") + value = known.get(str(port)) or known.get(port) + return normalise_fingerprint(value) if value else None + + +def _refused_fingerprint(exc: BaseException) -> str | None: + """The fingerprint urllib3 saw when it refused a pinned connection, in *exc*'s chain.""" + seen: set[int] = set() + stack: list[BaseException | None] = [exc] + while stack: + current = stack.pop() + if current is None or id(current) in seen: + continue + seen.add(id(current)) + match = _URLLIB3_MISMATCH.search(str(current)) + if match: + return _format(match.group(1)) + stack.extend([current.__cause__, current.__context__]) + return None + + +class CertificateCheck: + """How one driver connection verifies its host's TLS certificate. + + *optional_args* are the driver's (both keys optional), *host* and *port* the + address the connection goes to. + """ + + def __init__(self, optional_args: Mapping[str, Any] | None, host: str, port: int) -> None: + args = optional_args or {} + self.host = host + self.port = int(port) + self.expected = _expected(args.get(KNOWN_FINGERPRINTS_ARG), self.port) + self._observer = args.get(OBSERVER_ARG) + self._verified: CertificateObservation | None = None + + def verify_server( + self, + timeout: float, + *, + connect: Callable[..., socket.socket] = socket.create_connection, + ) -> CertificateObservation: + """Take the certificate from a handshake of its own and :meth:`verify` it.""" + return self.verify(fetch_certificate(self.host, self.port, timeout, connect=connect)) + + def verify(self, der: bytes) -> CertificateObservation: + """Judge the DER certificate *der*: a refused one is reported and raises. + + An accepted one is reported by :meth:`observe`, once the login stands. + """ + observation = CertificateObservation(self.host, self.port, fingerprint(der), self.expected) + if not observation.accepted: + self._report(observation) + raise TlsCertificateMismatch(observation) + self._verified = observation + return observation + + def pin(self, session: Any) -> None: + """Every HTTPS connection of the ``requests`` *session* must present the verified one.""" + verified = self._require_verified() + session.mount("https://", _pinned_adapter(_hex(verified.seen))) + + @contextmanager + def refusing(self) -> Iterator[None]: + """Around requests on a pinned session: a refused certificate becomes a mismatch.""" + try: + yield + except Exception as exc: + offered = _refused_fingerprint(exc) + if offered is None: + raise + trusted = self.expected or self._require_verified().seen + observation = CertificateObservation(self.host, self.port, offered, trusted) + self._report(observation) + raise TlsCertificateMismatch(observation) from exc + + def observe(self) -> CertificateObservation: + """After the login: the verified certificate, reported.""" + verified = self._require_verified() + self._report(verified) + return verified + + def _require_verified(self) -> CertificateObservation: + if self._verified is None: + raise RuntimeError("verify the certificate first (verify_server or verify)") + return self._verified + + def _report(self, observation: CertificateObservation) -> None: + if not callable(self._observer): + return + try: + self._observer(observation.as_dict()) + except Exception: # the store is the caller's; a connection does not hang on it + logger.warning( + "Could not report the TLS certificate of %s:%s", + self.host, + self.port, + exc_info=True, + ) + + +def _pinned_adapter(hex_digest: str) -> Any: + """A ``requests`` adapter whose connections, direct or through a proxy, must match *hex_digest*. + + ``requests`` is imported here: it comes with napalm, and a module that only + verifies a fingerprint does not need it loaded. + """ + from requests.adapters import HTTPAdapter + + class _PinnedAdapter(HTTPAdapter): # type: ignore[misc,unused-ignore] + def init_poolmanager(self, *args: Any, **kwargs: Any) -> None: + kwargs["assert_fingerprint"] = hex_digest + super().init_poolmanager(*args, **kwargs) + + def proxy_manager_for(self, proxy: str, **kwargs: Any) -> Any: + kwargs["assert_fingerprint"] = hex_digest + return super().proxy_manager_for(proxy, **kwargs) + + return _PinnedAdapter() diff --git a/pyproject.toml b/pyproject.toml index 06800cb..ee6a8e7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,7 @@ classifiers = [ ] dependencies = [ "napalm>=5.1", # older releases import pkg_resources, which a 3.12 venv lacks + "requests>=2.25.0", "typing-extensions>=4.0", ] diff --git a/tests/test_tls_certificates.py b/tests/test_tls_certificates.py new file mode 100644 index 0000000..ab0f817 --- /dev/null +++ b/tests/test_tls_certificates.py @@ -0,0 +1,285 @@ +"""TLS certificates: trust on first use by fingerprint, a changed one refused before any login. + +The layer above keeps the store (NetOrk/netork#1167, NAPALM/napalm-hpe-aruba-procurve#17). +A driver gets the fingerprint it should expect through ``optional_args["tls_fingerprints"]`` +(by port) and reports every certificate it sees through +``optional_args["tls_certificate_observer"]``. The handshake tests run real TLS against an +in-process server on a socket pair: what matters is the certificate the server presents in +a real handshake, which a fake could not show. +""" + +from __future__ import annotations + +import datetime +import hashlib +import socket +import ssl +import threading + +import pytest +import requests +from cryptography import x509 +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import ec +from cryptography.x509.oid import NameOID +from napalm.base.exceptions import ConnectionException + +from napalm_device_types.tls_certificates import ( + KNOWN_FINGERPRINTS_ARG, + OBSERVER_ARG, + CertificateCheck, + TlsCertificateMismatch, + fetch_certificate, + fingerprint, + normalise_fingerprint, +) + +DER = b"not really a certificate, but bytes are bytes" +DER_HEX = hashlib.sha256(DER).hexdigest().upper() +DER_FP = "SHA256:" + ":".join(DER_HEX[i : i + 2] for i in range(0, 64, 2)) +OTHER_FP = "SHA256:" + ":".join(["AB"] * 32) + + +def _self_signed(tmp_path, name: str = "switch") -> tuple[bytes, str, str]: + """A self-signed certificate, as a ProCurve ships one: (DER, cert file, key file).""" + key = ec.generate_private_key(ec.SECP256R1()) + subject = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, name)]) + now = datetime.datetime.now(datetime.UTC) + cert = ( + x509.CertificateBuilder() + .subject_name(subject) + .issuer_name(subject) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(now - datetime.timedelta(days=1)) + .not_valid_after(now + datetime.timedelta(days=1)) + .sign(key, hashes.SHA256()) + ) + cert_file, key_file = tmp_path / f"{name}.crt", tmp_path / f"{name}.key" + cert_file.write_bytes(cert.public_bytes(serialization.Encoding.PEM)) + key_file.write_bytes( + key.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + serialization.NoEncryption(), + ) + ) + return cert.public_bytes(serialization.Encoding.DER), str(cert_file), str(key_file) + + +class _TlsServer: + """A TLS server on one end of a socket pair; it handshakes and then waits.""" + + def __init__(self, cert_file: str, key_file: str) -> None: + self.client_sock, server_sock = socket.socketpair() + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain(cert_file, key_file) + self._server_sock = server_sock + self._context = context + self.received = b"" + self._thread = threading.Thread(target=self._serve, daemon=True) + self._thread.start() + + def _serve(self) -> None: + try: + tls = self._context.wrap_socket(self._server_sock, server_side=True) + self.received = tls.recv(1024) + except (OSError, ssl.SSLError): # the client hung up after the handshake + pass + + def connect(self, address, timeout=None): + return self.client_sock + + def close(self) -> None: + self.client_sock.close() + self._server_sock.close() + self._thread.join(timeout=5) + + +@pytest.fixture +def certificate(tmp_path): + return _self_signed(tmp_path) + + +@pytest.fixture +def server(certificate): + _der, cert_file, key_file = certificate + s = _TlsServer(cert_file, key_file) + yield s + s.close() + + +def _check(expected: str | None, seen: list, port: int = 443) -> CertificateCheck: + args: dict = {OBSERVER_ARG: seen.append} + if expected is not None: + args[KNOWN_FINGERPRINTS_ARG] = {str(port): expected} + return CertificateCheck(args, "10.0.0.5", port) + + +class TestFingerprint: + def test_is_sha256_of_the_der_certificate_in_colon_hex(self): + assert fingerprint(DER) == DER_FP + + @pytest.mark.parametrize( + "value", + [ + DER_FP, + DER_FP.lower(), + DER_HEX, + DER_HEX.lower(), + DER_FP.removeprefix("SHA256:"), + " ".join(DER_HEX[i : i + 2] for i in range(0, 64, 2)), + "sha256:" + DER_HEX, + ], + ) + def test_a_pin_may_be_written_as_openssl_or_a_browser_shows_it(self, value): + assert normalise_fingerprint(value) == DER_FP + + @pytest.mark.parametrize( + "value", ["", "AB:CD", "SHA1:" + DER_HEX, "ZZ" * 32, DER_HEX + "AB", 42] + ) + def test_anything_else_is_refused(self, value): + with pytest.raises(ValueError): + normalise_fingerprint(value) + + +class TestKnownFingerprints: + def test_the_pin_is_the_one_for_the_port_connected_to(self): + args = {KNOWN_FINGERPRINTS_ARG: {"443": DER_HEX}} + assert CertificateCheck(args, "10.0.0.5", 443).expected == DER_FP + assert CertificateCheck(args, "10.0.0.5", 8443).expected is None + + def test_a_port_may_be_given_as_a_number(self): + args = {KNOWN_FINGERPRINTS_ARG: {8443: DER_FP}} + assert CertificateCheck(args, "10.0.0.5", 8443).expected == DER_FP + + def test_a_malformed_mapping_is_refused(self): + with pytest.raises(ValueError): + CertificateCheck({KNOWN_FINGERPRINTS_ARG: [DER_FP]}, "10.0.0.5", 443) + + def test_without_optional_args_it_is_first_use(self): + assert CertificateCheck(None, "10.0.0.5", 443).expected is None + + +class TestVerify: + def test_first_use_accepts_and_reports_only_once_observed(self): + seen: list = [] + check = _check(None, seen) + observation = check.verify(DER) + assert observation.accepted is True + assert seen == [] + check.observe() + assert seen == [ + { + "host": "10.0.0.5", + "port": 443, + "seen": {"fingerprint": DER_FP}, + "expected": None, + "accepted": True, + } + ] + + def test_the_pinned_certificate_is_accepted(self): + seen: list = [] + check = _check(DER_HEX.lower(), seen) + assert check.verify(DER).accepted is True + check.observe() + assert seen[0]["expected"] == {"fingerprint": DER_FP} + assert seen[0]["accepted"] is True + + def test_another_certificate_is_refused_reported_and_named(self): + seen: list = [] + with pytest.raises(TlsCertificateMismatch) as caught: + _check(OTHER_FP, seen).verify(DER) + assert caught.value.observation.seen == DER_FP + assert caught.value.observation.expected == OTHER_FP + assert seen == [ + { + "host": "10.0.0.5", + "port": 443, + "seen": {"fingerprint": DER_FP}, + "expected": {"fingerprint": OTHER_FP}, + "accepted": False, + } + ] + + def test_the_mismatch_is_a_connection_failure_that_names_both(self): + with pytest.raises(ConnectionException) as caught: + _check(OTHER_FP, []).verify(DER) + message = str(caught.value) + assert "10.0.0.5:443" in message + assert DER_FP in message + assert OTHER_FP in message + + def test_a_failing_observer_does_not_break_the_connection(self): + def boom(_observation): + raise RuntimeError("store down") + + check = CertificateCheck({OBSERVER_ARG: boom}, "10.0.0.5", 443) + check.verify(DER) + check.observe() + + def test_observe_before_verify_is_a_mistake(self): + with pytest.raises(RuntimeError): + _check(None, []).observe() + + +class TestHandshake: + def test_fetch_returns_the_certificate_the_server_presents(self, server, certificate): + der, _cert, _key = certificate + assert fetch_certificate("10.0.0.5", 443, 5, connect=server.connect) == der + + def test_verify_server_refuses_another_certificate_before_anything_is_sent( + self, server, tmp_path + ): + other_der, _c, _k = _self_signed(tmp_path, "impostor") + seen: list = [] + check = _check(fingerprint(other_der), seen) + with pytest.raises(TlsCertificateMismatch): + check.verify_server(5, connect=server.connect) + server.close() + assert server.received == b"" + assert seen[0]["accepted"] is False + + def test_verify_server_accepts_on_first_use(self, server, certificate): + der, _cert, _key = certificate + observation = _check(None, []).verify_server(5, connect=server.connect) + assert observation.seen == fingerprint(der) + + +class TestRequests: + def test_pin_makes_every_connection_present_the_verified_certificate(self): + check = _check(None, []) + check.verify(DER) + session = requests.Session() + check.pin(session) + adapter = session.get_adapter("https://10.0.0.5/rest/v7/") + assert adapter.poolmanager.connection_pool_kw["assert_fingerprint"] == DER_HEX.lower() + proxy = adapter.proxy_manager_for("http://proxy.example:3128") + assert proxy.connection_pool_kw["assert_fingerprint"] == DER_HEX.lower() + + def test_pin_before_verify_is_a_mistake(self): + with pytest.raises(RuntimeError): + _check(None, []).pin(requests.Session()) + + def test_a_changed_certificate_on_a_pinned_session_is_a_mismatch(self): + from urllib3.util.ssl_ import assert_fingerprint + + seen: list = [] + check = _check(DER_FP, seen) + check.verify(DER) + impostor = b"another certificate" + with pytest.raises(TlsCertificateMismatch) as caught, check.refusing(): + try: + assert_fingerprint(impostor, DER_HEX) + except Exception as exc: + raise requests.exceptions.SSLError(exc) from exc + assert caught.value.observation.seen == fingerprint(impostor) + assert caught.value.observation.expected == DER_FP + assert seen[-1]["accepted"] is False + + def test_refusing_lets_other_errors_through(self): + check = _check(None, []) + check.verify(DER) + with pytest.raises(requests.exceptions.ConnectionError), check.refusing(): + raise requests.exceptions.ConnectionError("no route to host")